tpt_serverless 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf37a0d47bf872ffbb9633925e4ddb37d6bfc38a570f84f75f3f61f492b8f100
-  data.tar.gz: 1038ef65a0ed86399a412fac44fcdb4ef037abfc491307dc3d92554f3a474424
+  metadata.gz: c1f06b4fad5f9e34e05e818977304236568122d0a604ecd88e8cf6da4b331d5a
+  data.tar.gz: 2b54c154a4457028ed90778f699aa159acef24ba4abe6d40f01f80deb0926393
 SHA512:
-  metadata.gz: 89ff6fca675780c16c77fbcbe976fcbbc67daeeaf01513a15f674ec97344047f8084fc7ae61ac62ec5c19f5c69f47830f319a00ffd578e433a9ff8950ef3ca20
-  data.tar.gz: d4c7ec90b2c42414f43391f1f7af4bab4f76f55521f3c42fd9810247bd1cdc4e04a06bf61861de57cf790bf4c76fb7dc1a20fcc0819c3fd5e19bb534cc99303a
+  metadata.gz: bec09d6328b91ac709832540773484e01773d2da113166b592aea0cf2070823d69376bf32b5bf442cf5cbb9e367fda385bf7fdb62774d8c716b6de28faf97507
+  data.tar.gz: 91f0d1ce2681a15b8a5602cfa3736a700e79ab5016e4c9c542fb81dd250da2e98a0c351cf295515f61549bf4746471a84c4ef107ca6a4a00ce361536cdc03e23

data/lib/tpt_serverless/datadog_log_forwarder.rb

@@ -15,6 +15,11 @@ Example of how to add this to your `serverless.yml`:
       DATADOG_SOURCE: cwl-aws-lambda         # default
       DATADOG_SERVICE: ${self:service.name}  # optional
       DATADOG_TAGS: stage:${self:provider.stage},team:platform  # optional
+      # If set to a truthy value (1, true, yes, or on; case/whitespace-insensitive),
+      # filters AWS Lambda platform lifecycle boilerplate logs like
+      # INIT_START / START / END / REPORT before forwarding to Datadog.
+      # Defaults to false.
+      DATADOG_FILTER_LIFECYCLE_LOGS: "true"  # optional
     events:
       - cloudwatchLog: /aws/lambda/${self:service.name}-functionOne-${self:provider.stage}
       - cloudwatchLog: /aws/lambda/${self:service.name}-functionTwo-${self:provider.stage}
@@ -28,9 +33,12 @@ require 'json'
 require 'net/http'
 require 'uri'
 
-TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
-
 class DatadogLogForwarder
+  TIME_REGEX ||= /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
+
+  # Matches AWS Lambda platform lifecycle lines that are usually noisy in Datadog
+  LAMBDA_LIFECYCLE_AFTER_TIMESTAMP_REGEX ||= /\A(?:INIT_START\b|START RequestId:|END RequestId:|REPORT RequestId:)/
+
   class << self
     # This handler receives CloudWatch log events, parses the events and forwards the extracted logs to
     # Datadog.
@@ -58,7 +66,16 @@ class DatadogLogForwarder
         return
       end
 
-      puts "message_count=#{log_events.length}"
+      normalized_log_events = normalize_messages(log_events)
+
+      filtered_log_events =
+        if should_filter_lifecycle_logs?
+          normalized_log_events.reject { |e| lambda_lifecycle_line?(e[:message]) }
+        else
+          normalized_log_events
+        end
+
+      puts "message_count=#{log_events.length} forwarded_count=#{filtered_log_events.length}"
 
       region = 'unknown'
       account_id = 'unknown'
@@ -72,11 +89,9 @@ class DatadogLogForwarder
       function_name = log_group&.split('/')&.last
       function_arn = "arn:aws:lambda:#{region}:#{account_id}:function:#{function_name}"
 
-      normalized_data = log_events.map do |log_event|
-        normalized_msg = normalize_messages([log_event]).first
-
+      normalized_data = filtered_log_events.map do |log_event|
         {
-          message: normalized_msg,
+          message: log_event[:message],
           ddsource: ENV['DATADOG_SOURCE'] || 'cwl-aws-lambda',
           service: ENV['DATADOG_SERVICE'] || function_name,
           hostname: function_arn,
@@ -87,10 +102,13 @@ class DatadogLogForwarder
               logStream: log_stream
             }
           },
-          id: log_event['id']
+          id: log_event[:id]
         }
       end
 
+      # If we filtered everything out, do nothing.
+      return if normalized_data.empty?
+
       begin
         send_json_to_datadog(normalized_data.to_json)
         puts "Datadog upload successful: sent #{normalized_data.length} log(s) for #{function_name}"
@@ -106,21 +124,38 @@ class DatadogLogForwarder
 
     def normalize_messages(log_events)
       log_events.map do |log_event|
+        id = log_event.fetch('id')
         message = log_event.fetch('message')
-        timestamp = Integer(log_event.fetch('timestamp'))
+        timestamp_ms = Integer(log_event.fetch('timestamp'))
 
-        # Ensure the message starts with a time.
-        # E.g. START/END/REPORT log events don't.
+        # Ensure the message starts with an ISO timestamp (same intent as Node)
         if TIME_REGEX !~ message
-          time_string = Time.at(timestamp/1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          time_string = Time.at(timestamp_ms / 1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
           message = "#{time_string} #{message}"
         end
 
         # AWS replaces newlines with carriage returns in Lambda logs
-        message.gsub(/\r(?!\n)/, "\n")
+        message = message.gsub(/\r(?!\n)/, "\n")
+
+        { id: id, message: message }
       end
     end
 
+    def should_filter_lifecycle_logs?
+      truthy_env?(ENV['DATADOG_FILTER_LIFECYCLE_LOGS'])
+    end
+
+    def truthy_env?(value)
+      return false if value.nil?
+      %w[1 true yes on].include?(value.strip.downcase)
+    end
+
+    def lambda_lifecycle_line?(message_with_timestamp)
+      # Strip the leading ISO timestamp and match on the remainder.
+      after_timestamp = message_with_timestamp.sub(TIME_REGEX, '').lstrip
+      LAMBDA_LIFECYCLE_AFTER_TIMESTAMP_REGEX.match?(after_timestamp)
+    end
+
     def send_json_to_datadog(data)
       datadog_site = ENV['DATADOG_SITE'] || 'datadoghq.com'
       datadog_url = URI("https://http-intake.logs.#{datadog_site}/api/v2/logs")
@@ -138,7 +173,6 @@ class DatadogLogForwarder
         )
 
         req.body = data
-
         http.request(req)
       end
 

data/lib/tpt_serverless/sumo_log_forwarder.rb

@@ -23,9 +23,9 @@ require 'json'
 require 'net/http'
 require 'uri'
 
-TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
-
 class SumoLogForwarder
+  TIME_REGEX ||= /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
+
   class << self
     # This handler receives CloudWatch log events, parses the events and forwards the extracted logs to
     # Sumo Logic.

data/lib/tpt_serverless/version.rb

@@ -1,3 +1,3 @@
 module TptServerless
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tpt_serverless
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - TpT