tpt_serverless 0.4.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f873d98101203807387f9ad586751351d2a3144b62069ddb2543a171f25c31b3
-  data.tar.gz: 817156c5725a80e393e778acb2cf5a1e73bc28718c1a39d07282e0a7776120f3
+  metadata.gz: eaf679b74f62f413493556247cd9111725a55dfc8346363b30700dd4f44e32a1
+  data.tar.gz: 7d0d3e2f07fc6c8d10cc0e7eb801506fe2c7e0f3066361679058643d07221d0a
 SHA512:
-  metadata.gz: 543cf9dad7a1c194148c1a46a9daa50af26b67fe78ae302c3c7fb412a4600dd1564b188ecbfb0ddab173dcda6f63bb984be28ad756886987c76e1ca1df4446b5
-  data.tar.gz: 8d9208dde57f413777eb9ed073feca37ec0b8e18578d1f90ade3c1c2cb1230018b724be92b23fd9567972ad4c9b03f14152f702beb6b6d198ac83be67732bc60
+  metadata.gz: 9d52cc7aab4a205c3041575bb0f2ada2815ecfee469c3b19e4f627716f315582cf7ebccd6183ff322f5bc5a85d12fd926c7f3d85ae66fcee447ff505648344a1
+  data.tar.gz: 533fcc9a0611b56650c5ac97624fca8620eb3c6013b88196b18e7b4c64381daa7375ae83162c39c95dab0ced51b380c8414062c7568df00013e87497d6b0dc08

data/lib/tpt_serverless/datadog_log_forwarder.rb

@@ -1,5 +1,5 @@
 =begin
-Receives logs from CloudWatch and forwards them to Datadog.
+Receives logs from CloudWatch and forwards them to Datadog Logs intake.
 
 Example of how to add this to your `serverless.yml`:
 
@@ -9,12 +9,18 @@ Example of how to add this to your `serverless.yml`:
     handler: /opt/ruby/3.4.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/datadog_log_forwarder.DatadogLogForwarder.handler
     timeout: 10 # seconds
     environment:
-      DD_API_KEY: ${self:custom.datadogApiKey}
+      DD_API_KEY: ${ssm:/path/to/DD_API_KEY}
       # Optional:
-      DATADOG_SITE: datadoghq.com            # default
-      DATADOG_SOURCE: cwl-aws-lambda         # default
-      DATADOG_SERVICE: ${self:service.name}  # optional
-      DATADOG_TAGS: stage:${self:provider.stage},team:platform  # optional
+      DD_SITE: datadoghq.com            # default
+      DD_SOURCE: cwl-aws-lambda         # default
+      DD_ENV: ${self:provider.stage}    # default
+      DD_SERVICE: ${self:service.name}  # optional
+      DD_TAGS: stage:${self:provider.stage},team:platform  # optional
+
+      # If true, filters AWS Lambda platform lifecycle boilerplate logs like
+      # INIT_START / START / END / REPORT before forwarding to Datadog.
+      # Defaults to false.
+      DD_FILTER_LIFECYCLE_LOGS: "true"  # optional
     events:
       - cloudwatchLog: /aws/lambda/${self:service.name}-functionOne-${self:provider.stage}
       - cloudwatchLog: /aws/lambda/${self:service.name}-functionTwo-${self:provider.stage}
@@ -31,17 +37,16 @@ require 'uri'
 class DatadogLogForwarder
   TIME_REGEX ||= /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
 
+  # Matches AWS Lambda platform lifecycle lines that are usually noisy in Datadog
+  LAMBDA_LIFECYCLE_AFTER_TIMESTAMP_REGEX ||= /\A(?:INIT_START\b|START RequestId:|END RequestId:|REPORT RequestId:)/
+
   class << self
     # This handler receives CloudWatch log events, parses the events and forwards the extracted logs to
     # Datadog.
-    #
-    # It performs a few helpful cleanup/prep functions as well.
     def handler(event:, context:)
-      dd_api_key = ENV['DD_API_KEY']
-
-      if dd_api_key.nil? || dd_api_key.strip.empty?
+      unless ENV['DD_API_KEY']
         puts 'ERROR: DD_API_KEY is not set. Skipping log forwarding.'
-        return
+        return nil
       end
 
       raw_data = event.fetch('awslogs').fetch('data')
@@ -55,15 +60,26 @@ class DatadogLogForwarder
 
       if message_type == 'CONTROL_MESSAGE'
         puts 'skipping control message'
-        return
+        return nil
       end
 
-      puts "message_count=#{log_events.length}"
+      # Normalize first so lifecycle filtering works even if the raw message didn't have a timestamp
+      normalized_log_events = normalize_messages(log_events)
+
+      # Drop AWS Lambda platform lifecycle boilerplate
+      filtered_log_events =
+        if should_filter_lifecycle_logs?
+          normalized_log_events.reject { |e| lambda_lifecycle_line?(e[:message]) }
+        else
+          normalized_log_events
+        end
+
+      puts "message_count=#{log_events.length} forwarded_count=#{filtered_log_events.length}"
 
       region = 'unknown'
       account_id = 'unknown'
 
-      if context&.respond_to?(:invoked_function_arn) && !context.invoked_function_arn.empty?
+      if context&.respond_to?(:invoked_function_arn) && !context.invoked_function_arn.to_s.empty?
         arn_parts = context.invoked_function_arn.split(':')
         region = arn_parts[3] if arn_parts[3]
         account_id = arn_parts[4] if arn_parts[4]
@@ -72,25 +88,35 @@ class DatadogLogForwarder
       function_name = log_group&.split('/')&.last
       function_arn = "arn:aws:lambda:#{region}:#{account_id}:function:#{function_name}"
 
-      normalized_data = log_events.map do |log_event|
-        normalized_msg = normalize_messages([log_event]).first
+      name_parts = (function_name || '').split('-')
+      # Extract the stage (everything from index 2 onwards e.g., "dev" or "dev-ivan")
+      extracted_env = name_parts.length > 2 ? name_parts[2..-1].join('-') : 'unknown'
+
+      normalized_data = filtered_log_events.map do |log_event|
+        ddtags_array = get_datadog_reserved_tags(
+          fallback_env: extracted_env,
+          fallback_service: function_name
+        )
 
         {
-          message: normalized_msg,
-          ddsource: ENV['DATADOG_SOURCE'] || 'cwl-aws-lambda',
-          service: ENV['DATADOG_SERVICE'] || function_name,
+          message: log_event[:message],
+          ddsource: ENV['DD_SOURCE'] || 'cwl-aws-lambda',
+          service: ENV['DD_SERVICE'] || ENV['APP_SERVICE_NAME'] || function_name,
           hostname: function_arn,
-          ddtags: [ENV['DATADOG_TAGS'], "region:#{region}"].compact.reject(&:empty?).join(','),
+          ddtags: (ddtags_array + ["region:#{region}"]).reject { |t| t.nil? || t.empty? }.join(','),
           aws: {
             awslogs: {
               logGroup: log_group,
               logStream: log_stream
             }
           },
-          id: log_event['id']
+          id: log_event[:id]
         }
       end
 
+      # If we filtered everything out, do nothing.
+      return nil if normalized_data.empty?
+
       begin
         send_json_to_datadog(normalized_data.to_json)
         puts "Datadog upload successful: sent #{normalized_data.length} log(s) for #{function_name}"
@@ -102,28 +128,65 @@ class DatadogLogForwarder
       nil
     end
 
+    # Computes the Datadog unified service tagging reserved tags from environment
+    # variables, using the following precedence:
+    #
+    # - env:     DD_ENV     → APP_STAGE
+    # - service: DD_SERVICE → APP_SERVICE_NAME
+    # - version: DD_VERSION
+    # - preset:  DD_TAGS
+    def get_datadog_reserved_tags(fallback_env: nil, fallback_service: nil)
+      env = ENV['DD_ENV'] || ENV['APP_STAGE'] || fallback_env
+      service = ENV['DD_SERVICE'] || ENV['APP_SERVICE_NAME'] || fallback_service
+      version = ENV['DD_VERSION']
+      preset = ENV['DD_TAGS'] ? ENV['DD_TAGS'].split(',').map(&:strip).reject(&:empty?) : []
+
+      tags = []
+      tags << "env:#{env}" if env
+      tags << "service:#{service}" if service
+      tags << "version:#{version}" if version
+      tags + preset
+    end
+
     private
 
     def normalize_messages(log_events)
       log_events.map do |log_event|
+        id = log_event.fetch('id')
         message = log_event.fetch('message')
-        timestamp = Integer(log_event.fetch('timestamp'))
+        timestamp_ms = Integer(log_event.fetch('timestamp'))
 
-        # Ensure the message starts with a time.
-        # E.g. START/END/REPORT log events don't.
+        # Ensure the message starts with an ISO timestamp
         if TIME_REGEX !~ message
-          time_string = Time.at(timestamp/1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          time_string = Time.at(timestamp_ms / 1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
           message = "#{time_string} #{message}"
         end
 
         # AWS replaces newlines with carriage returns in Lambda logs
-        message.gsub(/\r(?!\n)/, "\n")
+        message = message.gsub(/\r(?!\n)/, "\n")
+
+        { id: id, message: message }
       end
     end
 
+    def should_filter_lifecycle_logs?
+      truthy_env?(ENV['DD_FILTER_LIFECYCLE_LOGS'])
+    end
+
+    def truthy_env?(value)
+      return false if value.nil?
+      %w[1 true yes on].include?(value.strip.downcase)
+    end
+
+    def lambda_lifecycle_line?(message_with_timestamp)
+      # Strip the leading ISO timestamp and match on the remainder.
+      after_timestamp = message_with_timestamp.sub(TIME_REGEX, '').lstrip
+      LAMBDA_LIFECYCLE_AFTER_TIMESTAMP_REGEX.match?(after_timestamp)
+    end
+
     def send_json_to_datadog(data)
-      datadog_site = ENV['DATADOG_SITE'] || 'datadoghq.com'
-      datadog_url = URI("https://http-intake.logs.#{datadog_site}/api/v2/logs")
+      dd_site = ENV['DD_SITE'] || 'datadoghq.com'
+      datadog_url = URI("https://http-intake.logs.#{dd_site}/api/v2/logs")
 
       response = Net::HTTP.start(datadog_url.host, datadog_url.port, use_ssl: datadog_url.scheme == 'https') do |http|
         http.open_timeout = 3
@@ -133,17 +196,16 @@ class DatadogLogForwarder
           datadog_url,
           {
             'Content-Type' => 'application/json',
-            'DD-API-KEY' => ENV['DD_API_KEY'],
+            'DD-API-KEY' => ENV['DD_API_KEY']
           }
         )
 
         req.body = data
-
         http.request(req)
       end
 
-      unless ['200', '202'].include?(response.code)
-        raise "Datadog upload failed: #{response.code} - #{response.body}"
+      unless %w[200 202].include?(response.code)
+        raise "Datadog upload failed with status #{response.code}: #{response.body}"
       end
 
       response

data/lib/tpt_serverless/version.rb

@@ -1,3 +1,3 @@
 module TptServerless
-  VERSION = "0.4.2"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tpt_serverless
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.6.0
 platform: ruby
 authors:
 - TpT