tpt_serverless 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6887c4fd9a67c788a18671e57b5710e5e9de0e83057af75f3b2ba9989f947ce
-  data.tar.gz: d8bb6a3f64a4e6b61b56bdf1c634867de17caa493de9ea6d344a8e0e6c598131
+  metadata.gz: bf37a0d47bf872ffbb9633925e4ddb37d6bfc38a570f84f75f3f61f492b8f100
+  data.tar.gz: 1038ef65a0ed86399a412fac44fcdb4ef037abfc491307dc3d92554f3a474424
 SHA512:
-  metadata.gz: f7b21869a41a38fd9a24d32a8c649bfe6914255e0092ebf35f5519b254ef1cd67f113597b38b305696905d67c2206a5221cc6c3a3d7677f1773c3bafdd6d49c5
-  data.tar.gz: 3c1b176c3cad4e4f17d592a7ee967d310c0aa0d4f077ab90f94d339279589ecc3eb301372c85aed14af5d4521fe12c7bedfc009f63ee5bf234c37ccffe312e16
+  metadata.gz: 89ff6fca675780c16c77fbcbe976fcbbc67daeeaf01513a15f674ec97344047f8084fc7ae61ac62ec5c19f5c69f47830f319a00ffd578e433a9ff8950ef3ca20
+  data.tar.gz: d4c7ec90b2c42414f43391f1f7af4bab4f76f55521f3c42fd9810247bd1cdc4e04a06bf61861de57cf790bf4c76fb7dc1a20fcc0819c3fd5e19bb534cc99303a

data/lib/tpt_serverless/datadog_log_forwarder.rb

@@ -0,0 +1,152 @@
+=begin
+Receives logs from CloudWatch and forwards them to Datadog.
+
+Example of how to add this to your `serverless.yml`:
+
+  forwardLogs:
+    name: ${self:service.name}-forwardLogs-${self:provider.stage}
+    # NOTE: Ensure that you update the path below if you update the tpt_serverless gem
+    handler: /opt/ruby/3.4.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/datadog_log_forwarder.DatadogLogForwarder.handler
+    timeout: 10 # seconds
+    environment:
+      DD_API_KEY: ${self:custom.datadogApiKey}
+      # Optional:
+      DATADOG_SITE: datadoghq.com            # default
+      DATADOG_SOURCE: cwl-aws-lambda         # default
+      DATADOG_SERVICE: ${self:service.name}  # optional
+      DATADOG_TAGS: stage:${self:provider.stage},team:platform  # optional
+    events:
+      - cloudwatchLog: /aws/lambda/${self:service.name}-functionOne-${self:provider.stage}
+      - cloudwatchLog: /aws/lambda/${self:service.name}-functionTwo-${self:provider.stage}
+      …etc…
+    reservedConcurrency: ${self:custom.datadogConcurrency}
+=end
+
+require 'zlib'
+require 'base64'
+require 'json'
+require 'net/http'
+require 'uri'
+
+TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
+
+class DatadogLogForwarder
+  class << self
+    # This handler receives CloudWatch log events, parses the events and forwards the extracted logs to
+    # Datadog.
+    #
+    # It performs a few helpful cleanup/prep functions as well.
+    def handler(event:, context:)
+      dd_api_key = ENV['DD_API_KEY']
+
+      if dd_api_key.nil? || dd_api_key.strip.empty?
+        puts 'ERROR: DD_API_KEY is not set. Skipping log forwarding.'
+        return
+      end
+
+      raw_data = event.fetch('awslogs').fetch('data')
+      unzipped_data = Zlib.gunzip(Base64.decode64(raw_data))
+      data = JSON.parse(unzipped_data)
+
+      message_type = data.fetch('messageType')
+      log_group = data.fetch('logGroup')
+      log_stream = data.fetch('logStream')
+      log_events = data.fetch('logEvents')
+
+      if message_type == 'CONTROL_MESSAGE'
+        puts 'skipping control message'
+        return
+      end
+
+      puts "message_count=#{log_events.length}"
+
+      region = 'unknown'
+      account_id = 'unknown'
+
+      if context&.respond_to?(:invoked_function_arn) && !context.invoked_function_arn.empty?
+        arn_parts = context.invoked_function_arn.split(':')
+        region = arn_parts[3] if arn_parts[3]
+        account_id = arn_parts[4] if arn_parts[4]
+      end
+
+      function_name = log_group&.split('/')&.last
+      function_arn = "arn:aws:lambda:#{region}:#{account_id}:function:#{function_name}"
+
+      normalized_data = log_events.map do |log_event|
+        normalized_msg = normalize_messages([log_event]).first
+
+        {
+          message: normalized_msg,
+          ddsource: ENV['DATADOG_SOURCE'] || 'cwl-aws-lambda',
+          service: ENV['DATADOG_SERVICE'] || function_name,
+          hostname: function_arn,
+          ddtags: [ENV['DATADOG_TAGS'], "region:#{region}"].compact.reject(&:empty?).join(','),
+          aws: {
+            awslogs: {
+              logGroup: log_group,
+              logStream: log_stream
+            }
+          },
+          id: log_event['id']
+        }
+      end
+
+      begin
+        send_json_to_datadog(normalized_data.to_json)
+        puts "Datadog upload successful: sent #{normalized_data.length} log(s) for #{function_name}"
+      rescue => e
+        puts e.message
+        puts "Error details: #{e.backtrace.first}"
+      end
+
+      nil
+    end
+
+    private
+
+    def normalize_messages(log_events)
+      log_events.map do |log_event|
+        message = log_event.fetch('message')
+        timestamp = Integer(log_event.fetch('timestamp'))
+
+        # Ensure the message starts with a time.
+        # E.g. START/END/REPORT log events don't.
+        if TIME_REGEX !~ message
+          time_string = Time.at(timestamp/1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          message = "#{time_string} #{message}"
+        end
+
+        # AWS replaces newlines with carriage returns in Lambda logs
+        message.gsub(/\r(?!\n)/, "\n")
+      end
+    end
+
+    def send_json_to_datadog(data)
+      datadog_site = ENV['DATADOG_SITE'] || 'datadoghq.com'
+      datadog_url = URI("https://http-intake.logs.#{datadog_site}/api/v2/logs")
+
+      response = Net::HTTP.start(datadog_url.host, datadog_url.port, use_ssl: datadog_url.scheme == 'https') do |http|
+        http.open_timeout = 3
+        http.read_timeout = 3
+
+        req = Net::HTTP::Post.new(
+          datadog_url,
+          {
+            'Content-Type' => 'application/json',
+            'DD-API-KEY' => ENV['DD_API_KEY'],
+          }
+        )
+
+        req.body = data
+
+        http.request(req)
+      end
+
+      unless ['200', '202'].include?(response.code)
+        raise "Datadog upload failed: #{response.code} - #{response.body}"
+      end
+
+      response
+    end
+  end
+end

data/lib/tpt_serverless/sumo_log_forwarder.rb

@@ -6,7 +6,7 @@ Example of how to add this to your `serverless.yml`:
   forwardLogs:
     name: ${self:service.name}-forwardLogs-${self:provider.stage}
     # NOTE: Ensure that you update the path below if you update the tpt_serverless gem
-    handler: /opt/ruby/2.5.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/sumo_log_forwarder.SumoLogForwarder.handler
+    handler: /opt/ruby/3.4.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/sumo_log_forwarder.SumoLogForwarder.handler
     timeout: 10 # seconds
     environment:
       SUMO_ENDPOINT: ${self:custom.sumoEndpoint}
@@ -19,6 +19,9 @@ Example of how to add this to your `serverless.yml`:
 
 require 'zlib'
 require 'base64'
+require 'json'
+require 'net/http'
+require 'uri'
 
 TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
 

data/lib/tpt_serverless/version.rb

@@ -1,3 +1,3 @@
 module TptServerless
-  VERSION = "0.3.0"
+  VERSION = "0.4.1"
 end

data/lib/tpt_serverless.rb

@@ -1,5 +1,6 @@
 require "tpt_serverless/version"
 require 'tpt_serverless/sumo_log_forwarder'
+require 'tpt_serverless/datadog_log_forwarder'
 require 'tpt_serverless/datadog'
 
 module TptServerless

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: tpt_serverless
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - TpT
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-27 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dogapi
@@ -24,20 +23,18 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.45'
-description:
-email:
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/tpt_serverless.rb
 - lib/tpt_serverless/datadog.rb
+- lib/tpt_serverless/datadog_log_forwarder.rb
 - lib/tpt_serverless/sumo_log_forwarder.rb
 - lib/tpt_serverless/version.rb
 homepage: https://github.com/TeachersPayTeachers/tpt_serverless-ruby
 licenses: []
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -52,8 +49,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: TpT serverless utils for Ruby
 test_files: []