tpt_serverless 0.2.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e40167211120d2403b0f6c91770b43384ff2d9781cb0282e2fad5c34f0a1459
-  data.tar.gz: 790516d5026b0a26123d929ddab13424e46ce56e27eb7de1dcd05d135cd7dba5
+  metadata.gz: bf37a0d47bf872ffbb9633925e4ddb37d6bfc38a570f84f75f3f61f492b8f100
+  data.tar.gz: 1038ef65a0ed86399a412fac44fcdb4ef037abfc491307dc3d92554f3a474424
 SHA512:
-  metadata.gz: 4731b9cf978c054b73779578280757be12d8e7ecc08305bdf5e87a6b3d272ac3f6101eddd6fdf3f9d60371126b227603a6eb218156d73a05e2856a72fcf83d22
-  data.tar.gz: f0f048fb5a0b0f02b34c2a06e5a157b8727ae13cfdb69c8fa2a3ebced4132c939e4a3b833a7d438c800fc00830e58f31279c1b2d75086c1ba6399a5694b51711
+  metadata.gz: 89ff6fca675780c16c77fbcbe976fcbbc67daeeaf01513a15f674ec97344047f8084fc7ae61ac62ec5c19f5c69f47830f319a00ffd578e433a9ff8950ef3ca20
+  data.tar.gz: d4c7ec90b2c42414f43391f1f7af4bab4f76f55521f3c42fd9810247bd1cdc4e04a06bf61861de57cf790bf4c76fb7dc1a20fcc0819c3fd5e19bb534cc99303a

data/lib/tpt_serverless/datadog.rb

@@ -0,0 +1,60 @@
+require "dogapi"
+require "timeout"
+
+# TODO: Look into using Lambda Extensions for Datadog (and logging) instead to avoid blocking or
+#       affecting our application code.
+module TptServerless
+  class Datadog
+    API_KEY = (ENV["DD_API_KEY"] && !ENV["DD_API_KEY"].strip.empty?) ? ENV["DD_API_KEY"] : nil
+    DATADOG_TIMEOUT = 2 # seconds
+    OUR_TIMEOUT = 3 # seconds. slightly longer than DATADOG_TIMEOUT.
+
+    def initialize(api_key: API_KEY, datadog_timeout: DATADOG_TIMEOUT, our_timeout: OUR_TIMEOUT)
+      @app_name = ENV.fetch("APP_SERVICE_NAME", "unknownService")
+      @stage = ENV.fetch("APP_STAGE", "unknown-stage")
+      @host = ENV.fetch("DD_HOSTNAME", nil)
+      @our_timeout = our_timeout
+
+      @client = if api_key
+        Dogapi::Client.new(
+          api_key,         # api key
+          nil,             # application_key, not required here
+          @host,           # host
+          nil,             # device
+          true,            # silent
+          datadog_timeout, # timeout in seconds
+        )
+      end
+    end
+
+    def inc(metric, amount: 1, tags: [])
+      # prefix the metric with the app name
+      metric = "#{@app_name}.#{metric}"
+
+      tags += ["env:#{@stage}"]
+      # change characters not supported by datadog to underscores
+      tags.map! { |t| t.gsub(%r([^\w:./-]), "_") }
+
+      # log the metric so that we can correlate between metrics & logs more easily
+      logger.info("metric=#{metric} amount=#{amount} tags=#{tags.join(",")}")
+
+      if @client
+        begin
+          Timeout.timeout(@our_timeout) do
+            @client.emit_point(metric, amount, type: "count", tags: tags, host: "aws.lambda")
+          end
+        rescue Exception => e
+          logger.info("WARNING: Failed to emit metric to datadog: #{e.message}")
+        end
+      end
+
+      nil
+    end
+
+    private
+
+    def logger
+      ServerlessLogger.logger
+    end
+  end
+end

data/lib/tpt_serverless/datadog_log_forwarder.rb

@@ -0,0 +1,152 @@
+=begin
+Receives logs from CloudWatch and forwards them to Datadog.
+
+Example of how to add this to your `serverless.yml`:
+
+  forwardLogs:
+    name: ${self:service.name}-forwardLogs-${self:provider.stage}
+    # NOTE: Ensure that you update the path below if you update the tpt_serverless gem
+    handler: /opt/ruby/3.4.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/datadog_log_forwarder.DatadogLogForwarder.handler
+    timeout: 10 # seconds
+    environment:
+      DD_API_KEY: ${self:custom.datadogApiKey}
+      # Optional:
+      DATADOG_SITE: datadoghq.com            # default
+      DATADOG_SOURCE: cwl-aws-lambda         # default
+      DATADOG_SERVICE: ${self:service.name}  # optional
+      DATADOG_TAGS: stage:${self:provider.stage},team:platform  # optional
+    events:
+      - cloudwatchLog: /aws/lambda/${self:service.name}-functionOne-${self:provider.stage}
+      - cloudwatchLog: /aws/lambda/${self:service.name}-functionTwo-${self:provider.stage}
+      …etc…
+    reservedConcurrency: ${self:custom.datadogConcurrency}
+=end
+
+require 'zlib'
+require 'base64'
+require 'json'
+require 'net/http'
+require 'uri'
+
+TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
+
+class DatadogLogForwarder
+  class << self
+    # This handler receives CloudWatch log events, parses the events and forwards the extracted logs to
+    # Datadog.
+    #
+    # It performs a few helpful cleanup/prep functions as well.
+    def handler(event:, context:)
+      dd_api_key = ENV['DD_API_KEY']
+
+      if dd_api_key.nil? || dd_api_key.strip.empty?
+        puts 'ERROR: DD_API_KEY is not set. Skipping log forwarding.'
+        return
+      end
+
+      raw_data = event.fetch('awslogs').fetch('data')
+      unzipped_data = Zlib.gunzip(Base64.decode64(raw_data))
+      data = JSON.parse(unzipped_data)
+
+      message_type = data.fetch('messageType')
+      log_group = data.fetch('logGroup')
+      log_stream = data.fetch('logStream')
+      log_events = data.fetch('logEvents')
+
+      if message_type == 'CONTROL_MESSAGE'
+        puts 'skipping control message'
+        return
+      end
+
+      puts "message_count=#{log_events.length}"
+
+      region = 'unknown'
+      account_id = 'unknown'
+
+      if context&.respond_to?(:invoked_function_arn) && !context.invoked_function_arn.empty?
+        arn_parts = context.invoked_function_arn.split(':')
+        region = arn_parts[3] if arn_parts[3]
+        account_id = arn_parts[4] if arn_parts[4]
+      end
+
+      function_name = log_group&.split('/')&.last
+      function_arn = "arn:aws:lambda:#{region}:#{account_id}:function:#{function_name}"
+
+      normalized_data = log_events.map do |log_event|
+        normalized_msg = normalize_messages([log_event]).first
+
+        {
+          message: normalized_msg,
+          ddsource: ENV['DATADOG_SOURCE'] || 'cwl-aws-lambda',
+          service: ENV['DATADOG_SERVICE'] || function_name,
+          hostname: function_arn,
+          ddtags: [ENV['DATADOG_TAGS'], "region:#{region}"].compact.reject(&:empty?).join(','),
+          aws: {
+            awslogs: {
+              logGroup: log_group,
+              logStream: log_stream
+            }
+          },
+          id: log_event['id']
+        }
+      end
+
+      begin
+        send_json_to_datadog(normalized_data.to_json)
+        puts "Datadog upload successful: sent #{normalized_data.length} log(s) for #{function_name}"
+      rescue => e
+        puts e.message
+        puts "Error details: #{e.backtrace.first}"
+      end
+
+      nil
+    end
+
+    private
+
+    def normalize_messages(log_events)
+      log_events.map do |log_event|
+        message = log_event.fetch('message')
+        timestamp = Integer(log_event.fetch('timestamp'))
+
+        # Ensure the message starts with a time.
+        # E.g. START/END/REPORT log events don't.
+        if TIME_REGEX !~ message
+          time_string = Time.at(timestamp/1000.0).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          message = "#{time_string} #{message}"
+        end
+
+        # AWS replaces newlines with carriage returns in Lambda logs
+        message.gsub(/\r(?!\n)/, "\n")
+      end
+    end
+
+    def send_json_to_datadog(data)
+      datadog_site = ENV['DATADOG_SITE'] || 'datadoghq.com'
+      datadog_url = URI("https://http-intake.logs.#{datadog_site}/api/v2/logs")
+
+      response = Net::HTTP.start(datadog_url.host, datadog_url.port, use_ssl: datadog_url.scheme == 'https') do |http|
+        http.open_timeout = 3
+        http.read_timeout = 3
+
+        req = Net::HTTP::Post.new(
+          datadog_url,
+          {
+            'Content-Type' => 'application/json',
+            'DD-API-KEY' => ENV['DD_API_KEY'],
+          }
+        )
+
+        req.body = data
+
+        http.request(req)
+      end
+
+      unless ['200', '202'].include?(response.code)
+        raise "Datadog upload failed: #{response.code} - #{response.body}"
+      end
+
+      response
+    end
+  end
+end

data/lib/tpt_serverless/sumo_log_forwarder.rb

@@ -6,7 +6,7 @@ Example of how to add this to your `serverless.yml`:
   forwardLogs:
     name: ${self:service.name}-forwardLogs-${self:provider.stage}
     # NOTE: Ensure that you update the path below if you update the tpt_serverless gem
-    handler: /opt/ruby/2.5.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/sumo_log_forwarder.SumoLogForwarder.handler
+    handler: /opt/ruby/3.4.0/gems/tpt_serverless-X.X.X/lib/tpt_serverless/sumo_log_forwarder.SumoLogForwarder.handler
     timeout: 10 # seconds
     environment:
       SUMO_ENDPOINT: ${self:custom.sumoEndpoint}
@@ -19,6 +19,9 @@ Example of how to add this to your `serverless.yml`:
 
 require 'zlib'
 require 'base64'
+require 'json'
+require 'net/http'
+require 'uri'
 
 TIME_REGEX = /^20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ/
 

data/lib/tpt_serverless/version.rb

@@ -1,3 +1,3 @@
 module TptServerless
-  VERSION = "0.2.1"
+  VERSION = "0.4.1"
 end

data/lib/tpt_serverless.rb

@@ -1,5 +1,7 @@
 require "tpt_serverless/version"
 require 'tpt_serverless/sumo_log_forwarder'
+require 'tpt_serverless/datadog_log_forwarder'
+require 'tpt_serverless/datadog'
 
 module TptServerless
   class Error < StandardError; end

metadata

@@ -1,28 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: tpt_serverless
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.4.1
 platform: ruby
 authors:
 - TpT
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-02 00:00:00.000000000 Z
-dependencies: []
-description: 
-email: 
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dogapi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.45'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.45'
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/tpt_serverless.rb
+- lib/tpt_serverless/datadog.rb
+- lib/tpt_serverless/datadog_log_forwarder.rb
 - lib/tpt_serverless/sumo_log_forwarder.rb
 - lib/tpt_serverless/version.rb
 homepage: https://github.com/TeachersPayTeachers/tpt_serverless-ruby
 licenses: []
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -37,8 +49,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: TpT serverless utils for Ruby
 test_files: []