jets 1.8.4 → 1.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6951f100727eef4fb0aa630b30111f1fae7d7c0411912c3c5b79e16829437658
-  data.tar.gz: 14cd101d0fdd239cdef08f8e73bd8acb3a589f134e39f4991be3f9eac97cf6fa
+  metadata.gz: b05de5f3ebf24e2ad8440c08d68efb0c3f08a6e5b2186571ae4cf35661c143c0
+  data.tar.gz: bd530e2693844b0578e869eb40c5d6a79a868d8fd6043c40949d7209aa732d81
 SHA512:
-  metadata.gz: 3ca52303ea7b97b6decb817f912da58be9af3d55b86cb795a9eafd67b5cd0105a628edea783be3300f9503bf7b653d80950f65e3f1877bdef928a7f85d8f6a60
-  data.tar.gz: b75112b19e8e13331819cd5067f122058a6f399c6ed06aa1e8ed09a3c4d0c02b8ae56527a0abced2c785e46d22a442bb22eb49f5aa7bc047c8090bb722b321b4
+  metadata.gz: 88825c727bddcdacdc805bcf7d9577b794f2b45fe71f2baf4af92e377db6318dd1b46629fea3cefa38f3003b54df721dae385d7086c4e8c9728c14dcfc8bc5b7
+  data.tar.gz: 14b1980cce7aa8e2e1126e0114e0ab133a25e8bf6657e231a271549775540315e392374b5f9228b2ec7226bf020aa5d77c394db62f361fd2e8a6d098bc71a19e

data/CHANGELOG.md

@@ -1,7 +1,11 @@
 # Change Log
 
 All notable changes to this project will be documented in this file.
-This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
+This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
+
+## [1.8.5]
+- #198 DynamoDB Stream Event Support
+- #199 Kinesis Event support
 
 ## [1.8.4]
 - rename s3_event_message to s3_event helper

data/jets.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk-cloudformation"
   spec.add_dependency "aws-sdk-cloudwatchlogs"
   spec.add_dependency "aws-sdk-dynamodb"
+  spec.add_dependency "aws-sdk-kinesis"
   spec.add_dependency "aws-sdk-lambda"
   spec.add_dependency "aws-sdk-s3"
   spec.add_dependency "aws-sdk-sns"

data/lib/jets/aws_services.rb

@@ -1,6 +1,7 @@
 require "aws-sdk-apigateway"
 require "aws-sdk-cloudformation"
 require "aws-sdk-cloudwatchlogs"
+require "aws-sdk-dynamodb"
 require "aws-sdk-lambda"
 require "aws-sdk-s3"
 require "aws-sdk-sts"
@@ -25,6 +26,11 @@ module Jets::AwsServices
   end
   memoize :cfn
 
+  def dynamodb
+    Aws::DynamoDB::Client.new
+  end
+  memoize :dynamodb
+
   def lambda
     Aws::Lambda::Client.new
   end

data/lib/jets/job/base.rb

@@ -6,15 +6,15 @@ require 'json'
 # Both Jets::Job::Base and Jets::Lambda::Functions have Dsl modules included.
 # So the Jets::Job::Dsl overrides some of the Jets::Lambda::Functions behavior.
 class Jets::Job
-  autoload :LogEventHelper, "jets/job/log_event_helper"
-  autoload :S3EventHelper, "jets/job/s3_event_helper"
+  autoload :Helpers, "jets/job/helpers"
 
   class Base < Jets::Lambda::Functions
     include Dsl
 
     # non-DSL methods
-    include LogEventHelper
-    include S3EventHelper
+    include Helpers::KinesisEventHelper
+    include Helpers::LogEventHelper
+    include Helpers::S3EventHelper
 
     # Tracks bucket each time an s3_event is declared
     # Map of bucket_name => stack_name (nested part)

data/lib/jets/job/dsl.rb

@@ -9,8 +9,10 @@
 module Jets::Job::Dsl
   extend ActiveSupport::Concern
   autoload :CloudwatchEvent, "jets/job/dsl/cloudwatch_event"
+  autoload :DynamodbEvent, "jets/job/dsl/dynamodb_event"
   autoload :EventSourceMapping, "jets/job/dsl/event_source_mapping" # base for sqs_event, etc
   autoload :IotEvent, "jets/job/dsl/iot_event"
+  autoload :KinesisEvent, "jets/job/dsl/kinesis_event"
   autoload :LogEvent, "jets/job/dsl/log_event"
   autoload :S3Event, "jets/job/dsl/s3_event"
   autoload :SnsEvent, "jets/job/dsl/sns_event"
@@ -18,9 +20,13 @@ module Jets::Job::Dsl
 
   included do
     class << self
+      include Jets::AwsServices
+
       include CloudwatchEvent
+      include DynamodbEvent
       include EventSourceMapping
       include IotEvent
+      include KinesisEvent
       include LogEvent
       include S3Event
       include SnsEvent

data/lib/jets/job/dsl/dynamodb_event.rb

@@ -0,0 +1,65 @@
+module Jets::Job::Dsl
+  module DynamodbEvent
+    def dynamodb_event(table_name, options={})
+      stream_arn = full_dynamodb_stream_arn(table_name)
+      default_iam_policy = default_dynamodb_stream_policy(stream_arn)
+
+      # Create iam policy allows access to queue
+      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
+      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
+      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+
+      props = options # by this time options only has EventSourceMapping properties
+      default = {
+        event_source_arn: stream_arn,
+        starting_position: "TRIM_HORIZON",
+      }
+      props = default.merge(props)
+
+      event_source_mapping(props)
+    end
+
+
+    # Expands table name to the full stream arn. Example:
+    #
+    #   test-table
+    # To:
+    #   arn:aws:dynamodb:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
+    #
+    # Note, this does not check if the stream has been disabled.
+    def full_dynamodb_stream_arn(table_name)
+      return table_name if table_name.include?("arn:aws:dynamodb") # assume full stream arn
+
+      begin
+        resp = dynamodb.describe_table(table_name: table_name)
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+        puts e.message
+        puts "ERROR: Was not able to find the DynamoDB table: #{table_name}.".color(:red)
+        code_line = caller.grep(%r{/app/jobs}).first
+        puts "Please check: #{code_line}"
+        puts "Exiting"
+        exit 1
+      end
+      stream_arn = resp.table.latest_stream_arn
+      return stream_arn if stream_arn
+    end
+
+    def default_dynamodb_stream_policy(stream_name_arn='*')
+      stream = {
+        action: ["dynamodb:GetRecords",
+                 "dynamodb:GetShardIterator",
+                 "dynamodb:DescribeStream",
+                 "dynamodb:ListStreams"],
+        effect: "Allow",
+        resource: stream_name_arn,
+      }
+      table_name_arn = stream_name_arn.gsub(%r{/stream/20.*},'')
+      table = {
+        action: ["dynamodb:DescribeTable"],
+        effect: "Allow",
+        resource: table_name_arn,
+      }
+      [stream, table]
+    end
+  end
+end

data/lib/jets/job/dsl/kinesis_event.rb

@@ -0,0 +1,46 @@
+module Jets::Job::Dsl
+  module KinesisEvent
+    def kinesis_event(stream_name, options={})
+      stream_arn = full_kinesis_stream_arn(stream_name)
+      default_iam_policy = default_kinesis_stream_policy(stream_arn)
+
+      # Create iam policy allows access to queue
+      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
+      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
+      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+
+      props = options # by this time options only has EventSourceMapping properties
+      default = {
+        event_source_arn: stream_arn,
+        starting_position: "LATEST",
+      }
+      props = default.merge(props)
+
+      event_source_mapping(props)
+    end
+
+    # Expands table name to the full stream arn. Example:
+    #
+    #   test-table
+    # To:
+    #   arn:aws:kinesis:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
+    #
+    # Note, this does not check if the stream has been disabled.
+    def full_kinesis_stream_arn(stream_name)
+      return stream_name if stream_name.include?("arn:aws:kinesis") # assume full stream arn
+
+      "arn:aws:kinesis:#{Jets.aws.region}:#{Jets.aws.account}:stream/#{stream_name}"
+    end
+
+    def default_kinesis_stream_policy(stream_name_arn='*')
+      {
+        action: ["kinesis:GetRecords",
+                 "kinesis:GetShardIterator",
+                 "kinesis:DescribeStream",
+                 "kinesis:ListStreams"],
+        effect: "Allow",
+        resource: stream_name_arn,
+      }
+    end
+  end
+end

data/lib/jets/job/helpers.rb

@@ -0,0 +1,5 @@
+module Jets::Job::Helpers
+  autoload :KinesisEventHelper, "jets/job/helpers/kinesis_event_helper"
+  autoload :LogEventHelper, "jets/job/helpers/log_event_helper"
+  autoload :S3EventHelper, "jets/job/helpers/s3_event_helper"
+end

data/lib/jets/job/helpers/kinesis_event_helper.rb

@@ -0,0 +1,13 @@
+require 'base64'
+
+module Jets::Job::Helpers
+  module KinesisEventHelper
+    def kinesis_data
+      records = event["Records"]
+      records.map do |record|
+        encoded = record["kinesis"]["data"]
+        Base64.decode64(encoded) # data
+      end
+    end
+  end
+end

data/lib/jets/job/{log_event_helper.rb → helpers/log_event_helper.rb}

@@ -3,7 +3,7 @@ require 'json'
 require 'stringio'
 require 'zlib'
 
-class Jets::Job
+module Jets::Job::Helpers
   module LogEventHelper
     def log_event
       encoded = event["awslogs"]["data"]

data/lib/jets/job/{s3_event_helper.rb → helpers/s3_event_helper.rb}

@@ -1,4 +1,4 @@
-class Jets::Job
+module Jets::Job::Helpers
   module S3EventHelper
     def s3_event
       message = event["Records"][0]["Sns"]["Message"]

data/lib/jets/version.rb

@@ -1,3 +1,3 @@
 module Jets
-  VERSION = "1.8.4"
+  VERSION = "1.8.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jets
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 1.8.5
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-15 00:00:00.000000000 Z
+date: 2019-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionmailer
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-kinesis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-lambda
   requirement: !ruby/object:Gem::Requirement
@@ -688,14 +702,18 @@ files:
 - lib/jets/job/base.rb
 - lib/jets/job/dsl.rb
 - lib/jets/job/dsl/cloudwatch_event.rb
+- lib/jets/job/dsl/dynamodb_event.rb
 - lib/jets/job/dsl/event_source_mapping.rb
 - lib/jets/job/dsl/iot_event.rb
+- lib/jets/job/dsl/kinesis_event.rb
 - lib/jets/job/dsl/log_event.rb
 - lib/jets/job/dsl/s3_event.rb
 - lib/jets/job/dsl/sns_event.rb
 - lib/jets/job/dsl/sqs_event.rb
-- lib/jets/job/log_event_helper.rb
-- lib/jets/job/s3_event_helper.rb
+- lib/jets/job/helpers.rb
+- lib/jets/job/helpers/kinesis_event_helper.rb
+- lib/jets/job/helpers/log_event_helper.rb
+- lib/jets/job/helpers/s3_event_helper.rb
 - lib/jets/klass.rb
 - lib/jets/lambda.rb
 - lib/jets/lambda/dsl.rb