jets 1.8.4 → 1.8.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -1
- data/jets.gemspec +1 -0
- data/lib/jets/aws_services.rb +6 -0
- data/lib/jets/job/base.rb +4 -4
- data/lib/jets/job/dsl.rb +6 -0
- data/lib/jets/job/dsl/dynamodb_event.rb +65 -0
- data/lib/jets/job/dsl/kinesis_event.rb +46 -0
- data/lib/jets/job/helpers.rb +5 -0
- data/lib/jets/job/helpers/kinesis_event_helper.rb +13 -0
- data/lib/jets/job/{log_event_helper.rb → helpers/log_event_helper.rb} +1 -1
- data/lib/jets/job/{s3_event_helper.rb → helpers/s3_event_helper.rb} +1 -1
- data/lib/jets/version.rb +1 -1
- metadata +22 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b05de5f3ebf24e2ad8440c08d68efb0c3f08a6e5b2186571ae4cf35661c143c0
|
4
|
+
data.tar.gz: bd530e2693844b0578e869eb40c5d6a79a868d8fd6043c40949d7209aa732d81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 88825c727bddcdacdc805bcf7d9577b794f2b45fe71f2baf4af92e377db6318dd1b46629fea3cefa38f3003b54df721dae385d7086c4e8c9728c14dcfc8bc5b7
|
7
|
+
data.tar.gz: 14b1980cce7aa8e2e1126e0114e0ab133a25e8bf6657e231a271549775540315e392374b5f9228b2ec7226bf020aa5d77c394db62f361fd2e8a6d098bc71a19e
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,11 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
4
|
-
This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/)
|
4
|
+
This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
|
5
|
+
|
6
|
+
## [1.8.5]
|
7
|
+
- #198 DynamoDB Stream Event Support
|
8
|
+
- #199 Kinesis Event support
|
5
9
|
|
6
10
|
## [1.8.4]
|
7
11
|
- rename s3_event_message to s3_event helper
|
data/jets.gemspec
CHANGED
@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
|
|
36
36
|
spec.add_dependency "aws-sdk-cloudformation"
|
37
37
|
spec.add_dependency "aws-sdk-cloudwatchlogs"
|
38
38
|
spec.add_dependency "aws-sdk-dynamodb"
|
39
|
+
spec.add_dependency "aws-sdk-kinesis"
|
39
40
|
spec.add_dependency "aws-sdk-lambda"
|
40
41
|
spec.add_dependency "aws-sdk-s3"
|
41
42
|
spec.add_dependency "aws-sdk-sns"
|
data/lib/jets/aws_services.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require "aws-sdk-apigateway"
|
2
2
|
require "aws-sdk-cloudformation"
|
3
3
|
require "aws-sdk-cloudwatchlogs"
|
4
|
+
require "aws-sdk-dynamodb"
|
4
5
|
require "aws-sdk-lambda"
|
5
6
|
require "aws-sdk-s3"
|
6
7
|
require "aws-sdk-sts"
|
@@ -25,6 +26,11 @@ module Jets::AwsServices
|
|
25
26
|
end
|
26
27
|
memoize :cfn
|
27
28
|
|
29
|
+
def dynamodb
|
30
|
+
Aws::DynamoDB::Client.new
|
31
|
+
end
|
32
|
+
memoize :dynamodb
|
33
|
+
|
28
34
|
def lambda
|
29
35
|
Aws::Lambda::Client.new
|
30
36
|
end
|
data/lib/jets/job/base.rb
CHANGED
@@ -6,15 +6,15 @@ require 'json'
|
|
6
6
|
# Both Jets::Job::Base and Jets::Lambda::Functions have Dsl modules included.
|
7
7
|
# So the Jets::Job::Dsl overrides some of the Jets::Lambda::Functions behavior.
|
8
8
|
class Jets::Job
|
9
|
-
autoload :
|
10
|
-
autoload :S3EventHelper, "jets/job/s3_event_helper"
|
9
|
+
autoload :Helpers, "jets/job/helpers"
|
11
10
|
|
12
11
|
class Base < Jets::Lambda::Functions
|
13
12
|
include Dsl
|
14
13
|
|
15
14
|
# non-DSL methods
|
16
|
-
include
|
17
|
-
include
|
15
|
+
include Helpers::KinesisEventHelper
|
16
|
+
include Helpers::LogEventHelper
|
17
|
+
include Helpers::S3EventHelper
|
18
18
|
|
19
19
|
# Tracks bucket each time an s3_event is declared
|
20
20
|
# Map of bucket_name => stack_name (nested part)
|
data/lib/jets/job/dsl.rb
CHANGED
@@ -9,8 +9,10 @@
|
|
9
9
|
module Jets::Job::Dsl
|
10
10
|
extend ActiveSupport::Concern
|
11
11
|
autoload :CloudwatchEvent, "jets/job/dsl/cloudwatch_event"
|
12
|
+
autoload :DynamodbEvent, "jets/job/dsl/dynamodb_event"
|
12
13
|
autoload :EventSourceMapping, "jets/job/dsl/event_source_mapping" # base for sqs_event, etc
|
13
14
|
autoload :IotEvent, "jets/job/dsl/iot_event"
|
15
|
+
autoload :KinesisEvent, "jets/job/dsl/kinesis_event"
|
14
16
|
autoload :LogEvent, "jets/job/dsl/log_event"
|
15
17
|
autoload :S3Event, "jets/job/dsl/s3_event"
|
16
18
|
autoload :SnsEvent, "jets/job/dsl/sns_event"
|
@@ -18,9 +20,13 @@ module Jets::Job::Dsl
|
|
18
20
|
|
19
21
|
included do
|
20
22
|
class << self
|
23
|
+
include Jets::AwsServices
|
24
|
+
|
21
25
|
include CloudwatchEvent
|
26
|
+
include DynamodbEvent
|
22
27
|
include EventSourceMapping
|
23
28
|
include IotEvent
|
29
|
+
include KinesisEvent
|
24
30
|
include LogEvent
|
25
31
|
include S3Event
|
26
32
|
include SnsEvent
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Jets::Job::Dsl
|
2
|
+
module DynamodbEvent
|
3
|
+
def dynamodb_event(table_name, options={})
|
4
|
+
stream_arn = full_dynamodb_stream_arn(table_name)
|
5
|
+
default_iam_policy = default_dynamodb_stream_policy(stream_arn)
|
6
|
+
|
7
|
+
# Create iam policy allows access to queue
|
8
|
+
# Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
|
9
|
+
iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
|
10
|
+
iam_policy(iam_policy_props) unless iam_policy_props == :disable
|
11
|
+
|
12
|
+
props = options # by this time options only has EventSourceMapping properties
|
13
|
+
default = {
|
14
|
+
event_source_arn: stream_arn,
|
15
|
+
starting_position: "TRIM_HORIZON",
|
16
|
+
}
|
17
|
+
props = default.merge(props)
|
18
|
+
|
19
|
+
event_source_mapping(props)
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
# Expands table name to the full stream arn. Example:
|
24
|
+
#
|
25
|
+
# test-table
|
26
|
+
# To:
|
27
|
+
# arn:aws:dynamodb:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
|
28
|
+
#
|
29
|
+
# Note, this does not check if the stream has been disabled.
|
30
|
+
def full_dynamodb_stream_arn(table_name)
|
31
|
+
return table_name if table_name.include?("arn:aws:dynamodb") # assume full stream arn
|
32
|
+
|
33
|
+
begin
|
34
|
+
resp = dynamodb.describe_table(table_name: table_name)
|
35
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
|
36
|
+
puts e.message
|
37
|
+
puts "ERROR: Was not able to find the DynamoDB table: #{table_name}.".color(:red)
|
38
|
+
code_line = caller.grep(%r{/app/jobs}).first
|
39
|
+
puts "Please check: #{code_line}"
|
40
|
+
puts "Exiting"
|
41
|
+
exit 1
|
42
|
+
end
|
43
|
+
stream_arn = resp.table.latest_stream_arn
|
44
|
+
return stream_arn if stream_arn
|
45
|
+
end
|
46
|
+
|
47
|
+
def default_dynamodb_stream_policy(stream_name_arn='*')
|
48
|
+
stream = {
|
49
|
+
action: ["dynamodb:GetRecords",
|
50
|
+
"dynamodb:GetShardIterator",
|
51
|
+
"dynamodb:DescribeStream",
|
52
|
+
"dynamodb:ListStreams"],
|
53
|
+
effect: "Allow",
|
54
|
+
resource: stream_name_arn,
|
55
|
+
}
|
56
|
+
table_name_arn = stream_name_arn.gsub(%r{/stream/20.*},'')
|
57
|
+
table = {
|
58
|
+
action: ["dynamodb:DescribeTable"],
|
59
|
+
effect: "Allow",
|
60
|
+
resource: table_name_arn,
|
61
|
+
}
|
62
|
+
[stream, table]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Jets::Job::Dsl
|
2
|
+
module KinesisEvent
|
3
|
+
def kinesis_event(stream_name, options={})
|
4
|
+
stream_arn = full_kinesis_stream_arn(stream_name)
|
5
|
+
default_iam_policy = default_kinesis_stream_policy(stream_arn)
|
6
|
+
|
7
|
+
# Create iam policy allows access to queue
|
8
|
+
# Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
|
9
|
+
iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
|
10
|
+
iam_policy(iam_policy_props) unless iam_policy_props == :disable
|
11
|
+
|
12
|
+
props = options # by this time options only has EventSourceMapping properties
|
13
|
+
default = {
|
14
|
+
event_source_arn: stream_arn,
|
15
|
+
starting_position: "LATEST",
|
16
|
+
}
|
17
|
+
props = default.merge(props)
|
18
|
+
|
19
|
+
event_source_mapping(props)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Expands table name to the full stream arn. Example:
|
23
|
+
#
|
24
|
+
# test-table
|
25
|
+
# To:
|
26
|
+
# arn:aws:kinesis:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
|
27
|
+
#
|
28
|
+
# Note, this does not check if the stream has been disabled.
|
29
|
+
def full_kinesis_stream_arn(stream_name)
|
30
|
+
return stream_name if stream_name.include?("arn:aws:kinesis") # assume full stream arn
|
31
|
+
|
32
|
+
"arn:aws:kinesis:#{Jets.aws.region}:#{Jets.aws.account}:stream/#{stream_name}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def default_kinesis_stream_policy(stream_name_arn='*')
|
36
|
+
{
|
37
|
+
action: ["kinesis:GetRecords",
|
38
|
+
"kinesis:GetShardIterator",
|
39
|
+
"kinesis:DescribeStream",
|
40
|
+
"kinesis:ListStreams"],
|
41
|
+
effect: "Allow",
|
42
|
+
resource: stream_name_arn,
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/jets/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jets
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.8.
|
4
|
+
version: 1.8.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tung Nguyen
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-02-
|
11
|
+
date: 2019-02-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: actionmailer
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: aws-sdk-kinesis
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: aws-sdk-lambda
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -688,14 +702,18 @@ files:
|
|
688
702
|
- lib/jets/job/base.rb
|
689
703
|
- lib/jets/job/dsl.rb
|
690
704
|
- lib/jets/job/dsl/cloudwatch_event.rb
|
705
|
+
- lib/jets/job/dsl/dynamodb_event.rb
|
691
706
|
- lib/jets/job/dsl/event_source_mapping.rb
|
692
707
|
- lib/jets/job/dsl/iot_event.rb
|
708
|
+
- lib/jets/job/dsl/kinesis_event.rb
|
693
709
|
- lib/jets/job/dsl/log_event.rb
|
694
710
|
- lib/jets/job/dsl/s3_event.rb
|
695
711
|
- lib/jets/job/dsl/sns_event.rb
|
696
712
|
- lib/jets/job/dsl/sqs_event.rb
|
697
|
-
- lib/jets/job/
|
698
|
-
- lib/jets/job/
|
713
|
+
- lib/jets/job/helpers.rb
|
714
|
+
- lib/jets/job/helpers/kinesis_event_helper.rb
|
715
|
+
- lib/jets/job/helpers/log_event_helper.rb
|
716
|
+
- lib/jets/job/helpers/s3_event_helper.rb
|
699
717
|
- lib/jets/klass.rb
|
700
718
|
- lib/jets/lambda.rb
|
701
719
|
- lib/jets/lambda/dsl.rb
|