nypl_ruby_util 0.0.5 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/deploy_helper.rb +136 -136
- data/lib/kinesis_client.rb +90 -22
- data/lib/kms_client.rb +6 -6
- data/lib/nypl_avro.rb +1 -1
- data/lib/platform_api_client.rb +4 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05c91cd9d8cd4ffd258f3a2c1f9b3cb41c2c6cf9b5df78d21f1483510e34c544
|
4
|
+
data.tar.gz: 55e61c0097b9721728b147b6201e0856e815c3de8631e3f41ed78b00c12d11bb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c2042cc492affeb2ae6d60d6c96fb6d0db3ea69c23f1ad6a31da3ff5025c52cd4d5f669112e174153e88b84dde7395f59b23b12182017c079f1ce6a024807f6
|
7
|
+
data.tar.gz: 832f11059c1267393afb6b2f5140dd5a8f98950f9e69c495e6fa1cfa31af947c78fbf4cd740016214b4fe1dc4bce939a6f57de295039a3673aa195da0ce2be2c
|
data/lib/deploy_helper.rb
CHANGED
@@ -1,152 +1,152 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "aws-sdk-lambda"
|
2
|
+
require "aws-sdk-cloudwatchevents"
|
3
|
+
require "yaml"
|
4
4
|
|
5
5
|
# Utility class for running rake methods
|
6
6
|
class DeployHelper
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
7
|
+
attr_reader(
|
8
|
+
:travis_branch,
|
9
|
+
:aws_access_key_id,
|
10
|
+
:aws_secret_access_key,
|
11
|
+
:aws_configuration,
|
12
|
+
:region,
|
13
|
+
:lambda_client,
|
14
|
+
:yaml,
|
15
|
+
:lambda_config,
|
16
|
+
:function_name,
|
17
|
+
:event
|
18
|
+
)
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@travis_branch = ENV["TRAVIS_BRANCH"].upcase
|
22
|
+
@travis_branch = ["MAIN", "MASTER"].include?(@travis_branch) ? "PRODUCTION" : @travis_branch
|
23
|
+
@aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
|
24
|
+
@aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
|
25
|
+
@yaml = YAML.safe_load(File.read(".travis.yml"))
|
26
|
+
@lambda_config = yaml["deploy"].find { |conf| name_matches_branch?(conf["function_name"], travis_branch) }
|
27
|
+
@region = @lambda_config["region"]
|
28
|
+
@function_name = @lambda_config["function_name"]
|
29
|
+
@aws_configuration = {
|
30
|
+
region: region,
|
31
|
+
access_key_id: aws_access_key_id,
|
32
|
+
secret_access_key: aws_secret_access_key
|
33
|
+
}
|
34
|
+
p("using configuration: ", aws_configuration)
|
35
|
+
p("lambda config: ", lambda_config)
|
36
|
+
@lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
|
37
|
+
end
|
38
|
+
|
39
|
+
def configured?
|
40
|
+
aws_access_key_id && aws_secret_access_key && region
|
41
|
+
end
|
42
|
+
|
43
|
+
def name_matches_branch?(name, branch)
|
44
|
+
downcase_name = name.downcase
|
45
|
+
downcase_branch = branch.downcase
|
46
|
+
variants = [
|
47
|
+
["dev", "development"],
|
48
|
+
["qa"],
|
49
|
+
["main", "master", "production", "prod"],
|
50
|
+
]
|
51
|
+
variants.any? do |group|
|
52
|
+
group.any? { |variant| downcase_name.include?(variant) }\
|
53
|
+
&& group.any? { |variant| downcase_branch.include?(variant) }
|
37
54
|
end
|
55
|
+
end
|
38
56
|
|
39
|
-
|
40
|
-
|
57
|
+
def update_lambda_configuration
|
58
|
+
unless configured? && lambda_config
|
59
|
+
p("insufficient configuration")
|
60
|
+
return nil
|
41
61
|
end
|
42
62
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
63
|
+
updated_lambda_configuration = {
|
64
|
+
function_name: function_name,
|
65
|
+
vpc_config: lambda_config["vpc_config"],
|
66
|
+
environment: lambda_config["environment"],
|
67
|
+
layers: lambda_config["layers"]
|
68
|
+
}
|
69
|
+
updated_lambda_configuration[:function_name] = function_name
|
70
|
+
p("updating_function_configuration with: ", updated_lambda_configuration)
|
71
|
+
update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
|
72
|
+
p("update_configuration_resp: ", update_configuration_resp)
|
73
|
+
end
|
74
|
+
|
75
|
+
def update_event
|
76
|
+
unless lambda_config["event"]
|
77
|
+
p("no event config")
|
78
|
+
return nil
|
55
79
|
end
|
56
80
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
updated_lambda_configuration = {
|
64
|
-
function_name: function_name,
|
65
|
-
vpc_config: lambda_config['vpc_config'],
|
66
|
-
environment: lambda_config['environment'],
|
67
|
-
layers: lambda_config['layers']
|
68
|
-
}
|
69
|
-
updated_lambda_configuration[:function_name] = function_name
|
70
|
-
p 'updating_function_configuration with: ', updated_lambda_configuration
|
71
|
-
update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
|
72
|
-
p 'update_configuration_resp: ', update_configuration_resp
|
81
|
+
@event = lambda_config["event"]
|
82
|
+
if event["event_source_arn"]
|
83
|
+
add_event_source
|
84
|
+
elsif event["schedule_expression"]
|
85
|
+
add_cron
|
73
86
|
end
|
87
|
+
end
|
74
88
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
|
81
|
-
@event = lambda_config['event']
|
82
|
-
if event['event_source_arn']
|
83
|
-
add_event_source
|
84
|
-
elsif event['schedule_expression']
|
85
|
-
add_cron
|
86
|
-
end
|
87
|
-
end
|
89
|
+
def add_event_source
|
90
|
+
existing_events = lambda_client.list_event_source_mappings({
|
91
|
+
function_name: function_name
|
92
|
+
}).event_source_mappings
|
88
93
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
94
|
+
existing_events.each do |existing_event|
|
95
|
+
p("deleting event with uuid: ", existing_event.uuid, "and arn: ", existing_event.event_source_arn)
|
96
|
+
lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
|
97
|
+
end
|
98
|
+
event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
|
99
|
+
event_to_create[:function_name] = function_name
|
100
|
+
p("creating event: ", event_to_create)
|
101
|
+
create_resp = lambda_client.create_event_source_mapping(event_to_create)
|
102
|
+
p("created: ", create_resp)
|
103
|
+
end
|
104
|
+
|
105
|
+
def add_cron
|
106
|
+
## create the event
|
107
|
+
events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
|
108
|
+
schedule_expression = event["schedule_expression"]
|
109
|
+
rule_name = "#{function_name}-rule"
|
110
|
+
p("rule_name: ", rule_name, "schedule_expression: ", schedule_expression)
|
111
|
+
events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
|
112
|
+
|
113
|
+
## next we have to connect the event to the lambda
|
114
|
+
## the first step is to get the lambda
|
115
|
+
|
116
|
+
return "missing function_name" unless function_name
|
117
|
+
|
118
|
+
p("getting lambda with function name", function_name)
|
119
|
+
lambda_resp = lambda_client.get_function(function_name: function_name).configuration
|
120
|
+
arn = lambda_resp.function_arn
|
121
|
+
|
122
|
+
## next figure out if the lambda already has granted cloudwatch
|
123
|
+
## permission to invoke
|
124
|
+
begin
|
125
|
+
policy_resp = lambda_client.get_policy(function_name: function_name)
|
126
|
+
if policy_resp.policy.include?("#{function_name}-permission")
|
127
|
+
p("lambda already has permission")
|
128
|
+
else
|
129
|
+
add_policy = true
|
130
|
+
end
|
131
|
+
rescue Aws::Lambda::Errors::ResourceNotFoundException
|
132
|
+
add_policy = true
|
133
|
+
p("no policy")
|
103
134
|
end
|
104
135
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
## the first step is to get the lambda
|
115
|
-
|
116
|
-
return 'missing function_name' unless function_name
|
117
|
-
|
118
|
-
p 'getting lambda with function name', function_name
|
119
|
-
lambda_resp = lambda_client.get_function(function_name: function_name).configuration
|
120
|
-
arn = lambda_resp.function_arn
|
121
|
-
|
122
|
-
## next figure out if the lambda already has granted cloudwatch
|
123
|
-
## permission to invoke
|
124
|
-
begin
|
125
|
-
policy_resp = lambda_client.get_policy(function_name: function_name)
|
126
|
-
unless policy_resp.policy.include?("#{function_name}-permission")
|
127
|
-
add_policy = true
|
128
|
-
else
|
129
|
-
p 'lambda already has permission'
|
130
|
-
end
|
131
|
-
rescue Aws::Lambda::Errors::ResourceNotFoundException
|
132
|
-
add_policy = true
|
133
|
-
p 'no policy'
|
134
|
-
end
|
135
|
-
|
136
|
-
## if not, add permission to invoke
|
137
|
-
if add_policy
|
138
|
-
permission = lambda_client.add_permission({
|
139
|
-
function_name: function_name,
|
140
|
-
principal: 'events.amazonaws.com',
|
141
|
-
statement_id: "#{function_name}-permission",
|
142
|
-
action: 'lambda:InvokeFunction'
|
143
|
-
})
|
144
|
-
p 'permission: ', permission
|
145
|
-
end
|
146
|
-
|
147
|
-
## finally we can tell the event to invoke the lambda
|
148
|
-
target_id = "#{function_name}-lambda"
|
149
|
-
p 'putting targets ', 'rule: ', rule_name, 'target_id: ', target_id, 'arn: ', arn
|
150
|
-
events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
|
136
|
+
## if not, add permission to invoke
|
137
|
+
if add_policy
|
138
|
+
permission = lambda_client.add_permission({
|
139
|
+
function_name: function_name,
|
140
|
+
principal: "events.amazonaws.com",
|
141
|
+
statement_id: "#{function_name}-permission",
|
142
|
+
action: "lambda:InvokeFunction"
|
143
|
+
})
|
144
|
+
p("permission: ", permission)
|
151
145
|
end
|
146
|
+
|
147
|
+
## finally we can tell the event to invoke the lambda
|
148
|
+
target_id = "#{function_name}-lambda"
|
149
|
+
p("putting targets ", "rule: ", rule_name, "target_id: ", target_id, "arn: ", arn)
|
150
|
+
events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
|
151
|
+
end
|
152
152
|
end
|
data/lib/kinesis_client.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require_relative
|
4
|
-
require_relative
|
1
|
+
require "securerandom"
|
2
|
+
require "aws-sdk-kinesis"
|
3
|
+
require_relative "nypl_avro"
|
4
|
+
require_relative "errors"
|
5
5
|
# Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
|
6
6
|
|
7
7
|
class KinesisClient
|
@@ -9,40 +9,108 @@ class KinesisClient
|
|
9
9
|
|
10
10
|
def initialize(config)
|
11
11
|
@config = config
|
12
|
+
@stream_name = @config[:stream_name]
|
12
13
|
@avro = nil
|
14
|
+
@batch_size = @config[:batch_size] || 1
|
15
|
+
@client_options = set_config(config)
|
16
|
+
@batch_count = 0
|
17
|
+
@records = []
|
18
|
+
@automatically_push = !(@config[:automatically_push] == false)
|
19
|
+
@client = Aws::Kinesis::Client.new(@client_options)
|
13
20
|
|
14
|
-
if config[:schema_string]
|
15
|
-
@avro = NYPLAvro.by_name(config[:schema_string])
|
16
|
-
end
|
21
|
+
@avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
|
17
22
|
|
23
|
+
@shovel_method = @batch_size > 1 ? :push_to_records : :push_record
|
18
24
|
end
|
19
25
|
|
20
|
-
def
|
26
|
+
def set_config(config)
|
27
|
+
if config[:profile]
|
28
|
+
{ profile: config[:profile] }
|
29
|
+
elsif config[:custom_aws_config]
|
30
|
+
config[:custom_aws_config]
|
31
|
+
else
|
32
|
+
{}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def convert_to_record(json_message)
|
21
37
|
if config[:schema_string]
|
22
38
|
message = avro.encode(json_message, false)
|
23
39
|
else
|
24
40
|
message = json_message
|
25
41
|
end
|
26
42
|
|
27
|
-
client = Aws::Kinesis::Client.new
|
28
43
|
partition_key = (config[:partition_key] ? json_message[config[:partition_key]] : SecureRandom.hex(20)).hash.to_s
|
29
|
-
|
30
|
-
resp = client.put_record({
|
31
|
-
stream_name: config[:stream_name],
|
44
|
+
{
|
32
45
|
data: message,
|
33
46
|
partition_key: partition_key
|
34
|
-
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def <<(json_message)
|
51
|
+
send(@shovel_method, json_message)
|
52
|
+
end
|
53
|
+
|
54
|
+
#This method is broken
|
55
|
+
def push_record(json_message)
|
56
|
+
record = convert_to_record(json_message)
|
57
|
+
record[:stream_name] = @stream_name
|
58
|
+
|
59
|
+
@client.put_record(record)
|
60
|
+
|
61
|
+
return_hash = {}
|
35
62
|
|
36
|
-
|
63
|
+
if resp.successful?
|
64
|
+
return_hash["code"] = "200"
|
65
|
+
return_hash["message"] = json_message, resp
|
66
|
+
$logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
|
67
|
+
else
|
68
|
+
$logger.error("message" => "FAILED to send message to #{@stream_name} #{json_message}, #{resp}.") if $logger
|
69
|
+
raise(NYPLError.new(json_message, resp))
|
70
|
+
end
|
71
|
+
return_hash
|
72
|
+
end
|
73
|
+
|
74
|
+
def push_to_records(json_message)
|
75
|
+
begin
|
76
|
+
@records << convert_to_record(json_message)
|
77
|
+
rescue AvroError => e
|
78
|
+
$logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
|
79
|
+
end
|
80
|
+
push_records if @automatically_push && @records.length >= @batch_size
|
81
|
+
end
|
37
82
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
83
|
+
def push_batch(batch)
|
84
|
+
resp = @client.put_records({
|
85
|
+
records: batch.to_a,
|
86
|
+
stream_name: @stream_name
|
87
|
+
})
|
88
|
+
|
89
|
+
if resp.failed_record_count > 0
|
90
|
+
failure_message = {
|
91
|
+
failures: resp.failed_record_count,
|
92
|
+
failures_data: filter_failures(resp)
|
93
|
+
}
|
94
|
+
$logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
|
95
|
+
else
|
96
|
+
$logger.info("Batch sent to #{config[:stream_name]} successfully")
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def push_records
|
101
|
+
if @records.length > 0
|
102
|
+
@records.each_slice(@batch_size) do |slice|
|
103
|
+
push_batch(slice)
|
104
|
+
@batch_count += 1
|
45
105
|
end
|
46
|
-
|
106
|
+
@records = []
|
107
|
+
@batch_count = 0
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def filter_failures(resp)
|
112
|
+
resp.records.filter_map.with_index do |record, i|
|
113
|
+
avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
|
114
|
+
end
|
47
115
|
end
|
48
116
|
end
|
data/lib/kms_client.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "aws-sdk-kms"
|
2
|
+
require "base64"
|
3
3
|
|
4
4
|
class KmsClient
|
5
5
|
@@kms = nil
|
@@ -11,15 +11,15 @@ class KmsClient
|
|
11
11
|
|
12
12
|
def decrypt(cipher)
|
13
13
|
# Assume value is base64 encoded:
|
14
|
-
decoded = Base64.decode64
|
15
|
-
decrypted = @kms.decrypt
|
14
|
+
decoded = Base64.decode64(cipher)
|
15
|
+
decrypted = @kms.decrypt(ciphertext_blob: decoded)
|
16
16
|
decrypted[:plaintext]
|
17
17
|
end
|
18
18
|
|
19
19
|
def self.aws_kms_client(options)
|
20
20
|
params = {
|
21
|
-
region:
|
22
|
-
stub_responses: ENV[
|
21
|
+
region: "us-east-1",
|
22
|
+
stub_responses: ENV["APP_ENV"] == "test"
|
23
23
|
}.merge(options)
|
24
24
|
@@kms = Aws::KMS::Client.new(params) if @@kms.nil?
|
25
25
|
@@kms
|
data/lib/nypl_avro.rb
CHANGED
@@ -38,7 +38,7 @@ class NYPLAvro
|
|
38
38
|
begin
|
39
39
|
bin_encoder.write(decoded_data, encoder)
|
40
40
|
rescue Avro::IO::AvroTypeError => e
|
41
|
-
raise AvroError.new(e), "Error encoding data using #{@schema.name} schema due to #{e.message}"
|
41
|
+
raise AvroError.new(e), "Error encoding data #{decoded_data} using #{@schema.name} schema due to #{e.message}"
|
42
42
|
end
|
43
43
|
|
44
44
|
buffer.rewind
|
data/lib/platform_api_client.rb
CHANGED
@@ -61,6 +61,7 @@ class PlatformApiClient
|
|
61
61
|
|
62
62
|
# Authorizes the request.
|
63
63
|
def authenticate!
|
64
|
+
$logger.debug('authenticating')
|
64
65
|
# NOOP if we've already authenticated
|
65
66
|
return nil if ! access_token.nil?
|
66
67
|
|
@@ -81,8 +82,10 @@ class PlatformApiClient
|
|
81
82
|
end
|
82
83
|
|
83
84
|
if response.code == '200'
|
84
|
-
access_token = JSON.parse(response.body)["access_token"]
|
85
|
+
self.access_token = JSON.parse(response.body)["access_token"]
|
86
|
+
$logger.debug('got token')
|
85
87
|
else
|
88
|
+
$logger.debug('no token')
|
86
89
|
nil
|
87
90
|
end
|
88
91
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nypl_ruby_util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Appel
|
@@ -141,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
141
141
|
- !ruby/object:Gem::Version
|
142
142
|
version: '0'
|
143
143
|
requirements: []
|
144
|
-
rubygems_version: 3.
|
144
|
+
rubygems_version: 3.1.4
|
145
145
|
signing_key:
|
146
146
|
specification_version: 4
|
147
147
|
summary: A repository of common utilities for NYPL Ruby application
|