nypl_ruby_util 0.0.3 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/deploy_helper.rb +152 -0
- data/lib/kinesis_client.rb +72 -18
- data/lib/nypl_avro.rb +1 -1
- data/lib/nypl_ruby_util.rb +4 -0
- data/lib/platform_api_client.rb +4 -1
- metadata +33 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5ff3e8b400c7989c57e1398a642631a8c415a1bf982feadef5e95a813d7ad971
|
4
|
+
data.tar.gz: 7818da3d0d4fc85ddd6f10b259375463bc23a6f17b34a57026d60b4d1d03f057
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2a0a99e281a95583fbe84f69fdbd066556f911167527be86dbdea2cffda000d3bb132edb6c90da4846e2b4d8129f7939594ee57b1a646f167955b2d5d1ed324
|
7
|
+
data.tar.gz: 0126f82acca76fd1e6c77afecce13a30c500766fd57f1ebecba075fcf85961c5c4b1c4e01d1641803cb23d86a9adadd21aed2f01d6e174f182b364e30dcc8fc6
|
@@ -0,0 +1,152 @@
|
|
1
|
+
require 'aws-sdk-lambda'
|
2
|
+
require 'aws-sdk-cloudwatchevents'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
# Utility class for running rake methods
|
6
|
+
class DeployHelper
|
7
|
+
attr_reader(
|
8
|
+
:travis_branch,
|
9
|
+
:aws_access_key_id,
|
10
|
+
:aws_secret_access_key,
|
11
|
+
:aws_configuration,
|
12
|
+
:region,
|
13
|
+
:lambda_client,
|
14
|
+
:yaml,
|
15
|
+
:lambda_config,
|
16
|
+
:function_name,
|
17
|
+
:event
|
18
|
+
)
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@travis_branch = ENV['TRAVIS_BRANCH'].upcase
|
22
|
+
@travis_branch = ['MAIN', 'MASTER'].include?(@travis_branch) ? 'PRODUCTION' : @travis_branch
|
23
|
+
@aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
|
24
|
+
@aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
|
25
|
+
@yaml = YAML.safe_load(File.read('.travis.yml'))
|
26
|
+
@lambda_config = yaml['deploy'].find { |conf| name_matches_branch?(conf['function_name'], travis_branch) }
|
27
|
+
@region = @lambda_config['region']
|
28
|
+
@function_name = @lambda_config['function_name']
|
29
|
+
@aws_configuration = {
|
30
|
+
region: region,
|
31
|
+
access_key_id: aws_access_key_id,
|
32
|
+
secret_access_key: aws_secret_access_key
|
33
|
+
}
|
34
|
+
p 'using configuration: ', aws_configuration
|
35
|
+
p 'lambda config: ', lambda_config
|
36
|
+
@lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
|
37
|
+
end
|
38
|
+
|
39
|
+
def configured?
|
40
|
+
aws_access_key_id && aws_secret_access_key && region
|
41
|
+
end
|
42
|
+
|
43
|
+
def name_matches_branch?(name, branch)
|
44
|
+
downcase_name = name.downcase
|
45
|
+
downcase_branch = branch.downcase
|
46
|
+
variants = [
|
47
|
+
['dev', 'development'],
|
48
|
+
['qa'],
|
49
|
+
['main', 'master', 'production', 'prod'],
|
50
|
+
]
|
51
|
+
variants.any? do |group|
|
52
|
+
group.any? { |variant| downcase_name.include? variant }\
|
53
|
+
&& group.any? { |variant| downcase_branch.include?(variant) }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def update_lambda_configuration
|
58
|
+
unless configured? && lambda_config
|
59
|
+
p 'insufficient configuration'
|
60
|
+
return nil
|
61
|
+
end
|
62
|
+
|
63
|
+
updated_lambda_configuration = {
|
64
|
+
function_name: function_name,
|
65
|
+
vpc_config: lambda_config['vpc_config'],
|
66
|
+
environment: lambda_config['environment'],
|
67
|
+
layers: lambda_config['layers']
|
68
|
+
}
|
69
|
+
updated_lambda_configuration[:function_name] = function_name
|
70
|
+
p 'updating_function_configuration with: ', updated_lambda_configuration
|
71
|
+
update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
|
72
|
+
p 'update_configuration_resp: ', update_configuration_resp
|
73
|
+
end
|
74
|
+
|
75
|
+
def update_event
|
76
|
+
unless lambda_config['event']
|
77
|
+
p 'no event config'
|
78
|
+
return nil
|
79
|
+
end
|
80
|
+
|
81
|
+
@event = lambda_config['event']
|
82
|
+
if event['event_source_arn']
|
83
|
+
add_event_source
|
84
|
+
elsif event['schedule_expression']
|
85
|
+
add_cron
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def add_event_source
|
90
|
+
existing_events = lambda_client.list_event_source_mappings({
|
91
|
+
function_name: function_name
|
92
|
+
}).event_source_mappings
|
93
|
+
|
94
|
+
existing_events.each do |existing_event|
|
95
|
+
p 'deleting event with uuid: ', existing_event.uuid, 'and arn: ', existing_event.event_source_arn
|
96
|
+
lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
|
97
|
+
end
|
98
|
+
event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
|
99
|
+
event_to_create[:function_name] = function_name
|
100
|
+
p 'creating event: ', event_to_create
|
101
|
+
create_resp = lambda_client.create_event_source_mapping(event_to_create)
|
102
|
+
p 'created: ', create_resp
|
103
|
+
end
|
104
|
+
|
105
|
+
def add_cron
|
106
|
+
## create the event
|
107
|
+
events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
|
108
|
+
schedule_expression = event['schedule_expression']
|
109
|
+
rule_name = "#{function_name}-rule"
|
110
|
+
p 'rule_name: ', rule_name, 'schedule_expression: ', schedule_expression
|
111
|
+
events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
|
112
|
+
|
113
|
+
## next we have to connect the event to the lambda
|
114
|
+
## the first step is to get the lambda
|
115
|
+
|
116
|
+
return 'missing function_name' unless function_name
|
117
|
+
|
118
|
+
p 'getting lambda with function name', function_name
|
119
|
+
lambda_resp = lambda_client.get_function(function_name: function_name).configuration
|
120
|
+
arn = lambda_resp.function_arn
|
121
|
+
|
122
|
+
## next figure out if the lambda already has granted cloudwatch
|
123
|
+
## permission to invoke
|
124
|
+
begin
|
125
|
+
policy_resp = lambda_client.get_policy(function_name: function_name)
|
126
|
+
unless policy_resp.policy.include?("#{function_name}-permission")
|
127
|
+
add_policy = true
|
128
|
+
else
|
129
|
+
p 'lambda already has permission'
|
130
|
+
end
|
131
|
+
rescue Aws::Lambda::Errors::ResourceNotFoundException
|
132
|
+
add_policy = true
|
133
|
+
p 'no policy'
|
134
|
+
end
|
135
|
+
|
136
|
+
## if not, add permission to invoke
|
137
|
+
if add_policy
|
138
|
+
permission = lambda_client.add_permission({
|
139
|
+
function_name: function_name,
|
140
|
+
principal: 'events.amazonaws.com',
|
141
|
+
statement_id: "#{function_name}-permission",
|
142
|
+
action: 'lambda:InvokeFunction'
|
143
|
+
})
|
144
|
+
p 'permission: ', permission
|
145
|
+
end
|
146
|
+
|
147
|
+
## finally we can tell the event to invoke the lambda
|
148
|
+
target_id = "#{function_name}-lambda"
|
149
|
+
p 'putting targets ', 'rule: ', rule_name, 'target_id: ', target_id, 'arn: ', arn
|
150
|
+
events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
|
151
|
+
end
|
152
|
+
end
|
data/lib/kinesis_client.rb
CHANGED
@@ -9,40 +9,94 @@ class KinesisClient
|
|
9
9
|
|
10
10
|
def initialize(config)
|
11
11
|
@config = config
|
12
|
+
@stream_name = @config[:stream_name]
|
12
13
|
@avro = nil
|
14
|
+
@batch_size = @config[:batch_size] || 1
|
15
|
+
@batch = []
|
16
|
+
@automatically_push = @config[:automatically_push] == false ? false : true
|
17
|
+
@client_options = config[:profile] ? { profile: config[:profile] } : {}
|
18
|
+
@client = Aws::Kinesis::Client.new @client_options
|
13
19
|
|
14
20
|
if config[:schema_string]
|
15
21
|
@avro = NYPLAvro.by_name(config[:schema_string])
|
16
22
|
end
|
17
23
|
|
24
|
+
@shovel_method = @batch_size > 1 ? :push_to_batch : :push_record
|
25
|
+
|
18
26
|
end
|
19
27
|
|
20
|
-
def
|
28
|
+
def convert_to_record(json_message)
|
21
29
|
if config[:schema_string]
|
22
30
|
message = avro.encode(json_message, false)
|
23
31
|
else
|
24
32
|
message = json_message
|
25
33
|
end
|
26
34
|
|
27
|
-
client = Aws::Kinesis::Client.new
|
28
35
|
partition_key = (config[:partition_key] ? json_message[config[:partition_key]] : SecureRandom.hex(20)).hash.to_s
|
29
|
-
|
30
|
-
resp = client.put_record({
|
31
|
-
stream_name: config[:stream_name],
|
36
|
+
{
|
32
37
|
data: message,
|
33
38
|
partition_key: partition_key
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
39
|
+
}
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
def <<(json_message)
|
44
|
+
send(@shovel_method, json_message)
|
45
|
+
end
|
46
|
+
|
47
|
+
def push_record(json_message)
|
48
|
+
record = convert_to_record json_message
|
49
|
+
record[:stream_name] = @stream_name
|
50
|
+
|
51
|
+
@client.put_record record
|
52
|
+
|
53
|
+
return_hash = {}
|
54
|
+
|
55
|
+
if resp.successful?
|
56
|
+
return_hash["code"] = "200"
|
57
|
+
return_hash["message"] = json_message, resp
|
58
|
+
$logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
|
59
|
+
else
|
60
|
+
$logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
|
61
|
+
raise NYPLError.new json_message, resp
|
62
|
+
end
|
63
|
+
return_hash
|
64
|
+
end
|
65
|
+
|
66
|
+
def push_to_batch(json_message)
|
67
|
+
begin
|
68
|
+
@batch << convert_to_record(json_message)
|
69
|
+
rescue AvroError => e
|
70
|
+
$logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
|
71
|
+
end
|
72
|
+
if @automatically_push && @batch.length >= @batch_size
|
73
|
+
push_records
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def push_batch(batch)
|
78
|
+
resp = @client.put_records({
|
79
|
+
records: batch.to_a,
|
80
|
+
stream_name: @stream_name
|
81
|
+
})
|
82
|
+
|
83
|
+
$logger.debug("Received #{resp} from #{@stream_name}")
|
84
|
+
|
85
|
+
return_message = {
|
86
|
+
failures: resp.failed_record_count,
|
87
|
+
error_messages: resp.records.map {|record| record.error_message }.compact
|
88
|
+
}
|
89
|
+
|
90
|
+
$logger.info("Message sent to #{config[:stream_name]} #{return_message}") if $logger
|
91
|
+
|
92
|
+
return {
|
93
|
+
"code": "200",
|
94
|
+
"message": return_message.to_json
|
95
|
+
}
|
96
|
+
end
|
97
|
+
|
98
|
+
def push_records
|
99
|
+
@batch.each_slice(@batch_size) {|slice| push_batch slice}
|
100
|
+
@batch = []
|
47
101
|
end
|
48
102
|
end
|
data/lib/nypl_avro.rb
CHANGED
@@ -38,7 +38,7 @@ class NYPLAvro
|
|
38
38
|
begin
|
39
39
|
bin_encoder.write(decoded_data, encoder)
|
40
40
|
rescue Avro::IO::AvroTypeError => e
|
41
|
-
raise AvroError.new(e), "Error encoding data using #{@schema.name} schema due to #{e.message}"
|
41
|
+
raise AvroError.new(e), "Error encoding data #{decoded_data} using #{@schema.name} schema due to #{e.message}"
|
42
42
|
end
|
43
43
|
|
44
44
|
buffer.rewind
|
data/lib/nypl_ruby_util.rb
CHANGED
@@ -6,6 +6,7 @@ require_relative 'directory'
|
|
6
6
|
require_relative 'nypl_avro'
|
7
7
|
require_relative 'errors'
|
8
8
|
require_relative 'kinesis_client'
|
9
|
+
require_relative 'deploy_helper'
|
9
10
|
|
10
11
|
class NYPLRubyUtil
|
11
12
|
class SierraApiClient < SierraApiClient
|
@@ -25,4 +26,7 @@ class NYPLRubyUtil
|
|
25
26
|
|
26
27
|
class KinesisClient < KinesisClient
|
27
28
|
end
|
29
|
+
|
30
|
+
class DeployHelper < DeployHelper
|
31
|
+
end
|
28
32
|
end
|
data/lib/platform_api_client.rb
CHANGED
@@ -61,6 +61,7 @@ class PlatformApiClient
|
|
61
61
|
|
62
62
|
# Authorizes the request.
|
63
63
|
def authenticate!
|
64
|
+
$logger.debug('authenticating')
|
64
65
|
# NOOP if we've already authenticated
|
65
66
|
return nil if ! access_token.nil?
|
66
67
|
|
@@ -81,8 +82,10 @@ class PlatformApiClient
|
|
81
82
|
end
|
82
83
|
|
83
84
|
if response.code == '200'
|
84
|
-
access_token = JSON.parse(response.body)["access_token"]
|
85
|
+
self.access_token = JSON.parse(response.body)["access_token"]
|
86
|
+
$logger.debug('got token')
|
85
87
|
else
|
88
|
+
$logger.debug('no token')
|
86
89
|
nil
|
87
90
|
end
|
88
91
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nypl_ruby_util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Appel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -52,6 +52,34 @@ dependencies:
|
|
52
52
|
- - '='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 1.36.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk-lambda
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: aws-sdk-cloudwatchevents
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
55
83
|
- !ruby/object:Gem::Dependency
|
56
84
|
name: nypl_log_formatter
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,6 +114,7 @@ executables: []
|
|
86
114
|
extensions: []
|
87
115
|
extra_rdoc_files: []
|
88
116
|
files:
|
117
|
+
- lib/deploy_helper.rb
|
89
118
|
- lib/directory.rb
|
90
119
|
- lib/errors.rb
|
91
120
|
- lib/kinesis_client.rb
|
@@ -93,7 +122,7 @@ files:
|
|
93
122
|
- lib/nypl_avro.rb
|
94
123
|
- lib/nypl_ruby_util.rb
|
95
124
|
- lib/platform_api_client.rb
|
96
|
-
homepage:
|
125
|
+
homepage: https://github.com/NYPL/NYPLRubyUtil
|
97
126
|
licenses:
|
98
127
|
- MIT
|
99
128
|
metadata: {}
|
@@ -112,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
141
|
- !ruby/object:Gem::Version
|
113
142
|
version: '0'
|
114
143
|
requirements: []
|
115
|
-
rubygems_version: 3.
|
144
|
+
rubygems_version: 3.2.3
|
116
145
|
signing_key:
|
117
146
|
specification_version: 4
|
118
147
|
summary: A repository of common utilities for NYPL Ruby application
|