nypl_ruby_util 0.0.3 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c6c6483ada755b5092548e42bd0bc6f699af6c642d8863e1488fabb78345df59
4
- data.tar.gz: 343534e1ca8672cb04552aa958a22e98211bfb00fb8996698b28e2ed78f97368
3
+ metadata.gz: 5ff3e8b400c7989c57e1398a642631a8c415a1bf982feadef5e95a813d7ad971
4
+ data.tar.gz: 7818da3d0d4fc85ddd6f10b259375463bc23a6f17b34a57026d60b4d1d03f057
5
5
  SHA512:
6
- metadata.gz: e3bdaa197c01456afae42a28dd4d6c8e95e4f9e7d402ebe3e54f4c4980f2daee7517b8be4c1fb19f62cea77f0a10b7bfafdcf03c04dfc1153f79ca576e5c7cc6
7
- data.tar.gz: 3a48402301714fa235c8ed865f99e1582aa19064791d099a2d47510cbcd99224852593d14aceb29a998643c37f97386cc94307beda6c336d5b4b6b31aac56c5b
6
+ metadata.gz: a2a0a99e281a95583fbe84f69fdbd066556f911167527be86dbdea2cffda000d3bb132edb6c90da4846e2b4d8129f7939594ee57b1a646f167955b2d5d1ed324
7
+ data.tar.gz: 0126f82acca76fd1e6c77afecce13a30c500766fd57f1ebecba075fcf85961c5c4b1c4e01d1641803cb23d86a9adadd21aed2f01d6e174f182b364e30dcc8fc6
@@ -0,0 +1,152 @@
1
+ require 'aws-sdk-lambda'
2
+ require 'aws-sdk-cloudwatchevents'
3
+ require 'yaml'
4
+
5
+ # Utility class for running rake methods
6
+ class DeployHelper
7
+ attr_reader(
8
+ :travis_branch,
9
+ :aws_access_key_id,
10
+ :aws_secret_access_key,
11
+ :aws_configuration,
12
+ :region,
13
+ :lambda_client,
14
+ :yaml,
15
+ :lambda_config,
16
+ :function_name,
17
+ :event
18
+ )
19
+
20
+ def initialize
21
+ @travis_branch = ENV['TRAVIS_BRANCH'].upcase
22
+ @travis_branch = ['MAIN', 'MASTER'].include?(@travis_branch) ? 'PRODUCTION' : @travis_branch
23
+ @aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
24
+ @aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
25
+ @yaml = YAML.safe_load(File.read('.travis.yml'))
26
+ @lambda_config = yaml['deploy'].find { |conf| name_matches_branch?(conf['function_name'], travis_branch) }
27
+ @region = @lambda_config['region']
28
+ @function_name = @lambda_config['function_name']
29
+ @aws_configuration = {
30
+ region: region,
31
+ access_key_id: aws_access_key_id,
32
+ secret_access_key: aws_secret_access_key
33
+ }
34
+ p 'using configuration: ', aws_configuration
35
+ p 'lambda config: ', lambda_config
36
+ @lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
37
+ end
38
+
39
+ def configured?
40
+ aws_access_key_id && aws_secret_access_key && region
41
+ end
42
+
43
+ def name_matches_branch?(name, branch)
44
+ downcase_name = name.downcase
45
+ downcase_branch = branch.downcase
46
+ variants = [
47
+ ['dev', 'development'],
48
+ ['qa'],
49
+ ['main', 'master', 'production', 'prod'],
50
+ ]
51
+ variants.any? do |group|
52
+ group.any? { |variant| downcase_name.include? variant }\
53
+ && group.any? { |variant| downcase_branch.include?(variant) }
54
+ end
55
+ end
56
+
57
+ def update_lambda_configuration
58
+ unless configured? && lambda_config
59
+ p 'insufficient configuration'
60
+ return nil
61
+ end
62
+
63
+ updated_lambda_configuration = {
64
+ function_name: function_name,
65
+ vpc_config: lambda_config['vpc_config'],
66
+ environment: lambda_config['environment'],
67
+ layers: lambda_config['layers']
68
+ }
69
+ updated_lambda_configuration[:function_name] = function_name
70
+ p 'updating_function_configuration with: ', updated_lambda_configuration
71
+ update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
72
+ p 'update_configuration_resp: ', update_configuration_resp
73
+ end
74
+
75
+ def update_event
76
+ unless lambda_config['event']
77
+ p 'no event config'
78
+ return nil
79
+ end
80
+
81
+ @event = lambda_config['event']
82
+ if event['event_source_arn']
83
+ add_event_source
84
+ elsif event['schedule_expression']
85
+ add_cron
86
+ end
87
+ end
88
+
89
+ def add_event_source
90
+ existing_events = lambda_client.list_event_source_mappings({
91
+ function_name: function_name
92
+ }).event_source_mappings
93
+
94
+ existing_events.each do |existing_event|
95
+ p 'deleting event with uuid: ', existing_event.uuid, 'and arn: ', existing_event.event_source_arn
96
+ lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
97
+ end
98
+ event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
99
+ event_to_create[:function_name] = function_name
100
+ p 'creating event: ', event_to_create
101
+ create_resp = lambda_client.create_event_source_mapping(event_to_create)
102
+ p 'created: ', create_resp
103
+ end
104
+
105
+ def add_cron
106
+ ## create the event
107
+ events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
108
+ schedule_expression = event['schedule_expression']
109
+ rule_name = "#{function_name}-rule"
110
+ p 'rule_name: ', rule_name, 'schedule_expression: ', schedule_expression
111
+ events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
112
+
113
+ ## next we have to connect the event to the lambda
114
+ ## the first step is to get the lambda
115
+
116
+ return 'missing function_name' unless function_name
117
+
118
+ p 'getting lambda with function name', function_name
119
+ lambda_resp = lambda_client.get_function(function_name: function_name).configuration
120
+ arn = lambda_resp.function_arn
121
+
122
+ ## next figure out if the lambda already has granted cloudwatch
123
+ ## permission to invoke
124
+ begin
125
+ policy_resp = lambda_client.get_policy(function_name: function_name)
126
+ unless policy_resp.policy.include?("#{function_name}-permission")
127
+ add_policy = true
128
+ else
129
+ p 'lambda already has permission'
130
+ end
131
+ rescue Aws::Lambda::Errors::ResourceNotFoundException
132
+ add_policy = true
133
+ p 'no policy'
134
+ end
135
+
136
+ ## if not, add permission to invoke
137
+ if add_policy
138
+ permission = lambda_client.add_permission({
139
+ function_name: function_name,
140
+ principal: 'events.amazonaws.com',
141
+ statement_id: "#{function_name}-permission",
142
+ action: 'lambda:InvokeFunction'
143
+ })
144
+ p 'permission: ', permission
145
+ end
146
+
147
+ ## finally we can tell the event to invoke the lambda
148
+ target_id = "#{function_name}-lambda"
149
+ p 'putting targets ', 'rule: ', rule_name, 'target_id: ', target_id, 'arn: ', arn
150
+ events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
151
+ end
152
+ end
@@ -9,40 +9,94 @@ class KinesisClient
9
9
 
10
10
  def initialize(config)
11
11
  @config = config
12
+ @stream_name = @config[:stream_name]
12
13
  @avro = nil
14
+ @batch_size = @config[:batch_size] || 1
15
+ @batch = []
16
+ @automatically_push = @config[:automatically_push] == false ? false : true
17
+ @client_options = config[:profile] ? { profile: config[:profile] } : {}
18
+ @client = Aws::Kinesis::Client.new @client_options
13
19
 
14
20
  if config[:schema_string]
15
21
  @avro = NYPLAvro.by_name(config[:schema_string])
16
22
  end
17
23
 
24
+ @shovel_method = @batch_size > 1 ? :push_to_batch : :push_record
25
+
18
26
  end
19
27
 
20
- def <<(json_message)
28
+ def convert_to_record(json_message)
21
29
  if config[:schema_string]
22
30
  message = avro.encode(json_message, false)
23
31
  else
24
32
  message = json_message
25
33
  end
26
34
 
27
- client = Aws::Kinesis::Client.new
28
35
  partition_key = (config[:partition_key] ? json_message[config[:partition_key]] : SecureRandom.hex(20)).hash.to_s
29
-
30
- resp = client.put_record({
31
- stream_name: config[:stream_name],
36
+ {
32
37
  data: message,
33
38
  partition_key: partition_key
34
- })
35
-
36
- return_hash = {}
37
-
38
- if resp.successful?
39
- return_hash["code"] = "200"
40
- return_hash["message"] = json_message, resp
41
- $logger.info("Message sent to HoldRequestResult #{json_message}, #{resp}") if $logger
42
- else
43
- $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
44
- raise NYPLError.new json_message, resp
45
- end
46
- return_hash
39
+ }
40
+
41
+ end
42
+
43
+ def <<(json_message)
44
+ send(@shovel_method, json_message)
45
+ end
46
+
47
+ def push_record(json_message)
48
+ record = convert_to_record json_message
49
+ record[:stream_name] = @stream_name
50
+
51
+ @client.put_record record
52
+
53
+ return_hash = {}
54
+
55
+ if resp.successful?
56
+ return_hash["code"] = "200"
57
+ return_hash["message"] = json_message, resp
58
+ $logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
59
+ else
60
+ $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
61
+ raise NYPLError.new json_message, resp
62
+ end
63
+ return_hash
64
+ end
65
+
66
+ def push_to_batch(json_message)
67
+ begin
68
+ @batch << convert_to_record(json_message)
69
+ rescue AvroError => e
70
+ $logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
71
+ end
72
+ if @automatically_push && @batch.length >= @batch_size
73
+ push_records
74
+ end
75
+ end
76
+
77
+ def push_batch(batch)
78
+ resp = @client.put_records({
79
+ records: batch.to_a,
80
+ stream_name: @stream_name
81
+ })
82
+
83
+ $logger.debug("Received #{resp} from #{@stream_name}")
84
+
85
+ return_message = {
86
+ failures: resp.failed_record_count,
87
+ error_messages: resp.records.map {|record| record.error_message }.compact
88
+ }
89
+
90
+ $logger.info("Message sent to #{config[:stream_name]} #{return_message}") if $logger
91
+
92
+ return {
93
+ "code": "200",
94
+ "message": return_message.to_json
95
+ }
96
+ end
97
+
98
+ def push_records
99
+ @batch.each_slice(@batch_size) {|slice| push_batch slice}
100
+ @batch = []
47
101
  end
48
102
  end
data/lib/nypl_avro.rb CHANGED
@@ -38,7 +38,7 @@ class NYPLAvro
38
38
  begin
39
39
  bin_encoder.write(decoded_data, encoder)
40
40
  rescue Avro::IO::AvroTypeError => e
41
- raise AvroError.new(e), "Error encoding data using #{@schema.name} schema due to #{e.message}"
41
+ raise AvroError.new(e), "Error encoding data #{decoded_data} using #{@schema.name} schema due to #{e.message}"
42
42
  end
43
43
 
44
44
  buffer.rewind
@@ -6,6 +6,7 @@ require_relative 'directory'
6
6
  require_relative 'nypl_avro'
7
7
  require_relative 'errors'
8
8
  require_relative 'kinesis_client'
9
+ require_relative 'deploy_helper'
9
10
 
10
11
  class NYPLRubyUtil
11
12
  class SierraApiClient < SierraApiClient
@@ -25,4 +26,7 @@ class NYPLRubyUtil
25
26
 
26
27
  class KinesisClient < KinesisClient
27
28
  end
29
+
30
+ class DeployHelper < DeployHelper
31
+ end
28
32
  end
@@ -61,6 +61,7 @@ class PlatformApiClient
61
61
 
62
62
  # Authorizes the request.
63
63
  def authenticate!
64
+ $logger.debug('authenticating')
64
65
  # NOOP if we've already authenticated
65
66
  return nil if ! access_token.nil?
66
67
 
@@ -81,8 +82,10 @@ class PlatformApiClient
81
82
  end
82
83
 
83
84
  if response.code == '200'
84
- access_token = JSON.parse(response.body)["access_token"]
85
+ self.access_token = JSON.parse(response.body)["access_token"]
86
+ $logger.debug('got token')
85
87
  else
88
+ $logger.debug('no token')
86
89
  nil
87
90
  end
88
91
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nypl_ruby_util
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Appel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-07-13 00:00:00.000000000 Z
11
+ date: 2020-09-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro
@@ -52,6 +52,34 @@ dependencies:
52
52
  - - '='
53
53
  - !ruby/object:Gem::Version
54
54
  version: 1.36.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: aws-sdk-lambda
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: aws-sdk-cloudwatchevents
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: nypl_log_formatter
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -86,6 +114,7 @@ executables: []
86
114
  extensions: []
87
115
  extra_rdoc_files: []
88
116
  files:
117
+ - lib/deploy_helper.rb
89
118
  - lib/directory.rb
90
119
  - lib/errors.rb
91
120
  - lib/kinesis_client.rb
@@ -93,7 +122,7 @@ files:
93
122
  - lib/nypl_avro.rb
94
123
  - lib/nypl_ruby_util.rb
95
124
  - lib/platform_api_client.rb
96
- homepage:
125
+ homepage: https://github.com/NYPL/NYPLRubyUtil
97
126
  licenses:
98
127
  - MIT
99
128
  metadata: {}
@@ -112,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
112
141
  - !ruby/object:Gem::Version
113
142
  version: '0'
114
143
  requirements: []
115
- rubygems_version: 3.1.2
144
+ rubygems_version: 3.2.3
116
145
  signing_key:
117
146
  specification_version: 4
118
147
  summary: A repository of common utilities for NYPL Ruby application