nypl_ruby_util 0.0.8 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kinesis_client.rb +35 -17
- data/lib/platform_api_client.rb +10 -9
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
|
4
|
+
data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
|
7
|
+
data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a
|
data/lib/kinesis_client.rb
CHANGED
@@ -5,6 +5,7 @@ require_relative "errors"
|
|
5
5
|
# Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
|
6
6
|
|
7
7
|
class KinesisClient
|
8
|
+
#note custom defined :failed_records method
|
8
9
|
attr_reader :config, :avro
|
9
10
|
|
10
11
|
def initialize(config)
|
@@ -12,10 +13,10 @@ class KinesisClient
|
|
12
13
|
@stream_name = @config[:stream_name]
|
13
14
|
@avro = nil
|
14
15
|
@batch_size = @config[:batch_size] || 1
|
15
|
-
@
|
16
|
+
@client_options = set_config(config)
|
16
17
|
@records = []
|
18
|
+
@failed_records = []
|
17
19
|
@automatically_push = !(@config[:automatically_push] == false)
|
18
|
-
@client_options = config[:profile] ? { profile: config[:profile] } : {}
|
19
20
|
@client = Aws::Kinesis::Client.new(@client_options)
|
20
21
|
|
21
22
|
@avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
|
@@ -23,6 +24,16 @@ class KinesisClient
|
|
23
24
|
@shovel_method = @batch_size > 1 ? :push_to_records : :push_record
|
24
25
|
end
|
25
26
|
|
27
|
+
def set_config(config)
|
28
|
+
if config[:profile]
|
29
|
+
{ profile: config[:profile] }
|
30
|
+
elsif config[:custom_aws_config]
|
31
|
+
config[:custom_aws_config]
|
32
|
+
else
|
33
|
+
{}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
26
37
|
def convert_to_record(json_message)
|
27
38
|
if config[:schema_string]
|
28
39
|
message = avro.encode(json_message, false)
|
@@ -41,6 +52,8 @@ class KinesisClient
|
|
41
52
|
send(@shovel_method, json_message)
|
42
53
|
end
|
43
54
|
|
55
|
+
#This method is broken
|
56
|
+
#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
|
44
57
|
def push_record(json_message)
|
45
58
|
record = convert_to_record(json_message)
|
46
59
|
record[:stream_name] = @stream_name
|
@@ -74,34 +87,39 @@ class KinesisClient
|
|
74
87
|
records: batch.to_a,
|
75
88
|
stream_name: @stream_name
|
76
89
|
})
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
return_message = {
|
82
|
-
failures: resp.failed_record_count,
|
83
|
-
failures_data: filter_failures(resp)
|
84
|
-
}
|
85
|
-
$logger.warn("Message sent to #{config[:stream_name]} #{return_message}") if $logger
|
90
|
+
if resp.failed_record_count > 0
|
91
|
+
failures = filter_failures(resp, batch)
|
92
|
+
$logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
|
93
|
+
failures.each{|failure| @failed_records << failure[:record]}
|
86
94
|
else
|
87
|
-
$logger.info("
|
95
|
+
$logger.info("Batch sent to #{config[:stream_name]} successfully")
|
88
96
|
end
|
89
97
|
end
|
90
98
|
|
91
99
|
def push_records
|
92
|
-
if @records.length > 0
|
100
|
+
if @records.length > 0
|
93
101
|
@records.each_slice(@batch_size) do |slice|
|
94
102
|
push_batch(slice)
|
95
|
-
@batch_count += 1
|
96
103
|
end
|
97
104
|
@records = []
|
98
|
-
@batch_count = 0
|
99
105
|
end
|
100
106
|
end
|
101
107
|
|
102
|
-
def filter_failures(resp)
|
108
|
+
def filter_failures(resp, batch)
|
103
109
|
resp.records.filter_map.with_index do |record, i|
|
104
|
-
|
110
|
+
{ record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
|
105
111
|
end
|
106
112
|
end
|
113
|
+
|
114
|
+
def retry_failed_records
|
115
|
+
unless @failed_records.empty?
|
116
|
+
@records = @failed_records
|
117
|
+
@failed_records = []
|
118
|
+
push_records
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def failed_records
|
123
|
+
@failed_records.map { |record| avro.decode(record) }
|
124
|
+
end
|
107
125
|
end
|
data/lib/platform_api_client.rb
CHANGED
@@ -21,7 +21,7 @@ class PlatformApiClient
|
|
21
21
|
@error_options = default_errors.merge(options[:errors] || {})
|
22
22
|
end
|
23
23
|
|
24
|
-
def get (path)
|
24
|
+
def get (path, transaction_data = {})
|
25
25
|
|
26
26
|
authenticate! if authenticated
|
27
27
|
|
@@ -38,7 +38,7 @@ class PlatformApiClient
|
|
38
38
|
|
39
39
|
$logger.debug "Got platform api response", { code: response.code, body: response.body }
|
40
40
|
|
41
|
-
parse_json_response response, path
|
41
|
+
parse_json_response response, path, transaction_data
|
42
42
|
|
43
43
|
rescue Exception => e
|
44
44
|
raise StandardError.new(e), "Failed to retrieve #{path} #{e.message}"
|
@@ -47,12 +47,12 @@ class PlatformApiClient
|
|
47
47
|
|
48
48
|
private
|
49
49
|
|
50
|
-
def parse_json_response (response, path)
|
50
|
+
def parse_json_response (response, path, transaction_data = {})
|
51
51
|
code = response.code.to_i
|
52
52
|
if code < 400
|
53
53
|
JSON.parse(response.body)
|
54
54
|
elsif error_options[code]
|
55
|
-
instance_exec(response, path, &error_options[code])
|
55
|
+
instance_exec(response, path, transaction_data, &error_options[code])
|
56
56
|
else
|
57
57
|
raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
|
58
58
|
{}
|
@@ -92,12 +92,13 @@ class PlatformApiClient
|
|
92
92
|
|
93
93
|
def default_errors
|
94
94
|
{
|
95
|
-
401 => lambda do |response, path|
|
96
|
-
|
95
|
+
401 => lambda do |response, path, transaction_data = {}|
|
96
|
+
transaction_data[:try_count] ||= 0
|
97
|
+
if transaction_data[:try_count] < 1
|
97
98
|
# Likely an expired access-token; Wipe it for next run
|
98
|
-
|
99
|
-
access_token = nil
|
100
|
-
get(path)
|
99
|
+
transaction_data[:try_count] += 1
|
100
|
+
self.access_token = nil
|
101
|
+
get(path, transaction_data)
|
101
102
|
else
|
102
103
|
raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
|
103
104
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nypl_ruby_util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Appel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|