nypl_ruby_util 0.0.8 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d2086b8b4b5bef52f3459a1245af09a7364cb35190ec40c0541a7bd72681a8f6
4
- data.tar.gz: 4e1c6a34f318b81c6659cca2f57f1930bfbf0e406040e950f681c4631177bec3
3
+ metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
4
+ data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
5
5
  SHA512:
6
- metadata.gz: 68670e8961ed473bb3854b2320cafc0c960b9418b37d336fdaac6f9cf41ae14acca33d131565986fad25f2d8eb6cb57f225828c93e5539c685bf497b80e6bab6
7
- data.tar.gz: 433b9dea26fa844a28a6711eff1d510a8f21df2f3f71e011d76e497f77dd2988714162973345084b681bb068771bc95433957f513e26553243cefbad521c70b3
6
+ metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
7
+ data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a
@@ -5,6 +5,7 @@ require_relative "errors"
5
5
  # Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
6
6
 
7
7
  class KinesisClient
8
+ #note custom defined :failed_records method
8
9
  attr_reader :config, :avro
9
10
 
10
11
  def initialize(config)
@@ -12,10 +13,10 @@ class KinesisClient
12
13
  @stream_name = @config[:stream_name]
13
14
  @avro = nil
14
15
  @batch_size = @config[:batch_size] || 1
15
- @batch_count = 0
16
+ @client_options = set_config(config)
16
17
  @records = []
18
+ @failed_records = []
17
19
  @automatically_push = !(@config[:automatically_push] == false)
18
- @client_options = config[:profile] ? { profile: config[:profile] } : {}
19
20
  @client = Aws::Kinesis::Client.new(@client_options)
20
21
 
21
22
  @avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
@@ -23,6 +24,16 @@ class KinesisClient
23
24
  @shovel_method = @batch_size > 1 ? :push_to_records : :push_record
24
25
  end
25
26
 
27
+ def set_config(config)
28
+ if config[:profile]
29
+ { profile: config[:profile] }
30
+ elsif config[:custom_aws_config]
31
+ config[:custom_aws_config]
32
+ else
33
+ {}
34
+ end
35
+ end
36
+
26
37
  def convert_to_record(json_message)
27
38
  if config[:schema_string]
28
39
  message = avro.encode(json_message, false)
@@ -41,6 +52,8 @@ class KinesisClient
41
52
  send(@shovel_method, json_message)
42
53
  end
43
54
 
55
+ #This method is broken
56
+ #TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
44
57
  def push_record(json_message)
45
58
  record = convert_to_record(json_message)
46
59
  record[:stream_name] = @stream_name
@@ -74,34 +87,39 @@ class KinesisClient
74
87
  records: batch.to_a,
75
88
  stream_name: @stream_name
76
89
  })
77
-
78
- $logger.debug("Received #{resp} from #{@stream_name}")
79
-
80
- if resp.failed_record_count > 0
81
- return_message = {
82
- failures: resp.failed_record_count,
83
- failures_data: filter_failures(resp)
84
- }
85
- $logger.warn("Message sent to #{config[:stream_name]} #{return_message}") if $logger
90
+ if resp.failed_record_count > 0
91
+ failures = filter_failures(resp, batch)
92
+ $logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
93
+ failures.each{|failure| @failed_records << failure[:record]}
86
94
  else
87
- $logger.info("Message sent to #{config[:stream_name]} successfully") if $logger
95
+ $logger.info("Batch sent to #{config[:stream_name]} successfully")
88
96
  end
89
97
  end
90
98
 
91
99
  def push_records
92
- if @records.length > 0
100
+ if @records.length > 0
93
101
  @records.each_slice(@batch_size) do |slice|
94
102
  push_batch(slice)
95
- @batch_count += 1
96
103
  end
97
104
  @records = []
98
- @batch_count = 0
99
105
  end
100
106
  end
101
107
 
102
- def filter_failures(resp)
108
+ def filter_failures(resp, batch)
103
109
  resp.records.filter_map.with_index do |record, i|
104
- avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
110
+ { record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
105
111
  end
106
112
  end
113
+
114
+ def retry_failed_records
115
+ unless @failed_records.empty?
116
+ @records = @failed_records
117
+ @failed_records = []
118
+ push_records
119
+ end
120
+ end
121
+
122
+ def failed_records
123
+ @failed_records.map { |record| avro.decode(record) }
124
+ end
107
125
  end
@@ -21,7 +21,7 @@ class PlatformApiClient
21
21
  @error_options = default_errors.merge(options[:errors] || {})
22
22
  end
23
23
 
24
- def get (path)
24
+ def get (path, transaction_data = {})
25
25
 
26
26
  authenticate! if authenticated
27
27
 
@@ -38,7 +38,7 @@ class PlatformApiClient
38
38
 
39
39
  $logger.debug "Got platform api response", { code: response.code, body: response.body }
40
40
 
41
- parse_json_response response, path
41
+ parse_json_response response, path, transaction_data
42
42
 
43
43
  rescue Exception => e
44
44
  raise StandardError.new(e), "Failed to retrieve #{path} #{e.message}"
@@ -47,12 +47,12 @@ class PlatformApiClient
47
47
 
48
48
  private
49
49
 
50
- def parse_json_response (response, path)
50
+ def parse_json_response (response, path, transaction_data = {})
51
51
  code = response.code.to_i
52
52
  if code < 400
53
53
  JSON.parse(response.body)
54
54
  elsif error_options[code]
55
- instance_exec(response, path, &error_options[code])
55
+ instance_exec(response, path, transaction_data, &error_options[code])
56
56
  else
57
57
  raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
58
58
  {}
@@ -92,12 +92,13 @@ class PlatformApiClient
92
92
 
93
93
  def default_errors
94
94
  {
95
- 401 => lambda do |response, path|
96
- if @try_count < 1
95
+ 401 => lambda do |response, path, transaction_data = {}|
96
+ transaction_data[:try_count] ||= 0
97
+ if transaction_data[:try_count] < 1
97
98
  # Likely an expired access-token; Wipe it for next run
98
- @try_count += 1
99
- access_token = nil
100
- get(path)
99
+ transaction_data[:try_count] += 1
100
+ self.access_token = nil
101
+ get(path, transaction_data)
101
102
  else
102
103
  raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
103
104
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nypl_ruby_util
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Appel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-08 00:00:00.000000000 Z
11
+ date: 2022-03-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro