nypl_ruby_util 0.0.10 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kinesis_client.rb +20 -11
- metadata +6 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
|
|
4
|
+
data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
|
|
7
|
+
data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a
|
data/lib/kinesis_client.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "errors"
|
|
|
5
5
|
# Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
|
|
6
6
|
|
|
7
7
|
class KinesisClient
|
|
8
|
+
#note custom defined :failed_records method
|
|
8
9
|
attr_reader :config, :avro
|
|
9
10
|
|
|
10
11
|
def initialize(config)
|
|
@@ -13,8 +14,8 @@ class KinesisClient
|
|
|
13
14
|
@avro = nil
|
|
14
15
|
@batch_size = @config[:batch_size] || 1
|
|
15
16
|
@client_options = set_config(config)
|
|
16
|
-
@batch_count = 0
|
|
17
17
|
@records = []
|
|
18
|
+
@failed_records = []
|
|
18
19
|
@automatically_push = !(@config[:automatically_push] == false)
|
|
19
20
|
@client = Aws::Kinesis::Client.new(@client_options)
|
|
20
21
|
|
|
@@ -52,6 +53,7 @@ class KinesisClient
|
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
#This method is broken
|
|
56
|
+
#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
|
|
55
57
|
def push_record(json_message)
|
|
56
58
|
record = convert_to_record(json_message)
|
|
57
59
|
record[:stream_name] = @stream_name
|
|
@@ -85,13 +87,10 @@ class KinesisClient
|
|
|
85
87
|
records: batch.to_a,
|
|
86
88
|
stream_name: @stream_name
|
|
87
89
|
})
|
|
88
|
-
|
|
89
90
|
if resp.failed_record_count > 0
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
}
|
|
94
|
-
$logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
|
|
91
|
+
failures = filter_failures(resp, batch)
|
|
92
|
+
$logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
|
|
93
|
+
failures.each{|failure| @failed_records << failure[:record]}
|
|
95
94
|
else
|
|
96
95
|
$logger.info("Batch sent to #{config[:stream_name]} successfully")
|
|
97
96
|
end
|
|
@@ -101,16 +100,26 @@ class KinesisClient
|
|
|
101
100
|
if @records.length > 0
|
|
102
101
|
@records.each_slice(@batch_size) do |slice|
|
|
103
102
|
push_batch(slice)
|
|
104
|
-
@batch_count += 1
|
|
105
103
|
end
|
|
106
104
|
@records = []
|
|
107
|
-
@batch_count = 0
|
|
108
105
|
end
|
|
109
106
|
end
|
|
110
107
|
|
|
111
|
-
def filter_failures(resp)
|
|
108
|
+
def filter_failures(resp, batch)
|
|
112
109
|
resp.records.filter_map.with_index do |record, i|
|
|
113
|
-
|
|
110
|
+
{ record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def retry_failed_records
|
|
115
|
+
unless @failed_records.empty?
|
|
116
|
+
@records = @failed_records
|
|
117
|
+
@failed_records = []
|
|
118
|
+
push_records
|
|
114
119
|
end
|
|
115
120
|
end
|
|
121
|
+
|
|
122
|
+
def failed_records
|
|
123
|
+
@failed_records.map { |record| avro.decode(record) }
|
|
124
|
+
end
|
|
116
125
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nypl_ruby_util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Appel
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-03-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: avro
|
|
@@ -109,7 +109,7 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: 1.0.3
|
|
111
111
|
description: A repository of common utilities for NYPL Ruby application
|
|
112
|
-
email:
|
|
112
|
+
email:
|
|
113
113
|
executables: []
|
|
114
114
|
extensions: []
|
|
115
115
|
extra_rdoc_files: []
|
|
@@ -126,7 +126,7 @@ homepage: https://github.com/NYPL/NYPLRubyUtil
|
|
|
126
126
|
licenses:
|
|
127
127
|
- MIT
|
|
128
128
|
metadata: {}
|
|
129
|
-
post_install_message:
|
|
129
|
+
post_install_message:
|
|
130
130
|
rdoc_options: []
|
|
131
131
|
require_paths:
|
|
132
132
|
- lib
|
|
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
142
142
|
version: '0'
|
|
143
143
|
requirements: []
|
|
144
144
|
rubygems_version: 3.1.2
|
|
145
|
-
signing_key:
|
|
145
|
+
signing_key:
|
|
146
146
|
specification_version: 4
|
|
147
147
|
summary: A repository of common utilities for NYPL Ruby application
|
|
148
148
|
test_files: []
|