nypl_ruby_util 0.0.10 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kinesis_client.rb +20 -11
- metadata +6 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
|
|
4
|
+
data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
|
|
7
|
+
data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a
|
data/lib/kinesis_client.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "errors"
|
|
|
5
5
|
# Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
|
|
6
6
|
|
|
7
7
|
class KinesisClient
|
|
8
|
+
#note custom defined :failed_records method
|
|
8
9
|
attr_reader :config, :avro
|
|
9
10
|
|
|
10
11
|
def initialize(config)
|
|
@@ -13,8 +14,8 @@ class KinesisClient
|
|
|
13
14
|
@avro = nil
|
|
14
15
|
@batch_size = @config[:batch_size] || 1
|
|
15
16
|
@client_options = set_config(config)
|
|
16
|
-
@batch_count = 0
|
|
17
17
|
@records = []
|
|
18
|
+
@failed_records = []
|
|
18
19
|
@automatically_push = !(@config[:automatically_push] == false)
|
|
19
20
|
@client = Aws::Kinesis::Client.new(@client_options)
|
|
20
21
|
|
|
@@ -52,6 +53,7 @@ class KinesisClient
|
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
#This method is broken
|
|
56
|
+
#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
|
|
55
57
|
def push_record(json_message)
|
|
56
58
|
record = convert_to_record(json_message)
|
|
57
59
|
record[:stream_name] = @stream_name
|
|
@@ -85,13 +87,10 @@ class KinesisClient
|
|
|
85
87
|
records: batch.to_a,
|
|
86
88
|
stream_name: @stream_name
|
|
87
89
|
})
|
|
88
|
-
|
|
89
90
|
if resp.failed_record_count > 0
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
}
|
|
94
|
-
$logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
|
|
91
|
+
failures = filter_failures(resp, batch)
|
|
92
|
+
$logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
|
|
93
|
+
failures.each{|failure| @failed_records << failure[:record]}
|
|
95
94
|
else
|
|
96
95
|
$logger.info("Batch sent to #{config[:stream_name]} successfully")
|
|
97
96
|
end
|
|
@@ -101,16 +100,26 @@ class KinesisClient
|
|
|
101
100
|
if @records.length > 0
|
|
102
101
|
@records.each_slice(@batch_size) do |slice|
|
|
103
102
|
push_batch(slice)
|
|
104
|
-
@batch_count += 1
|
|
105
103
|
end
|
|
106
104
|
@records = []
|
|
107
|
-
@batch_count = 0
|
|
108
105
|
end
|
|
109
106
|
end
|
|
110
107
|
|
|
111
|
-
def filter_failures(resp)
|
|
108
|
+
def filter_failures(resp, batch)
|
|
112
109
|
resp.records.filter_map.with_index do |record, i|
|
|
113
|
-
|
|
110
|
+
{ record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def retry_failed_records
|
|
115
|
+
unless @failed_records.empty?
|
|
116
|
+
@records = @failed_records
|
|
117
|
+
@failed_records = []
|
|
118
|
+
push_records
|
|
114
119
|
end
|
|
115
120
|
end
|
|
121
|
+
|
|
122
|
+
def failed_records
|
|
123
|
+
@failed_records.map { |record| avro.decode(record) }
|
|
124
|
+
end
|
|
116
125
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nypl_ruby_util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Appel
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-03-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: avro
|
|
@@ -109,7 +109,7 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: 1.0.3
|
|
111
111
|
description: A repository of common utilities for NYPL Ruby application
|
|
112
|
-
email:
|
|
112
|
+
email:
|
|
113
113
|
executables: []
|
|
114
114
|
extensions: []
|
|
115
115
|
extra_rdoc_files: []
|
|
@@ -126,7 +126,7 @@ homepage: https://github.com/NYPL/NYPLRubyUtil
|
|
|
126
126
|
licenses:
|
|
127
127
|
- MIT
|
|
128
128
|
metadata: {}
|
|
129
|
-
post_install_message:
|
|
129
|
+
post_install_message:
|
|
130
130
|
rdoc_options: []
|
|
131
131
|
require_paths:
|
|
132
132
|
- lib
|
|
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
142
142
|
version: '0'
|
|
143
143
|
requirements: []
|
|
144
144
|
rubygems_version: 3.1.2
|
|
145
|
-
signing_key:
|
|
145
|
+
signing_key:
|
|
146
146
|
specification_version: 4
|
|
147
147
|
summary: A repository of common utilities for NYPL Ruby application
|
|
148
148
|
test_files: []
|