nypl_ruby_util 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kinesis_client.rb +17 -8
- metadata +6 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 05c91cd9d8cd4ffd258f3a2c1f9b3cb41c2c6cf9b5df78d21f1483510e34c544
|
|
4
|
+
data.tar.gz: 55e61c0097b9721728b147b6201e0856e815c3de8631e3f41ed78b00c12d11bb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6c2042cc492affeb2ae6d60d6c96fb6d0db3ea69c23f1ad6a31da3ff5025c52cd4d5f669112e174153e88b84dde7395f59b23b12182017c079f1ce6a024807f6
|
|
7
|
+
data.tar.gz: 832f11059c1267393afb6b2f5140dd5a8f98950f9e69c495e6fa1cfa31af947c78fbf4cd740016214b4fe1dc4bce939a6f57de295039a3673aa195da0ce2be2c
|
data/lib/kinesis_client.rb
CHANGED
|
@@ -12,10 +12,10 @@ class KinesisClient
|
|
|
12
12
|
@stream_name = @config[:stream_name]
|
|
13
13
|
@avro = nil
|
|
14
14
|
@batch_size = @config[:batch_size] || 1
|
|
15
|
+
@client_options = set_config(config)
|
|
15
16
|
@batch_count = 0
|
|
16
17
|
@records = []
|
|
17
18
|
@automatically_push = !(@config[:automatically_push] == false)
|
|
18
|
-
@client_options = config[:profile] ? { profile: config[:profile] } : {}
|
|
19
19
|
@client = Aws::Kinesis::Client.new(@client_options)
|
|
20
20
|
|
|
21
21
|
@avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
|
|
@@ -23,6 +23,16 @@ class KinesisClient
|
|
|
23
23
|
@shovel_method = @batch_size > 1 ? :push_to_records : :push_record
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
def set_config(config)
|
|
27
|
+
if config[:profile]
|
|
28
|
+
{ profile: config[:profile] }
|
|
29
|
+
elsif config[:custom_aws_config]
|
|
30
|
+
config[:custom_aws_config]
|
|
31
|
+
else
|
|
32
|
+
{}
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
26
36
|
def convert_to_record(json_message)
|
|
27
37
|
if config[:schema_string]
|
|
28
38
|
message = avro.encode(json_message, false)
|
|
@@ -41,6 +51,7 @@ class KinesisClient
|
|
|
41
51
|
send(@shovel_method, json_message)
|
|
42
52
|
end
|
|
43
53
|
|
|
54
|
+
#This method is broken
|
|
44
55
|
def push_record(json_message)
|
|
45
56
|
record = convert_to_record(json_message)
|
|
46
57
|
record[:stream_name] = @stream_name
|
|
@@ -75,21 +86,19 @@ class KinesisClient
|
|
|
75
86
|
stream_name: @stream_name
|
|
76
87
|
})
|
|
77
88
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
if resp.failed_record_count > 0
|
|
81
|
-
return_message = {
|
|
89
|
+
if resp.failed_record_count > 0
|
|
90
|
+
failure_message = {
|
|
82
91
|
failures: resp.failed_record_count,
|
|
83
92
|
failures_data: filter_failures(resp)
|
|
84
93
|
}
|
|
85
|
-
$logger.warn("
|
|
94
|
+
$logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
|
|
86
95
|
else
|
|
87
|
-
$logger.info("
|
|
96
|
+
$logger.info("Batch sent to #{config[:stream_name]} successfully")
|
|
88
97
|
end
|
|
89
98
|
end
|
|
90
99
|
|
|
91
100
|
def push_records
|
|
92
|
-
if @records.length > 0
|
|
101
|
+
if @records.length > 0
|
|
93
102
|
@records.each_slice(@batch_size) do |slice|
|
|
94
103
|
push_batch(slice)
|
|
95
104
|
@batch_count += 1
|
metadata
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nypl_ruby_util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Appel
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
11
|
date: 2020-09-08 00:00:00.000000000 Z
|
|
@@ -109,7 +109,7 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: 1.0.3
|
|
111
111
|
description: A repository of common utilities for NYPL Ruby application
|
|
112
|
-
email:
|
|
112
|
+
email:
|
|
113
113
|
executables: []
|
|
114
114
|
extensions: []
|
|
115
115
|
extra_rdoc_files: []
|
|
@@ -126,7 +126,7 @@ homepage: https://github.com/NYPL/NYPLRubyUtil
|
|
|
126
126
|
licenses:
|
|
127
127
|
- MIT
|
|
128
128
|
metadata: {}
|
|
129
|
-
post_install_message:
|
|
129
|
+
post_install_message:
|
|
130
130
|
rdoc_options: []
|
|
131
131
|
require_paths:
|
|
132
132
|
- lib
|
|
@@ -141,8 +141,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
141
141
|
- !ruby/object:Gem::Version
|
|
142
142
|
version: '0'
|
|
143
143
|
requirements: []
|
|
144
|
-
rubygems_version: 3.1.
|
|
145
|
-
signing_key:
|
|
144
|
+
rubygems_version: 3.1.4
|
|
145
|
+
signing_key:
|
|
146
146
|
specification_version: 4
|
|
147
147
|
summary: A repository of common utilities for NYPL Ruby application
|
|
148
148
|
test_files: []
|