nypl_ruby_util 0.0.9 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05c91cd9d8cd4ffd258f3a2c1f9b3cb41c2c6cf9b5df78d21f1483510e34c544
-  data.tar.gz: 55e61c0097b9721728b147b6201e0856e815c3de8631e3f41ed78b00c12d11bb
+  metadata.gz: 653b400784892750f42ddb4202e68fe2935174730b8f655b086e440c1ea2c31e
+  data.tar.gz: 215cddb81d35593ebb0cedef604bec3577ba2cac8d9d568bc85f7845157d7421
 SHA512:
-  metadata.gz: 6c2042cc492affeb2ae6d60d6c96fb6d0db3ea69c23f1ad6a31da3ff5025c52cd4d5f669112e174153e88b84dde7395f59b23b12182017c079f1ce6a024807f6
-  data.tar.gz: 832f11059c1267393afb6b2f5140dd5a8f98950f9e69c495e6fa1cfa31af947c78fbf4cd740016214b4fe1dc4bce939a6f57de295039a3673aa195da0ce2be2c
+  metadata.gz: 7d7a49c8009dcef64ef7668e182c447214dadf91bebae23f33c4195559286dadb81d01567bd7a60d64ae3aae992f553f204c9e4df7a6bdd0fcb15f65d0b25d45
+  data.tar.gz: 7da7c0f9adf0eba65cdf0e653c2663f4ec2b58cc5001b04aec27170fa758bb0601c00b01c1616eab9be82d74cd54c13fe94bbbc78da5836bf15a9d554bc102f5

data/lib/kinesis_client.rb

@@ -5,6 +5,7 @@ require_relative "errors"
 # Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
 
 class KinesisClient
+  #note custom defined :failed_records method
   attr_reader :config, :avro
 
   def initialize(config)
@@ -13,8 +14,8 @@ class KinesisClient
     @avro = nil
     @batch_size = @config[:batch_size] || 1
     @client_options = set_config(config)
-    @batch_count = 0
     @records = []
+    @failed_records = []
     @automatically_push = !(@config[:automatically_push] == false)
     @client = Aws::Kinesis::Client.new(@client_options)
 
@@ -52,6 +53,7 @@ class KinesisClient
   end
 
 #This method is broken
+#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
   def push_record(json_message)
     record = convert_to_record(json_message)
     record[:stream_name] = @stream_name
@@ -85,13 +87,10 @@ class KinesisClient
       records: batch.to_a,
       stream_name: @stream_name
     })
-
     if resp.failed_record_count > 0
-      failure_message = {
-        failures: resp.failed_record_count,
-        failures_data: filter_failures(resp)
-      }
-      $logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
+      failures = filter_failures(resp, batch) 
+      $logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
+      failures.each{|failure| @failed_records << failure[:record]}
     else
       $logger.info("Batch sent to #{config[:stream_name]} successfully")
     end
@@ -101,16 +100,26 @@ class KinesisClient
     if @records.length > 0
       @records.each_slice(@batch_size) do |slice|
         push_batch(slice)
-        @batch_count += 1
       end
       @records = []
-      @batch_count = 0
     end
   end
 
-  def filter_failures(resp)
+  def filter_failures(resp, batch)
     resp.records.filter_map.with_index do |record, i|
-      avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
+      { record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
+    end
+  end
+
+  def retry_failed_records
+    unless @failed_records.empty?
+      @records = @failed_records
+      @failed_records = []
+      push_records
     end
   end
+
+  def failed_records
+    @failed_records.map { |record| avro.decode(record) }
+  end
 end

data/lib/platform_api_client.rb

@@ -21,7 +21,7 @@ class PlatformApiClient
     @error_options = default_errors.merge(options[:errors] || {})
   end
 
-  def get (path)
+  def get (path, transaction_data = {})
 
     authenticate! if authenticated
 
@@ -38,7 +38,7 @@ class PlatformApiClient
 
       $logger.debug "Got platform api response", { code: response.code, body: response.body }
 
-      parse_json_response response, path
+      parse_json_response response, path, transaction_data
 
     rescue Exception => e
       raise StandardError.new(e), "Failed to retrieve #{path} #{e.message}"
@@ -47,12 +47,12 @@ class PlatformApiClient
 
   private
 
-  def parse_json_response (response, path)
+  def parse_json_response (response, path, transaction_data = {})
     code = response.code.to_i
     if code < 400
       JSON.parse(response.body)
     elsif error_options[code]
-      instance_exec(response, path, &error_options[code])
+      instance_exec(response, path, transaction_data, &error_options[code])
     else
       raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
       {}
@@ -92,12 +92,13 @@ class PlatformApiClient
 
   def default_errors
     {
-      401 => lambda do |response, path|
-        if @try_count < 1
+      401 => lambda do |response, path, transaction_data = {}|
+        transaction_data[:try_count] ||= 0
+        if transaction_data[:try_count] < 1
           # Likely an expired access-token; Wipe it for next run
-          @try_count += 1
-          access_token = nil
-          get(path)
+          transaction_data[:try_count] += 1
+          self.access_token = nil
+          get(path, transaction_data)
         else
           raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Daniel Appel
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2022-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.3
 description: A repository of common utilities for NYPL Ruby application
-email:
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -126,7 +126,7 @@ homepage: https://github.com/NYPL/NYPLRubyUtil
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -141,8 +141,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.1.2
+signing_key: 
 specification_version: 4
 summary: A repository of common utilities for NYPL Ruby application
 test_files: []