nypl_ruby_util 0.0.10 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c6574f7bf8717173b11a9635daacec8d49be69a7fc35e5584d713f8ef37c673
-  data.tar.gz: 5035a2920d8c8f012c711050604f1ecd12832b4e8df3661eb0a16f707735c9d6
+  metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
+  data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
 SHA512:
-  metadata.gz: 9349fa7cc85b2429daef2ecd358392f58425a107516f1392d8fbbdb8861e460ec4dd48fab3d4163f58e351dc3001a1981efcaa86b76e5758442510e06ccb7ac3
-  data.tar.gz: 7a1e9057927bad0ff9425617f534abf17e63e2cb25cabb651bc2dd10fe234c5ca9c0d80c8d4dc54c51539a8cede92a82ce289f35e10dd64031d0c9c7a34d4332
+  metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
+  data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a

data/lib/kinesis_client.rb

@@ -5,6 +5,7 @@ require_relative "errors"
 # Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
 
 class KinesisClient
+  #note custom defined :failed_records method
   attr_reader :config, :avro
 
   def initialize(config)
@@ -13,8 +14,8 @@ class KinesisClient
     @avro = nil
     @batch_size = @config[:batch_size] || 1
     @client_options = set_config(config)
-    @batch_count = 0
     @records = []
+    @failed_records = []
     @automatically_push = !(@config[:automatically_push] == false)
     @client = Aws::Kinesis::Client.new(@client_options)
 
@@ -52,6 +53,7 @@ class KinesisClient
   end
 
 #This method is broken
+#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
   def push_record(json_message)
     record = convert_to_record(json_message)
     record[:stream_name] = @stream_name
@@ -85,13 +87,10 @@ class KinesisClient
       records: batch.to_a,
       stream_name: @stream_name
     })
-
     if resp.failed_record_count > 0
-      failure_message = {
-        failures: resp.failed_record_count,
-        failures_data: filter_failures(resp)
-      }
-      $logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
+      failures = filter_failures(resp, batch) 
+      $logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
+      failures.each{|failure| @failed_records << failure[:record]}
     else
       $logger.info("Batch sent to #{config[:stream_name]} successfully")
     end
@@ -101,16 +100,26 @@ class KinesisClient
     if @records.length > 0
       @records.each_slice(@batch_size) do |slice|
         push_batch(slice)
-        @batch_count += 1
       end
       @records = []
-      @batch_count = 0
     end
   end
 
-  def filter_failures(resp)
+  def filter_failures(resp, batch)
     resp.records.filter_map.with_index do |record, i|
-      avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
+      { record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
+    end
+  end
+
+  def retry_failed_records
+    unless @failed_records.empty?
+      @records = @failed_records
+      @failed_records = []
+      push_records
     end
   end
+
+  def failed_records
+    @failed_records.map { |record| avro.decode(record) }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.1.0
 platform: ruby
 authors:
 - Daniel Appel
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2022-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.3
 description: A repository of common utilities for NYPL Ruby application
-email:
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -126,7 +126,7 @@ homepage: https://github.com/NYPL/NYPLRubyUtil
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key:
+signing_key: 
 specification_version: 4
 summary: A repository of common utilities for NYPL Ruby application
 test_files: []