nypl_ruby_util 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2086b8b4b5bef52f3459a1245af09a7364cb35190ec40c0541a7bd72681a8f6
-  data.tar.gz: 4e1c6a34f318b81c6659cca2f57f1930bfbf0e406040e950f681c4631177bec3
+  metadata.gz: 15e9f3f7cc3ceb996f43d1934a8f59b4a22953b0872d1a5a6d4a14590f733184
+  data.tar.gz: 8d9393c91419e241e1f91b81e6ad14570295a8851b534d0fad234c5368ab4b6d
 SHA512:
-  metadata.gz: 68670e8961ed473bb3854b2320cafc0c960b9418b37d336fdaac6f9cf41ae14acca33d131565986fad25f2d8eb6cb57f225828c93e5539c685bf497b80e6bab6
-  data.tar.gz: 433b9dea26fa844a28a6711eff1d510a8f21df2f3f71e011d76e497f77dd2988714162973345084b681bb068771bc95433957f513e26553243cefbad521c70b3
+  metadata.gz: '048284f206fa02fcfa1ed33b6199cf20c85ac981cff0773dd4b4223f6a0184f656502cac540d83e97fff000ed3358460eef732f5a829186873b5988597c971d6'
+  data.tar.gz: 0d1c259a9b5c170d4a194e580bd9061fe10e82fa674d45aee9d97fd3699c82b0d11db92f2124470df30c769a95f20af4221cfe0199d7f8be79e2ca1bde7b5f0a

data/lib/kinesis_client.rb

@@ -5,6 +5,7 @@ require_relative "errors"
 # Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
 
 class KinesisClient
+  #note custom defined :failed_records method
   attr_reader :config, :avro
 
   def initialize(config)
@@ -12,10 +13,10 @@ class KinesisClient
     @stream_name = @config[:stream_name]
     @avro = nil
     @batch_size = @config[:batch_size] || 1
-    @batch_count = 0
+    @client_options = set_config(config)
     @records = []
+    @failed_records = []
     @automatically_push = !(@config[:automatically_push] == false)
-    @client_options = config[:profile] ? { profile: config[:profile] } : {}
     @client = Aws::Kinesis::Client.new(@client_options)
 
     @avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
@@ -23,6 +24,16 @@ class KinesisClient
     @shovel_method = @batch_size > 1 ? :push_to_records : :push_record
   end
 
+  def set_config(config)
+    if config[:profile]
+      { profile: config[:profile] }
+    elsif config[:custom_aws_config]
+      config[:custom_aws_config]
+    else
+      {}
+    end
+  end
+
   def convert_to_record(json_message)
     if config[:schema_string]
       message = avro.encode(json_message, false)
@@ -41,6 +52,8 @@ class KinesisClient
     send(@shovel_method, json_message)
   end
 
+#This method is broken
+#TO DO: figure out how to determine successful or failed record, successful? is not a method on the object
   def push_record(json_message)
     record = convert_to_record(json_message)
     record[:stream_name] = @stream_name
@@ -74,34 +87,39 @@ class KinesisClient
       records: batch.to_a,
       stream_name: @stream_name
     })
-
-    $logger.debug("Received #{resp} from #{@stream_name}")
-
-    if resp.failed_record_count > 0 
-      return_message = {
-        failures: resp.failed_record_count,
-        failures_data: filter_failures(resp)
-      }
-      $logger.warn("Message sent to #{config[:stream_name]} #{return_message}") if $logger
+    if resp.failed_record_count > 0
+      failures = filter_failures(resp, batch) 
+      $logger.warn("Batch sent to #{config[:stream_name]} with #{failures.length} failures: #{failures}")
+      failures.each{|failure| @failed_records << failure[:record]}
     else
-      $logger.info("Message sent to #{config[:stream_name]} successfully") if $logger
+      $logger.info("Batch sent to #{config[:stream_name]} successfully")
     end
   end
 
   def push_records
-    if @records.length > 0 
+    if @records.length > 0
       @records.each_slice(@batch_size) do |slice|
         push_batch(slice)
-        @batch_count += 1
       end
       @records = []
-      @batch_count = 0
     end
   end
 
-  def filter_failures(resp)
+  def filter_failures(resp, batch)
     resp.records.filter_map.with_index do |record, i|
-      avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
+      { record: batch[i], error_message: record.error_message } if record.responds_to?(:error_message)
     end
   end
+
+  def retry_failed_records
+    unless @failed_records.empty?
+      @records = @failed_records
+      @failed_records = []
+      push_records
+    end
+  end
+
+  def failed_records
+    @failed_records.map { |record| avro.decode(record) }
+  end
 end

data/lib/platform_api_client.rb

@@ -21,7 +21,7 @@ class PlatformApiClient
     @error_options = default_errors.merge(options[:errors] || {})
   end
 
-  def get (path)
+  def get (path, transaction_data = {})
 
     authenticate! if authenticated
 
@@ -38,7 +38,7 @@ class PlatformApiClient
 
       $logger.debug "Got platform api response", { code: response.code, body: response.body }
 
-      parse_json_response response, path
+      parse_json_response response, path, transaction_data
 
     rescue Exception => e
       raise StandardError.new(e), "Failed to retrieve #{path} #{e.message}"
@@ -47,12 +47,12 @@ class PlatformApiClient
 
   private
 
-  def parse_json_response (response, path)
+  def parse_json_response (response, path, transaction_data = {})
     code = response.code.to_i
     if code < 400
       JSON.parse(response.body)
     elsif error_options[code]
-      instance_exec(response, path, &error_options[code])
+      instance_exec(response, path, transaction_data, &error_options[code])
     else
       raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
       {}
@@ -92,12 +92,13 @@ class PlatformApiClient
 
   def default_errors
     {
-      401 => lambda do |response, path|
-        if @try_count < 1
+      401 => lambda do |response, path, transaction_data = {}|
+        transaction_data[:try_count] ||= 0
+        if transaction_data[:try_count] < 1
           # Likely an expired access-token; Wipe it for next run
-          @try_count += 1
-          access_token = nil
-          get(path)
+          transaction_data[:try_count] += 1
+          self.access_token = nil
+          get(path, transaction_data)
         else
           raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - Daniel Appel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2022-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro