nypl_ruby_util 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7792a53b4410092c9b4c215da151d9bdf6dbe37e37af60c35be4bc07b2ec419e
-  data.tar.gz: 864bbf4ae3f196c2810884779b91f0f912871e9cc3913bb19c608926681f7560
+  metadata.gz: 5ff3e8b400c7989c57e1398a642631a8c415a1bf982feadef5e95a813d7ad971
+  data.tar.gz: 7818da3d0d4fc85ddd6f10b259375463bc23a6f17b34a57026d60b4d1d03f057
 SHA512:
-  metadata.gz: 330aabe2c15a245efac358c65ae96ed79ef6928d6f1b2cf92a8b136df54429dd68dcc0e5b6e77b7677cbbd716db76fc81daed2e9cd54c44bfa008d22c8a2a9cb
-  data.tar.gz: a95be077f61ec0c17d0a4b1c7956e35a66b98e04a4c1cd4093db9ab349bf81bc4d9d986c441cd5f801921aa3c1bd4f300bbd52774ecfb0e9809dc4640792f71b
+  metadata.gz: a2a0a99e281a95583fbe84f69fdbd066556f911167527be86dbdea2cffda000d3bb132edb6c90da4846e2b4d8129f7939594ee57b1a646f167955b2d5d1ed324
+  data.tar.gz: 0126f82acca76fd1e6c77afecce13a30c500766fd57f1ebecba075fcf85961c5c4b1c4e01d1641803cb23d86a9adadd21aed2f01d6e174f182b364e30dcc8fc6

data/lib/kinesis_client.rb

@@ -9,40 +9,94 @@ class KinesisClient
 
   def initialize(config)
     @config = config
+    @stream_name = @config[:stream_name]
     @avro = nil
+    @batch_size = @config[:batch_size] || 1
+    @batch = []
+    @automatically_push = @config[:automatically_push] == false ? false : true
+    @client_options = config[:profile] ? { profile: config[:profile] } : {}
+    @client = Aws::Kinesis::Client.new @client_options
 
     if config[:schema_string]
       @avro = NYPLAvro.by_name(config[:schema_string])
     end
 
+    @shovel_method = @batch_size > 1 ? :push_to_batch : :push_record
+
   end
 
-  def <<(json_message)
+  def convert_to_record(json_message)
     if config[:schema_string]
       message = avro.encode(json_message, false)
     else
       message = json_message
     end
 
-    client = Aws::Kinesis::Client.new
     partition_key = (config[:partition_key] ? json_message[config[:partition_key]] : SecureRandom.hex(20)).hash.to_s
-
-    resp = client.put_record({
-      stream_name: config[:stream_name],
+    {
       data: message,
       partition_key: partition_key
-      })
-
-      return_hash = {}
-
-      if resp.successful?
-        return_hash["code"] = "200"
-        return_hash["message"] = json_message, resp
-        $logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
-      else
-        $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
-        raise NYPLError.new json_message, resp
-      end
-      return_hash
+    }
+
+  end
+
+  def <<(json_message)
+    send(@shovel_method, json_message)
+  end
+
+  def push_record(json_message)
+    record = convert_to_record json_message
+    record[:stream_name] = @stream_name
+
+    @client.put_record record
+
+    return_hash = {}
+
+    if resp.successful?
+      return_hash["code"] = "200"
+      return_hash["message"] = json_message, resp
+      $logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
+    else
+      $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
+      raise NYPLError.new json_message, resp
+    end
+    return_hash
+  end
+
+  def push_to_batch(json_message)
+    begin
+      @batch << convert_to_record(json_message)
+    rescue AvroError => e
+      $logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
+    end
+    if @automatically_push && @batch.length >= @batch_size
+      push_records
+    end
+  end
+
+  def push_batch(batch)
+    resp = @client.put_records({
+      records: batch.to_a,
+      stream_name: @stream_name
+    })
+
+    $logger.debug("Received #{resp} from #{@stream_name}")
+
+    return_message = {
+      failures: resp.failed_record_count,
+      error_messages: resp.records.map {|record| record.error_message }.compact
+    }
+
+    $logger.info("Message sent to #{config[:stream_name]} #{return_message}") if $logger
+
+    return {
+      "code": "200",
+      "message": return_message.to_json
+    }
+  end
+
+  def push_records
+    @batch.each_slice(@batch_size) {|slice| push_batch slice}
+    @batch = []
   end
 end

data/lib/nypl_avro.rb

@@ -38,7 +38,7 @@ class NYPLAvro
     begin
       bin_encoder.write(decoded_data, encoder)
     rescue Avro::IO::AvroTypeError => e
-      raise AvroError.new(e), "Error encoding data using #{@schema.name} schema due to #{e.message}"
+      raise AvroError.new(e), "Error encoding data #{decoded_data} using #{@schema.name} schema due to #{e.message}"
     end
 
     buffer.rewind

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Daniel Appel