nypl_ruby_util 0.0.7 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ff3e8b400c7989c57e1398a642631a8c415a1bf982feadef5e95a813d7ad971
-  data.tar.gz: 7818da3d0d4fc85ddd6f10b259375463bc23a6f17b34a57026d60b4d1d03f057
+  metadata.gz: 5c6574f7bf8717173b11a9635daacec8d49be69a7fc35e5584d713f8ef37c673
+  data.tar.gz: 5035a2920d8c8f012c711050604f1ecd12832b4e8df3661eb0a16f707735c9d6
 SHA512:
-  metadata.gz: a2a0a99e281a95583fbe84f69fdbd066556f911167527be86dbdea2cffda000d3bb132edb6c90da4846e2b4d8129f7939594ee57b1a646f167955b2d5d1ed324
-  data.tar.gz: 0126f82acca76fd1e6c77afecce13a30c500766fd57f1ebecba075fcf85961c5c4b1c4e01d1641803cb23d86a9adadd21aed2f01d6e174f182b364e30dcc8fc6
+  metadata.gz: 9349fa7cc85b2429daef2ecd358392f58425a107516f1392d8fbbdb8861e460ec4dd48fab3d4163f58e351dc3001a1981efcaa86b76e5758442510e06ccb7ac3
+  data.tar.gz: 7a1e9057927bad0ff9425617f534abf17e63e2cb25cabb651bc2dd10fe234c5ca9c0d80c8d4dc54c51539a8cede92a82ce289f35e10dd64031d0c9c7a34d4332

data/lib/deploy_helper.rb

@@ -1,152 +1,152 @@
-require 'aws-sdk-lambda'
-require 'aws-sdk-cloudwatchevents'
-require 'yaml'
+require "aws-sdk-lambda"
+require "aws-sdk-cloudwatchevents"
+require "yaml"
 
 # Utility class for running rake methods
 class DeployHelper
-    attr_reader(
-        :travis_branch,
-        :aws_access_key_id,
-        :aws_secret_access_key,
-        :aws_configuration,
-        :region,
-        :lambda_client,
-        :yaml,
-        :lambda_config,
-        :function_name,
-        :event
-    )
-
-    def initialize
-        @travis_branch = ENV['TRAVIS_BRANCH'].upcase
-        @travis_branch = ['MAIN', 'MASTER'].include?(@travis_branch) ? 'PRODUCTION' : @travis_branch
-        @aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
-        @aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
-        @yaml = YAML.safe_load(File.read('.travis.yml'))
-        @lambda_config = yaml['deploy'].find { |conf| name_matches_branch?(conf['function_name'], travis_branch) }
-        @region = @lambda_config['region']
-        @function_name = @lambda_config['function_name']
-        @aws_configuration = {
-            region: region,
-            access_key_id: aws_access_key_id,
-            secret_access_key: aws_secret_access_key
-        }
-        p 'using configuration: ', aws_configuration
-        p 'lambda config: ', lambda_config
-        @lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
+  attr_reader(
+    :travis_branch,
+    :aws_access_key_id,
+    :aws_secret_access_key,
+    :aws_configuration,
+    :region,
+    :lambda_client,
+    :yaml,
+    :lambda_config,
+    :function_name,
+    :event
+  )
+
+  def initialize
+    @travis_branch = ENV["TRAVIS_BRANCH"].upcase
+    @travis_branch = ["MAIN", "MASTER"].include?(@travis_branch) ? "PRODUCTION" : @travis_branch
+    @aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
+    @aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
+    @yaml = YAML.safe_load(File.read(".travis.yml"))
+    @lambda_config = yaml["deploy"].find { |conf| name_matches_branch?(conf["function_name"], travis_branch) }
+    @region = @lambda_config["region"]
+    @function_name = @lambda_config["function_name"]
+    @aws_configuration = {
+      region: region,
+      access_key_id: aws_access_key_id,
+      secret_access_key: aws_secret_access_key
+    }
+    p("using configuration: ", aws_configuration)
+    p("lambda config: ", lambda_config)
+    @lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
+  end
+
+  def configured?
+    aws_access_key_id && aws_secret_access_key && region
+  end
+
+  def name_matches_branch?(name, branch)
+    downcase_name = name.downcase
+    downcase_branch = branch.downcase
+    variants = [
+      ["dev", "development"],
+      ["qa"],
+      ["main", "master", "production", "prod"],
+    ]
+    variants.any? do |group|
+      group.any? { |variant| downcase_name.include?(variant) }\
+      && group.any? { |variant| downcase_branch.include?(variant) }
     end
+  end
 
-    def configured?
-        aws_access_key_id && aws_secret_access_key && region
+  def update_lambda_configuration
+    unless configured? && lambda_config
+      p("insufficient configuration")
+      return nil
     end
 
-    def name_matches_branch?(name, branch)
-        downcase_name = name.downcase
-        downcase_branch = branch.downcase
-        variants = [
-            ['dev', 'development'],
-            ['qa'],
-            ['main', 'master', 'production', 'prod'],
-        ]
-        variants.any? do |group|
-            group.any? { |variant| downcase_name.include? variant }\
-            && group.any? { |variant| downcase_branch.include?(variant) }
-        end
+    updated_lambda_configuration = {
+      function_name: function_name,
+      vpc_config: lambda_config["vpc_config"],
+      environment: lambda_config["environment"],
+      layers: lambda_config["layers"]
+    }
+    updated_lambda_configuration[:function_name] = function_name
+    p("updating_function_configuration with: ", updated_lambda_configuration)
+    update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
+    p("update_configuration_resp: ", update_configuration_resp)
+  end
+
+  def update_event
+    unless lambda_config["event"]
+      p("no event config")
+      return nil
     end
 
-    def update_lambda_configuration
-        unless configured? && lambda_config
-            p 'insufficient configuration'
-            return nil
-        end
-
-        updated_lambda_configuration = {
-            function_name: function_name,
-            vpc_config: lambda_config['vpc_config'],
-            environment: lambda_config['environment'],
-            layers: lambda_config['layers']
-        }
-        updated_lambda_configuration[:function_name] = function_name
-        p 'updating_function_configuration with: ', updated_lambda_configuration
-        update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
-        p 'update_configuration_resp: ', update_configuration_resp
+    @event = lambda_config["event"]
+    if event["event_source_arn"]
+      add_event_source
+    elsif event["schedule_expression"]
+      add_cron
     end
+  end
 
-    def update_event
-        unless lambda_config['event']
-            p 'no event config'
-            return nil
-        end
-
-        @event = lambda_config['event']
-        if event['event_source_arn']
-            add_event_source
-        elsif event['schedule_expression']
-            add_cron
-        end
-    end
+  def add_event_source
+    existing_events = lambda_client.list_event_source_mappings({
+      function_name: function_name
+    }).event_source_mappings
 
-    def add_event_source
-        existing_events = lambda_client.list_event_source_mappings({
-            function_name: function_name
-        }).event_source_mappings
-
-        existing_events.each do |existing_event|
-            p 'deleting event with uuid: ', existing_event.uuid, 'and arn: ', existing_event.event_source_arn
-            lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
-        end
-        event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
-        event_to_create[:function_name] = function_name
-        p 'creating event: ', event_to_create
-        create_resp = lambda_client.create_event_source_mapping(event_to_create)
-        p 'created: ', create_resp
+    existing_events.each do |existing_event|
+      p("deleting event with uuid: ", existing_event.uuid, "and arn: ", existing_event.event_source_arn)
+      lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
+    end
+    event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
+    event_to_create[:function_name] = function_name
+    p("creating event: ", event_to_create)
+    create_resp = lambda_client.create_event_source_mapping(event_to_create)
+    p("created: ", create_resp)
+  end
+
+  def add_cron
+    ## create the event
+    events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
+    schedule_expression = event["schedule_expression"]
+    rule_name = "#{function_name}-rule"
+    p("rule_name: ", rule_name, "schedule_expression: ", schedule_expression)
+    events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
+
+    ## next we have to connect the event to the lambda
+    ## the first step is to get the lambda
+
+    return "missing function_name" unless function_name
+
+    p("getting lambda with function name", function_name)
+    lambda_resp = lambda_client.get_function(function_name: function_name).configuration
+    arn = lambda_resp.function_arn
+
+    ## next figure out if the lambda already has granted cloudwatch
+    ## permission to invoke
+    begin
+      policy_resp = lambda_client.get_policy(function_name: function_name)
+      if policy_resp.policy.include?("#{function_name}-permission")
+        p("lambda already has permission")
+      else
+        add_policy = true
+      end
+    rescue Aws::Lambda::Errors::ResourceNotFoundException
+      add_policy = true
+      p("no policy")
     end
 
-    def add_cron
-        ## create the event
-        events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
-        schedule_expression = event['schedule_expression']
-        rule_name = "#{function_name}-rule"
-        p 'rule_name: ', rule_name, 'schedule_expression: ', schedule_expression
-        events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
-
-        ## next we have to connect the event to the lambda
-        ## the first step is to get the lambda
-
-        return 'missing function_name' unless function_name
-
-        p 'getting lambda with function name', function_name
-        lambda_resp = lambda_client.get_function(function_name: function_name).configuration
-        arn = lambda_resp.function_arn
-
-        ## next figure out if the lambda already has granted cloudwatch
-        ## permission to invoke
-        begin
-            policy_resp = lambda_client.get_policy(function_name: function_name)
-            unless policy_resp.policy.include?("#{function_name}-permission")
-                add_policy = true
-            else
-                p 'lambda already has permission'
-            end
-        rescue Aws::Lambda::Errors::ResourceNotFoundException
-            add_policy = true
-            p 'no policy'
-        end
-
-        ## if not, add permission to invoke
-        if add_policy
-            permission = lambda_client.add_permission({
-                function_name: function_name,
-                principal: 'events.amazonaws.com',
-                statement_id: "#{function_name}-permission",
-                action: 'lambda:InvokeFunction'
-            })
-            p 'permission: ', permission
-        end
-
-        ## finally we can tell the event to invoke the lambda
-        target_id = "#{function_name}-lambda"
-        p 'putting targets ', 'rule: ', rule_name, 'target_id: ', target_id, 'arn: ', arn
-        events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
+    ## if not, add permission to invoke
+    if add_policy
+      permission = lambda_client.add_permission({
+        function_name: function_name,
+        principal: "events.amazonaws.com",
+        statement_id: "#{function_name}-permission",
+        action: "lambda:InvokeFunction"
+      })
+      p("permission: ", permission)
     end
+
+    ## finally we can tell the event to invoke the lambda
+    target_id = "#{function_name}-lambda"
+    p("putting targets ", "rule: ", rule_name, "target_id: ", target_id, "arn: ", arn)
+    events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
+  end
 end

data/lib/kinesis_client.rb

@@ -1,7 +1,7 @@
-require 'securerandom'
-require 'aws-sdk-kinesis'
-require_relative 'nypl_avro'
-require_relative 'errors'
+require "securerandom"
+require "aws-sdk-kinesis"
+require_relative "nypl_avro"
+require_relative "errors"
 # Model representing the result message posted to Kinesis stream about everything that has gone on here -- good, bad, or otherwise.
 
 class KinesisClient
@@ -12,17 +12,25 @@ class KinesisClient
     @stream_name = @config[:stream_name]
     @avro = nil
     @batch_size = @config[:batch_size] || 1
-    @batch = []
-    @automatically_push = @config[:automatically_push] == false ? false : true
-    @client_options = config[:profile] ? { profile: config[:profile] } : {}
-    @client = Aws::Kinesis::Client.new @client_options
+    @client_options = set_config(config)
+    @batch_count = 0
+    @records = []
+    @automatically_push = !(@config[:automatically_push] == false)
+    @client = Aws::Kinesis::Client.new(@client_options)
 
-    if config[:schema_string]
-      @avro = NYPLAvro.by_name(config[:schema_string])
-    end
+    @avro = NYPLAvro.by_name(config[:schema_string]) if config[:schema_string]
 
-    @shovel_method = @batch_size > 1 ? :push_to_batch : :push_record
+    @shovel_method = @batch_size > 1 ? :push_to_records : :push_record
+  end
 
+  def set_config(config)
+    if config[:profile]
+      { profile: config[:profile] }
+    elsif config[:custom_aws_config]
+      config[:custom_aws_config]
+    else
+      {}
+    end
   end
 
   def convert_to_record(json_message)
@@ -37,18 +45,18 @@ class KinesisClient
       data: message,
       partition_key: partition_key
     }
-
   end
 
   def <<(json_message)
     send(@shovel_method, json_message)
   end
 
+#This method is broken
   def push_record(json_message)
-    record = convert_to_record json_message
+    record = convert_to_record(json_message)
     record[:stream_name] = @stream_name
 
-    @client.put_record record
+    @client.put_record(record)
 
     return_hash = {}
 
@@ -57,21 +65,19 @@ class KinesisClient
       return_hash["message"] = json_message, resp
       $logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
     else
-      $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
-      raise NYPLError.new json_message, resp
+      $logger.error("message" => "FAILED to send message to #{@stream_name} #{json_message}, #{resp}.") if $logger
+      raise(NYPLError.new(json_message, resp))
     end
     return_hash
   end
 
-  def push_to_batch(json_message)
+  def push_to_records(json_message)
     begin
-      @batch << convert_to_record(json_message)
+      @records << convert_to_record(json_message)
     rescue AvroError => e
       $logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
     end
-    if @automatically_push && @batch.length >= @batch_size
-      push_records
-    end
+    push_records if @automatically_push && @records.length >= @batch_size
   end
 
   def push_batch(batch)
@@ -80,23 +86,31 @@ class KinesisClient
       stream_name: @stream_name
     })
 
-    $logger.debug("Received #{resp} from #{@stream_name}")
-
-    return_message = {
-      failures: resp.failed_record_count,
-      error_messages: resp.records.map {|record| record.error_message }.compact
-    }
-
-    $logger.info("Message sent to #{config[:stream_name]} #{return_message}") if $logger
-
-    return {
-      "code": "200",
-      "message": return_message.to_json
-    }
+    if resp.failed_record_count > 0
+      failure_message = {
+        failures: resp.failed_record_count,
+        failures_data: filter_failures(resp)
+      }
+      $logger.warn("Batch sent to #{config[:stream_name]} with failures: #{failure_message}")
+    else
+      $logger.info("Batch sent to #{config[:stream_name]} successfully")
+    end
   end
 
   def push_records
-    @batch.each_slice(@batch_size) {|slice| push_batch slice}
-    @batch = []
+    if @records.length > 0
+      @records.each_slice(@batch_size) do |slice|
+        push_batch(slice)
+        @batch_count += 1
+      end
+      @records = []
+      @batch_count = 0
+    end
+  end
+
+  def filter_failures(resp)
+    resp.records.filter_map.with_index do |record, i|
+      avro.decode(@records[i + @batch_size * @batch_count]) if record.responds_to?(:error_message)
+    end
   end
 end

data/lib/kms_client.rb

@@ -1,5 +1,5 @@
-require 'aws-sdk-kms'
-require 'base64'
+require "aws-sdk-kms"
+require "base64"
 
 class KmsClient
   @@kms = nil
@@ -11,15 +11,15 @@ class KmsClient
 
   def decrypt(cipher)
     # Assume value is base64 encoded:
-    decoded = Base64.decode64 cipher
-    decrypted = @kms.decrypt ciphertext_blob: decoded
+    decoded = Base64.decode64(cipher)
+    decrypted = @kms.decrypt(ciphertext_blob: decoded)
     decrypted[:plaintext]
   end
 
   def self.aws_kms_client(options)
     params = {
-      region: 'us-east-1',
-      stub_responses: ENV['APP_ENV'] == 'test'
+      region: "us-east-1",
+      stub_responses: ENV["APP_ENV"] == "test"
     }.merge(options)
     @@kms = Aws::KMS::Client.new(params) if @@kms.nil?
     @@kms

data/lib/platform_api_client.rb

@@ -21,7 +21,7 @@ class PlatformApiClient
     @error_options = default_errors.merge(options[:errors] || {})
   end
 
-  def get (path)
+  def get (path, transaction_data = {})
 
     authenticate! if authenticated
 
@@ -38,7 +38,7 @@ class PlatformApiClient
 
       $logger.debug "Got platform api response", { code: response.code, body: response.body }
 
-      parse_json_response response, path
+      parse_json_response response, path, transaction_data
 
     rescue Exception => e
       raise StandardError.new(e), "Failed to retrieve #{path} #{e.message}"
@@ -47,12 +47,12 @@ class PlatformApiClient
 
   private
 
-  def parse_json_response (response, path)
+  def parse_json_response (response, path, transaction_data = {})
     code = response.code.to_i
     if code < 400
       JSON.parse(response.body)
     elsif error_options[code]
-      instance_exec(response, path, &error_options[code])
+      instance_exec(response, path, transaction_data, &error_options[code])
     else
       raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
       {}
@@ -92,12 +92,13 @@ class PlatformApiClient
 
   def default_errors
     {
-      401 => lambda do |response, path|
-        if @try_count < 1
+      401 => lambda do |response, path, transaction_data = {}|
+        transaction_data[:try_count] ||= 0
+        if transaction_data[:try_count] < 1
           # Likely an expired access-token; Wipe it for next run
-          @try_count += 1
-          access_token = nil
-          get(path)
+          transaction_data[:try_count] += 1
+          self.access_token = nil
+          get(path, transaction_data)
         else
           raise "Error interpretting response for path #{path}: (#{response.code}): #{response.body}"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.10
 platform: ruby
 authors:
 - Daniel Appel
@@ -141,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: A repository of common utilities for NYPL Ruby application