nypl_ruby_util 0.0.3 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6c6483ada755b5092548e42bd0bc6f699af6c642d8863e1488fabb78345df59
-  data.tar.gz: 343534e1ca8672cb04552aa958a22e98211bfb00fb8996698b28e2ed78f97368
+  metadata.gz: 5ff3e8b400c7989c57e1398a642631a8c415a1bf982feadef5e95a813d7ad971
+  data.tar.gz: 7818da3d0d4fc85ddd6f10b259375463bc23a6f17b34a57026d60b4d1d03f057
 SHA512:
-  metadata.gz: e3bdaa197c01456afae42a28dd4d6c8e95e4f9e7d402ebe3e54f4c4980f2daee7517b8be4c1fb19f62cea77f0a10b7bfafdcf03c04dfc1153f79ca576e5c7cc6
-  data.tar.gz: 3a48402301714fa235c8ed865f99e1582aa19064791d099a2d47510cbcd99224852593d14aceb29a998643c37f97386cc94307beda6c336d5b4b6b31aac56c5b
+  metadata.gz: a2a0a99e281a95583fbe84f69fdbd066556f911167527be86dbdea2cffda000d3bb132edb6c90da4846e2b4d8129f7939594ee57b1a646f167955b2d5d1ed324
+  data.tar.gz: 0126f82acca76fd1e6c77afecce13a30c500766fd57f1ebecba075fcf85961c5c4b1c4e01d1641803cb23d86a9adadd21aed2f01d6e174f182b364e30dcc8fc6

data/lib/deploy_helper.rb

@@ -0,0 +1,152 @@
+require 'aws-sdk-lambda'
+require 'aws-sdk-cloudwatchevents'
+require 'yaml'
+
+# Utility class for running rake methods
+class DeployHelper
+    attr_reader(
+        :travis_branch,
+        :aws_access_key_id,
+        :aws_secret_access_key,
+        :aws_configuration,
+        :region,
+        :lambda_client,
+        :yaml,
+        :lambda_config,
+        :function_name,
+        :event
+    )
+
+    def initialize
+        @travis_branch = ENV['TRAVIS_BRANCH'].upcase
+        @travis_branch = ['MAIN', 'MASTER'].include?(@travis_branch) ? 'PRODUCTION' : @travis_branch
+        @aws_access_key_id = ENV["AWS_ACCESS_KEY_ID_#{travis_branch}"]
+        @aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY_#{travis_branch}"]
+        @yaml = YAML.safe_load(File.read('.travis.yml'))
+        @lambda_config = yaml['deploy'].find { |conf| name_matches_branch?(conf['function_name'], travis_branch) }
+        @region = @lambda_config['region']
+        @function_name = @lambda_config['function_name']
+        @aws_configuration = {
+            region: region,
+            access_key_id: aws_access_key_id,
+            secret_access_key: aws_secret_access_key
+        }
+        p 'using configuration: ', aws_configuration
+        p 'lambda config: ', lambda_config
+        @lambda_client = Aws::Lambda::Client.new(aws_configuration) if configured?
+    end
+
+    def configured?
+        aws_access_key_id && aws_secret_access_key && region
+    end
+
+    def name_matches_branch?(name, branch)
+        downcase_name = name.downcase
+        downcase_branch = branch.downcase
+        variants = [
+            ['dev', 'development'],
+            ['qa'],
+            ['main', 'master', 'production', 'prod'],
+        ]
+        variants.any? do |group|
+            group.any? { |variant| downcase_name.include? variant }\
+            && group.any? { |variant| downcase_branch.include?(variant) }
+        end
+    end
+
+    def update_lambda_configuration
+        unless configured? && lambda_config
+            p 'insufficient configuration'
+            return nil
+        end
+
+        updated_lambda_configuration = {
+            function_name: function_name,
+            vpc_config: lambda_config['vpc_config'],
+            environment: lambda_config['environment'],
+            layers: lambda_config['layers']
+        }
+        updated_lambda_configuration[:function_name] = function_name
+        p 'updating_function_configuration with: ', updated_lambda_configuration
+        update_configuration_resp = lambda_client.update_function_configuration(updated_lambda_configuration)
+        p 'update_configuration_resp: ', update_configuration_resp
+    end
+
+    def update_event
+        unless lambda_config['event']
+            p 'no event config'
+            return nil
+        end
+
+        @event = lambda_config['event']
+        if event['event_source_arn']
+            add_event_source
+        elsif event['schedule_expression']
+            add_cron
+        end
+    end
+
+    def add_event_source
+        existing_events = lambda_client.list_event_source_mappings({
+            function_name: function_name
+        }).event_source_mappings
+
+        existing_events.each do |existing_event|
+            p 'deleting event with uuid: ', existing_event.uuid, 'and arn: ', existing_event.event_source_arn
+            lambda_client.delete_event_source_mapping({ uuid: existing_event.uuid })
+        end
+        event_to_create = event.map { |k, v| [k.to_sym, v] }.to_h
+        event_to_create[:function_name] = function_name
+        p 'creating event: ', event_to_create
+        create_resp = lambda_client.create_event_source_mapping(event_to_create)
+        p 'created: ', create_resp
+    end
+
+    def add_cron
+        ## create the event
+        events_client = Aws::CloudWatchEvents::Client.new(aws_configuration)
+        schedule_expression = event['schedule_expression']
+        rule_name = "#{function_name}-rule"
+        p 'rule_name: ', rule_name, 'schedule_expression: ', schedule_expression
+        events_client.put_rule(name: rule_name, schedule_expression: schedule_expression)
+
+        ## next we have to connect the event to the lambda
+        ## the first step is to get the lambda
+
+        return 'missing function_name' unless function_name
+
+        p 'getting lambda with function name', function_name
+        lambda_resp = lambda_client.get_function(function_name: function_name).configuration
+        arn = lambda_resp.function_arn
+
+        ## next figure out if the lambda already has granted cloudwatch
+        ## permission to invoke
+        begin
+            policy_resp = lambda_client.get_policy(function_name: function_name)
+            unless policy_resp.policy.include?("#{function_name}-permission")
+                add_policy = true
+            else
+                p 'lambda already has permission'
+            end
+        rescue Aws::Lambda::Errors::ResourceNotFoundException
+            add_policy = true
+            p 'no policy'
+        end
+
+        ## if not, add permission to invoke
+        if add_policy
+            permission = lambda_client.add_permission({
+                function_name: function_name,
+                principal: 'events.amazonaws.com',
+                statement_id: "#{function_name}-permission",
+                action: 'lambda:InvokeFunction'
+            })
+            p 'permission: ', permission
+        end
+
+        ## finally we can tell the event to invoke the lambda
+        target_id = "#{function_name}-lambda"
+        p 'putting targets ', 'rule: ', rule_name, 'target_id: ', target_id, 'arn: ', arn
+        events_client.put_targets(rule: rule_name, targets: [{ id: target_id, arn: arn }])
+    end
+end

data/lib/kinesis_client.rb

@@ -9,40 +9,94 @@ class KinesisClient
 
   def initialize(config)
     @config = config
+    @stream_name = @config[:stream_name]
     @avro = nil
+    @batch_size = @config[:batch_size] || 1
+    @batch = []
+    @automatically_push = @config[:automatically_push] == false ? false : true
+    @client_options = config[:profile] ? { profile: config[:profile] } : {}
+    @client = Aws::Kinesis::Client.new @client_options
 
     if config[:schema_string]
       @avro = NYPLAvro.by_name(config[:schema_string])
     end
 
+    @shovel_method = @batch_size > 1 ? :push_to_batch : :push_record
+
   end
 
-  def <<(json_message)
+  def convert_to_record(json_message)
     if config[:schema_string]
       message = avro.encode(json_message, false)
     else
       message = json_message
     end
 
-    client = Aws::Kinesis::Client.new
     partition_key = (config[:partition_key] ? json_message[config[:partition_key]] : SecureRandom.hex(20)).hash.to_s
-
-    resp = client.put_record({
-      stream_name: config[:stream_name],
+    {
       data: message,
       partition_key: partition_key
-      })
-
-      return_hash = {}
-
-      if resp.successful?
-        return_hash["code"] = "200"
-        return_hash["message"] = json_message, resp
-        $logger.info("Message sent to HoldRequestResult #{json_message}, #{resp}") if $logger
-      else
-        $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
-        raise NYPLError.new json_message, resp
-      end
-      return_hash
+    }
+
+  end
+
+  def <<(json_message)
+    send(@shovel_method, json_message)
+  end
+
+  def push_record(json_message)
+    record = convert_to_record json_message
+    record[:stream_name] = @stream_name
+
+    @client.put_record record
+
+    return_hash = {}
+
+    if resp.successful?
+      return_hash["code"] = "200"
+      return_hash["message"] = json_message, resp
+      $logger.info("Message sent to #{config[:stream_name]} #{json_message}, #{resp}") if $logger
+    else
+      $logger.error("message" => "FAILED to send message to HoldRequestResult #{json_message}, #{resp}.") if $logger
+      raise NYPLError.new json_message, resp
+    end
+    return_hash
+  end
+
+  def push_to_batch(json_message)
+    begin
+      @batch << convert_to_record(json_message)
+    rescue AvroError => e
+      $logger.error("message" => "Avro encoding error #{e.message} for #{json_message}")
+    end
+    if @automatically_push && @batch.length >= @batch_size
+      push_records
+    end
+  end
+
+  def push_batch(batch)
+    resp = @client.put_records({
+      records: batch.to_a,
+      stream_name: @stream_name
+    })
+
+    $logger.debug("Received #{resp} from #{@stream_name}")
+
+    return_message = {
+      failures: resp.failed_record_count,
+      error_messages: resp.records.map {|record| record.error_message }.compact
+    }
+
+    $logger.info("Message sent to #{config[:stream_name]} #{return_message}") if $logger
+
+    return {
+      "code": "200",
+      "message": return_message.to_json
+    }
+  end
+
+  def push_records
+    @batch.each_slice(@batch_size) {|slice| push_batch slice}
+    @batch = []
   end
 end

data/lib/nypl_avro.rb

@@ -38,7 +38,7 @@ class NYPLAvro
     begin
       bin_encoder.write(decoded_data, encoder)
     rescue Avro::IO::AvroTypeError => e
-      raise AvroError.new(e), "Error encoding data using #{@schema.name} schema due to #{e.message}"
+      raise AvroError.new(e), "Error encoding data #{decoded_data} using #{@schema.name} schema due to #{e.message}"
     end
 
     buffer.rewind

data/lib/nypl_ruby_util.rb

@@ -6,6 +6,7 @@ require_relative 'directory'
 require_relative 'nypl_avro'
 require_relative 'errors'
 require_relative 'kinesis_client'
+require_relative 'deploy_helper'
 
 class NYPLRubyUtil
   class SierraApiClient < SierraApiClient
@@ -25,4 +26,7 @@ class NYPLRubyUtil
 
   class KinesisClient < KinesisClient
   end
+
+  class DeployHelper < DeployHelper
+  end
 end

data/lib/platform_api_client.rb

@@ -61,6 +61,7 @@ class PlatformApiClient
 
   # Authorizes the request.
   def authenticate!
+    $logger.debug('authenticating')
     # NOOP if we've already authenticated
     return nil if ! access_token.nil?
 
@@ -81,8 +82,10 @@ class PlatformApiClient
     end
 
     if response.code == '200'
-      access_token = JSON.parse(response.body)["access_token"]
+      self.access_token = JSON.parse(response.body)["access_token"]
+      $logger.debug('got token')
     else
+      $logger.debug('no token')
       nil
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nypl_ruby_util
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.7
 platform: ruby
 authors:
 - Daniel Appel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-13 00:00:00.000000000 Z
+date: 2020-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -52,6 +52,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.36.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-lambda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchevents
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nypl_log_formatter
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +114,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/deploy_helper.rb
 - lib/directory.rb
 - lib/errors.rb
 - lib/kinesis_client.rb
@@ -93,7 +122,7 @@ files:
 - lib/nypl_avro.rb
 - lib/nypl_ruby_util.rb
 - lib/platform_api_client.rb
-homepage:
+homepage: https://github.com/NYPL/NYPLRubyUtil
 licenses:
 - MIT
 metadata: {}
@@ -112,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: A repository of common utilities for NYPL Ruby application