lambda_convert 0.0.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ab417e94470f8f933b2d90b7010d3050cbdf608d
-  data.tar.gz: c2748ee3e263683c0b7aadf00d4a2da2e5b5883d
+SHA256:
+  metadata.gz: dae0a87a2bc2e45653d677a270f8ff84b605f4175a9250b5241d309673a52180
+  data.tar.gz: 7d809ea4f044bbd9eae311ea7909f3f129baf1d32bc47a31b6c073577dcfa79c
 SHA512:
-  metadata.gz: 30120d3f06649b62c0bdf9b3bcf44ca68fbf4a4d6efcc7a14b198187f054ff86d2a101a7eb1fec24a4ec84726812ca26b2425649344a6a587681ec3ffda0b6b3
-  data.tar.gz: 68052dd61c6afdc2da6d303105169eb302376b0d871d2ffcda3bdf2fdd8ccec4ad872578cb85fbcfb2651893b8b595c4a5047bf90e8ab53b145eaf77bfc5c403
+  metadata.gz: 799f4822b0010635000df3f6256908fccaef8faf966d9ed30088b1388e5f2e0d97b75d716c09f67a8040161f9f65a5c33535f00fb64b2ca824c05c42b45662fc
+  data.tar.gz: 6517f828ceddd7a726acb56c321ccd281ea1eafa9c3f9a52df67a1a1e77f348ad1632276e3776d886362c88f670bb1149eede818b72949bccbc7a6c4ad11afc1

data/README.md

@@ -19,11 +19,15 @@ To eliminate the big image file uploading issue and the security risk, the idea
 
 ## Environment variables
 
- - **CONVERT_S3_REGION** - AWS region for S3, default value will be reading from `AWS_REGION` if this environment variable is not set. (required)
- - **CONVERT_LAMBDA_REGION** - AWS region for Lambda, default value will be reading from `AWS_REGION` if this environment variable is not set. (required)
- - **CONVERT_ACCESS_KEY** - AWS access key, default value will be reading from `AWS_ACCESS_KEY_ID` if this environment variable is not set. (required)
- - **CONVERT_SECRET_ACCESS_KEY** - AWS secret key, default value will be reading from `AWS_SECRET_ACCESS_KEY` if this environment variable is not set. (required)
+ - **CONVERT_S3_REGION** - AWS region for S3, default value will be read from `AWS_REGION` if this environment variable is not set.
+ - **CONVERT_LAMBDA_REGION** - AWS region for Lambda, default value will be read from `AWS_REGION` if this environment variable is not set.
+ - **CONVERT_ACCESS_KEY** - AWS access key, default value will follow standard `aws-sdk` credential lookup sequence
+ - **CONVERT_SECRET_ACCESS_KEY** - AWS secret key, default value will follow standard `aws-sdk` credential lookup sequence
  - **CONVERT_S3_BUCKET** - AWS S3 bucket. (required)
  - **CONVERT_S3_KEY_PREFIX** - AWS S3 temporary file uploading prefix, default value is `_convert_tmp/`
  - **CONVERT_LAMBDA_FUNCTION** - Name of the AWS Lambda function to invoke, default value is `image-convert-prod`
  - **CONVERT_DISABLE_FALLBACK** - By default, this command line tool fallbacks to local `convert` command if remote operation fails. Set this value to 1 to disable the fallback behavior.
+
+## The AWS Lambda function
+
+The AWS Lambda function for running ImageMagick can be found here at https://github.com/envoy/envoy-convert

data/lib/lambda_convert/cli.rb

@@ -1,103 +1,110 @@
-require 'rubygems'
 require 'json'
+require 'securerandom'
 require 'English'
 
-require 'aws-sdk'
+require 'aws-sdk-lambda'
+require 'aws-sdk-s3'
+require 'forwardable'
 
 module LambdaConvert
   # `convert` command line tool implementation
   module CLI
-    class <<self
+    class << self
+      extend Forwardable
+      def_delegator :LambdaConvert, :config
       attr_accessor :logger
-    end
-
-    def self.lambda_convert
-      s3_region = ENV['CONVERT_S3_REGION'] || ENV['AWS_REGION']
-      lambda_region = ENV['CONVERT_LAMBDA_REGION'] || ENV['AWS_REGION']
-      aws_credentials = Aws::Credentials.new(
-        ENV['CONVERT_ACCESS_KEY'] || ENV['AWS_ACCESS_KEY_ID'],
-        ENV['CONVERT_SECRET_ACCESS_KEY'] || ENV['AWS_SECRET_ACCESS_KEY']
-      )
-      s3_bucket = ENV['CONVERT_S3_BUCKET']
-      s3_key_prefix = ENV['CONVERT_S3_KEY_PREFIX'] || '_convert_tmp/'
-      lambda_function = ENV['CONVERT_LAMBDA_FUNCTION'] || 'image-convert-prod'
 
-      s3 = Aws::S3::Client.new(
-        region: s3_region,
-        credentials: aws_credentials
-      )
-      aws_lambda = Aws::Lambda::Client.new(
-        region: lambda_region,
-        credentials: aws_credentials
-      )
-
-      abort('Invalid arguments') if ARGV.count < 2
+      def aws_credentials
+        if config.access_key && config.secret_key
+          Aws::Credentials.new(config.access_key, config.secret_key)
+        end
+      end
 
-      input_file, input_selecting = LambdaConvert::Utils.parse_input_path(
-        ARGV[0]
-      )
-      # Notice: there is also special output file syntax for convert command,
-      # but we are not supporting them now, as we probably won't use it
-      output_file = ARGV[-1]
+      def s3_client
+        @s3_client ||= Aws::S3::Client.new(
+          region: config.s3_region,
+          credentials: aws_credentials
+        )
+      end
 
-      input_key = "#{s3_key_prefix}#{SecureRandom.uuid}"
-      output_key = "#{s3_key_prefix}#{SecureRandom.uuid}"
+      def lambda_client
+        @aws_lambda ||= Aws::Lambda::Client.new(
+          region: config.lambda_region,
+          credentials: aws_credentials
+        )
+      end
+    end
 
-      logger.info("Uploading file to s3://#{s3_bucket}/#{input_key}")
+    def self.upload_file(input_file, input_key)
+      logger.info("Uploading file to s3://#{config.s3_bucket}/#{input_key}")
       File.open(input_file, 'rb') do |file|
-        s3.put_object(bucket: s3_bucket, key: input_key, body: file)
+        s3_client.put_object(bucket: config.s3_bucket, key: input_key, body: file)
       end
+    end
+
+    def self.invoke_lambda(input_key, input_selecting, args, output_key)
       source = '{source}'
       source += "[#{input_selecting}]" unless input_selecting.nil?
       instruction = {
         schema: 'envoy-convert-instruction',
         original: input_key,
-        bucket: s3_bucket,
-        write_options: {
-          acl: 'private'
-        },
+        bucket: config.s3_bucket,
+        write_options: {},
         key: output_key,
-        args: [source] + ARGV[1..-2] + ['{dest}']
+        args: [source] + args + ['{dest}']
       }
       logger.info("Invoking lambda with instruction #{instruction}")
 
-      resp = aws_lambda.invoke(
-        function_name: lambda_function,
+      resp = lambda_client.invoke(
+        function_name: config.lambda_function,
         invocation_type: 'RequestResponse',
         payload: JSON.dump(instruction)
       )
       logger.info("Get response of invoke #{resp}")
       raise 'Failed to run convert on Lambda' if resp.status_code != 200
+    end
 
+    def self.download_file(output_key, output_file)
       logger.info(
-        "Downloading file from s3://#{s3_bucket}/#{output_key} to " \
+        "Downloading file from s3://#{config.s3_bucket}/#{output_key} to " \
         "#{output_file}"
       )
-      s3.get_object(
+      s3_client.get_object(
         response_target: output_file,
-        bucket: s3_bucket,
+        bucket: config.s3_bucket,
         key: output_key
       )
+    end
+
+    def self.delete_files(keys)
+      logger.info("Delete files #{keys} from #{config.s3_bucket}")
+      s3_client.delete_objects(
+        bucket: config.s3_bucket,
+        delete: {
+          objects: keys.map { |key| { key: key } },
+          quiet: true
+        }
+      )
+    end
+
+    def self.lambda_convert
+      input_file, input_selecting = LambdaConvert::Utils.parse_input_path(
+        ARGV[0]
+      )
+      # Notice: there is also special output file syntax for convert command,
+      # but we are not supporting them now, as we probably won't use it
+      output_file = ARGV[-1]
+      input_key = File.join config.s3_key_prefix.to_s, SecureRandom.uuid
+      output_key = File.join config.s3_key_prefix.to_s, SecureRandom.uuid
+
+      upload_file(input_file, input_key)
+      invoke_lambda(input_key, input_selecting, ARGV[1..-2], output_key)
+      download_file(output_key, output_file)
+
       logger.info('Done')
     ensure
-      if !s3.nil? && !input_key.nil? && !output_key.nil?
-        logger.info(
-          "Delete files #{input_key} and #{output_key} from #{s3_bucket}"
-        )
-        s3.delete_objects(
-          bucket: s3_bucket,
-          delete: {
-            objects: [
-              {
-                key: input_key
-              },
-              {
-                key: output_key
-              }
-            ],
-            quiet: true
-          }
-        )
+      if !input_key.nil? && !output_key.nil?
+        delete_files([input_key, output_key])
       end
     end
 
@@ -117,9 +124,9 @@ module LambdaConvert
 
     def self.main
       abort('Recursive call') if ENV['CONVERT_RECURSIVE_FLAG'] == '1'
+      abort('Invalid arguments') if ARGV.count < 2
       lambda_convert
     rescue StandardError => e
-
       logger.warn("Failed to convert via lambda, error=#{e}")
       fallback_disabled = (ENV['CONVERT_DISABLE_FALLBACK'].to_i != 0) || false
       if fallback_disabled

data/lib/lambda_convert/config.rb

@@ -0,0 +1,22 @@
+module LambdaConvert
+  class Config
+    attr_accessor :access_key,
+                  :lambda_function,
+                  :lambda_region,
+                  :s3_bucket,
+                  :s3_key_prefix,
+                  :s3_region,
+                  :secret_key
+
+    # Set some defaults
+    def initialize
+      @access_key      = ??
+      @secret_key      = ??
+      @lambda_function = "image-convert-dev"
+      @lambda_region   = "us-east-1"
+      @s3_region       = "us-east-1"
+      @s3_bucket       = "envoy-development-staging-2"
+      @s3_key_prefix   = "_convert_tmp"
+    end
+  end
+end

data/lib/lambda_convert.rb

@@ -1,2 +1,13 @@
 require 'lambda_convert/cli'
+require 'lambda_convert/config'
 require 'lambda_convert/utils'
+
+module LambdaConvert
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure(&_block)
+    yield config
+  end
+end

metadata

@@ -1,48 +1,62 @@
 --- !ruby/object:Gem::Specification
 name: lambda_convert
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 1.0.0
 platform: ruby
 authors:
 - Fang-Pen Lin
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2017-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-lambda
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
-description: 
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description:
 email: fang@envoy.com
 executables:
 - convert
 extensions: []
 extra_rdoc_files: []
 files:
-- "./lib/.DS_Store"
 - "./lib/lambda_convert.rb"
 - "./lib/lambda_convert/cli.rb"
+- "./lib/lambda_convert/config.rb"
 - "./lib/lambda_convert/utils.rb"
 - LICENSE
 - README.md
 - bin/convert
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -57,11 +71,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
-signing_key: 
+rubygems_version: 3.4.22
+signing_key:
 specification_version: 4
 summary: AWS Lambda powered drop-in replacement for ImageMagick convert command line
   tool
 test_files: []
-has_rdoc: