sfn-lambda 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 687938d6c0855b30342de6b52eb87a837043aabe
+  data.tar.gz: e38d4701d422e4a108a045b9fbeb5c2c8e439a98
+SHA512:
+  metadata.gz: 5c76cb6f390565b252cbd115de9073cf2033171617c754fd65c2f99e4dca2cd30ce76841cd5dae27f1c15643ff0a2b6e8d2bd014dae45705229b50d06f9325c9
+  data.tar.gz: 3135cd6699333754d1825d00ddc1633e4d702e118ae737e1b6dfbd89f75c42c392d157ee70883fe738b48d300c11833bf8bdb202abf785635da62a82a45886ed

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Chris Roberts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,215 @@
+# AWS Lambda for SparkleFormation
+
+Lets make lambda functions easier to manage in CloudFormation!
+
+## Design
+
+This SparkleFormation Callback adds a new helper method to AWS based
+SparkleFormation templates called `lambda!`. This helper method will
+insert an `AWS::Lambda::Function` resource into the template using
+source files contained within configured directories.
+
+## Features
+
+* Individual source files for lambda functions
+* Automatic resource creation within templates
+* Automatic code setup for resource
+ * Acceptable functions will be defined inline
+ * S3 storage will be used when inline is unacceptable
+ * S3 versioning will be used when bucket configured for versioning
+ * Automatic asset builds (for `java8` runtime targets)
+
+## Usage
+
+### Setup
+
+First add the `sfn-lambda` gem to the local bundle (in the `./Gemfile`):
+
+```ruby
+group :sfn do
+  gem 'sfn-lambda'
+end
+```
+
+Now enable the `sfn-lambda` callback in the `.sfn` configuration file:
+
+```ruby
+Configuration.new do
+  ...
+  callbacks do
+    default ['lambda']
+  end
+  ...
+end
+```
+
+_NOTE: If using the `java8` runtime for lambda functions, `maven` must
+be installed with `mvn` being available within the user's PATH._
+
+### Configuration
+
+#### Lambda function files directory
+
+By default the `sfn-lambda` callback will search the `./lambda` directory
+for lambda function files. A custom directory path can be used by modifying
+the configuration:
+
+```ruby
+Configuration.new do
+  lambda do
+    directory './my-lambdas'
+  end
+end
+```
+
+#### S3 lambda function file storage
+
+By default the `sfn-lambda` callback will use the bucket name provided by
+the `nesting_bucket` configuration item. This can be customized to use a
+different bucket by modifying the configuration:
+
+```ruby
+Configuration.new do
+  lambda do
+    upload do
+      bucket 'my-custom-bucket'
+    end
+  end
+end
+```
+
+### Lambda function files
+
+The path of lambda function files is important. The path is used to determine
+the proper handler for running the lambda function, as well as providing the
+identifier to reference the function. The path structure is as follows:
+
+```
+./lambda/RUNTIME/FUNCTION_NAME.extension
+```
+
+The `RUNTIME` defines the runtime used for handling the lambda function. At
+the time of writing this, that value can be one of:
+
+* `nodejs`
+* `nodejs4.3`
+* `java8`
+* `python2.7`
+
+_NOTE: Runtime values are not validated which allows new runtimes to be used
+as they are made available._
+
+The `FUNCTION_NAME` is used for two purposes:
+
+1. It identifies the function name lambda should use
+2. It is used in combination with the `RUNTIME` to identify the lambda in the template
+
+### Example
+
+Using the python example described within the lambda documentation:
+
+* http://docs.aws.amazon.com/lambda/latest/dg/python-programming-model-handler-types.html
+
+we can define our handler code:
+
+* `./lambda/python2.7/my_handler.py`
+
+```python
+def my_handler(event, context):
+    message = 'Hello {} {}!'.format(event['first_name'], event['last_name'])
+    return {
+        'message' : message
+    }
+```
+
+Now, using a new helper method, lambda resources can be created within a SparkleFormation template using
+the newly created file:
+
+* `./sparkleformation/lambda_test.rb`
+
+```ruby
+SparkleFormation.new(:lambda_test) do
+  lambda!(:my_handler)
+end
+```
+
+When the template is printed a lambda resource is shown with the function properly inlined:
+
+```
+$ sfn print --file lambda_test
+{
+  "Resources": {
+    "MyHandlerLambdaFunction": {
+      "Type": "AWS::Lambda::Function",
+      "Properties": {
+        "Handler": "python2.7",
+        "FunctionName": "my_handler",
+        "ZipFile": "def my_handler(event, context):\n    message = 'Hello {} {}!'.format(event['first_name'], event['last_name'])\n    return {\n        'message' : message\n    }\n\n"
+      }
+    }
+  }
+}
+```
+
+If the name of a lambda function is shared across multiple runtimes, the desired runtime
+can be specified within the call:
+
+```ruby
+SparkleFormation.new(:lambda_test) do
+  lambda!(:my_handler, :runtime => 'python2.7')
+end
+```
+
+If a lambda function is to be used for creating multiple resources within a template, a
+custom name can be added as well:
+
+```ruby
+SparkleFormation.new(:lambda_test) do
+  lambda!(:my_handler, :first, :runtime => 'python2.7')
+  lambda!(:my_handler, :second, :runtime => 'python2.7')
+end
+```
+
+### Special Cases
+
+### S3 Storage
+
+When the size of the lambda function is greater than the defined max size (4096 default),
+the function will be stored on S3. If the bucket configured for storage has versioning
+enabled, versioning information will be automatically set within the resource. If no
+versioning information is available, a checksum will be attached to the generated key name.
+
+### Builds
+
+For lambda functions utilizing the `java8` runtime, the `sfn-lambda` callback will behave
+slightly different. When discovering available lambda functions, the directory names under the
+`./lambda/java8` directory will be used. This allows for the collection of required files to
+be stored within the directory.
+
+Using the example here: http://docs.aws.amazon.com/lambda/latest/dg/java-create-jar-pkg-maven-no-ide.html
+
+The defined directory structure would be:
+
+```
+$ cd ./lambda
+$ tree
+.
+|____java8
+| |____hello
+| | |____src
+| | | |____main
+| | | | |____java
+| | | | | |____example
+| | | | | | |____Hello.java
+| | |____pom.xml
+```
+
+When the `hello` lambda function is used within a template, `sfn-lambda` will automatically generate
+the required jar file using Maven and store the resulting asset on S3.
+
+_NOTE: Maven is required to be installed when using the `java8` runtime_
+
+## Info
+
+* Repository: https://github.com/sparkleformation/sfn-lambda
+* IRC: Freenode @ #sparkleformation

data/lib/sfn-lambda.rb

@@ -0,0 +1,5 @@
+require 'sfn'
+require 'sfn-lambda/setup'
+require 'sfn-lambda/control'
+require 'sfn-lambda/inject'
+require 'sfn-lambda/version'

data/lib/sfn-lambda/control.rb

@@ -0,0 +1,234 @@
+require 'singleton'
+
+module SfnLambda
+
+  # @return [Control]
+  def self.control
+    Control.instance
+  end
+
+  class Control
+
+    include Singleton
+
+    DEFAULTS = {
+      :INLINE_MAX_SIZE => 4096,
+      :INLINE_RESTRICTED => ['java8'].freeze,
+      :BUILD_REQUIRED => {
+        'java8' => {
+          :build_command => 'mvn package',
+          :output_directory => './target',
+          :asset_extension => '.jar'
+        }.freeze
+      }.freeze
+    }.freeze
+
+    attr_reader :functions
+    attr_accessor :callback
+
+    # Create a new control instance
+    #
+    # @return [self]
+    def initialize
+      @functions = Smash.new
+    end
+
+    # @return [Array<String>] paths to lambda storage directories
+    def lambda_directories
+      paths = [callback.config.fetch(:lambda, :directory, 'lambda')].flatten.compact.uniq.map do |path|
+        File.expand_path(path)
+      end
+      invalids = paths.find_all do |path|
+        !File.directory?(path)
+      end
+      unless(invalids.empty?)
+        raise "Invalid lambda directory paths provided: #{invalids.join(', ')}"
+      end
+      paths
+    end
+
+    # Get path to lambda function
+    #
+    # @param name [String] name of lambda function
+    # @param runtime [String] runtime of lambda function
+    # @return [Hash] {:path, :runtime}
+    def get(name, runtime=nil)
+      unless(runtime)
+        runtime = functions.keys.find_all do |r_name|
+          functions[r_name].keys.include?(name.to_s)
+        end
+        if(runtime.empty?)
+          raise "Failed to locate requested lambda function `#{name}`"
+        elsif(runtime.size > 1)
+          raise "Multiple lambda function matches for `#{name}`. (In runtimes: `#{runtime.sort.join('`, `')}`)"
+        end
+        runtime = runtime.first
+      end
+      result = functions.get(runtime, name)
+      if(result.nil?)
+        raise "Failed to locate requested lambda function `#{name}`"
+      else
+        Smash.new(:path => result, :runtime => runtime, :name => name)
+      end
+    end
+
+    # Format lambda function content to use within template. Will provide raw
+    # source when function can be inlined within the template. If inline is not
+    # available, it will store within S3
+    #
+    # @param info [Hash] content information
+    # @option info [String] :path path to lambda function
+    # @option info [String] :runtime runtime of lambda function
+    # @option info [String] :name name of lambda function
+    # @return [Smash] content information
+    def format_content(info)
+      if(can_inline?(info))
+        Smash.new(:raw => File.read(info[:path]))
+      else
+        apply_build!(info)
+        key_name = generate_key_name(info)
+        io = File.open(info[:path], 'rb')
+        file = bucket.files.build
+        file.name = key_name
+        file.body = io
+        file.save
+        io.close
+        if(versioning_enabled?)
+          result = s3.request(
+            :path => s3.file_path(file),
+            :endpoint => s3.bucket_endpoint(file.bucket),
+            :method => :head
+          )
+          version = result[:headers][:x_amz_version_id]
+        end
+        Smash(:bucket => storage_bucket, :key => key_name, :version => version)
+      end
+    end
+
+    # Build the lambda asset if building is a requirement
+    #
+    # @param info [Hash]
+    # @return [TrueClass, FalseClass] build was performed
+    def apply_build!(info)
+      if(build_info = self[:build_required][info[:runtime]])
+        Open3.popen3(build_info[:build_command], :chdir => info[:path]) do |stdin, stdout, stderr, wait_thread|
+          exit_status = wait_thread.value
+          unless(exit_status.success?)
+            callback.ui.error "Failed to build lambda assets for storage from path: #{info[:path]}"
+            callback.ui.debug "Build command used which generated failure: `#{build_info[:build_command]}`"
+            callback.ui.debug "STDOUT: #{stdout.read}"
+            callback.ui.debug "STDERR: #{stderr.read}"
+            raise "Failed to build lambda asset for storage! (path: `#{info[:path]}`)"
+          end
+        end
+        file = Dir.glob(File.join(info[:path], build_info[:output_directory], "*.#{build_config[:asset_extension]}")).first
+        if(file)
+          info[:path] = file
+          true
+        else
+          debug "Glob pattern used for build asset detection: `#{File.join(info[:path], build_info[:output_directory], "*.#{build_config[:asset_extension]}")}`"
+          raise "Failed to locate generated build asset for storage! (path: `#{info[:path]}`)"
+        end
+      else
+        false
+      end
+    end
+
+    # @return [Miasma::Models::Storage::Bucket]
+    def bucket
+      storage_bucket = callback.config.fetch(:lambda, :upload, :bucket, callback.config[:nesting_bucket])
+      if(storage_bucket)
+        s3 = api.connection.api_for(:storage)
+        l_bucket = s3.buckets.get(storage_bucket)
+      end
+      unless(l_bucket)
+        raise "Failed to locate configured bucket for lambda storage (#{storage_bucket})"
+      else
+        l_bucket
+      end
+    end
+
+    # @return [TrueClass, FalseClass] bucket has versioning enabled
+    def versioning_enabled?
+      unless(@versioned.nil?)
+        s3 = api.connection.api_for(:storage)
+        result = s3.request(
+          :path => '/',
+          :params => {
+            :versioning => true
+          },
+          :endpoint => s3.bucket_endpoint(bucket)
+        )
+        @versioned = result.get(:body, 'VersioningConfiguration', 'Status') == 'Enabled'
+      end
+      @versioned
+    end
+
+    # Generate key name based on state
+    #
+    # @param info [Hash]
+    # @return [String] key name
+    def generate_key_name(info)
+      if(versioning_enabled?)
+        "sfn.lambda/#{info[:runtime]}/#{File.basename(info[:path])}"
+      else
+        checksum = Digest::SHA256.new
+        File.open(info[:path], 'rb') do |file|
+          while(content = file.read(2048))
+            checksum << content
+          end
+        end
+        "sfn.lambda/#{info[:runtime]}/#{File.basename(info[:path])}-#{checksum.base64digest}"
+      end
+    end
+
+    # Determine if function can be defined inline within template
+    #
+    # @param info [Hash]
+    # @return [TrueClass, FalseClass]
+    def can_inline?(info)
+      !self[:inline_restricted].include?(info[:runtime]) && File.size(info[:path]) <= self[:inline_max_size]
+    end
+
+    # Get configuration value for control via sfn configuration and fall back
+    # to defined defaults if not set
+    #
+    # @return [Object] configuration value
+    def [](key)
+      callback.config.fetch(:lambda, :config, key.to_s.downcase, DEFAULTS[key.to_s.upcase.to_sym])
+    end
+
+    # Discover all defined lambda functions available in directories provided
+    # via configuration
+    #
+    # @return [NilClass]
+    def discover_functions!
+      core_paths = lambda_directories
+      core_paths.each do |path|
+        Dir.new(path).each do |dir_item|
+          next if dir_item.start_with?('.')
+          next unless File.directory?(File.join(path, dir_item))
+          if(self[:build_required].keys.include?(dir_item))
+            Dir.new(File.join(path, dir_item)).each do |item|
+              next if item.start_with?('.')
+              full_item = File.join(path, dir_item, item)
+              next unless File.directory?(full_item)
+              functions.set(dir_item, item, full_item)
+            end
+          else
+            items = Dir.glob(File.join(path, dir_item, '**', '**', '*'))
+            items.each do |full_item|
+              next unless File.file?(full_item)
+              item = full_item.sub(File.join(path, dir_item, ''), '').gsub(File::SEPARATOR, '')
+              item = item.sub(/#{Regexp.escape(File.extname(item))}$/, '')
+              functions.set(dir_item, item, full_item)
+            end
+          end
+        end
+      end
+      nil
+    end
+
+  end
+
+end

data/lib/sfn-lambda/inject.rb

@@ -0,0 +1,42 @@
+class SparkleFormation
+  module SparkleAttribute
+    module Aws
+
+      def _lambda(*fn_args)
+        if(fn_args.size > 2)
+          fn_name, fn_uniq_name, fn_opts = fn_args
+        else
+          fn_name, fn_uniq_name = fn_args
+        end
+        __t_stringish(fn_name)
+        __t_stringish(fn_uniq_name) unless fn_uniq_name.is_a?(::NilClass)
+        if(fn_opts)
+          fn_runtime = fn_opts[:runtime] if fn_opts[:runtime]
+        end
+        unless(fn_runtime.is_a?(::NilClass))
+          __t_stringish(fn_runtime)
+        end
+        lookup = ::SfnLambda.control.get(fn_name, fn_runtime)
+        new_fn = _dynamic(:aws_lambda_function,
+          [fn_name, fn_uniq_name].compact.map(&:to_s).join('_'),
+          :resource_name_suffix => :lambda_function
+        )
+        new_fn.properties.handler lookup[:runtime]
+        new_fn.properties.function_name fn_name
+        content = ::SfnLambda.control.format_content(lookup)
+        if(content[:raw])
+          new_fn.properties.zip_file content[:raw]
+        else
+          new_fn.properties.s3_bucket content[:bucket]
+          new_fn.properties.s3_key content[:key]
+          if(content[:version])
+            new_fn.properties.s3_object_version content[:version]
+          end
+        end
+        new_fn
+      end
+      alias_method :lambda!, :_lambda
+
+    end
+  end
+end

data/lib/sfn-lambda/setup.rb

@@ -0,0 +1,18 @@
+require 'sfn-lambda'
+
+module Sfn
+  class Callback
+    class Lambda < Callback
+
+      def quiet
+        ENV['DEBUG']
+      end
+
+      def after_config(*_)
+        SfnLambda.control.callback = self
+        SfnLambda.control.discover_functions!
+      end
+
+    end
+  end
+end

data/lib/sfn-lambda/version.rb

@@ -0,0 +1,3 @@
+module SfnLambda
+  VERSION = Gem::Version.new('0.1.0')
+end

data/sfn-lambda.gemspec

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'sfn-lambda/version'
+Gem::Specification.new do |s|
+  s.name = 'sfn-lambda'
+  s.version = SfnLambda::VERSION.to_s
+  s.summary = 'AWS Lambda integration for SparkleFormation'
+  s.author = 'Chris Roberts'
+  s.email = 'chrisroberts.code@gmail.com'
+  s.homepage = 'http://github.com/spox/sfn-lambda'
+  s.description = 'AWS Lambda integration for SparkleFormation'
+  s.license = 'MIT'
+  s.require_path = 'lib'
+  s.add_runtime_dependency 'sparkle_formation', '>= 2.1.0'
+  s.files = Dir['{lib,docs}/**/*'] + %w(sfn-lambda.gemspec README.md CHANGELOG.md LICENSE)
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: sfn-lambda
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sparkle_formation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+description: AWS Lambda integration for SparkleFormation
+email: chrisroberts.code@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/sfn-lambda.rb
+- lib/sfn-lambda/control.rb
+- lib/sfn-lambda/inject.rb
+- lib/sfn-lambda/setup.rb
+- lib/sfn-lambda/version.rb
+- sfn-lambda.gemspec
+homepage: http://github.com/spox/sfn-lambda
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: AWS Lambda integration for SparkleFormation
+test_files: []