modulator 0.1.1

data/lib/lambda/aws_stack_builder.rb

@@ -0,0 +1,223 @@
+require 'humidifier'
+require_relative 'aws_stack_uploader'
+
+module AwsStackBuilder
+  module_function
+
+  RUBY_VERSION = 'ruby2.5'
+  GEM_PATH = '/opt/ruby/2.5.0'
+  LAMBDA_HANDLER_FILE_NAME = 'lambda-handler'
+
+  class << self
+    attr_accessor :app_name, :stack, :api_gateway_deployment, :gateway_id, :app_path, :app_dir, :bucket, :stack_opts
+  end
+
+  def init(app_name:, bucket:, **stack_opts)
+    puts 'Initializing stack'
+    @app_name   = app_name.camelize
+    @bucket     = bucket
+    @app_path   = Pathname.getwd
+    @app_dir    = app_path.basename.to_s
+    @stack_opts = stack_opts
+    @stack      = Humidifier::Stack.new(name: @app_name, aws_template_format_version: '2010-09-09')
+
+    # api stage
+    @stack.add_parameter('ApiGatewayStageName', description: 'Gateway deployment stage', type: 'String', default: 'v1')
+
+    add_api_gateway
+    add_api_gateway_deployment
+    add_lambda_iam_role
+    upload_files
+    extend_stack_instance(@stack)
+    @stack
+  end
+
+  def upload_files
+    upload_lambda_handler
+    puts 'Generating layers'
+    app_path.join('.modulator').mkpath
+    upload_gems_layer
+    upload_app_layer
+  end
+
+  # helpers
+  def extend_stack_instance(stack)
+    stack.instance_eval do
+      def add_lambda_endpoint(**opts) # gateway:, mod:, wrapper: {}, env: {}, settings: {}
+        # add lambda
+        lambda = AwsStackBuilder.add_lambda(opts)
+        # add api resources
+        AwsStackBuilder.add_api_gateway_resources(gateway: opts[:gateway], lambda: lambda)
+      end
+    end
+  end
+
+  # gateway
+  def add_api_gateway
+    @gateway_id = 'ApiGateway'
+    @stack.add(gateway_id, Humidifier::ApiGateway::RestApi.new(name: app_name, description: app_name + ' API'))
+  end
+
+  # gateway deployment
+  def add_api_gateway_deployment
+    @api_gateway_deployment = Humidifier::ApiGateway::Deployment.new(
+      rest_api_id: Humidifier.ref(gateway_id),
+      stage_name: Humidifier.ref("ApiGatewayStageName")
+    )
+    @stack.add('ApiGatewayDeployment', @api_gateway_deployment)
+    @stack.add_output('ApiGatewayInvokeURL',
+        value: Humidifier.fn.sub("https://${#{gateway_id}}.execute-api.${AWS::Region}.amazonaws.com/${ApiGatewayStageName}"),
+        description: 'API root url',
+        export_name: app_name + 'RootUrl'
+    )
+    @api_gateway_deployment.depends_on = []
+  end
+
+  # lambda function
+  def add_lambda(gateway:, mod:, wrapper: {}, env: {}, settings: {})
+    lambda_config = {}
+    name_parts = mod[:name].split('::')
+    {gateway: gateway, module: mod, wrapper: wrapper}.each do |env_group_prefix, env_group|
+      env_group.each{|env_key, env_value| lambda_config["#{env_group_prefix}_#{env_key}"] = env_value}
+    end
+
+    lambda_function = Humidifier::Lambda::Function.new(
+      description: "Lambda for #{mod[:name]}.#{mod[:method]}",
+      function_name: [app_name.dasherize, name_parts, mod[:method]].flatten.map(&:downcase).join('-'),
+      handler: "#{LAMBDA_HANDLER_FILE_NAME}.AwsLambdaHandler.call",
+      environment: {
+        variables: env
+          .reduce({}){|env_as_string, (k, v)| env_as_string.update(k.to_s => v.to_s)}
+          .merge(lambda_config)
+          .merge('GEM_PATH' => GEM_PATH, 'app_dir' => app_dir)
+      },
+      role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
+      timeout: settings[:timeout] || stack_opts[:timeout] || 15,
+      memory_size: settings[:memory_size] || stack_opts[:memory_size] || 128,
+      runtime: RUBY_VERSION,
+      code: {
+        s3_bucket: bucket,
+        s3_key: LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
+      },
+      layers: [
+        Humidifier.ref(app_name + 'Layer'),
+        Humidifier.ref(app_name + 'GemsLayer')
+      ]
+    )
+    id = ['Lambda', name_parts, mod[:method].capitalize].join
+    @stack.add(id, lambda_function)
+    add_lambda_invoke_permission(id: id, gateway: gateway)
+    id
+  end
+
+  # invoke permission
+  def add_lambda_invoke_permission(id:, gateway:)
+    arn_path_matcher = gateway[:path].split('/').each_with_object([]) do |fragment, matcher|
+      fragment = '*' if fragment.start_with?(':')
+      matcher << fragment
+    end.join('/')
+    @stack.add('LambdaPermission', Humidifier::Lambda::Permission.new(
+        action: "lambda:InvokeFunction",
+        function_name: Humidifier.fn.get_att([id, 'Arn']),
+        principal: "apigateway.amazonaws.com",
+        source_arn: Humidifier.fn.sub("arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${#{gateway_id}}/*/#{gateway[:verb]}/#{arn_path_matcher}")
+      )
+    )
+  end
+
+  # gateway method
+  def add_api_gateway_resources(gateway:, lambda:)
+
+    # example: calculator/algebra/:x/:y/sum -> module name, args, method name
+    path = gateway[:path].split('/')
+
+    # root resource
+    root_resource = path.shift
+    @stack.add(root_resource.camelize, Humidifier::ApiGateway::Resource.new(
+        rest_api_id: Humidifier.ref(AwsStackBuilder.gateway_id),
+        parent_id: Humidifier.fn.get_att(["ApiGateway", "RootResourceId"]),
+        path_part: root_resource
+      )
+    )
+
+    # args and method name are nested resources
+    parent_resource = root_resource.camelize
+    path.each do |fragment|
+      if fragment.start_with?(':')
+        fragment = fragment[1..-1]
+        dynamic_fragment = "{#{fragment}}"
+      end
+      @stack.add(parent_resource + fragment.camelize, Humidifier::ApiGateway::Resource.new(
+          rest_api_id: Humidifier.ref(AwsStackBuilder.gateway_id),
+          parent_id: Humidifier.ref(parent_resource),
+          path_part: dynamic_fragment || fragment
+        )
+      )
+      parent_resource = parent_resource + fragment.camelize
+    end
+
+    # attach lambda to last resource
+    id = 'EndpointFor' + (gateway[:path].gsub(':', '').gsub('/', '_')).camelize
+    @stack.add(id, Humidifier::ApiGateway::Method.new(
+        authorization_type: 'NONE',
+        http_method: gateway[:verb].to_s.upcase,
+        integration: {
+          integration_http_method: 'POST',
+          type: "AWS_PROXY",
+          uri: Humidifier.fn.sub([
+            "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations",
+            'lambdaArn' => Humidifier.fn.get_att([lambda, 'Arn'])
+          ])
+        },
+        rest_api_id: Humidifier.ref(gateway_id),
+        resource_id: Humidifier.ref(parent_resource) # last evaluated resource
+      )
+    )
+
+    # deployment depends on the method
+    @api_gateway_deployment.depends_on << id
+  end
+
+  def add_lambda_iam_role(function_name: nil)
+    @stack.add('LambdaRole', Humidifier::IAM::Role.new(
+        assume_role_policy_document: {
+          'Version' => "2012-10-17",
+          'Statement' => [
+            {
+              'Action' => ["sts:AssumeRole"],
+              'Effect' => "Allow",
+              'Principal' => {
+                'Service' => ["lambda.amazonaws.com"]
+              }
+            }
+          ]
+        },
+        policies: [
+          {
+            'policy_document' => {
+              'Version' => "2012-10-17",
+              'Statement' => [
+                {
+                  'Action' => [
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                  ],
+                  'Effect' => "Allow",
+                  'Resource' => Humidifier.fn.sub("arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*")
+                },
+                {
+                  'Action' => [
+                    "logs:CreateLogGroup",
+                  ],
+                  'Effect' => "Allow",
+                  'Resource' => "*"
+                }
+              ]
+            },
+            'policy_name' => "cloud-watch-access"
+          }
+        ]
+      )
+    )
+  end
+end

data/lib/lambda/aws_stack_uploader.rb

@@ -0,0 +1,147 @@
+require 'aws-sdk-s3'
+require 'digest'
+require 'bundler'
+
+module AwsStackBuilder
+  module_function
+
+  S3Client = Aws::S3::Client.new
+
+  def upload_lambda_handler
+    bucket_name = Humidifier.ref("S3Bucket").reference
+    lambda_handler_key = LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
+    source = <<~SOURCE
+      # see handler AwsApiGatewayEventHandler.call(event: event, context: context) in required file
+      require 'lambda/aws_lambda_handler'
+      Dir.chdir('/opt/ruby/lib/' + ENV['app_dir'])
+    SOURCE
+
+    existing_handler = S3Client.get_object(
+      bucket: bucket,
+      key: lambda_handler_key
+    ) rescue false # not found
+
+    existing_source = Zip::InputStream.open(existing_handler.body) do |zip_file|
+      zip_file.get_next_entry
+      zip_file.read
+    end if existing_handler
+
+    if existing_source != source
+      puts '- uploading generic lambda handler'
+      source_zip_file = Zip::OutputStream.write_buffer do |zip|
+        zip.put_next_entry LAMBDA_HANDLER_FILE_NAME + '.rb'
+        zip.print source
+      end
+      S3Client.put_object(
+        bucket: bucket,
+        key: lambda_handler_key,
+        body: source_zip_file.tap(&:rewind).read
+      )
+    end
+  end
+
+  def upload_gems_layer
+    if !app_path.join('Gemfile').exist?
+      puts '- no Gemfile detected'
+      return
+    end
+
+    # calculate Gemfile checksum
+    checksum_path = app_path.join('.modulator/gemfile_checksum')
+    old_checksum  = (checksum_path.read rescue nil)
+    new_checksum  = Digest::MD5.hexdigest(File.read(app_path.join('Gemfile.lock')))
+
+    zip_file_name = app_dir + '_gems.zip'
+    gems_path = app_path.join('.modulator/gems')
+    gems_zip_path = gems_path.parent.join(zip_file_name)
+
+    if old_checksum != new_checksum
+      puts '- uploading gems layer'
+      checksum_path.write(new_checksum)
+
+      # bundle gems
+      Bundler.with_clean_env do
+        Dir.chdir(app_path) do
+          `bundle install --path=./.modulator/gems --clean`
+        end
+      end
+      ZipFileGenerator.new(gems_path, gems_zip_path).write
+
+      # upload zipped file
+      gem_layer = S3Client.put_object(
+        bucket: bucket,
+        key: zip_file_name,
+        body: gems_zip_path.read
+      )
+      # delete zipped file
+      FileUtils.remove_dir(gems_path)
+      gems_zip_path.delete
+    else
+      puts '- using existing gems layer'
+      gem_layer = S3Client.get_object(bucket: bucket, key: zip_file_name)
+    end
+
+    add_layer(
+      name: app_name + 'Gems',
+      description: "App gems",
+      s3_key: zip_file_name,
+      s3_object_version: gem_layer.version_id
+    )
+  end
+
+  def upload_app_layer
+    wd = Pathname.getwd
+    zip_file_name = app_dir + '.zip'
+    app_zip_path  = wd.join(zip_file_name)
+
+    # calculate checksum for app folder
+    checksum_path = app_path.join('.modulator/app_checksum')
+    old_checksum  = (checksum_path.read rescue nil)
+    new_checksum  = checksum(app_path)
+
+    if old_checksum != new_checksum
+      puts '- uploading app layer'
+      checksum_path.write(new_checksum)
+      ZipFileGenerator.new(app_path, app_zip_path).write
+      # upload zipped file
+      app_layer = S3Client.put_object(
+        bucket: bucket,
+        key: zip_file_name,
+        body: app_zip_path.read
+      )
+      # delete zipped file
+      app_zip_path.delete
+    else
+      puts '- using existing app layer'
+      app_layer = S3Client.get_object(bucket: bucket, key: zip_file_name)
+    end
+
+    add_layer(
+      name: app_name,
+      description: "App source. MD5: #{new_checksum}",
+      s3_key: zip_file_name,
+      s3_object_version: app_layer.version_id
+    )
+  end
+
+  # add layer
+  def add_layer(name:, description:, s3_key:, s3_object_version:)
+    stack.add(name + 'Layer', Humidifier::Lambda::LayerVersion.new(
+        compatible_runtimes: [RUBY_VERSION],
+        layer_name: name,
+        description: description,
+        content: {
+          s3_bucket: bucket,
+          s3_key: s3_key,
+          s3_object_version: s3_object_version
+        }
+      )
+    )
+  end
+
+  def checksum(dir)
+    files = Dir["#{dir}/**/*"].reject{|f| File.directory?(f)}
+    content = files.map{|f| File.read(f)}.join
+    Digest::MD5.hexdigest(content)
+  end
+end

data/lib/modulator.rb

@@ -0,0 +1,114 @@
+require 'json'
+
+require 'lambda/aws_lambda_handler'
+require 'lambda/aws_stack_builder'
+require 'utils'
+
+module Modulator
+  module_function
+  LAMBDAS   = {}
+
+  def add_lambda(lambda_def, **opts) # opts are for overrides
+    if lambda_def.is_a?(Hash)
+      add_lambda_from_hash(lambda_def)
+    else
+      add_lambda_from_module(lambda_def, **opts)
+    end
+  end
+
+  def add_lambda_from_hash(hash)
+    LAMBDAS[hash[:name]] = {
+      name: hash[:name],
+      gateway: hash[:gateway],
+      module: hash[:module],
+      wrapper: hash[:wrapper] || {},
+      env: hash[:env] || {},
+      settings: hash[:settings] || {}
+    }
+  end
+
+  def add_lambda_from_module(mod, **opts)
+    mod.singleton_methods.sort.each do |module_method|
+      module_name = mod.to_s
+      module_names = module_name.split('::').map(&:downcase)
+      verb = 'GET'
+      path_fragments = module_names.dup
+
+      # process parameters
+      # method(a, b, c = 1, *args, d:, e: 2, **opts)
+      # [[:req, :a], [:req, :b], [:opt, :c], [:rest, :args], [:keyreq, :d], [:key, :e], [:keyrest, :opts]]
+      mod.method(module_method).parameters.each do |param|
+        param_type = param[0]
+        param_name = param[1]
+
+        # collect required params
+        path_fragments << ":#{param_name}" if param_type == :req
+
+        # post if we have optional key param, ie. pet: {}
+        verb = 'POST' if param_type == :key
+      end
+
+      # delete is special case based on method name
+      verb = 'DELETE' if %w[destroy delete remove implode].include? module_method.to_s
+
+      # finalize path
+      path_fragments << module_method
+      path = path_fragments.join('/')
+
+      add_lambda_from_hash(
+        {
+          name: "#{module_names.join('-')}-#{module_method}",
+          gateway: {
+            verb: opts.dig(module_method, :gateway, :verb) || verb,
+            path: opts.dig(module_method, :gateway, :path) || path
+          },
+          module: {
+            name: module_name,
+            method: module_method.to_s,
+            path: module_names.join('/') # file name
+          },
+          wrapper: opts.dig(module_method, :wrapper) || opts[:wrapper],
+          env: opts.dig(module_method, :env),
+          settings: opts.dig(module_method, :settings)
+        }
+      )
+    end
+  end
+
+  def set_env(lambda_def)
+    # remove wrapper if already set
+    %w(name method path).each{|name| ENV.delete('wrapper_' + name)}
+    # set env values
+    lambda_def[:module].each{|name, value| ENV['module_' + name.to_s] = value.to_s}
+    lambda_def[:gateway].each{|name, value| ENV['gateway_' + name.to_s] = value.to_s}
+    lambda_def[:wrapper]&.each{|name, value| ENV['wrapper_' + name.to_s] = value.to_s}
+    lambda_def[:env]&.each{|name, value| ENV[name.to_s] = value.to_s} # custom values
+  end
+
+  def init_stack(app_name:, bucket:, **stack_opts)
+    stack = AwsStackBuilder.init({
+        app_name: app_name.camelize,
+        bucket: bucket,
+      }.merge(stack_opts))
+
+    # add lambdas to stack
+    puts 'Generating endpoints'
+    LAMBDAS.each do |name, config|
+      puts "- adding #{config.dig(:module, :name)}.#{config.dig(:module, :method)} to #{config.dig(:gateway, :path)}"
+      stack.add_lambda_endpoint(
+        gateway: config[:gateway],
+        mod: config[:module],
+        wrapper: config[:wrapper] || {},
+        env: config[:env] || {},
+        settings: config[:settings] || {}
+      )
+    end
+
+    # validate stack
+    # puts 'Validating stack'
+    # puts '- it is valid' if stack.valid?
+
+    # return humidifier instance
+    stack
+  end
+end

data/lib/utils.rb

@@ -0,0 +1,101 @@
+require 'zip'
+
+# NOTE: this file is not required while running your code
+# the patched classes are used only in tests and tools
+class String
+  def camelize
+    split('_').collect do |word|
+      word[0] = word[0].upcase
+      word
+    end.join
+  end
+
+  def underscore
+    gsub(/([A-Z]+)([0-9]|[A-Z]|\z)/){"#{$1.capitalize}#{$2}"}
+      .gsub(/(.)([A-Z])/, '\1_\2')
+      .downcase
+  end
+
+  def dasherize
+    underscore.gsub('_', '-')
+  end
+end
+
+class Object
+  def symbolize_keys
+    case self
+    when Hash
+      hash = {}
+      each {|k, v| hash[k.to_sym] = v.symbolize_keys}
+      hash
+    when Array
+      map {|x| x.symbolize_keys}
+    else
+      self
+    end
+  end
+
+
+  def stringify_keys
+    case self
+    when Hash
+      hash = {}
+      each {|k, v| hash[k.to_s] = v.stringify_keys}
+      hash
+    when Array
+      map {|x| x.stringify_keys}
+    else
+      self
+    end
+  end
+end
+
+module Utils
+  module_function
+
+  def load_json(path)
+    JSON.parse(File.read(path))
+  end
+end
+
+# NOTE: this code is taken from https://github.com/rubyzip/rubyzip examples
+# Usage:
+#   directoryToZip = "/tmp/input"
+#   outputFile = "/tmp/out.zip"
+#   zf = ZipFileGenerator.new(directoryToZip, outputFile)
+#   zf.write()
+class ZipFileGenerator
+
+  # Initialize with the directory to zip and the location of the output archive.
+  def initialize(inputDir, outputFile)
+    @inputDir = inputDir
+    @outputFile = outputFile
+  end
+
+  # Zip the input directory.
+  def write()
+    entries = Dir.entries(@inputDir); entries.delete("."); entries.delete("..")
+    io = Zip::File.open(@outputFile, Zip::File::CREATE);
+
+    writeEntries(entries, "", io)
+    io.close();
+  end
+
+  # A helper method to make the recursion work.
+  private
+  def writeEntries(entries, path, io)
+
+    entries.each { |e|
+      zipFilePath = path == "" ? e : File.join(path, e)
+      diskFilePath = File.join(@inputDir, zipFilePath)
+      # puts "Deflating " + diskFilePath
+      if File.directory?(diskFilePath)
+        io.mkdir(zipFilePath)
+        subdir =Dir.entries(diskFilePath); subdir.delete("."); subdir.delete("..")
+        writeEntries(subdir, zipFilePath, io)
+      else
+        io.get_output_stream(zipFilePath) { |f| f.puts(File.open(diskFilePath, "rb").read())}
+      end
+    }
+  end
+end

data/lib/version.rb

@@ -0,0 +1,3 @@
+module Modulator
+  VERSION = "0.1.1"
+end

data/modulator.gemspec

@@ -0,0 +1,42 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "modulator"
+  spec.version       = Modulator::VERSION
+  spec.authors       = ["Damir Roso"]
+  spec.email         = ["damir.roso@webteh.us"]
+
+  spec.summary       = %q{Publish ruby methods as aws lambdas}
+  spec.description   = %q{Publish ruby methods as aws lambdas}
+  spec.homepage      = "https://github.com/damir/modulator"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rack-test", "~> 1.1"
+
+  # stack builder
+  # custom source is not supported by bundler, it is added to gemfile
+  # spec.add_dependency "humidifier", github: 'damir/humidifier'
+  spec.add_dependency "aws-sdk-s3"
+  spec.add_dependency "aws-sdk-cloudformation"
+  spec.add_dependency "rubyzip"
+
+  # local gateway
+  spec.add_dependency "puma"
+  spec.add_dependency "roda"
+  spec.add_dependency "rerun"
+end