modulator 0.2.2 → 0.3.0

data/lib/modulator/stack/policies.rb

@@ -0,0 +1,99 @@
+module StackBuilder
+  module LambdaPolicy
+    module_function
+
+    # add inline iam role to lambda, NOTE: use the same role for all lambdas for now
+    def add_lambda_iam_role(function_name: nil)
+      StackBuilder.stack.add('LambdaRole', Humidifier::IAM::Role.new(
+          assume_role_policy_document: {
+            "Version" => "2012-10-17",
+            'Statement' => [
+              {
+                "Action" => ["sts:AssumeRole"],
+                "Effect" => "Allow",
+                'Principal' => {
+                  'Service' => ["lambda.amazonaws.com"]
+                }
+              }
+            ]
+          },
+          policies: []
+        )
+      )
+    end
+
+    def add_policy(policy, **opts)
+      StackBuilder.stack.resources['LambdaRole'].properties['policies'] << send(policy, opts)
+    end
+
+    # policy to access cloudwatch
+    def cloudwatch(**opts)
+      {
+        "policy_document" => {
+          "Version" => "2012-10-17",
+          'Statement' => [
+            {
+              "Sid" => "AllowLogCreation",
+              "Action" => [
+                "logs:CreateLogStream",
+                "logs:PutLogEvents",
+              ],
+              "Effect" => "Allow",
+              "Resource" => Humidifier.fn.sub("arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*")
+            },
+            {
+              "Sid" => "AllowLogGroupCreation",
+              "Action" => [
+                "logs:CreateLogGroup",
+              ],
+              "Effect" => "Allow",
+              "Resource" => "*"
+            }
+          ]
+        },
+        "policy_name" => "cloud-watch-access"
+      }
+    end
+
+    # policy to access prefixed dynamo tables
+    def dynamo_db(**opts)
+      prefixes = opts[:prefixes] || []
+      prefix_separator = opts[:prefix_separator] || '-'
+      wildcard = '*'
+      if prefixes.any?
+        prefixes.map!{|prefix| prefix == :app_name ? StackBuilder.stack.app_name.dasherize.split('-') : prefix}
+        wildcard = "#{(prefixes << '*').join(prefix_separator)}"
+      end
+      {
+        "policy_document" => {
+          "Version" => "2012-10-17",
+          'Statement' => [
+            {
+              "Sid" => "AllowAllActionsOnPrefixedTable",
+              "Effect" => "Allow",
+              "Action" => [
+                  "dynamodb:*"
+              ],
+              "Resource" => Humidifier.fn.sub("arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/#{wildcard}")
+            },
+            {
+              "Sid" => "AdditionalPrivileges",
+              "Effect" => "Allow",
+              "Action" => [
+                  "dynamodb:ListTables",
+                  "dynamodb:DescribeTable"
+              ],
+              "Resource" => "*"
+            }
+          ]
+        },
+        "policy_name" => "dynamo-db-access"
+      }
+    end
+
+    # TODO: add access to named secrets
+    def secret_manager(**opts)
+
+    end
+  end
+end

data/lib/modulator/{lambda/aws_stack_uploader.rb → stack/uploader.rb}

@@ -2,28 +2,49 @@ require 'aws-sdk-s3'
 require 'digest'
 require 'bundler'
 
-module AwsStackBuilder
+module StackBuilder
   module_function
 
   S3Client = Aws::S3::Client.new
 
-  def upload_lambda_handler
-    bucket_name = Humidifier.ref("S3Bucket").reference
+  # bundle gems and upload all for simple lambda apps
+  def upload_lambda_files
+    puts '- bundling dependencies'
+    Bundler.with_clean_env do
+      Dir.chdir(app_path) do
+        `bundle install`
+        `bundle install --deployment --without development`
+      end
+    end
+    FileUtils.remove_dir(app_path.join("vendor/bundle/ruby/#{GEM_PATH_RUBY_VERSION}/cache")) # remove cache dir
+    upload_app_layer(sub_dirs: '', add_layer_to_stack: false) # reuse layer upload
+    FileUtils.remove_dir(app_path.join('.bundle'))
+    FileUtils.remove_dir(app_path.join('vendor'))
+  end
+
+  # generic handler for all lambda
+  def upload_generic_lambda_handler
     lambda_handler_key = LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
+    modulator_handler_source = Pathname.new(__FILE__).dirname.parent.join('lambda_handler.rb').read
     source = <<~SOURCE
-      require 'modulator/lambda/aws_lambda_handler'
+      # DO NOT EDIT THIS FILE
+
+      #{modulator_handler_source}
       Dir.chdir('/opt/ruby/lib')
     SOURCE
 
     existing_handler = S3Client.get_object(
-      bucket: bucket,
+      bucket: s3_bucket,
       key: lambda_handler_key
     ) rescue false # not found
 
-    existing_source = Zip::InputStream.open(existing_handler.body) do |zip_file|
-      zip_file.get_next_entry
-      zip_file.read
-    end if existing_handler
+    if existing_handler
+      existing_source = Zip::InputStream.open(existing_handler.body) do |zip_file|
+        zip_file.get_next_entry
+        zip_file.read
+      end
+      self.lambda_handler_s3_object_version = existing_handler.version_id
+    end
 
     if existing_source != source
       puts '- uploading generic lambda handler'
@@ -31,11 +52,12 @@ module AwsStackBuilder
         zip.put_next_entry LAMBDA_HANDLER_FILE_NAME + '.rb'
         zip.print source
       end
-      S3Client.put_object(
-        bucket: bucket,
+      new_handler = S3Client.put_object(
+        bucket: s3_bucket,
         key: lambda_handler_key,
         body: source_zip_file.tap(&:rewind).read
       )
+      self.lambda_handler_s3_object_version = new_handler.version_id
     end
   end
 
@@ -51,7 +73,7 @@ module AwsStackBuilder
     new_checksum  = Digest::MD5.hexdigest(File.read(app_path.join('Gemfile.lock')))
 
     zip_file_name = app_dir + '_gems.zip'
-    gems_path = app_path.join(hidden_dir, 'gems')
+    gems_path     = app_path.join(hidden_dir, 'gems')
     gems_zip_path = app_path.join(hidden_dir, zip_file_name)
 
     if old_checksum != new_checksum
@@ -61,14 +83,14 @@ module AwsStackBuilder
       # bundle gems
       Bundler.with_clean_env do
         Dir.chdir(app_path) do
-          `bundle install --path=./#{hidden_dir}/gems --clean`
+          `bundle install --path=./#{hidden_dir}/gems --clean --without development`
         end
       end
       ZipFileGenerator.new(gems_path, gems_zip_path).write
 
       # upload zipped file
       gem_layer = S3Client.put_object(
-        bucket: bucket,
+        bucket: s3_bucket,
         key: zip_file_name,
         body: gems_zip_path.read
       )
@@ -77,7 +99,7 @@ module AwsStackBuilder
       gems_zip_path.delete
     else
       puts '- using existing gems layer'
-      gem_layer = S3Client.get_object(bucket: bucket, key: zip_file_name)
+      gem_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
     end
 
     add_layer(
@@ -88,48 +110,53 @@ module AwsStackBuilder
     )
   end
 
-  def upload_app_layer
+  def upload_app_layer(sub_dirs: 'ruby/lib', add_layer_to_stack: true)
     zip_file_name = app_dir + '.zip'
     app_zip_path  = app_path.join(hidden_dir, zip_file_name)
 
-    # copy app code to ruby/lib in use outside temp dir
+    # copy app code to ruby/lib in outside temp dir
     temp_dir_name = '.modulator_temp'
-    ruby_lib_dirs = 'ruby/lib'
+    temp_sub_dirs = sub_dirs
     temp_path     = app_path.parent.join(temp_dir_name)
-    temp_path.join(ruby_lib_dirs).mkpath
-    FileUtils.copy_entry app_path, temp_path.join(ruby_lib_dirs)
+    temp_path.join(temp_sub_dirs).mkpath
+    FileUtils.copy_entry app_path, temp_path.join(temp_sub_dirs)
 
     # calculate checksum for app folder
     checksum_path = app_path.join(hidden_dir, 'app_checksum')
     old_checksum  = (checksum_path.read rescue nil)
-    new_checksum  = checksum(app_path)
+    new_checksum  = Utils.checksum(app_path)
 
     if old_checksum != new_checksum
-      puts '- uploading app layer'
+      puts '- uploading app files'
       checksum_path.write(new_checksum)
       ZipFileGenerator.new(temp_path, app_zip_path).write
       # upload zipped file
       app_layer = S3Client.put_object(
-        bucket: bucket,
+        bucket: s3_bucket,
         key: zip_file_name,
         body: app_zip_path.read
       )
       # delete zipped file
       app_zip_path.delete
     else
-      puts '- using existing app layer'
-      app_layer = S3Client.get_object(bucket: bucket, key: zip_file_name)
+      puts '- using existing app files'
+      app_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
     end
 
     # delete temp dir
     FileUtils.remove_dir(temp_path)
 
-    add_layer(
-      name: app_name,
-      description: "App source. MD5: #{new_checksum}",
-      s3_key: zip_file_name,
-      s3_object_version: app_layer.version_id
-    )
+    if add_layer_to_stack
+      add_layer(
+        name: app_name,
+        description: "App source. MD5: #{new_checksum}",
+        s3_key: zip_file_name,
+        s3_object_version: app_layer.version_id
+      )
+    else # for simple lambda apps
+      self.lambda_handler_s3_key = zip_file_name
+      self.lambda_handler_s3_object_version = app_layer.version_id
+    end
   end
 
   # add layer
@@ -139,17 +166,11 @@ module AwsStackBuilder
         layer_name: name,
         description: description,
         content: {
-          s3_bucket: bucket,
+          s3_bucket: s3_bucket,
           s3_key: s3_key,
           s3_object_version: s3_object_version
         }
       )
     )
   end
-
-  def checksum(dir)
-    files = Dir["#{dir}/**/*"].reject{|f| File.directory?(f)}
-    content = files.map{|f| File.read(f)}.join
-    Digest::MD5.hexdigest(content)
-  end
 end

data/lib/modulator/version.rb

@@ -1,3 +1,3 @@
 module Modulator
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/utils.rb

@@ -4,7 +4,7 @@ require 'zip'
 # the patched classes are used only in tests and tools
 class String
   def camelize
-    split('_').collect do |word|
+    gsub('-', '_').split('_').collect do |word|
       word[0] = word[0].upcase
       word
     end.join
@@ -56,6 +56,12 @@ module Utils
   def load_json(path)
     JSON.parse(File.read(path))
   end
+
+  def checksum(dir)
+    files = Dir["#{dir}/**/*"].reject{|f| File.directory?(f)}
+    content = files.map{|f| File.read(f)}.join
+    Digest::MD5.hexdigest(content)
+  end
 end
 
 # NOTE: this code is taken from https://github.com/rubyzip/rubyzip examples

data/modulator.gemspec

@@ -29,10 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test", "~> 1.1"
 
   # stack builder
-  # custom source is not supported by bundler, it is added to gemfile
-  # spec.add_dependency "humidifier", github: 'damir/humidifier'
-  spec.add_dependency "aws-sdk-s3"
-  spec.add_dependency "aws-sdk-cloudformation"
+  spec.add_dependency "humidifier"
   spec.add_dependency "rubyzip"
 
   # local gateway

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: modulator
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Damir Roso
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-01 00:00:00.000000000 Z
+date: 2019-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,21 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-s3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: aws-sdk-cloudformation
+  name: humidifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -160,6 +146,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -171,9 +158,11 @@ files:
 - lib/modulator.rb
 - lib/modulator/gateway/gateway.rb
 - lib/modulator/gateway/routes/console.rb
-- lib/modulator/lambda/aws_lambda_handler.rb
-- lib/modulator/lambda/aws_stack_builder.rb
-- lib/modulator/lambda/aws_stack_uploader.rb
+- lib/modulator/gateway_event.json
+- lib/modulator/lambda_handler.rb
+- lib/modulator/stack/builder.rb
+- lib/modulator/stack/policies.rb
+- lib/modulator/stack/uploader.rb
 - lib/modulator/version.rb
 - lib/utils.rb
 - modulator.gemspec

data/lib/modulator/lambda/aws_stack_builder.rb

@@ -1,225 +0,0 @@
-require 'humidifier'
-require_relative 'aws_stack_uploader'
-
-module AwsStackBuilder
-  module_function
-
-  RUBY_VERSION = 'ruby2.5'
-  GEM_PATH = '/opt/ruby/2.5.0'
-  LAMBDA_HANDLER_FILE_NAME = 'lambda-handler'
-
-  class << self
-    attr_accessor :app_name, :stack, :api_gateway_deployment, :gateway_id
-    attr_accessor :app_path, :app_dir, :hidden_dir, :bucket, :stack_opts
-  end
-
-  def init(app_name:, bucket:, **stack_opts)
-    puts 'Initializing stack'
-    @app_name   = app_name.camelize
-    @bucket     = bucket
-    @app_path   = Pathname.getwd
-    @app_dir    = app_path.basename.to_s
-    @hidden_dir = '.modulator'
-    @stack_opts = stack_opts
-    @stack      = Humidifier::Stack.new(name: @app_name, aws_template_format_version: '2010-09-09')
-
-    # api stage
-    @stack.add_parameter('ApiGatewayStageName', description: 'Gateway deployment stage', type: 'String', default: 'v1')
-
-    add_api_gateway
-    add_api_gateway_deployment
-    add_lambda_iam_role
-    upload_files
-    extend_stack_instance(@stack)
-    @stack
-  end
-
-  def upload_files
-    upload_lambda_handler
-    puts 'Generating layers'
-    app_path.join(hidden_dir).mkpath
-    upload_gems_layer
-    upload_app_layer
-  end
-
-  # helpers
-  def extend_stack_instance(stack)
-    stack.instance_eval do
-      def add_lambda_endpoint(**opts) # gateway:, mod:, wrapper: {}, env: {}, settings: {}
-        # add lambda
-        lambda = AwsStackBuilder.add_lambda(opts)
-        # add api resources
-        AwsStackBuilder.add_api_gateway_resources(gateway: opts[:gateway], lambda: lambda)
-      end
-    end
-  end
-
-  # gateway
-  def add_api_gateway
-    @gateway_id = 'ApiGateway'
-    @stack.add(gateway_id, Humidifier::ApiGateway::RestApi.new(name: app_name, description: app_name + ' API'))
-  end
-
-  # gateway deployment
-  def add_api_gateway_deployment
-    @api_gateway_deployment = Humidifier::ApiGateway::Deployment.new(
-      rest_api_id: Humidifier.ref(gateway_id),
-      stage_name: Humidifier.ref("ApiGatewayStageName")
-    )
-    @stack.add('ApiGatewayDeployment', @api_gateway_deployment)
-    @stack.add_output('ApiGatewayInvokeURL',
-        value: Humidifier.fn.sub("https://${#{gateway_id}}.execute-api.${AWS::Region}.amazonaws.com/${ApiGatewayStageName}"),
-        description: 'API root url',
-        export_name: app_name + 'RootUrl'
-    )
-    @api_gateway_deployment.depends_on = []
-  end
-
-  # lambda function
-  def add_lambda(gateway:, mod:, wrapper: {}, env: {}, settings: {})
-    lambda_config = {}
-    name_parts = mod[:name].split('::')
-    {gateway: gateway, module: mod, wrapper: wrapper}.each do |env_group_prefix, env_group|
-      env_group.each{|env_key, env_value| lambda_config["#{env_group_prefix}_#{env_key}"] = env_value}
-    end
-
-    lambda_function = Humidifier::Lambda::Function.new(
-      description: "Lambda for #{mod[:name]}.#{mod[:method]}",
-      function_name: [app_name.dasherize, name_parts, mod[:method]].flatten.map(&:downcase).join('-'),
-      handler: "#{LAMBDA_HANDLER_FILE_NAME}.AwsLambdaHandler.call",
-      environment: {
-        variables: env
-          .reduce({}){|env_as_string, (k, v)| env_as_string.update(k.to_s => v.to_s)}
-          .merge(lambda_config)
-          .merge('GEM_PATH' => GEM_PATH, 'app_dir' => app_dir)
-      },
-      role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
-      timeout: settings[:timeout] || stack_opts[:timeout] || 15,
-      memory_size: settings[:memory_size] || stack_opts[:memory_size] || 128,
-      runtime: RUBY_VERSION,
-      code: {
-        s3_bucket: bucket,
-        s3_key: LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
-      },
-      layers: [
-        Humidifier.ref(app_name + 'Layer'),
-        Humidifier.ref(app_name + 'GemsLayer')
-      ]
-    )
-    id = ['Lambda', name_parts, mod[:method].capitalize].join
-    @stack.add(id, lambda_function)
-    add_lambda_invoke_permission(id: id, gateway: gateway)
-    id
-  end
-
-  # invoke permission
-  def add_lambda_invoke_permission(id:, gateway:)
-    arn_path_matcher = gateway[:path].split('/').each_with_object([]) do |fragment, matcher|
-      fragment = '*' if fragment.start_with?(':')
-      matcher << fragment
-    end.join('/')
-    @stack.add('LambdaPermission', Humidifier::Lambda::Permission.new(
-        action: "lambda:InvokeFunction",
-        function_name: Humidifier.fn.get_att([id, 'Arn']),
-        principal: "apigateway.amazonaws.com",
-        source_arn: Humidifier.fn.sub("arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${#{gateway_id}}/*/#{gateway[:verb]}/#{arn_path_matcher}")
-      )
-    )
-  end
-
-  # gateway method
-  def add_api_gateway_resources(gateway:, lambda:)
-
-    # example: calculator/algebra/:x/:y/sum -> module name, args, method name
-    path = gateway[:path].split('/')
-
-    # root resource
-    root_resource = path.shift
-    @stack.add(root_resource.camelize, Humidifier::ApiGateway::Resource.new(
-        rest_api_id: Humidifier.ref(AwsStackBuilder.gateway_id),
-        parent_id: Humidifier.fn.get_att(["ApiGateway", "RootResourceId"]),
-        path_part: root_resource
-      )
-    )
-
-    # args and method name are nested resources
-    parent_resource = root_resource.camelize
-    path.each do |fragment|
-      if fragment.start_with?(':')
-        fragment = fragment[1..-1]
-        dynamic_fragment = "{#{fragment}}"
-      end
-      @stack.add(parent_resource + fragment.camelize, Humidifier::ApiGateway::Resource.new(
-          rest_api_id: Humidifier.ref(AwsStackBuilder.gateway_id),
-          parent_id: Humidifier.ref(parent_resource),
-          path_part: dynamic_fragment || fragment
-        )
-      )
-      parent_resource = parent_resource + fragment.camelize
-    end
-
-    # attach lambda to last resource
-    id = 'EndpointFor' + (gateway[:path].gsub(':', '').gsub('/', '_')).camelize
-    @stack.add(id, Humidifier::ApiGateway::Method.new(
-        authorization_type: 'NONE',
-        http_method: gateway[:verb].to_s.upcase,
-        integration: {
-          integration_http_method: 'POST',
-          type: "AWS_PROXY",
-          uri: Humidifier.fn.sub([
-            "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations",
-            'lambdaArn' => Humidifier.fn.get_att([lambda, 'Arn'])
-          ])
-        },
-        rest_api_id: Humidifier.ref(gateway_id),
-        resource_id: Humidifier.ref(parent_resource) # last evaluated resource
-      )
-    )
-
-    # deployment depends on the method
-    @api_gateway_deployment.depends_on << id
-  end
-
-  def add_lambda_iam_role(function_name: nil)
-    @stack.add('LambdaRole', Humidifier::IAM::Role.new(
-        assume_role_policy_document: {
-          'Version' => "2012-10-17",
-          'Statement' => [
-            {
-              'Action' => ["sts:AssumeRole"],
-              'Effect' => "Allow",
-              'Principal' => {
-                'Service' => ["lambda.amazonaws.com"]
-              }
-            }
-          ]
-        },
-        policies: [
-          {
-            'policy_document' => {
-              'Version' => "2012-10-17",
-              'Statement' => [
-                {
-                  'Action' => [
-                    "logs:CreateLogStream",
-                    "logs:PutLogEvents",
-                  ],
-                  'Effect' => "Allow",
-                  'Resource' => Humidifier.fn.sub("arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*")
-                },
-                {
-                  'Action' => [
-                    "logs:CreateLogGroup",
-                  ],
-                  'Effect' => "Allow",
-                  'Resource' => "*"
-                }
-              ]
-            },
-            'policy_name' => "cloud-watch-access"
-          }
-        ]
-      )
-    )
-  end
-end