modulator 0.2.2 → 0.3.0

data/lib/modulator/gateway/gateway.rb

@@ -12,7 +12,7 @@ class Gateway < Roda
 
   my_dir = Pathname.new(__FILE__).dirname
   my_dir.glob('routes/*.rb').each{|file| require_relative file}
-  DUMMY_AWS_EVENT = Utils.load_json my_dir.parent.join('../../spec/lambda/aws/event.json')
+  DUMMY_AWS_EVENT = Utils.load_json my_dir.parent.join('gateway_event.json')
 
   before do
     @time = Time.now
@@ -32,7 +32,7 @@ class Gateway < Roda
   after do |res|
     puts "Matched path: #{request.matched_path}"
     puts "Status: #{response.status}"
-    puts ("Took: #{Time.now - @time} seconds")
+    puts "Took: #{Time.now - @time} seconds"
   end
 
   plugin :error_handler do |e|
@@ -51,11 +51,9 @@ class Gateway < Roda
 
     # process lambda configs
     Modulator::LAMBDAS.each do |lambda_name, lambda_config|
-      # puts "* Registering #{lambda_name}"
-      # pp lambda_config
 
-      # module and wrapper config
-      Modulator.set_env lambda_config
+      # copy config to env
+      Modulator.set_env_values lambda_config
 
       # build route
       @path_params = {}

data/lib/modulator/gateway/routes/console.rb

@@ -1,11 +1,5 @@
-require 'aws-sdk-cloudformation'
-
-# console API
 Gateway.route('console') do |r|
 
-  client    = Aws::CloudFormation::Client.new
-  app_name  = (opts[:app_dir] || Pathname.getwd.basename.to_s).camelize
-
   # helpers
   def capture_output
     previous_stdout, $stdout = $stdout, StringIO.new
@@ -18,18 +12,29 @@ Gateway.route('console') do |r|
       $stderr = previous_stderr
   end
 
-  def render_command_result(command_result, command_output)
-    if command_result
-      command_result.to_hash
+  # cf call return value or its capture from stdout
+  def render_cf_call_output(cf_call_result, cf_call_output, cf_call_name = 'aws cf sdk call')
+    if cf_call_result
+      cf_call_result.to_hash
     else
-      {'aws-sdk-cloudformation': command_output.split("\n").first}
+      {cf_call_name => cf_call_output.split("\n").first}
     end
   end
 
+  # list registered lambdas
+  r.on 'lambdas' do
+    r.get 'list' do
+      Modulator::LAMBDAS
+    end
+  end
+
+  # stack operations
   r.on 'stack' do
-    bucket_name = ENV['S3BUCKET'] || 'modulator-lambdas'
-    payload = request.params.symbolize_keys
-    command_result = nil
+    client    = Aws::CloudFormation::Client.new
+    app_name  = (opts[:app_dir] || Pathname.getwd.basename.to_s).camelize
+    s3_bucket = opts[:s3_bucket] || ENV['MODULATOR_S3_BUCKET'] || 'modulator-apps'
+    payload   = request.params.symbolize_keys
+    cf_call_result = nil
 
     r.get 'events' do
       resp = client.describe_stack_events(stack_name: app_name, next_token: @headers['X-Next-Token'])
@@ -37,6 +42,7 @@ Gateway.route('console') do |r|
       resp.stack_events.map(&:to_hash)
     end
 
+    # initialize stack
     r.on 'init' do
       serializer = :yaml
       content_type = 'text/html'
@@ -47,42 +53,34 @@ Gateway.route('console') do |r|
         r.pass
       end
 
-      # init stack
       stack = Modulator.init_stack(
-        app_name: app_name,
-        bucket: bucket_name,
+        s3_bucket: s3_bucket,
         timeout: 15
       )
 
-      # add stack parameters if found in payload
-      bucket_name[:parameters]&.each do |param|
-        stack.add_parameter(param[:key],
-          description: param[:description],
-          type: param[:type],
-          value: param[:value]
-        )
-      end
-
-      r.post 'valid' do
-        command_output = capture_output do
-          command_result = stack.valid?
+      # validate stack
+      r.post 'validate' do
+        cf_call_output = capture_output do
+          cf_call_result = stack.valid?
         end
-        render_command_result(command_result, command_output)
+        render_cf_call_output(cf_call_result, cf_call_output, 'stack validation call')
       end
 
+      # deploy stack
       r.post 'deploy' do
-        command_output = capture_output do
-          command_result = stack.deploy(
-            parameters: bucket_name[:parameters]&.map{|param| {parameter_key: param[:key], parameter_value: param[:value]}},
+        cf_call_output = capture_output do
+          cf_call_result = stack.deploy(
+            parameters: s3_bucket[:parameters]&.map{|param| {parameter_key: param[:key], parameter_value: param[:value]}},
             capabilities: ['CAPABILITY_IAM']
           )
         end
-        render_command_result(command_result, command_output)
+        render_cf_call_output(cf_call_result, cf_call_output)
       end
 
+      # print template
       r.post do
         response['Content-Type'] = content_type
-        template = stack.to_cf(serializer)
+        stack.to_cf(serializer)
       end
     end
   end

data/lib/modulator/gateway_event.json

@@ -0,0 +1,60 @@
+{
+  "body": "{\"x\": 123}",
+  "isBase64Encoded": false,
+
+  "resource": "/calc/{proxy+}",
+  "path": "/calc/1/add/2",
+  "httpMethod": "POST",
+
+  "multiValueHeaders": {
+    "x-custom": [
+      "abc"
+    ]
+  },
+  "queryStringParameters": {
+    "aa": "11",
+    "bb": "22"
+  },
+  "multiValueQueryStringParameters": {
+    "aa": [
+      "11"
+    ],
+    "bb": [
+      "22"
+    ]
+  },
+  "pathParameters": {
+    "id": "1",
+    "other_id": "2"
+  },
+  "stageVariables": null,
+
+  "headers": {
+    "Host": "1234567890.execute-api.us-east-1.amazonaws.com",
+    "User-Agent": "Custom User Agent String",
+    "X-Forwarded-For": "127.0.0.1, 127.0.0.2",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "123456",
+    "stage": "prod",
+    "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef",
+    "requestTime": "09/Apr/2015:12:34:56 +0000",
+    "requestTimeEpoch": 1428582896000,
+    "identity": {
+      "cognitoIdentityPoolId": null,
+      "accountId": null,
+      "cognitoIdentityId": null,
+      "caller": null,
+      "accessKey": null,
+      "sourceIp": "127.0.0.1",
+      "cognitoAuthenticationType": null,
+      "cognitoAuthenticationProvider": null,
+      "userArn": null,
+      "userAgent": "Custom User Agent String",
+      "user": null
+    }
+  }
+}

data/lib/modulator/{lambda/aws_lambda_handler.rb → lambda_handler.rb}

@@ -1,5 +1,6 @@
 require 'pathname'
 
+# NOTE: aws has LambdaHandler already defined as a class so we need an aws prefix here
 module AwsLambdaHandler
   module_function
 
@@ -8,20 +9,6 @@ module AwsLambdaHandler
     # TODO: implement handlers for other event types based on some event key, like AwsS3EventHandler
     AwsApiGatewayEventHandler.call(event: event, context: context)
   end
-
-  # helpers
-  def symbolize_keys(obj)
-    case obj
-    when Hash
-      hash = {}
-      obj.each {|k, v| hash[k.to_sym] = symbolize_keys(v)}
-      hash
-    when Array
-      obj.map {|x| symbolize_keys(x)}
-    else
-      obj
-    end
-  end
 end
 
 module AwsApiGatewayEventHandler
@@ -96,7 +83,7 @@ module AwsApiGatewayEventHandler
       mod.send(mod_method, *path_params.values)
 
     elsif verb == 'POST'
-      payload = AwsLambdaHandler.symbolize_keys(JSON.parse(event['body']))
+      payload = JSON.parse(event['body'], symbolize_names: true)
       method_signature.each do |arg_type, arg_name|         # [[:req, :id], [:key, :pet]]
         payload = {arg_name => payload} if arg_type == :key # scope payload to first named argument
       end

data/lib/modulator/stack/builder.rb

@@ -0,0 +1,256 @@
+require 'forwardable'
+require 'humidifier'
+require_relative 'uploader'
+require_relative 'policies'
+
+module StackBuilder
+  module_function
+
+  RUBY_VERSION = 'ruby2.5'
+  GEM_PATH_RUBY_VERSION = '2.5.0'
+  GEM_PATH = "/opt/ruby/#{GEM_PATH_RUBY_VERSION}"
+  LAMBDA_HANDLER_FILE_NAME = 'modulator-lambda-handler'
+
+  class << self
+    attr_accessor :stack, :stack_opts, :app_name, :app_path, :app_dir
+    attr_accessor :hidden_dir, :s3_bucket, :lambda_handler_s3_object_version
+    attr_accessor :api_gateway_deployment, :api_gateway_id, :lambda_policies
+    attr_accessor :lambda_handlers, :lambda_handler_s3_key
+  end
+
+  def init(app_name:, s3_bucket:, **stack_opts)
+    puts 'Initializing stack'
+    @app_name   = app_name.camelize
+    @s3_bucket  = s3_bucket
+    @app_path   = Pathname.getwd
+    @app_dir    = app_path.basename.to_s
+    @hidden_dir = '.modulator'
+    @stack_opts = stack_opts
+    @lambda_handlers = stack_opts[:lambda_handlers] || []
+    @lambda_policies = Array(stack_opts[:lambda_policies]) << :cloudwatch
+
+    # create hidden dir for build artifacts
+    app_path.join(hidden_dir).mkpath
+
+    # init stack instance
+    self.stack = Humidifier::Stack.new(name: app_name, aws_template_format_version: '2010-09-09')
+
+    # app environment -  test, development, production ...
+    app_envs = stack_opts[:app_envs] || ['development']
+    stack.add_parameter('AppEnvironment', description: 'Application environment', type: 'String', allowed_values: app_envs, constraint_description: "Must be one of #{app_envs.join(', ')}")
+
+    if lambda_handlers.empty?
+      # api stage
+      stack.add_parameter('ApiGatewayStageName', description: 'Gateway deployment stage', type: 'String', default: 'v1')
+
+      # add gateway
+      stack.add_api_gateway
+      stack.add_api_gateway_deployment
+    end
+
+    # add role
+    stack.add_lambda_iam_role
+
+    # add policies to role
+    stack.lambda_policies.each do |policy|
+      stack.add_policy(policy) if policy.is_a?(Symbol)
+      stack.add_policy(policy[:name], **policy) if policy.is_a?(Hash)
+    end
+
+    # simple lambda app
+    if lambda_handlers.any?
+      stack.upload_lambda_files
+      lambda_handlers.each do |handler|
+        stack.add_lambda(handler: handler, env: stack_opts[:env] || {}, settings: stack_opts[:settings] || {})
+      end
+    else
+      # upload handlers and layers
+      stack.upload_files
+    end
+
+    # return humidifier instance
+    stack
+  end
+
+  def upload_files
+    if stack_opts[:skip_upload]
+      puts 'Skipping upload'
+      return
+    end
+    stack.upload_generic_lambda_handler
+    puts 'Generating layers'
+    stack.upload_gems_layer
+    stack.upload_app_layer
+  end
+
+  def add_lambda_endpoint(**opts) # gateway:, mod:, wrapper: {}, env: {}, settings: {}
+    # add api resources and its lambda
+    stack.add_api_gateway_resources(gateway: opts[:gateway], lambda: stack.add_generic_lambda(opts))
+  end
+
+  # gateway
+  def add_api_gateway
+    self.api_gateway_id = 'ApiGateway'
+    stack.add(api_gateway_id, Humidifier::ApiGateway::RestApi.new(name: app_name, description: app_name + ' API'))
+  end
+
+  # gateway deployment
+  def add_api_gateway_deployment
+    self.api_gateway_deployment = Humidifier::ApiGateway::Deployment.new(
+      rest_api_id: Humidifier.ref(api_gateway_id),
+      stage_name: Humidifier.ref("ApiGatewayStageName")
+    )
+    stack.add('ApiGatewayDeployment', api_gateway_deployment)
+    stack.add_output('ApiGatewayInvokeURL',
+        value: Humidifier.fn.sub("https://${#{api_gateway_id}}.execute-api.${AWS::Region}.amazonaws.com/${ApiGatewayStageName}"),
+        description: 'API root url',
+        export_name: app_name + 'RootUrl'
+    )
+    api_gateway_deployment.depends_on = []
+  end
+
+  # custom lambda function
+  def add_lambda(handler:, env: {}, settings: {})
+    lambda_resource = generate_lambda_resource(
+      description: "Lambda for #{handler}",
+      function_name: ([app_name] << handler.split('.')).flatten.join('-').dasherize,
+      handler: handler,
+      s3_key: lambda_handler_s3_key,
+      env_vars: env.merge('app_env' => Humidifier.ref('AppEnvironment')),
+      role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
+      settings: settings
+    )
+    stack.add(handler.gsub('.', '_').camelize, lambda_resource)
+  end
+
+  # generic lambda function for gateway
+  def add_generic_lambda(gateway: {}, mod: {}, wrapper: {}, env: {}, settings: {})
+    lambda_config = {}
+    name_parts = mod[:name].split('::')
+    {gateway: gateway, module: mod, wrapper: wrapper}.each do |env_group_prefix, env_group|
+      env_group.each{|env_key, env_value| lambda_config["#{env_group_prefix}_#{env_key}"] = env_value}
+    end
+    env_vars = env
+        .reduce({}){|env_as_string, (k, v)| env_as_string.update(k.to_s => v.to_s)}
+        .merge(lambda_config)
+        .merge(
+          'GEM_PATH' => GEM_PATH,
+          'app_dir'  => app_dir,
+          'app_env'  => Humidifier.ref('AppEnvironment')
+        )
+
+    lambda_resource = generate_lambda_resource(
+      description: "Lambda for #{mod[:name]}.#{mod[:method]}",
+      function_name: [app_name, name_parts, mod[:method]].flatten.join('-').dasherize,
+      handler: "#{LAMBDA_HANDLER_FILE_NAME}.AwsLambdaHandler.call",
+      s3_key: LAMBDA_HANDLER_FILE_NAME + '.rb.zip',
+      env_vars: env_vars,
+      role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
+      settings: settings,
+      layers: [Humidifier.ref(app_name + 'Layer'), Humidifier.ref(app_name + 'GemsLayer')]
+    )
+
+    # add to stack
+    ['Lambda', name_parts, mod[:method].capitalize].join.tap do |id|
+      stack.add(id, lambda_resource)
+      stack.add_lambda_invoke_permission(id: id, gateway: gateway)
+    end
+  end
+
+  def generate_lambda_resource(description:, function_name:, handler:, s3_key:, env_vars:, role:, settings:, layers: [])
+    lambda_function = Humidifier::Lambda::Function.new(
+      description: description,
+      function_name: function_name,
+      handler: handler,
+      environment: {variables: env_vars},
+      role: role,
+      timeout: settings[:timeout] || stack_opts[:timeout] || 15,
+      memory_size: settings[:memory_size] || stack_opts[:memory_size] || 128,
+      runtime: RUBY_VERSION,
+      code: {
+        s3_bucket: s3_bucket,
+        s3_key: s3_key,
+        s3_object_version: lambda_handler_s3_object_version
+      },
+      layers: layers
+    )
+  end
+
+  # invoke permission
+  def add_lambda_invoke_permission(id:, gateway:)
+    arn_path_matcher = gateway[:path].split('/').each_with_object([]) do |fragment, matcher|
+      fragment = '*' if fragment.start_with?(':')
+      matcher << fragment
+    end.join('/')
+    stack.add(id + 'InvokePermission' , Humidifier::Lambda::Permission.new(
+        action: "lambda:InvokeFunction",
+        function_name: Humidifier.fn.get_att([id, 'Arn']),
+        principal: "apigateway.amazonaws.com",
+        source_arn: Humidifier.fn.sub("arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${#{api_gateway_id}}/*/#{gateway[:verb]}/#{arn_path_matcher}")
+      )
+    )
+  end
+
+  # gateway method
+  def add_api_gateway_resources(gateway:, lambda:)
+
+    # example: calculator/algebra/:x/:y/sum -> module name, args, method name
+    path = gateway[:path].split('/')
+
+    # root resource
+    root_resource = path.shift
+    stack.add(root_resource.camelize, Humidifier::ApiGateway::Resource.new(
+        rest_api_id: Humidifier.ref(api_gateway_id),
+        parent_id: Humidifier.fn.get_att(["ApiGateway", "RootResourceId"]),
+        path_part: root_resource
+      )
+    )
+
+    # args and method name are nested resources
+    parent_resource = root_resource.camelize
+    path.each do |fragment|
+      if fragment.start_with?(':')
+        fragment = fragment[1..-1]
+        dynamic_fragment = "{#{fragment}}"
+      end
+      stack.add(parent_resource + fragment.camelize, Humidifier::ApiGateway::Resource.new(
+          rest_api_id: Humidifier.ref(api_gateway_id),
+          parent_id: Humidifier.ref(parent_resource),
+          path_part: dynamic_fragment || fragment
+        )
+      )
+      parent_resource = parent_resource + fragment.camelize
+    end
+
+    # attach lambda to last resource
+    id = 'EndpointFor' + (gateway[:path].gsub(':', '').gsub('/', '_')).camelize
+    stack.add(id, Humidifier::ApiGateway::Method.new(
+        authorization_type: 'NONE',
+        http_method: gateway[:verb].to_s.upcase,
+        integration: {
+          integration_http_method: 'POST',
+          type: "AWS_PROXY",
+          uri: Humidifier.fn.sub([
+            "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations",
+            'lambdaArn' => Humidifier.fn.get_att([lambda, 'Arn'])
+          ])
+        },
+        rest_api_id: Humidifier.ref(api_gateway_id),
+        resource_id: Humidifier.ref(parent_resource) # last evaluated resource
+      )
+    )
+
+    # deployment depends on each endpoint
+    api_gateway_deployment.depends_on << id
+  end
+end
+
+# delegate from stack instance to our module
+module Humidifier
+  class Stack
+    extend Forwardable
+    [StackBuilder, StackBuilder::LambdaPolicy].each do |mod|
+      def_delegators mod.to_s.to_sym, *mod.singleton_methods
+    end
+  end
+end