modulator 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2702e54f413b5629b9dd99073b1367e8837991782974fc5451f8e0505f0beb0c
-  data.tar.gz: 4fba3f0efa51536a050d05ce5b5c7874d0d1c1576050463e132f2dc4b33d0129
+  metadata.gz: b1a99c918b27ea88351705275d5657942b07efb8b9e87662318e4e15e2e5da08
+  data.tar.gz: f78e0d6e3cd7ac9e1d0767e4e91b2b3bfa475dc3d0569d32f94c4349cd249dda
 SHA512:
-  metadata.gz: 9314246a7e0398717674b4a535d88f705766cb5e36c94b472cfdb7583da0c5971f8b8a46aab7da7d0168168b2601f3eca04e609c50a512c37a5f4a75705fe044
-  data.tar.gz: 1b969a118871276f3be2c41b7e38ee5cc9063524047efecb918b507e2179787ddf3dcbe795a87066161da9e2e4aaaafdf72c220cf6febc60bad736a5a8137daf
+  metadata.gz: 3177cbde9f6c198c23dec0216cbbe8b8acc6b74a866c3225e0cf11a1494f1752da82d3011cc990d7fb0e4715a2b47f7c9b87f9da60d9d7f0f389d3c17cc26e79
+  data.tar.gz: 3e89b602c5a48491f267efc3dd88b8f6b5a6d98ac154936dc2c1763bde164fa9358c5126f6687862c5c2f78d61b4bb80c2a988b1539cb18e00a8948a06ed30b2

data/.gitignore

@@ -10,3 +10,9 @@
 
 # rspec failure tracking
 .rspec_status
+
+# build artifacts
+.bundle
+.modulator
+
+

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.3.0] - 2019-06-07
+
+### Added
+
+- This file
+- Modulator#wrap_with
+
+### Changed
+
+- Use symbolize_names when parsing json instead custom helper
+- Updated readme file with examples

data/Gemfile

@@ -1,7 +1,3 @@
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}"}
-
-gem 'humidifier', github: 'damir/humidifier'
-
 gemspec

data/Gemfile.lock

@@ -1,15 +1,8 @@
-GIT
-  remote: https://github.com/damir/humidifier
-  revision: 81a75bbd4b47aadaaed5f967fb3527a2175dec82
-  specs:
-    humidifier (2.15.0)
-
 PATH
   remote: .
   specs:
-    modulator (0.2.2)
-      aws-sdk-cloudformation
-      aws-sdk-s3
+    modulator (0.3.0)
+      humidifier
       puma
       rerun
       roda
@@ -19,26 +12,34 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.0.3)
-    aws-partitions (1.156.0)
-    aws-sdk-cloudformation (1.19.0)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-partitions (1.173.0)
+    aws-sdk-cloudformation (1.22.0)
+      aws-sdk-core (~> 3, >= 3.53.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.49.0)
+    aws-sdk-core (3.54.2)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.17.0)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-sdk-kms (1.21.0)
+      aws-sdk-core (~> 3, >= 3.53.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.36.1)
-      aws-sdk-core (~> 3, >= 3.48.2)
+    aws-sdk-s3 (1.42.0)
+      aws-sdk-core (~> 3, >= 3.53.0)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.0)
+      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
     diff-lcs (1.3)
-    ffi (1.10.0)
+    fast_underscore (0.3.1)
+    ffi (1.11.1)
+    hollaback (0.1.0)
+    humidifier (3.3.0)
+      aws-sdk-cloudformation (~> 1.19)
+      aws-sdk-s3 (~> 1.36)
+      fast_underscore (~> 0.3)
+      thor (~> 0.20)
+      thor-hollaback (~> 0.1)
     jmespath (1.4.0)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
@@ -54,7 +55,7 @@ GEM
       ffi (~> 1.0)
     rerun (0.13.0)
       listen (~> 3.0)
-    roda (3.19.0)
+    roda (3.20.0)
       rack
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
@@ -70,14 +71,17 @@ GEM
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
     ruby_dep (1.5.0)
-    rubyzip (1.2.2)
+    rubyzip (1.2.3)
+    thor (0.20.3)
+    thor-hollaback (0.1.2)
+      hollaback (~> 0.1.0)
+      thor (~> 0.20, >= 0.19.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 2.0)
-  humidifier!
   modulator!
   rack-test (~> 1.1)
   rake (~> 10.0)

data/README.md

@@ -1,28 +1,264 @@
 # Modulator
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/modulator`. To experiment with that code, run `bin/console` for an interactive prompt.
+Modulator is a tool for adding HTTP layer on top of your application using AWS Lambda and API Gateway services. You register the methods you want to publish and run the deploy script. CloudFormation engine will then provision the necessary infrastructure and deploy your application in seconds.
 
-TODO: Delete this and the text above, and describe your gem
+Because your application is isolated form HTTP handling you will write regular Ruby code without polluting it with framework or HTTP specific details. This is possible by reflecting on method signatures to construct API Gateway endpoints and consuming its events in predictable way.
+
+Code is deployed in two lambda layers, one for the gems and one for the application code. You will need a writable bucket to store the files and ability to manage CloudFormation stacks.
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'modulator'
+gem 'modulator', group: :development
 ```
 
 And then execute:
 
     $ bundle
 
-Or install it yourself as:
-
-    $ gem install modulator
+NOTE: do not put modulator entry outside the group, the tool will bundle default group for deployment and the gem is not required in lambda runtime
 
 ## Usage
 
-TODO: Write usage instructions here
+### Quick example
+
+Write Ruby application:
+
+```ruby
+# calculator/algebra.rb:
+
+module Calculator
+  module Algebra
+    def self.sum(x, y)
+      {
+        x: x,
+        y: y,
+        sum: x + y
+      }
+    end
+
+    def self.square(x, ip = nil)
+      {
+        x: x,
+        ip: ip,
+        square: x * x
+      }
+    end
+  end
+end
+```
+
+Add deploy script to working directory:
+
+```ruby
+# stack.rb
+
+require 'modulator'
+require_relative 'calculator/algebra'
+
+# register module methods
+Modulator.register(Calculator::Algebra)
+
+# initialize and deploy the stack
+stack = Modulator.init_stack s3_bucket: 'my-modulator-apps' # bucket for code and gems layers
+puts stack.valid?
+puts stack.to_cf(:yaml)
+puts stack.deploy_and_wait capabilities: ['CAPABILITY_IAM'], parameters: [
+  {parameter_key: 'AppEnvironment', parameter_value: 'development'},
+  {parameter_key: 'ApiGatewayStageName', parameter_value: 'v1'}
+]
+```
+
+Run the script then visit CloudFormation page in AWS console and navigate to created stack, click on Outputs tab and copy ApiGatewayInvokeURL:
+
+- https://some-api-id.execute-api.us-east-1.amazonaws.com/v1
+
+Then Invoke your methods using the browser or postman:
+
+- https://some-api-id.execute-api.us-east-1.amazonaws.com/v1/algebra/2/square
+- https://some-api-id.execute-api.us-east-1.amazonaws.com/v1/algebra/2/3/sum
+
+These URLs are also available in lambda page when clicking on API Gateway icon:
+
+```
+ModulatorGatewayApp
+arn:aws:execute-api:us-east-1:your-account-id:some-api-id/*/GET/calculator/algebra/*/square
+
+Details
+API endpoint: https://some-api-id.execute-api.us-east-1.amazonaws.com/v1/calculator/algebra/{x}/square
+Authorization: NONE
+Method: GET
+Resource path: /calculator/algebra/{x}/square
+Stage: v1
+```
+
+You can save CF template to a file by capturing output of stack.to\_cf(:yaml) or stack.to\_cf(:json.) 
+
+### Wrapping the method to get data from lambda event and context
+
+Data from the request can be extracted using wrapper method which will pass the values to wrapped method as optional arguments. This example will wrap Calculator::Algebra#square with Wrappers::Authorizer#call to autorize request and provide optional ip argument:
+
+```ruby
+# wrappers/authorizer.rb
+
+module Wrappers
+  module Authorizer
+    module_function
+
+    def call(event:, context:)
+      token = event.dig('headers', 'Authorization').to_s.split(' ').last
+      if token == 'block'
+        {status: 401, body: {error: 'Blocking token'}}
+      elsif token == 'pass'
+        {ip: event.dig('requestContext', 'identity', 'sourceIp')}
+      else
+        # block with generic 403
+      end
+    end
+  end
+end
+```
+
+```ruby
+# stack.rb
+require_relative 'wrappers/authorizer'
+Modulator.register(Calculator::Algebra).wrap_with(Wrappers::Authorizer, only: :square)
+```
+
+This method can be invoked only when Aurhorization header is set to 'pass', otherwise it will print explcit 401 with custom message when the value is 'block', or will default to 403 with generic message.
+
+Available options are :only and :except where value is the method name or an array of names.
+
+### Registering and configuring methods
+
+Registering module will add configuration entry to Modulator::LAMBDAS. Each entry is a plain hash which can be overriden. From this configuration a CloudFormation template is generated with all necessary resources for API Gateway endpoints and their lambdas, including function policies and execution roles.
+
+Consider this example:
+
+```ruby
+Modulator
+  .register(Calculator::Algebra, sum: {
+      gateway: {path: 'calc/:x/add/:y'},
+      settings: {timeout: 1, memory_size: 256},
+      env: {custom_var: 123}
+    }
+  )
+  .wrap_with(Wrappers::Authorizer, only: [:square])
+```
+
+For that example Modulator::LAMBDAS will print this configuration:
+
+```ruby
+{"calculator-algebra-square"=>
+  {:name=>"calculator-algebra-square",
+   :gateway=>{:verb=>"GET", :path=>"calculator/algebra/:x/square"},
+   :module=>
+    {:name=>"Calculator::Algebra",
+     :method=>"square",
+     :path=>"calculator/algebra"},
+   :wrapper=>
+    {:name=>"Wrappers::Authorizer",
+     :path=>"wrappers/authorizer",
+     :method=>"call"},
+   :env=>{},
+   :settings=>{}},
+ "calculator-algebra-sum"=>
+  {:name=>"calculator-algebra-sum",
+   :gateway=>{:verb=>"GET", :path=>"calc/:x/add/:y"},
+   :module=>
+    {:name=>"Calculator::Algebra",
+     :method=>"sum",
+     :path=>"calculator/algebra"},
+   :wrapper=>{},
+   :env=>{:custom_var=>123},
+   :settings=>{:timeout=>1, :memory_size=>256}}}
+```
+
+- :gateway is used to construct API Gateway endpoint, it has :path key from which the URL is constructed and :verb which sets the HTTP method for that URL
+- :wrapper defines wrapping method, :name is the module namespace, :method is the method name from that namespace and :path is the relative file path where the code is
+- :settings holds lambda settings values, :timeout and :memory_size
+- :env will add extra environment variables to lambda runtime
+
+Any value can be changed manualy during or after the config is generated if you want to override defaults. For example you can change :verb from GET to DELETE or you can rearrange static and dynamic URL path fragments.
+
+### Rules for mapping URL paths and HTTP methods to method signatures
+
+- Methods will be invoked with GET unless:
+	- the method name is delete, remove, or destroy for which DELETE is set
+	- the method has optional key paramater for which POST is set and payload is passed as its value
+- Required positional parameters are mapped as dinamic URL fragments, numbers are type casted to ruby classes
+- Module namespace and method name are mapped as static URL fragments
+
+For examples please check the spec folder and the sample application code there.
+
+### Local API gateway for development
+
+It is possible to run code locally as it would run in the cloud. You need to add config.ru and register some modules:
+
+```ruby
+# config.ru
+
+require 'modulator/gateway/gateway'
+require_relative 'calculator/algebra'
+require_relative 'wrappers/authorizer'
+Modulator.register(Calculator::Algebra).wrap_with(Wrappers::Authorizer, only: [:square])
+```
+
+Then start the server with this command:
+
+	rerun -- puma gateway.ru
+
+NOTE: rerun will restart the server when code changes.
+
+Visiting localhost:9292/calculator/algebra/2/square should give this result:
+
+```ruby
+{
+    "x": 2,
+    "ip": "127.0.0.1",
+    "square": 4
+}
+``` 
+
+Server log will print detailed information about request and method invocation:
+
+```
+Method: GET
+Path: /calculator/algebra/2/square
+Headers: {
+	"Accept"=>"*/*", "Accept-Encoding"=>"gzip, deflate", "Authorization"=>"pass",
+	"Cache-Control"=>"no-cache", "Connection"=>"keep-alive", "Host"=>"localhost:9292",
+	"Postman-Token"=>"8708ee60-a156-4ca2-9937-f1f408a568e2",
+	"User-Agent"=>"PostmanRuntime/7.13.0", "Version"=>"HTTP/1.1"}
+Path params: {"x"=>"2"}
+Calling wrapper Wrappers::Authorizer.call
+Resolving GET calculator/algebra/:x/square to Calculator::Algebra.square with [[:req, :x], [:opt, :ip]]
+Matched path: /calculator/algebra/2/square
+Status: 200
+Took: 0.00117 seconds
+```
+
+Local gateway is implemented with [Roda](https://github.com/jeremyevans/roda).
+
+### Manipulating generated CloudFormation template
+
+Modulator#init_stack will return [Humidifier](https://github.com/kddeisz/humidifier) instance which allows for easy manipulation of generated CF template. If you need to add more resources or tweak existing ones please consult its documentation. 
+
+One example of extending the template is Modulator#add_policy which adds extra policies to lambdas by passing optional values to init method:
+
+```ruby
+Modulator.init_stack(
+  lambda_policies: [{name: :dynamo_db, prefixes: ['my-app']}],
+)
+```
+
+This will give lambdas an access to DynamoDB tables prefixed by 'my-app'. Alternatively you could do it directly by providing your own policy:
+
+```ruby
+stack.resources['LambdaRole'].properties['policies'] << my_policy_template
+```
 
 ## Development
 

data/lib/modulator.rb

@@ -1,38 +1,56 @@
 require 'json'
 
-require 'modulator/lambda/aws_lambda_handler'
-require 'modulator/lambda/aws_stack_builder'
+require 'modulator/lambda_handler'
+require 'modulator/stack/builder'
 require 'utils'
 
 module Modulator
   module_function
-  LAMBDAS   = {}
+  LAMBDAS = {}
 
-  def add_lambda(lambda_def, **opts) # opts are for overrides
+  class << self
+    attr_accessor :stack, :registering_module
+  end
+
+  def register(lambda_def, **opts) # opts are for overrides
     if lambda_def.is_a?(Hash)
-      add_lambda_from_hash(lambda_def)
+      register_from_hash(lambda_def)
     else
-      add_lambda_from_module(lambda_def, **opts)
+      register_from_module(lambda_def, **opts)
+      self.registering_module = lambda_def
+      self
+    end
+  end
+
+  def wrap_with(wrapper_mod, only: nil, except: nil)
+    registering_module.singleton_methods.sort.each do |module_method|
+      next if only && !Array(only).include?(module_method)
+      next if except && Array(except).include?(module_method)
+      LAMBDAS[calculate_registry_key(registering_module, module_method)][:wrapper] = {
+        name:   wrapper_mod.to_s,
+        path:   wrapper_mod.to_s.split('::').map(&:downcase).join('/'), # file name
+        method: 'call'
+      }
     end
   end
 
-  def add_lambda_from_hash(hash)
+  def register_from_hash(hash)
     LAMBDAS[hash[:name]] = {
-      name: hash[:name],
-      gateway: hash[:gateway],
-      module: hash[:module],
-      wrapper: hash[:wrapper] || {},
-      env: hash[:env] || {},
+      name:     hash[:name],
+      gateway:  hash[:gateway],
+      module:   hash[:module],
+      wrapper:  hash[:wrapper]  || {},
+      env:      hash[:env]      || {},
       settings: hash[:settings] || {}
     }
   end
 
-  def add_lambda_from_module(mod, **opts)
+  def register_from_module(mod, **opts)
     mod.singleton_methods.sort.each do |module_method|
-      module_name = mod.to_s
-      module_names = module_name.split('::').map(&:downcase)
-      verb = 'GET'
-      path_fragments = module_names.dup
+      module_name     = mod.to_s
+      module_names    = module_name.split('::').map(&:downcase)
+      verb            = 'GET'
+      path_fragments  = module_names.dup
 
       # process parameters
       # method(a, b, c = 1, *args, d:, e: 2, **opts)
@@ -48,67 +66,85 @@ module Modulator
         verb = 'POST' if param_type == :key
       end
 
-      # delete is special case based on method name
+      # delete is a special case based on method name
       verb = 'DELETE' if %w[destroy delete remove implode].include? module_method.to_s
 
       # finalize path
       path_fragments << module_method
       path = path_fragments.join('/')
+      registry_key = calculate_registry_key(mod, module_method)
 
-      add_lambda_from_hash(
+      register_from_hash(
         {
-          name: "#{module_names.join('-')}-#{module_method}",
+          name: registry_key,
           gateway: {
             verb: opts.dig(module_method, :gateway, :verb) || verb,
             path: opts.dig(module_method, :gateway, :path) || path
           },
           module: {
-            name: module_name,
+            name:   module_name,
             method: module_method.to_s,
-            path: module_names.join('/') # file name
+            path:   module_names.join('/') # file name
           },
-          wrapper: opts.dig(module_method, :wrapper) || opts[:wrapper],
-          env: opts.dig(module_method, :env),
+          wrapper:  opts.dig(module_method, :wrapper) || opts[:wrapper],
+          env:      opts.dig(module_method, :env),
           settings: opts.dig(module_method, :settings)
         }
       )
     end
   end
 
-  def set_env(lambda_def)
+  # lambda definition reference
+  def calculate_registry_key(mod, module_method)
+    module_names = mod.to_s.split('::').map(&:downcase)
+    "#{module_names.join('-')}-#{module_method}"
+  end
+
+  # local gateway helper
+  def set_env_values(lambda_def)
     # remove wrapper if already set
-    %w(name method path).each{|name| ENV.delete('wrapper_' + name)}
-    # set env values
-    lambda_def[:module].each{|name, value| ENV['module_' + name.to_s] = value.to_s}
-    lambda_def[:gateway].each{|name, value| ENV['gateway_' + name.to_s] = value.to_s}
-    lambda_def[:wrapper]&.each{|name, value| ENV['wrapper_' + name.to_s] = value.to_s}
-    lambda_def[:env]&.each{|name, value| ENV[name.to_s] = value.to_s} # custom values
+    %i(name method path).each{|key| ENV.delete("wrapper_#{key}")}
+
+    # set env for each group
+    %i(module gateway wrapper env).each do |group_key|
+      lambda_def[group_key]&.each do |key, value|
+        key = "#{group_key}_#{key}" if group_key != :env
+        ENV[key.to_s] = value.to_s
+      end
+    end
   end
 
-  def init_stack(app_name:, bucket:, **stack_opts)
-    stack = AwsStackBuilder.init({
-        app_name: app_name.camelize,
-        bucket: bucket,
+  # init humidifier stack instance
+  def init_stack(app_name: Pathname.getwd.basename.to_s, s3_bucket:, **stack_opts)
+    stack = StackBuilder.init({
+        app_name:   app_name.camelize,
+        s3_bucket:  s3_bucket,
       }.merge(stack_opts))
 
+    # validate stack
+    # puts 'Validating stack'
+    # puts '- it is valid' if stack.valid?
+
+    self.stack = stack
+    generate_endoints if LAMBDAS.any?
+
+    # return instance
+    stack
+  end
+
+  # delegate to stack instance
+  def generate_endoints
     # add lambdas to stack
     puts 'Generating endpoints'
     LAMBDAS.each do |name, config|
       puts "- adding #{config.dig(:module, :name)}.#{config.dig(:module, :method)} to #{config.dig(:gateway, :path)}"
       stack.add_lambda_endpoint(
-        gateway: config[:gateway],
-        mod: config[:module],
-        wrapper: config[:wrapper] || {},
-        env: config[:env] || {},
+        gateway:  config[:gateway],
+        mod:      config[:module],
+        wrapper:  config[:wrapper]  || {},
+        env:      config[:env]      || {},
         settings: config[:settings] || {}
       )
     end
-
-    # validate stack
-    # puts 'Validating stack'
-    # puts '- it is valid' if stack.valid?
-
-    # return humidifier instance
-    stack
   end
 end