modulator 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e7dc8b70cbed5a8393011232c81278535c22415e6f2cef565b68f1f74a50b69f
+  data.tar.gz: d68ee3491a171bbe19a717ea8cd85e5cbdb1376d6abb27c8294dc987aee54b6a
+SHA512:
+  metadata.gz: ddd88590bb797341ee9c0b27f1ecfe5e76650740b660581ee0f1be5a89ced5fc3aee8fe957de1938dceb163c6d6d8e450da6102cb25688b17c6ae61f57e39507
+  data.tar.gz: 8376dd8b6ba36dfa66813d1319273b0c85dffedf482c15bc0e1bd55a84f8d437c317d90813f4c0f8f87b715e75e365c73d6d80aeca7b011c72c84cb5b71ed67c

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 2.0.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at damir.roso@nih.gov. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}"}
+
+gem 'humidifier', github: 'damir/humidifier'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,87 @@
+GIT
+  remote: https://github.com/damir/humidifier
+  revision: 81a75bbd4b47aadaaed5f967fb3527a2175dec82
+  specs:
+    humidifier (2.15.0)
+
+PATH
+  remote: .
+  specs:
+    modulator (0.1.1)
+      aws-sdk-cloudformation
+      aws-sdk-s3
+      puma
+      rerun
+      roda
+      rubyzip
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.0.3)
+    aws-partitions (1.154.0)
+    aws-sdk-cloudformation (1.19.0)
+      aws-sdk-core (~> 3, >= 3.48.2)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-core (3.48.6)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.17.0)
+      aws-sdk-core (~> 3, >= 3.48.2)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.36.1)
+      aws-sdk-core (~> 3, >= 3.48.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    diff-lcs (1.3)
+    ffi (1.10.0)
+    jmespath (1.4.0)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    puma (3.12.1)
+    rack (2.0.7)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rake (10.5.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.10.0)
+      ffi (~> 1.0)
+    rerun (0.13.0)
+      listen (~> 3.0)
+    roda (3.19.0)
+      rack
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby_dep (1.5.0)
+    rubyzip (1.2.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  humidifier!
+  modulator!
+  rack-test (~> 1.1)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Damir Roso
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Modulator
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/modulator`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'modulator'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install modulator
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/damir/modulator. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Modulator project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/modulator/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "modulator"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/gateway/gateway.rb

@@ -0,0 +1,123 @@
+require 'roda'
+require 'modulator'
+
+class Gateway < Roda
+  plugin :json
+  plugin :hooks
+  plugin :json_parser
+  plugin :pass
+  plugin :all_verbs
+  plugin :multi_route
+  plugin :default_headers, 'Content-Type'=>'application/json'
+
+  my_dir = Pathname.new(__FILE__).dirname
+  my_dir.glob('routes/*.rb').each{|file| require_relative file}
+  DUMMY_AWS_EVENT = Utils.load_json my_dir.parent.join('../spec/lambda/aws/event.json')
+
+  before do
+    @time = Time.now
+    puts
+    puts "Method: #{env['REQUEST_METHOD']}"
+    puts "Path: #{request.path}"
+    puts "Payload: #{request.params}" if request.params.any?
+    puts "Query string: #{env['QUERY_STRING']}" unless env['QUERY_STRING'].size.zero?
+    @headers = Hash[*env.select {|k,v| k.start_with? 'HTTP_'}
+      .collect {|k,v| [k.sub(/^HTTP_/, ''), v]}
+      .collect {|k,v| [k.split('_').collect(&:capitalize).join('-'), v]}
+      .sort
+      .flatten]
+    puts "Headers: #{@headers}"
+  end
+
+  after do |res|
+    puts "Matched path: #{request.matched_path}"
+    puts "Status: #{response.status}"
+    puts ("Took: #{Time.now - @time} seconds")
+  end
+
+  plugin :error_handler do |e|
+    puts
+    error = {error: e.class, message: e.message, backtrace: e.backtrace&.take(10)}
+    puts "FAILED: #{error}"
+    error
+  end
+
+  route do |r|
+    r.root do
+      {working_dir: Pathname.getwd}
+    end
+
+    r.multi_route # tools routes
+
+    # process lambda configs
+    Modulator::LAMBDAS.each do |lambda_name, lambda_config|
+      # puts "* Registering #{lambda_name}"
+      # pp lambda_config
+
+      # module and wrapper config
+      Modulator.set_env lambda_config
+
+      # build route
+      @path_params = {}
+      build_route(r, ENV['gateway_verb'], ENV['gateway_path'].split('/'))
+    end
+
+    # return 404 if no route is found, the loop will return 200 otherwise
+    r.halt
+  end
+
+  def build_route(r, verb, fragments)
+    # end of path, tail is gone: [] -> nil
+    return unless fragments
+
+    # execute when all fragments are processed
+    execute_lambda(r, verb) and return if fragments.empty?
+
+    # traverse fragments
+    fragment = fragments.first
+    tail = fragments[1..-1] || []
+
+    # dynamic fragment
+    if fragment.start_with?(':')
+      r.on String do |value|
+        fragment = fragment[1..-1]
+        @path_params[fragment] = value
+        build_route(r, verb, tail)
+        r.pass # continue loop
+      end
+
+    # static fragment
+    else
+      r.on fragment do
+        build_route(r, verb, tail)
+        r.pass # continue loop
+      end
+    end
+  end
+
+  def execute_lambda(r, verb)
+    r.send(verb.downcase) do
+      puts "Path params: #{@path_params}"
+
+      # build aws event
+      aws_event = DUMMY_AWS_EVENT.merge(
+        'pathParameters' => @path_params,
+        'queryStringParameters' => {},
+        'headers' => @headers,
+        'body' => request.params.to_json
+      )
+
+      # build aws context
+      aws_context = {}
+
+      # execute lambda
+      result = Dir.chdir(opts[:app_dir] || '.') do
+        AwsLambdaHandler.call(event: aws_event, context: aws_context)
+      end
+
+      # render
+      response.status = result[:statusCode]
+      result[:body]
+    end
+  end
+end

data/lib/gateway/routes/console.rb

@@ -0,0 +1,89 @@
+require 'aws-sdk-cloudformation'
+
+# console API
+Gateway.route('console') do |r|
+
+  client    = Aws::CloudFormation::Client.new
+  app_name  = (opts[:app_dir] || Pathname.getwd.basename.to_s).camelize
+
+  # helpers
+  def capture_output
+    previous_stdout, $stdout = $stdout, StringIO.new
+    previous_stderr, $stderr = $stderr, StringIO.new
+    yield
+    $stdout.string if $stdout.string.size > 0
+    $stderr.string if $stderr.string.size > 0
+    ensure
+      $stdout = previous_stdout
+      $stderr = previous_stderr
+  end
+
+  def render_command_result(command_result, command_output)
+    if command_result
+      command_result.to_hash
+    else
+      {'aws-sdk-cloudformation': command_output.split("\n").first}
+    end
+  end
+
+  r.on 'stack' do
+    bucket_name = ENV['S3BUCKET'] || 'modulator-lambdas'
+    payload = request.params.symbolize_keys
+    command_result = nil
+
+    r.get 'events' do
+      resp = client.describe_stack_events(stack_name: app_name, next_token: @headers['X-Next-Token'])
+      response['X-Next-Token'] = resp.next_token
+      resp.stack_events.map(&:to_hash)
+    end
+
+    r.on 'init' do
+      serializer = :yaml
+      content_type = 'text/html'
+
+      r.on 'json' do
+        serializer = :json
+        content_type = 'application/json'
+        r.pass
+      end
+
+      # init stack
+      stack = Modulator.init_stack(
+        app_name: app_name,
+        bucket: bucket_name,
+        timeout: 15
+      )
+
+      # add stack parameters if found in payload
+      bucket_name[:parameters]&.each do |param|
+        stack.add_parameter(param[:key],
+          description: param[:description],
+          type: param[:type],
+          value: param[:value]
+        )
+      end
+
+      r.post 'valid' do
+        command_output = capture_output do
+          command_result = stack.valid?
+        end
+        render_command_result(command_result, command_output)
+      end
+
+      r.post 'deploy' do
+        command_output = capture_output do
+          command_result = stack.deploy(
+            parameters: bucket_name[:parameters]&.map{|param| {parameter_key: param[:key], parameter_value: param[:value]}},
+            capabilities: ['CAPABILITY_IAM']
+          )
+        end
+        render_command_result(command_result, command_output)
+      end
+
+      r.post do
+        response['Content-Type'] = content_type
+        template = stack.to_cf(serializer)
+      end
+    end
+  end
+end

data/lib/lambda/aws_lambda_handler.rb

@@ -0,0 +1,140 @@
+require 'pathname'
+
+module AwsLambdaHandler
+  module_function
+
+  # select event handler
+  def call(event:, context:)
+    # TODO: implement handlers for other event types based on some event key, like AwsS3EventHandler
+    AwsApiGatewayEventHandler.call(event: event, context: context)
+  end
+
+  # helpers
+  def symbolize_keys(obj)
+    case obj
+    when Hash
+      hash = {}
+      obj.each {|k, v| hash[k.to_sym] = symbolize_keys(v)}
+      hash
+    when Array
+      obj.map {|x| symbolize_keys(x)}
+    else
+      obj
+    end
+  end
+end
+
+module AwsApiGatewayEventHandler
+  module_function
+
+  NUMBER_REGEX = /\A[+-]?\d+(\.[\d]+)?\z/
+
+  def call(event:, context:)
+    wrapper_params = {}
+
+    if ENV['wrapper_name']
+      # module init
+      require Pathname.getwd.join(ENV['wrapper_path']).to_s
+      wrapper = Object.const_get(ENV['wrapper_name'])
+      mod_method = ENV['wrapper_method']
+
+      # check arity
+      if wrapper.method(mod_method).parameters != [[:keyreq, :event], [:keyreq, :context]]
+        raise("#{wrapper}.#{mod_method} should accept event and context keyword arguments")
+      end
+
+      # print call info
+      puts "Calling wrapper #{wrapper}.#{mod_method}"
+      wrapper_result = wrapper.send(mod_method, event: event, context: context)
+
+      if wrapper_result.is_a?(Hash)
+        # block with custom status and body
+        return {
+          statusCode: wrapper_result[:status],
+          body: JSON.generate(wrapper_result[:body])
+        } if wrapper_result[:status]
+
+        # or set params
+        wrapper_params = wrapper_result
+
+      elsif !wrapper_result
+        # block if result is false/nil
+        return {
+          statusCode: 403,
+          body: JSON.generate(forbidden: "#{wrapper}.#{mod_method}")
+        } if !wrapper_result
+      end
+    end
+
+    # module init
+    require Pathname.getwd.join(ENV['module_path']).to_s
+    mod = Object.const_get(ENV['module_name'])
+    mod_method = ENV['module_method']
+
+    # gateway def
+    verb = ENV['gateway_verb']
+    path = ENV['gateway_path']
+
+    # print call info
+    method_signature = mod.method(mod_method).parameters
+    puts "Resolving #{verb.to_s.upcase} #{path} to #{mod}.#{mod_method} with #{method_signature}"
+
+    # cast path parameters to numbers
+    path_params = event['pathParameters'].each_with_object({}) do |(key, value), params|
+      if matcher = NUMBER_REGEX.match(value)
+        value = matcher[1] ? Float(value) : value.to_i
+      end
+      params[key] = value
+    end
+
+    # merge wrapper params
+    path_params.merge!(wrapper_params)
+
+    # call the module method
+    result =
+    if ['GET', 'DELETE'].include?(verb)
+      mod.send(mod_method, *path_params.values)
+
+    elsif verb == 'POST'
+      payload = AwsLambdaHandler.symbolize_keys(JSON.parse(event['body']))
+      method_signature.each do |arg_type, arg_name|         # [[:req, :id], [:key, :pet]]
+        payload = {arg_name => payload} if arg_type == :key # scope payload to first named argument
+      end
+
+      # we can override GET to POST without payload, ruby will break if **payload is resolved from {}
+      if payload.any?
+        mod.send(mod_method, *path_params.values, **payload)
+      else
+        mod.send(mod_method, *path_params.values)
+      end
+
+    else
+      raise 'Verb should be GET, POST or DELETE'
+    end
+
+    # set status and body values
+    if result.nil?
+      status  = 404
+      body    = nil # it will print json null string
+    else
+      status  = result[:status] ? result[:status] : 200
+      body    = result[:body] ? result[:body] : result
+    end
+
+    # return result
+    return {statusCode: status, body: JSON.generate(body)}
+
+    # print and return error
+    rescue => e
+      puts output = {
+        statusCode: 500,
+        body: JSON.generate(
+          error: {
+            class: e.class,
+            message: e.message
+          }.merge(ENV['debug'] ? {backtrace: e.backtrace.take(20)} : {})
+        )
+      }
+      output
+  end
+end