full360-sequencer 0.0.7 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 11689f8562fd17688883eca48d0e173a573d3823
-  data.tar.gz: 2328c3194fc48a4333bfeb4b7300dc839a9539a5
+SHA256:
+  metadata.gz: e39df18990fb5fdf6924e3e4b60b373f1400db68d6125be7a24b56b3401950f5
+  data.tar.gz: 400f3182c16e5f32dfa88883c0717f7ddecc88c1e8aec993a19abbdb2f5e613d
 SHA512:
-  metadata.gz: 163c12b36137fd5a09c96851fe43315f50f191afd793e14b32817fd5dc6fd260b1afdf05e2a694ca4c942883fcf6bf7bf902332552e168cf97c5c86986661cfd
-  data.tar.gz: e5ef1c88f6cad5d4c4c37f435b7e3be01b1460bdeb7c8cf1b84ad3b7f98eba5ec25968aff1af19162235f92faf972b82fd456be82a24ca7d60e21a78a7228f18
+  metadata.gz: 75181267e1df232193818cff99384d6ac2262f3b6059afdf71d1f445ab0b09d869ead4aed3882545a30b8242c0c7091661bbc301bc24410efc9a5d3e57e37459
+  data.tar.gz: a78938906d4c9c7cbf650a32f3932000c57349d00490b3779ba55fea428060f6ef3809f265d40ae81f26e75b4209a5032c75d58910007a6b34cfa5d8446a1271

data/.github/workflows/release.yml

@@ -0,0 +1,94 @@
+name: Release Gem
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout code if release was created
+      - uses: actions/checkout@v2
+
+      # Setup ruby if a release was created
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+          bundler-cache: true
+
+      # Publish
+      - name: publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
+
+  docker-jruby:
+    runs-on: ubuntu-latest
+    needs: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Extract tag name
+        id: tag
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build JRuby image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./Dockerfile.jruby
+          pull: true
+          push: true
+          tags: |
+            full360/sequencer:jruby-latest
+            full360/sequencer:${{ steps.tag.outputs.VERSION }}-jruby-latest
+
+  docker-ruby:
+    runs-on: ubuntu-latest
+    needs: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Extract tag name
+        id: tag
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build Ruby image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./Dockerfile.ruby
+          pull: true
+          push: true
+          tags: |
+            full360/sequencer:ruby-latest
+            full360/sequencer:${{ steps.tag.outputs.VERSION }}-ruby-latest

data/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: CI Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
+        ruby: ["2.7", "jruby-9.2.19.0", "jruby-9.3.0.0"]
+    runs-on: ${{ matrix.os }}
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Test ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - run: bundle exec rake

data/.gitignore

@@ -0,0 +1,17 @@
+.DS_Store
+.ruby-version
+
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# generated gems
+*.gem

data/CHANGELOG.md

@@ -0,0 +1,69 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic
+Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+### Changed
+### Removed
+
+## 0.2.6
+### Added
+### Changed
+### Removed
+- Remove the `github-script` action.
+
+## 0.2.5
+### Added
+### Changed
+- Multiple workflow changese to make it work.
+### Removed
+
+## 0.2.0
+### Added
+- Add Docker image for Ruby and JRuby that will get triggered after the release.
+### Changed
+- Refactor the code to make it easier to read and modify.
+- Update the AWS SDK for ECS to V3.
+- Fix a missing logger dependency in the bin/sequencer code.
+### Removed
+- Tests for Ruby 3.0 as they were failing for an unknown reason and I couldn't
+  reproduce locally.
+
+## 0.1.3
+### Added
+- Fix release workflow.
+### Changed
+### Removed
+
+## 0.1.2
+### Added
+- Fix release workflow.
+### Changed
+### Removed
+
+## 0.1.1
+### Added
+- Workflows dependable.
+### Changed
+### Removed
+
+## 0.1.0
+### Added
+- Working GitHub Actions workflow for tests and releases.
+- Add Gemfile to the Gem.
+### Changed
+- Update the .gemspec file and dependencies.
+- Update the gem file format.
+- Change from single quote to double quotes.
+- Fix deprecation warnings in tests
+### Removed
+
+## 0.0.7
+### Added
+- Working version.
+### Changed
+### Removed

data/Dockerfile.jruby

@@ -0,0 +1,23 @@
+FROM jruby:9.2-jre8
+
+# Should be more than sufficient for transforms without large recordset
+# operations.
+ENV JRUBY_OPTS=-J-Xmx1024m
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+  unzip \
+  ca-certificates \
+  awscli \
+  && update-ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+#set time zone
+RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
+
+RUN gem install full360-sequencer
+
+# this is the entry point
+ADD entrypoint.sh /usr/sbin/runner.sh
+RUN chmod 755 /usr/sbin/runner.sh
+ENTRYPOINT ["/usr/sbin/runner.sh"]

data/Dockerfile.ruby

@@ -0,0 +1,21 @@
+FROM ruby:2.7-alpine
+
+# Install required packages
+RUN apk add --no-cache \
+  unzip \
+  ca-certificates \
+  bash \
+  wget \
+  curl \
+  aws-cli \
+  && update-ca-certificates \
+
+#set time zone
+RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
+
+RUN gem install full360-sequencer
+
+# this is the entry point
+ADD entrypoint.sh /usr/sbin/runner.sh
+RUN chmod 755 /usr/sbin/runner.sh
+ENTRYPOINT ["/usr/sbin/runner.sh"]

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    full360-sequencer (0.2.6)
+      aws-sdk-ecs (~> 1.85)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.2.0)
+    aws-partitions (1.510.0)
+    aws-sdk-core (3.121.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-ecs (1.85.0)
+      aws-sdk-core (~> 3, >= 3.120.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.4.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+    coderay (1.1.3)
+    ffi (1.15.4-java)
+    jmespath (1.4.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry (0.14.1-java)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+      spoon (~> 0.0)
+    rake (12.3.3)
+    spoon (0.0.6)
+      ffi
+
+PLATFORMS
+  universal-java-11
+  universal-java-8
+  x86_64-darwin-18
+  x86_64-darwin-19
+  x86_64-linux
+
+DEPENDENCIES
+  full360-sequencer!
+  minitest (~> 5.9)
+  pry (~> 0.14)
+  rake (~> 12)
+
+BUNDLED WITH
+   2.2.28

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Full 360 Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# Full 360 Sequencer
+
+Sequencer is a tool that supports the synchronous execution of docker containers
+running in Amazon ECS for the purpose of batch processing.
+
+While there are many solutions for container orchestration, most of them are
+focused on managing and scaling long running microservices, where a group of
+containers need to kept alive and able to talk to each other. This model is
+quite different from the execution of a nightly ETL batch where a container
+performs a series of actions such as:
+
+* collect the data from source system
+* transform the data into the data lake
+* ingest the data into the database
+* transform the data inside the database with SQL
+* export data to downstream consumers such as third parties or BI tools
+
+AWS Batch service could do the above but it is currently only available in a
+single region.
+
+At one end of the spectrum your other options fall into the realm of shell
+scripts.. while the other end brings you powerful (and cool!) DAG based tools
+such as Nomad and Airflow.
+
+We consider the DAG based tools to be way overkill for a simple batch of
+synchronous jobs, so we opted to expand on the idea of a shell script.
+
+Putting a shell script into a container is easy enough, but it requires that you
+build and deploy a new container every time you make a change.
+
+Sequencer allows you to define a list of ECS tasks in a yaml file, the location
+of which is passed into the container at run time. This yaml file can sit in an
+S3 repository (git support forthcoming). Sequencer will synchronously run each
+task, passing in your override parameters as specified.
+
+## Installing Sequencer
+
+Sequencer is installed as a ruby gem:
+
+    gem install full360-sequencer
+
+Once installed... you can run the sequencer command from anywhere on your
+system. Simply pass the path to your YAML file as a parameter. Note that you
+will need to export some environment variables in order to use the AWS API.
+
+    $ export AWS_ACCESS_KEY_ID=aaaaaaaaayourkey
+    $ export AWS_SECRET_ACCESS_KEY=bbbbbbbbbbbbbbbbbbbbbbbbbbbbyoursecret
+    $ export AWS_REGION=us-west-2
+    $ sequencer mybatch.yml
+
+Sequencer will utilize AWS instance or container roles if appropriate, though
+you will have to provide `AWS_REGION` if you are not in us-east-1.
+
+While using the command line sequencer is just fine, we suggest that you use the
+full360/sequencer container available on Dockerhub.
+
+## YAML File Specification
+
+Sequencer requires a yaml file with the following structure:
+
+```yaml
+- first_task:
+    type: ecs_task
+    parameters:
+      cluster: yourcluster
+      task_definition: test-task-def:1
+      count: 1
+- first_task:
+    type: ecs_task
+    parameters:
+      cluster: yourcluster2
+      task_definition: test-task-def:2
+      count: 1
+```
+
+The top level structure is an array of tasks to be executed in the order shown.
+Each task can be provided a name. Below each task you must specify the task
+`type`. At this time, only ecs_task is supported as a task type.
+
+The `parameters` element correlates to the request provided to the ECS run_task
+API call. All of the elements are supported as long as you provide a YAML
+structure. Refer to the API [reference][ref] for more details.
+
+## Releasing
+
+Releasing a new version of the Gem requires a few steps:
+- Update the `CHANGELOG.md` file
+  - Add a new section for the [UNRELEASED] code
+  - Check that all changes are reflected for the current release version
+- Update the `version.rb` file
+- Do a bundle install to ensure we are locking the latest version
+- Commit all the changes
+  - Example: `git commit -m "Bump gem to version 1.0.0"`
+- Create a Git tag that matches the version number in `version.rb`
+  - Example: `git tag -m "Version 1.0.0" v1.0.0`
+
+## Running the Sequencer from Docker
+
+Pass the following environment variables to the container at runtime:
+
+* `SEQUENCER_YAML_S3_PATH` - s3:// path to the yaml configuration file.
+* `AWS_REGION` - AWS region in which you are running the container.
+
+[ref]: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+
+task :test do
+  Dir["./test/*_test.rb"].each { |file| require file}
+end
+
+task default: %w(test)

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pry"
+require "full360-sequencer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+Pry.start

data/bin/sequencer

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
-require 'full360-sequencer'
+require "logger"
+require "full360-sequencer"
 
 config_file = ARGV[0]
 
@@ -14,29 +15,29 @@ def logger
 end
 
 def sequencer_version
-  Gem.loaded_specs['full360-sequencer'].version
+  Full360::Sequencer::VERSION
 end
 
 begin
   logger.info("sequencer version #{sequencer_version}")
-  logger.level = ENV['SEQUENCER_LOG_DEBUG'] ? Logger::DEBUG : Logger::INFO
+  logger.level = ENV["SEQUENCER_LOG_DEBUG"] ? Logger::DEBUG : Logger::INFO
 
   if config_file == nil
-    logger.error('SEQUENCER_ERROR')
-    logger.error('YAML file not provided... exiting with error code 1')
+    logger.error("SEQUENCER_ERROR: YAML file not provided... exiting with error code 1")
     exit 1
   else
-    r = Full360::Sequencer::Runner.new(logger)
-    r.config_from_file(config_file)
-    r.sleep_between_checks = ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS'].to_i if ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS']
-    r.run
+    sleep_between_checks = ENV.fetch("SEQUENCER_SLEEP_BETWEEN_CHECKS", 5).to_i
+
+    Full360::Sequencer::Runner.new(sleep_between_checks, logger).tap do |runner|
+      runner.config_from_file(config_file)
+      runner.run
+    end
   end
-  logger.info('all steps succeeded... exiting with code 0')
+  logger.info("all steps succeeded... exiting with code 0")
   exit 0
 rescue => e
-  logger.error('SEQUENCER_ERROR')
-  logger.error(e.message)
+  logger.error("SEQUENCER_ERROR: #{e.message}")
   e.backtrace.each { |r| logger.error(r) }
-  logger.error('failure... exiting with code 1')
+  logger.error("failure... exiting with code 1")
   exit 1
-end
+end

data/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ -n "$AWS_REGION" ]
+then
+  aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml --region $AWS_REGION
+else
+  aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml
+fi
+
+sequencer /sequencer/file.yml

data/full360-sequencer.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "full360_sequencer/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "full360-sequencer"
+  spec.version     = Full360::Sequencer::VERSION
+  spec.date        = "2017-10-17"
+  spec.summary     = "Full 360 sequencer utility"
+  spec.description = "Automation for simple batch jobs run in AWS"
+  spec.authors     = ["Full 360 Group"]
+  spec.email       = "support@full360.com"
+  spec.files       = ["lib/full360-sequencer.rb"]
+  spec.homepage    = "https://full360.com"
+  spec.license     = "MIT"
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Make it Ruby 2+ only
+  spec.required_ruby_version = ">= 2.0"
+
+  spec.add_runtime_dependency "aws-sdk-ecs", "~> 1.85"
+
+  # development dependencies
+  spec.add_development_dependency "minitest", "~> 5.9"
+  spec.add_development_dependency "rake", "~> 12"
+  spec.add_development_dependency "pry", "~> 0.14"
+end

data/lib/full360-sequencer.rb

@@ -1,165 +1 @@
-#!/usr/bin/ruby
-require 'aws-sdk'
-require 'yaml'
-require 'logger'
-require 'pp'
-
-module Full360
-  module Sequencer
-    class Runner
-      attr_accessor :sleep_between_checks
-      attr_accessor :config
-
-      def initialize(logger = nil)
-        @logger = logger ? logger : Logger.new(STDOUT)
-
-        # default 5 seconds between completed? checks
-        @sleep_between_checks = 5
-      end
-
-      def config_from_file(yaml_path)
-        @config = parse_config_file(yaml_path)
-      end
-
-      def run_task_class(task_type_string)
-        case task_type_string
-        when 'ecs_task' then Full360::Sequencer::RunECSTask
-        else nil
-        end
-      end
-
-      def run
-        @config.each do |params|
-          this_task_name = task_name(params)
-          this_task = run_task_class(params[this_task_name]['type']).new(
-            this_task_name,
-            params[this_task_name]
-          )
-          this_task.run_task
-          until this_task.completed?
-            sleep @sleep_between_checks
-          end
-          raise "task failed error" unless this_task.success
-        end
-      rescue => e
-        @logger.error('SEQUENCER_ERROR')
-        @logger.error(e.message)
-        e.backtrace.each { |r| @logger.error(r) }
-        raise e
-      end
-
-      def task_name(params)
-        params.keys.first
-      end
-
-      def parse_config_file(yaml_path)
-        YAML.load_file(yaml_path)
-      end
-
-      def config_valid?(config)
-        return false unless config.is_a? Array
-        true
-      end
-    end
-
-    class RunTaskBase
-      attr_reader :success
-      attr_reader :exit_code
-
-      def run_task; end
-      def completed?; end
-      def kill_task; end #will be used for timeout
-    end
-
-    class RunECSTask < RunTaskBase
-      def initialize(task_name, params, logger = nil)
-        @logger = logger ? logger : Logger.new(STDOUT)
-        @task_name = task_name
-        @params = params['parameters']
-        @params = keys_to_symbol(@params)
-        @cluster = @params[:cluster]
-      end
-
-      def keys_to_symbol(params)
-        # replaces string keys with symbol keys
-        # required by AWS SDK
-        if params.is_a?(Hash)
-          params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
-        else
-          nil
-        end
-      end
-
-      def run_task
-        @logger.info("starting ECS task #{@task_name}")
-        resp = ecs_run_task
-        @task_arn = resp.tasks[0].task_arn
-        @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
-      end
-
-      def ecs_run_task
-        @logger.debug("creating AWS client for ECS task #{@task_name}...")
-        @ecs_client = ::Aws::ECS::Client.new
-        @logger.debug("running ECS task #{@task_name}...")
-        @start_time = Time.new
-        resp = @ecs_client.run_task(@params)
-        return resp
-      rescue => e
-        @logger.error('SEQUENCER_ERROR')
-        @logger.error("error creating ECS task...")
-        @logger.error("response from ECS: #{resp}")
-        raise e
-      end
-
-      def ecs_describe_tasks
-        @ecs_client.describe_tasks(
-          {
-            cluster: @cluster,
-            tasks: [@task_arn] # required
-          }
-        )
-      end
-
-      def completed?
-        retries ||= 0
-        resp = ecs_describe_tasks
-        status = last_task_status(resp)
-        @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
-        if status == 'STOPPED'
-          @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
-          # parse exit_code(s) and return completion
-          @success = determine_success(resp)
-          return true
-        end
-        false
-      rescue => e
-        @logger.warn(e.message)
-        @logger.warn("task completion check failed, trying again ##{ retries }")
-        sleep 10*retries
-        retry if (retries += 1) < 3
-
-        @logger.error('SEQUENCER_ERROR')
-        @logger.error(e.message)
-        e.backtrace.each { |r| @logger.error(r) }
-      end
-
-      # parses last status from aws API response
-      def last_task_status(resp)
-        resp.tasks[0].last_status
-      end
-
-      # success is determined by all containers having zero exit code
-      def determine_success(resp)
-        success = true
-        resp.tasks[0].containers.each do |c|
-          @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
-          if c.exit_code != 0
-            # we had a problem!
-            success = false
-          end
-        end
-        success
-      end
-    end
-  end
-end
+require "full360_sequencer"

data/lib/full360_sequencer/run_ecs_task.rb

@@ -0,0 +1,103 @@
+require "logger"
+require "aws-sdk-ecs"
+
+module Full360
+  module Sequencer
+    class RunECSTask < RunTaskBase
+      attr_accessor :task_name
+      attr_accessor :params
+      attr_accessor :ecs_client
+      attr_accessor :logger
+
+      attr_reader :cluster
+      attr_reader :task_arn
+      attr_reader :start_time
+
+      def initialize(task_name, params, ecs_client = nil, logger = nil)
+        @logger     = logger ||= Logger.new(STDOUT)
+        @ecs_client = ecs_client ||= Aws::ECS::Client.new
+        @task_name  = task_name
+        @params     = params[:parameters]
+        @cluster    = self.params[:cluster]
+      end
+
+      def run_task
+        logger.info("starting ECS task #{task_name}")
+
+        resp = ecs_run_task
+        @task_arn = resp.tasks.first.task_arn
+
+        logger.info("#{task_name} task created #{task_arn} on cluster #{cluster}")
+      end
+
+      def ecs_run_task
+        logger.debug("running ECS task #{task_name}...")
+        @start_time = Time.new.utc
+
+        resp = ecs_client.run_task(params)
+        resp
+      rescue => e
+        logger.error("SEQUENCER_ERROR: response from ECS #{resp}")
+        raise e
+      end
+
+      def ecs_describe_tasks
+        ecs_client.describe_tasks(
+          {
+            cluster: cluster,
+            tasks: [task_arn],
+          }
+        )
+      end
+
+      def completed?
+        retries ||= 0
+
+        resp = ecs_describe_tasks
+        status = last_task_status(resp)
+
+        logger.info("#{task_name}: #{task_arn} current status: #{status}")
+
+        completed = false
+
+        if status == "STOPPED"
+          logger.info("#{task_name} completed in #{Time.new.utc - start_time} seconds")
+          # parse exit_code(s) and return completion
+          @success = determine_success(resp)
+          completed = true
+        end
+
+        completed
+      rescue => e
+        logger.warn(e.message)
+        logger.warn("task completion check failed, trying again ##{retries}")
+
+        sleep 10*retries
+
+        retry if (retries += 1) < 3
+
+        logger.error("SEQUENCER_ERROR: #{e.message}")
+        e.backtrace.each { |r| logger.error(r) }
+      end
+
+      # parses last status from aws API response
+      def last_task_status(resp)
+        resp.tasks.first.last_status
+      end
+
+      # success is determined by all containers having zero exit code
+      def determine_success(resp)
+        success = true
+
+        resp.tasks.first.containers.each do |c|
+          logger.info("#{task_name}: container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
+
+          # we had a problem!
+          success = false if c.exit_code != 0
+        end
+
+        success
+      end
+    end
+  end
+end

data/lib/full360_sequencer/run_task_base.rb

@@ -0,0 +1,12 @@
+module Full360
+  module Sequencer
+    class RunTaskBase
+      attr_reader :success
+      attr_reader :exit_code
+
+      def run_task; end
+      def completed?; end
+      def kill_task; end # will be used for timeout
+    end
+  end
+end

data/lib/full360_sequencer/runner.rb

@@ -0,0 +1,79 @@
+require "yaml"
+require "logger"
+require "full360_sequencer/run_ecs_task"
+
+module Full360
+  module Sequencer
+    class Runner
+      attr_accessor :sleep_between_checks
+      attr_accessor :logger
+
+      attr_reader :config
+
+      # Initializes the class
+      # @params sleep_between_checks [Int]
+      # @params logger [Logger]
+      # @return [Full360::Sequencer::Runner]
+      def initialize(sleep_between_checks, logger = nil)
+        @sleep_between_checks = sleep_between_checks
+        @logger               = logger ||= Logger.new(STDOUT)
+      end
+
+      def config_from_file(yaml_path)
+        @config = parse_config_file(yaml_path)
+      end
+
+      def run_task_class(task_type_string)
+        case task_type_string
+        when "ecs_task"
+          Full360::Sequencer::RunECSTask
+        else
+          nil
+        end
+      end
+
+      def run
+        config.each do |params|
+          this_task_name = task_name(params)
+
+          this_task = run_task_class(params[this_task_name][:type]).new(
+            this_task_name,
+            params[this_task_name]
+          )
+
+          this_task.run_task
+
+          until this_task.completed?
+            sleep sleep_between_checks
+          end
+
+          raise "task failed error" unless this_task.success
+        end
+      rescue => e
+        logger.error("SEQUENCER_ERROR: #{e.message}")
+
+        e.backtrace.each { |r| logger.error(r) }
+        raise e
+      end
+
+      def task_name(params)
+        params.keys.first
+      end
+
+      def config_valid?(config)
+        return false unless config.is_a? Array
+        true
+      end
+
+      private
+
+      # parse_config_file reads and parses the configuration file and returns
+      # an array of hashes with symbolize keys ready to consume.
+      # @params yaml_path [String]
+      # @return [Array]
+      def parse_config_file(yaml_path)
+        YAML.safe_load(File.read(yaml_path), symbolize_names: true)
+      end
+    end
+  end
+end

data/lib/full360_sequencer/version.rb

@@ -0,0 +1,5 @@
+module Full360
+  module Sequencer
+    VERSION = "0.2.6".freeze
+  end
+end

data/lib/full360_sequencer.rb

@@ -0,0 +1,11 @@
+require "logger"
+
+# Autoload all gem classes
+module Full360
+  module Sequencer
+    autoload :VERSION,     "full360_sequencer/version"
+    autoload :Runner,      "full360_sequencer/runner"
+    autoload :RunECSTask,  "full360_sequencer/run_ecs_task"
+    autoload :RunTaskBase, "full360_sequencer/run_task_base"
+  end
+end

metadata

@@ -1,68 +1,101 @@
 --- !ruby/object:Gem::Specification
 name: full360-sequencer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.2.6
 platform: ruby
 authors:
-- pankaj batra
-- jeremy winters
+- Full 360 Group
 autorequire: 
 bindir: bin
 cert_chain: []
 date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logger
+  name: aws-sdk-ecs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.85'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.85'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
-  type: :runtime
+        version: '5.9'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '5.9'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
-description: automation for simple batch jobs run in AWS
-email: pankaj.batra@full360.com
+        version: '12'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+description: Automation for simple batch jobs run in AWS
+email: support@full360.com
 executables:
+- console
 - sequencer
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/release.yml"
+- ".github/workflows/test.yml"
+- ".gitignore"
+- CHANGELOG.md
+- Dockerfile.jruby
+- Dockerfile.ruby
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
 - bin/sequencer
+- entrypoint.sh
+- full360-sequencer.gemspec
 - lib/full360-sequencer.rb
-homepage: https://www.full360.com
+- lib/full360_sequencer.rb
+- lib/full360_sequencer/run_ecs_task.rb
+- lib/full360_sequencer/run_task_base.rb
+- lib/full360_sequencer/runner.rb
+- lib/full360_sequencer/version.rb
+homepage: https://full360.com
 licenses:
 - MIT
 metadata: {}
@@ -81,9 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
-summary: full360 sequencer utility
+summary: Full 360 sequencer utility
 test_files: []