full360-sequencer 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54bca4472a18632b312e84aff4c3ca5958053645dca8860f579c27e257689337
-  data.tar.gz: 024a6bad17c794fe021cfd31df13a03ada093aa4c0c12c3dfd7432e38b43a518
+  metadata.gz: b89f369df62f5fa73c8eef3b7597596bcbe052026761aed9e9c9b1b23bc0fb90
+  data.tar.gz: bb4a599e8df5d9cd2410fe5d91fe15a68648731f1bf33c3979708f6dc3f822ed
 SHA512:
-  metadata.gz: 04e30b01ccbb05f4622d1596c37cbff99ad15cabd7ab8e6bd8bc1906a46482a35e7f1cc5fc51588e66b281a71ea3e31b4ab945b01abe5650cbf30eb777117c16
-  data.tar.gz: 510b7447b7182e6a746d6584a48ea4949aeeeb320a222bc0605e4ebcae93729f56928dd301d2ae3fa01a54843364a9f5672c1f2526197374b4425f8a0fd57015
+  metadata.gz: 7df9df844dabb0d75c55971cc09da599b34ca4796fb670e6a49209ba652bfffdd81d290b9c35c516d9f8513d097a0cec8305eeb739eb68acfeb2a5ae5770c336
+  data.tar.gz: 1585bb1dadb01907286f2c7deab022d8f69d7fc2dec9ad51aeaf860885a74b624bffe48677ae159779eb8dd5faee383b19ed93f1f38c03b3b9978feb8c65e02d

data/.github/workflows/release.yml

@@ -1,17 +1,14 @@
 name: Release Gem
 
 on:
-  workflow_run:
-    workflows: ["CI Test"]
-    types: [completed]
-    branches:
-      - master
+  push:
     tags:
-      - v*
+      - "v*"
 
 jobs:
   release:
     runs-on: ubuntu-latest
+
     steps:
       # Checkout code if release was created
       - uses: actions/checkout@v2
@@ -33,3 +30,69 @@ jobs:
           gem push *.gem
         env:
           RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
+
+  docker-jruby:
+    runs-on: ubuntu-latest
+    needs: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Extract tag name
+        id: tag
+        uses: actions/github-script@0.2.0
+        with:
+          script: return context.payload.ref.replace(/\/refs\/tags\//, '')
+
+      - name: Build JRuby image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./Dockerfile.jruby
+          pull: true
+          push: true
+          tags: |
+            full360/sequencer:jruby-latest
+            full360/sequencer:${{ steps.tag.outputs.result }}-jruby-latest
+
+  docker-ruby:
+    runs-on: ubuntu-latest
+    needs: release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Extract tag name
+        id: tag
+        uses: actions/github-script@0.2.0
+        with:
+          script: return context.payload.ref.replace(/\/refs\/tags\//, '')
+
+      - name: Build Ruby image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./Dockerfile.ruby
+          pull: true
+          push: true
+          tags: |
+            full360/sequencer:ruby-latest
+            full360/sequencer:${{ steps.tag.outputs.result }}-ruby-latest

data/.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
-        ruby: ["2.7", "3.0", "jruby-9.2.19.0", "jruby-9.3.0.0"]
+        ruby: ["2.7", "jruby-9.2.19.0", "jruby-9.3.0.0"]
     runs-on: ${{ matrix.os }}
 
     if: "!contains(github.event.head_commit.message, '[ci skip]')"

data/CHANGELOG.md

@@ -10,6 +10,29 @@ Versioning](http://semver.org/spec/v2.0.0.html).
 ### Changed
 ### Removed
 
+## 0.2.0
+### Added
+- Add Docker image for Ruby and JRuby that will get triggered after the release.
+### Changed
+- Refactor the code to make it easier to read and modify.
+- Update the AWS SDK for ECS to V3.
+- Fix a missing logger dependency in the bin/sequencer code.
+### Removed
+- Tests for Ruby 3.0 as they were failing for an unknown reason and I couldn't
+  reproduce locally.
+
+## 0.1.3
+### Added
+- Fix release workflow.
+### Changed
+### Removed
+
+## 0.1.2
+### Added
+- Fix release workflow.
+### Changed
+### Removed
+
 ## 0.1.1
 ### Added
 - Workflows dependable.

data/Dockerfile.jruby

@@ -0,0 +1,23 @@
+FROM jruby:9.2-jre8
+
+# Should be more than sufficient for transforms without large recordset
+# operations.
+ENV JRUBY_OPTS=-J-Xmx1024m
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+  unzip \
+  ca-certificates \
+  awscli \
+  && update-ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+#set time zone
+RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
+
+RUN gem install full360-sequencer
+
+# this is the entry point
+ADD entrypoint.sh /usr/sbin/runner.sh
+RUN chmod 755 /usr/sbin/runner.sh
+ENTRYPOINT ["/usr/sbin/runner.sh"]

data/Dockerfile.ruby

@@ -0,0 +1,21 @@
+FROM ruby:2.7-alpine
+
+# Install required packages
+RUN apk add --no-cache \
+  unzip \
+  ca-certificates \
+  bash \
+  wget \
+  curl \
+  aws-cli \
+  && update-ca-certificates \
+
+#set time zone
+RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
+
+RUN gem install full360-sequencer
+
+# this is the entry point
+ADD entrypoint.sh /usr/sbin/runner.sh
+RUN chmod 755 /usr/sbin/runner.sh
+ENTRYPOINT ["/usr/sbin/runner.sh"]

data/Gemfile.lock

@@ -1,20 +1,22 @@
 PATH
   remote: .
   specs:
-    full360-sequencer (0.1.3)
-      aws-sdk (~> 2.9)
+    full360-sequencer (0.2.0)
+      aws-sdk-ecs (~> 1.85)
 
 GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.2.0)
-    aws-sdk (2.11.632)
-      aws-sdk-resources (= 2.11.632)
-    aws-sdk-core (2.11.632)
-      aws-sigv4 (~> 1.0)
+    aws-partitions (1.509.0)
+    aws-sdk-core (3.121.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.11.632)
-      aws-sdk-core (= 2.11.632)
+    aws-sdk-ecs (1.85.0)
+      aws-sdk-core (~> 3, >= 3.120.0)
+      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     coderay (1.1.3)
@@ -47,4 +49,4 @@ DEPENDENCIES
   rake (~> 12)
 
 BUNDLED WITH
-   2.2.27
+   2.2.28

data/README.md

@@ -94,4 +94,11 @@ Releasing a new version of the Gem requires a few steps:
 - Create a Git tag that matches the version number in `version.rb`
   - Example: `git tag -m "Version 1.0.0" v1.0.0`
 
+## Running the Sequencer from Docker
+
+Pass the following environment variables to the container at runtime:
+
+* `SEQUENCER_YAML_S3_PATH` - s3:// path to the yaml configuration file.
+* `AWS_REGION` - AWS region in which you are running the container.
+
 [ref]: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html

data/bin/sequencer

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require "logger"
 require "full360-sequencer"
 
 config_file = ARGV[0]
@@ -14,7 +15,7 @@ def logger
 end
 
 def sequencer_version
-  Gem.loaded_specs["full360-sequencer"].version
+  Full360::Sequencer::VERSION
 end
 
 begin
@@ -22,20 +23,20 @@ begin
   logger.level = ENV["SEQUENCER_LOG_DEBUG"] ? Logger::DEBUG : Logger::INFO
 
   if config_file == nil
-    logger.error("SEQUENCER_ERROR")
-    logger.error("YAML file not provided... exiting with error code 1")
+    logger.error("SEQUENCER_ERROR: YAML file not provided... exiting with error code 1")
     exit 1
   else
-    r = Full360::Sequencer::Runner.new(logger)
-    r.config_from_file(config_file)
-    r.sleep_between_checks = ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"].to_i if ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"]
-    r.run
+    sleep_between_checks = ENV.fetch("SEQUENCER_SLEEP_BETWEEN_CHECKS", 5).to_i
+
+    Full360::Sequencer::Runner.new(sleep_between_checks, logger).tap do |runner|
+      runner.config_from_file(config_file)
+      runner.run
+    end
   end
   logger.info("all steps succeeded... exiting with code 0")
   exit 0
 rescue => e
-  logger.error("SEQUENCER_ERROR")
-  logger.error(e.message)
+  logger.error("SEQUENCER_ERROR: #{e.message}")
   e.backtrace.each { |r| logger.error(r) }
   logger.error("failure... exiting with code 1")
   exit 1

data/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ -n "$AWS_REGION" ]
+then
+  aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml --region $AWS_REGION
+else
+  aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml
+fi
+
+sequencer /sequencer/file.yml

data/full360-sequencer.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   # Make it Ruby 2+ only
   spec.required_ruby_version = ">= 2.0"
 
-  spec.add_runtime_dependency "aws-sdk", "~> 2.9"
+  spec.add_runtime_dependency "aws-sdk-ecs", "~> 1.85"
 
   # development dependencies
   spec.add_development_dependency "minitest", "~> 5.9"

data/lib/full360_sequencer/run_ecs_task.rb

@@ -1,95 +1,101 @@
-require "aws-sdk"
 require "logger"
+require "aws-sdk-ecs"
 
 module Full360
   module Sequencer
     class RunECSTask < RunTaskBase
-      def initialize(task_name, params, logger = nil)
-        @logger = logger ? logger : Logger.new(STDOUT)
-        @task_name = task_name
-        @params = params["parameters"]
-        @params = keys_to_symbol(@params)
-        @cluster = @params[:cluster]
-      end
+      attr_accessor :task_name
+      attr_accessor :params
+      attr_accessor :ecs_client
+      attr_accessor :logger
 
-      def keys_to_symbol(params)
-        # replaces string keys with symbol keys
-        # required by AWS SDK
-        if params.is_a?(Hash)
-          params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
-        else
-          nil
-        end
+      attr_reader :cluster
+      attr_reader :task_arn
+      attr_reader :start_time
+
+      def initialize(task_name, params, ecs_client = nil, logger = nil)
+        @logger     = logger ||= Logger.new(STDOUT)
+        @ecs_client = ecs_client ||= Aws::ECS::Client.new
+        @task_name  = task_name
+        @params     = params[:parameters]
+        @cluster    = self.params[:cluster]
       end
 
       def run_task
-        @logger.info("starting ECS task #{@task_name}")
+        logger.info("starting ECS task #{task_name}")
+
         resp = ecs_run_task
-        @task_arn = resp.tasks[0].task_arn
-        @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
+        @task_arn = resp.tasks.first.task_arn
+
+        logger.info("#{task_name} task created #{task_arn} on cluster #{cluster}")
       end
 
       def ecs_run_task
-        @logger.debug("creating AWS client for ECS task #{@task_name}...")
-        @ecs_client = ::Aws::ECS::Client.new
-        @logger.debug("running ECS task #{@task_name}...")
-        @start_time = Time.new
-        resp = @ecs_client.run_task(@params)
-        return resp
+        logger.debug("running ECS task #{task_name}...")
+        @start_time = Time.new.utc
+
+        resp = ecs_client.run_task(params)
+        resp
       rescue => e
-        @logger.error("SEQUENCER_ERROR")
-        @logger.error("error creating ECS task...")
-        @logger.error("response from ECS: #{resp}")
+        logger.error("SEQUENCER_ERROR: response from ECS #{resp}")
         raise e
       end
 
       def ecs_describe_tasks
-        @ecs_client.describe_tasks(
+        ecs_client.describe_tasks(
           {
-            cluster: @cluster,
-            tasks: [@task_arn] # required
+            cluster: cluster,
+            tasks: [task_arn],
           }
         )
       end
 
       def completed?
         retries ||= 0
+
         resp = ecs_describe_tasks
         status = last_task_status(resp)
-        @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
+
+        logger.info("#{task_name}: #{task_arn} current status: #{status}")
+
+        completed = false
+
         if status == "STOPPED"
-          @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
+          logger.info("#{task_name} completed in #{Time.new.utc - start_time} seconds")
           # parse exit_code(s) and return completion
           @success = determine_success(resp)
-          return true
+          completed = true
         end
-        false
+
+        completed
       rescue => e
-        @logger.warn(e.message)
-        @logger.warn("task completion check failed, trying again ##{ retries }")
+        logger.warn(e.message)
+        logger.warn("task completion check failed, trying again ##{retries}")
+
         sleep 10*retries
+
         retry if (retries += 1) < 3
 
-        @logger.error("SEQUENCER_ERROR")
-        @logger.error(e.message)
-        e.backtrace.each { |r| @logger.error(r) }
+        logger.error("SEQUENCER_ERROR: #{e.message}")
+        e.backtrace.each { |r| logger.error(r) }
       end
 
       # parses last status from aws API response
       def last_task_status(resp)
-        resp.tasks[0].last_status
+        resp.tasks.first.last_status
       end
 
       # success is determined by all containers having zero exit code
       def determine_success(resp)
         success = true
-        resp.tasks[0].containers.each do |c|
-          @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
-          if c.exit_code != 0
-            # we had a problem!
-            success = false
-          end
+
+        resp.tasks.first.containers.each do |c|
+          logger.info("#{task_name}: container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
+
+          # we had a problem!
+          success = false if c.exit_code != 0
         end
+
         success
       end
     end

data/lib/full360_sequencer/runner.rb

@@ -1,17 +1,22 @@
 require "yaml"
 require "logger"
+require "full360_sequencer/run_ecs_task"
 
 module Full360
   module Sequencer
     class Runner
       attr_accessor :sleep_between_checks
-      attr_accessor :config
+      attr_accessor :logger
 
-      def initialize(logger = nil)
-        @logger = logger ? logger : Logger.new(STDOUT)
+      attr_reader :config
 
-        # default 5 seconds between completed? checks
-        @sleep_between_checks = 5
+      # Initializes the class
+      # @params sleep_between_checks [Int]
+      # @params logger [Logger]
+      # @return [Full360::Sequencer::Runner]
+      def initialize(sleep_between_checks, logger = nil)
+        @sleep_between_checks = sleep_between_checks
+        @logger               = logger ||= Logger.new(STDOUT)
       end
 
       def config_from_file(yaml_path)
@@ -20,28 +25,34 @@ module Full360
 
       def run_task_class(task_type_string)
         case task_type_string
-        when "ecs_task" then Full360::Sequencer::RunECSTask
-        else nil
+        when "ecs_task"
+          Full360::Sequencer::RunECSTask
+        else
+          nil
         end
       end
 
       def run
-        @config.each do |params|
+        config.each do |params|
           this_task_name = task_name(params)
-          this_task = run_task_class(params[this_task_name]["type"]).new(
+
+          this_task = run_task_class(params[this_task_name][:type]).new(
             this_task_name,
             params[this_task_name]
           )
+
           this_task.run_task
+
           until this_task.completed?
-            sleep @sleep_between_checks
+            sleep sleep_between_checks
           end
+
           raise "task failed error" unless this_task.success
         end
       rescue => e
-        @logger.error("SEQUENCER_ERROR")
-        @logger.error(e.message)
-        e.backtrace.each { |r| @logger.error(r) }
+        logger.error("SEQUENCER_ERROR: #{e.message}")
+
+        e.backtrace.each { |r| logger.error(r) }
         raise e
       end
 
@@ -49,14 +60,20 @@ module Full360
         params.keys.first
       end
 
-      def parse_config_file(yaml_path)
-        YAML.load_file(yaml_path)
-      end
-
       def config_valid?(config)
         return false unless config.is_a? Array
         true
       end
+
+      private
+
+      # parse_config_file reads and parses the configuration file and returns
+      # an array of hashes with symbolize keys ready to consume.
+      # @params yaml_path [String]
+      # @return [Array]
+      def parse_config_file(yaml_path)
+        YAML.safe_load(File.read(yaml_path), symbolize_names: true)
+      end
     end
   end
 end

data/lib/full360_sequencer/version.rb

@@ -1,5 +1,5 @@
 module Full360
   module Sequencer
-    VERSION = "0.1.3".freeze
+    VERSION = "0.2.0".freeze
   end
 end

data/lib/full360_sequencer.rb

@@ -1,3 +1,5 @@
+require "logger"
+
 # Autoload all gem classes
 module Full360
   module Sequencer

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full360-sequencer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Full 360 Group
@@ -11,19 +11,19 @@ cert_chain: []
 date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-ecs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '1.85'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '1.85'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,8 @@ files:
 - ".github/workflows/test.yml"
 - ".gitignore"
 - CHANGELOG.md
+- Dockerfile.jruby
+- Dockerfile.ruby
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -85,6 +87,7 @@ files:
 - Rakefile
 - bin/console
 - bin/sequencer
+- entrypoint.sh
 - full360-sequencer.gemspec
 - lib/full360-sequencer.rb
 - lib/full360_sequencer.rb