full360-sequencer 0.0.6 → 0.2.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 816056f91214a6885ee6279ccbe3598bf39af896ca551266881571e91362a8bb
4
- data.tar.gz: 87f1ebf3249d9cf08e41f7502cd6fed6d31033ce02b2675550f6c744a0032077
3
+ metadata.gz: 2847dd62aff2e2cd5540fd3abc9fdcf0b1087bb0e38c9adaeb501b04c4bddc8e
4
+ data.tar.gz: c0a54bc734e1e3c44aa2c3543e6a171839346c75b9e881028bd28580585f7c06
5
5
  SHA512:
6
- metadata.gz: ca93e5c5b5b063fa50a662f1724673a280222c3c64af200a73bd4ebaf21378ad9c11869a59ade63a99792be1186e077135f3b5773d427579e238e412c7ccf7ee
7
- data.tar.gz: 06c0903cd4588a3818862d4d5fef2f1f2498628cc84c6f6fd9849aac3b54d7c5af1cd8148df6ac378b07f0242b5626d83d71401ce348238098ff6c921cb59140
6
+ metadata.gz: f912e6cab2f0bed981e8a11d7fd9add676a1956c40fa6333b628f2ee24ad3e2ddb5e125a78ea0d22520841e39e76fc6c8d0b411e1510b998a04a2f99855a903b
7
+ data.tar.gz: e86a6e8eccc6296ddd7d58db583e4f9434c1fd8f82ff5f47011ccaa3412ebc9317b06e887c82ed53324bac4df82d9e33eb033a989a74aca1ccd54c49328d87fb
@@ -0,0 +1,98 @@
1
+ name: Release Gem
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+
8
+ jobs:
9
+ release:
10
+ runs-on: ubuntu-latest
11
+
12
+ steps:
13
+ # Checkout code if release was created
14
+ - uses: actions/checkout@v2
15
+
16
+ # Setup ruby if a release was created
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: 2.7
20
+ bundler-cache: true
21
+
22
+ # Publish
23
+ - name: publish gem
24
+ run: |
25
+ mkdir -p $HOME/.gem
26
+ touch $HOME/.gem/credentials
27
+ chmod 0600 $HOME/.gem/credentials
28
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
29
+ gem build *.gemspec
30
+ gem push *.gem
31
+ env:
32
+ RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
33
+
34
+ docker-jruby:
35
+ runs-on: ubuntu-latest
36
+ needs: release
37
+
38
+ steps:
39
+ - name: Checkout
40
+ uses: actions/checkout@v2
41
+
42
+ - name: Set up Docker Buildx
43
+ uses: docker/setup-buildx-action@v1
44
+
45
+ - name: Login to DockerHub
46
+ uses: docker/login-action@v1
47
+ with:
48
+ username: ${{ secrets.DOCKER_HUB_USERNAME }}
49
+ password: ${{ secrets.DOCKER_HUB_TOKEN }}
50
+
51
+ - name: Extract tag name
52
+ id: tag
53
+ uses: actions/github-script@0.2.0
54
+ with:
55
+ script: return context.payload.ref.replace(/\/refs\/tags\//, '')
56
+
57
+ - name: Build JRuby image
58
+ uses: docker/build-push-action@v2
59
+ with:
60
+ file: ./Dockerfile.jruby
61
+ pull: true
62
+ push: true
63
+ tags: |
64
+ full360/sequencer:jruby-latest
65
+ full360/sequencer:${{ steps.tag.outputs.result }}-jruby-latest
66
+
67
+ docker-ruby:
68
+ runs-on: ubuntu-latest
69
+ needs: release
70
+
71
+ steps:
72
+ - name: Checkout
73
+ uses: actions/checkout@v2
74
+
75
+ - name: Set up Docker Buildx
76
+ uses: docker/setup-buildx-action@v1
77
+
78
+ - name: Login to DockerHub
79
+ uses: docker/login-action@v1
80
+ with:
81
+ username: ${{ secrets.DOCKER_HUB_USERNAME }}
82
+ password: ${{ secrets.DOCKER_HUB_TOKEN }}
83
+
84
+ - name: Extract tag name
85
+ id: tag
86
+ uses: actions/github-script@0.2.0
87
+ with:
88
+ script: return context.payload.ref.replace(/\/refs\/tags\//, '')
89
+
90
+ - name: Build Ruby image
91
+ uses: docker/build-push-action@v2
92
+ with:
93
+ file: ./Dockerfile.ruby
94
+ pull: true
95
+ push: true
96
+ tags: |
97
+ full360/sequencer:ruby-latest
98
+ full360/sequencer:${{ steps.tag.outputs.result }}-ruby-latest
@@ -0,0 +1,26 @@
1
+ name: CI Test
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ strategy:
8
+ fail-fast: false
9
+ matrix:
10
+ os: [ubuntu-latest, macos-latest]
11
+ # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
12
+ ruby: ["2.7", "jruby-9.2.19.0", "jruby-9.3.0.0"]
13
+ runs-on: ${{ matrix.os }}
14
+
15
+ if: "!contains(github.event.head_commit.message, '[ci skip]')"
16
+
17
+ steps:
18
+ - name: Checkout
19
+ uses: actions/checkout@v2
20
+
21
+ - name: Test ${{ matrix.ruby }}
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby }}
25
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
26
+ - run: bundle exec rake
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ .DS_Store
2
+ .ruby-version
3
+
4
+ /.bundle/
5
+ /.yardoc
6
+ /_yardoc/
7
+ /coverage/
8
+ /doc/
9
+ /pkg/
10
+ /spec/reports/
11
+ /tmp/
12
+
13
+ # rspec failure tracking
14
+ .rspec_status
15
+
16
+ # generated gems
17
+ *.gem
data/CHANGELOG.md ADDED
@@ -0,0 +1,63 @@
1
+ # Changelog
2
+ All notable changes to this project will be documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
5
+ and this project adheres to [Semantic
6
+ Versioning](http://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+ ### Added
10
+ ### Changed
11
+ ### Removed
12
+
13
+ ## 0.2.5
14
+ ### Added
15
+ ### Changed
16
+ - Multiple workflow changese to make it work.
17
+ ### Removed
18
+
19
+ ## 0.2.0
20
+ ### Added
21
+ - Add Docker image for Ruby and JRuby that will get triggered after the release.
22
+ ### Changed
23
+ - Refactor the code to make it easier to read and modify.
24
+ - Update the AWS SDK for ECS to V3.
25
+ - Fix a missing logger dependency in the bin/sequencer code.
26
+ ### Removed
27
+ - Tests for Ruby 3.0 as they were failing for an unknown reason and I couldn't
28
+ reproduce locally.
29
+
30
+ ## 0.1.3
31
+ ### Added
32
+ - Fix release workflow.
33
+ ### Changed
34
+ ### Removed
35
+
36
+ ## 0.1.2
37
+ ### Added
38
+ - Fix release workflow.
39
+ ### Changed
40
+ ### Removed
41
+
42
+ ## 0.1.1
43
+ ### Added
44
+ - Workflows dependable.
45
+ ### Changed
46
+ ### Removed
47
+
48
+ ## 0.1.0
49
+ ### Added
50
+ - Working GitHub Actions workflow for tests and releases.
51
+ - Add Gemfile to the Gem.
52
+ ### Changed
53
+ - Update the .gemspec file and dependencies.
54
+ - Update the gem file format.
55
+ - Change from single quote to double quotes.
56
+ - Fix deprecation warnings in tests
57
+ ### Removed
58
+
59
+ ## 0.0.7
60
+ ### Added
61
+ - Working version.
62
+ ### Changed
63
+ ### Removed
data/Dockerfile.jruby ADDED
@@ -0,0 +1,23 @@
1
+ FROM jruby:9.2-jre8
2
+
3
+ # Should be more than sufficient for transforms without large recordset
4
+ # operations.
5
+ ENV JRUBY_OPTS=-J-Xmx1024m
6
+
7
+ RUN apt-get update \
8
+ && apt-get install -y --no-install-recommends \
9
+ unzip \
10
+ ca-certificates \
11
+ awscli \
12
+ && update-ca-certificates \
13
+ && rm -rf /var/lib/apt/lists/*
14
+
15
+ #set time zone
16
+ RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
17
+
18
+ RUN gem install full360-sequencer
19
+
20
+ # this is the entry point
21
+ ADD entrypoint.sh /usr/sbin/runner.sh
22
+ RUN chmod 755 /usr/sbin/runner.sh
23
+ ENTRYPOINT ["/usr/sbin/runner.sh"]
data/Dockerfile.ruby ADDED
@@ -0,0 +1,21 @@
1
+ FROM ruby:2.7-alpine
2
+
3
+ # Install required packages
4
+ RUN apk add --no-cache \
5
+ unzip \
6
+ ca-certificates \
7
+ bash \
8
+ wget \
9
+ curl \
10
+ aws-cli \
11
+ && update-ca-certificates \
12
+
13
+ #set time zone
14
+ RUN mv /etc/localtime /etc/localtime.bak ; ln -s /usr/share/zoneinfo/UTC /etc/localtime
15
+
16
+ RUN gem install full360-sequencer
17
+
18
+ # this is the entry point
19
+ ADD entrypoint.sh /usr/sbin/runner.sh
20
+ RUN chmod 755 /usr/sbin/runner.sh
21
+ ENTRYPOINT ["/usr/sbin/runner.sh"]
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,52 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ full360-sequencer (0.2.5)
5
+ aws-sdk-ecs (~> 1.85)
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ aws-eventstream (1.2.0)
11
+ aws-partitions (1.510.0)
12
+ aws-sdk-core (3.121.1)
13
+ aws-eventstream (~> 1, >= 1.0.2)
14
+ aws-partitions (~> 1, >= 1.239.0)
15
+ aws-sigv4 (~> 1.1)
16
+ jmespath (~> 1.0)
17
+ aws-sdk-ecs (1.85.0)
18
+ aws-sdk-core (~> 3, >= 3.120.0)
19
+ aws-sigv4 (~> 1.1)
20
+ aws-sigv4 (1.4.0)
21
+ aws-eventstream (~> 1, >= 1.0.2)
22
+ coderay (1.1.3)
23
+ ffi (1.15.4-java)
24
+ jmespath (1.4.0)
25
+ method_source (1.0.0)
26
+ minitest (5.14.4)
27
+ pry (0.14.1)
28
+ coderay (~> 1.1)
29
+ method_source (~> 1.0)
30
+ pry (0.14.1-java)
31
+ coderay (~> 1.1)
32
+ method_source (~> 1.0)
33
+ spoon (~> 0.0)
34
+ rake (12.3.3)
35
+ spoon (0.0.6)
36
+ ffi
37
+
38
+ PLATFORMS
39
+ universal-java-11
40
+ universal-java-8
41
+ x86_64-darwin-18
42
+ x86_64-darwin-19
43
+ x86_64-linux
44
+
45
+ DEPENDENCIES
46
+ full360-sequencer!
47
+ minitest (~> 5.9)
48
+ pry (~> 0.14)
49
+ rake (~> 12)
50
+
51
+ BUNDLED WITH
52
+ 2.2.28
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2017 Full 360 Inc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,104 @@
1
+ # Full 360 Sequencer
2
+
3
+ Sequencer is a tool that supports the synchronous execution of docker containers
4
+ running in Amazon ECS for the purpose of batch processing.
5
+
6
+ While there are many solutions for container orchestration, most of them are
7
+ focused on managing and scaling long running microservices, where a group of
8
+ containers need to kept alive and able to talk to each other. This model is
9
+ quite different from the execution of a nightly ETL batch where a container
10
+ performs a series of actions such as:
11
+
12
+ * collect the data from source system
13
+ * transform the data into the data lake
14
+ * ingest the data into the database
15
+ * transform the data inside the database with SQL
16
+ * export data to downstream consumers such as third parties or BI tools
17
+
18
+ AWS Batch service could do the above but it is currently only available in a
19
+ single region.
20
+
21
+ At one end of the spectrum your other options fall into the realm of shell
22
+ scripts.. while the other end brings you powerful (and cool!) DAG based tools
23
+ such as Nomad and Airflow.
24
+
25
+ We consider the DAG based tools to be way overkill for a simple batch of
26
+ synchronous jobs, so we opted to expand on the idea of a shell script.
27
+
28
+ Putting a shell script into a container is easy enough, but it requires that you
29
+ build and deploy a new container every time you make a change.
30
+
31
+ Sequencer allows you to define a list of ECS tasks in a yaml file, the location
32
+ of which is passed into the container at run time. This yaml file can sit in an
33
+ S3 repository (git support forthcoming). Sequencer will synchronously run each
34
+ task, passing in your override parameters as specified.
35
+
36
+ ## Installing Sequencer
37
+
38
+ Sequencer is installed as a ruby gem:
39
+
40
+ gem install full360-sequencer
41
+
42
+ Once installed... you can run the sequencer command from anywhere on your
43
+ system. Simply pass the path to your YAML file as a parameter. Note that you
44
+ will need to export some environment variables in order to use the AWS API.
45
+
46
+ $ export AWS_ACCESS_KEY_ID=aaaaaaaaayourkey
47
+ $ export AWS_SECRET_ACCESS_KEY=bbbbbbbbbbbbbbbbbbbbbbbbbbbbyoursecret
48
+ $ export AWS_REGION=us-west-2
49
+ $ sequencer mybatch.yml
50
+
51
+ Sequencer will utilize AWS instance or container roles if appropriate, though
52
+ you will have to provide `AWS_REGION` if you are not in us-east-1.
53
+
54
+ While using the command line sequencer is just fine, we suggest that you use the
55
+ full360/sequencer container available on Dockerhub.
56
+
57
+ ## YAML File Specification
58
+
59
+ Sequencer requires a yaml file with the following structure:
60
+
61
+ ```yaml
62
+ - first_task:
63
+ type: ecs_task
64
+ parameters:
65
+ cluster: yourcluster
66
+ task_definition: test-task-def:1
67
+ count: 1
68
+ - first_task:
69
+ type: ecs_task
70
+ parameters:
71
+ cluster: yourcluster2
72
+ task_definition: test-task-def:2
73
+ count: 1
74
+ ```
75
+
76
+ The top level structure is an array of tasks to be executed in the order shown.
77
+ Each task can be provided a name. Below each task you must specify the task
78
+ `type`. At this time, only ecs_task is supported as a task type.
79
+
80
+ The `parameters` element correlates to the request provided to the ECS run_task
81
+ API call. All of the elements are supported as long as you provide a YAML
82
+ structure. Refer to the API [reference][ref] for more details.
83
+
84
+ ## Releasing
85
+
86
+ Releasing a new version of the Gem requires a few steps:
87
+ - Update the `CHANGELOG.md` file
88
+ - Add a new section for the [UNRELEASED] code
89
+ - Check that all changes are reflected for the current release version
90
+ - Update the `version.rb` file
91
+ - Do a bundle install to ensure we are locking the latest version
92
+ - Commit all the changes
93
+ - Example: `git commit -m "Bump gem to version 1.0.0"`
94
+ - Create a Git tag that matches the version number in `version.rb`
95
+ - Example: `git tag -m "Version 1.0.0" v1.0.0`
96
+
97
+ ## Running the Sequencer from Docker
98
+
99
+ Pass the following environment variables to the container at runtime:
100
+
101
+ * `SEQUENCER_YAML_S3_PATH` - s3:// path to the yaml configuration file.
102
+ * `AWS_REGION` - AWS region in which you are running the container.
103
+
104
+ [ref]: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html
data/Rakefile ADDED
@@ -0,0 +1,7 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task :test do
4
+ Dir["./test/*_test.rb"].each { |file| require file}
5
+ end
6
+
7
+ task default: %w(test)
data/bin/console ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pry"
5
+ require "full360-sequencer"
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ Pry.start
data/bin/sequencer CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'full360-sequencer'
3
+ require "logger"
4
+ require "full360-sequencer"
4
5
 
5
6
  config_file = ARGV[0]
6
7
 
@@ -14,29 +15,29 @@ def logger
14
15
  end
15
16
 
16
17
  def sequencer_version
17
- Gem.loaded_specs['full360-sequencer'].version
18
+ Full360::Sequencer::VERSION
18
19
  end
19
20
 
20
21
  begin
21
22
  logger.info("sequencer version #{sequencer_version}")
22
- logger.level = ENV['SEQUENCER_LOG_DEBUG'] ? Logger::DEBUG : Logger::INFO
23
+ logger.level = ENV["SEQUENCER_LOG_DEBUG"] ? Logger::DEBUG : Logger::INFO
23
24
 
24
25
  if config_file == nil
25
- logger.error('SEQUENCER_ERROR')
26
- logger.error('YAML file not provided... exiting with error code 1')
26
+ logger.error("SEQUENCER_ERROR: YAML file not provided... exiting with error code 1")
27
27
  exit 1
28
28
  else
29
- r = Full360::Sequencer::Runner.new(logger)
30
- r.config_from_file(config_file)
31
- r.sleep_between_checks = ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS'].to_i if ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS']
32
- r.run
29
+ sleep_between_checks = ENV.fetch("SEQUENCER_SLEEP_BETWEEN_CHECKS", 5).to_i
30
+
31
+ Full360::Sequencer::Runner.new(sleep_between_checks, logger).tap do |runner|
32
+ runner.config_from_file(config_file)
33
+ runner.run
34
+ end
33
35
  end
34
- logger.info('all steps succeeded... exiting with code 0')
36
+ logger.info("all steps succeeded... exiting with code 0")
35
37
  exit 0
36
38
  rescue => e
37
- logger.error('SEQUENCER_ERROR')
38
- logger.error(e.message)
39
+ logger.error("SEQUENCER_ERROR: #{e.message}")
39
40
  e.backtrace.each { |r| logger.error(r) }
40
- logger.error('failure... exiting with code 1')
41
+ logger.error("failure... exiting with code 1")
41
42
  exit 1
42
- end
43
+ end
data/entrypoint.sh ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env bash
2
+
3
+ if [ -n "$AWS_REGION" ]
4
+ then
5
+ aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml --region $AWS_REGION
6
+ else
7
+ aws s3 cp $SEQUENCER_YAML_S3_PATH /sequencer/file.yml
8
+ fi
9
+
10
+ sequencer /sequencer/file.yml
@@ -0,0 +1,34 @@
1
+ lib = File.expand_path("../lib", __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require "full360_sequencer/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "full360-sequencer"
7
+ spec.version = Full360::Sequencer::VERSION
8
+ spec.date = "2017-10-17"
9
+ spec.summary = "Full 360 sequencer utility"
10
+ spec.description = "Automation for simple batch jobs run in AWS"
11
+ spec.authors = ["Full 360 Group"]
12
+ spec.email = "support@full360.com"
13
+ spec.files = ["lib/full360-sequencer.rb"]
14
+ spec.homepage = "https://full360.com"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ end
20
+
21
+ spec.bindir = "bin"
22
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
+ spec.require_paths = ["lib"]
24
+
25
+ # Make it Ruby 2+ only
26
+ spec.required_ruby_version = ">= 2.0"
27
+
28
+ spec.add_runtime_dependency "aws-sdk-ecs", "~> 1.85"
29
+
30
+ # development dependencies
31
+ spec.add_development_dependency "minitest", "~> 5.9"
32
+ spec.add_development_dependency "rake", "~> 12"
33
+ spec.add_development_dependency "pry", "~> 0.14"
34
+ end
@@ -1,159 +1 @@
1
- #!/usr/bin/ruby
2
- require 'aws-sdk'
3
- require 'yaml'
4
- require 'logger'
5
- require 'pp'
6
-
7
- module Full360
8
- module Sequencer
9
- class Runner
10
- attr_accessor :sleep_between_checks
11
- attr_accessor :config
12
-
13
- def initialize(logger = nil)
14
- @logger = logger ? logger : Logger.new(STDOUT)
15
-
16
- # default 5 seconds between completed? checks
17
- @sleep_between_checks = 5
18
- end
19
-
20
- def config_from_file(yaml_path)
21
- @config = parse_config_file(yaml_path)
22
- end
23
-
24
- def run_task_class(task_type_string)
25
- case task_type_string
26
- when 'ecs_task' then Full360::Sequencer::RunECSTask
27
- else nil
28
- end
29
- end
30
-
31
- def run
32
- @config.each do |params|
33
- this_task_name = task_name(params)
34
- this_task = run_task_class(params[this_task_name]['type']).new(
35
- this_task_name,
36
- params[this_task_name]
37
- )
38
- this_task.run_task
39
- until this_task.completed?
40
- sleep @sleep_between_checks
41
- end
42
- raise "task failed error" unless this_task.success
43
- end
44
- rescue => e
45
- @logger.error('SEQUENCER_ERROR')
46
- @logger.error(e.message)
47
- e.backtrace.each { |r| @logger.error(r) }
48
- raise e
49
- end
50
-
51
- def task_name(params)
52
- params.keys.first
53
- end
54
-
55
- def parse_config_file(yaml_path)
56
- YAML.load_file(yaml_path)
57
- end
58
-
59
- def config_valid?(config)
60
- return false unless config.is_a? Array
61
- true
62
- end
63
- end
64
-
65
- class RunTaskBase
66
- attr_reader :success
67
- attr_reader :exit_code
68
-
69
- def run_task; end
70
- def completed?; end
71
- def kill_task; end #will be used for timeout
72
- end
73
-
74
- class RunECSTask < RunTaskBase
75
- def initialize(task_name, params, logger = nil)
76
- @logger = logger ? logger : Logger.new(STDOUT)
77
- @task_name = task_name
78
- @params = params['parameters']
79
- @params = keys_to_symbol(@params)
80
- @cluster = @params[:cluster]
81
- end
82
-
83
- def keys_to_symbol(params)
84
- # replaces string keys with symbol keys
85
- # required by AWS SDK
86
- if params.is_a?(Hash)
87
- params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
88
- else
89
- nil
90
- end
91
- end
92
-
93
- def run_task
94
- @logger.info("starting ECS task #{@task_name}")
95
- resp = ecs_run_task
96
- @task_arn = resp.tasks[0].task_arn
97
- @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
98
- end
99
-
100
- def ecs_run_task
101
- @logger.debug("creating AWS client for ECS task #{@task_name}...")
102
- @ecs_client = ::Aws::ECS::Client.new
103
- @logger.debug("running ECS task #{@task_name}...")
104
- @start_time = Time.new
105
- resp = @ecs_client.run_task(@params)
106
- return resp
107
- rescue => e
108
- @logger.error('SEQUENCER_ERROR')
109
- @logger.error("error creating ECS task...")
110
- @logger.error("response from ECS: #{resp}")
111
- raise e
112
- end
113
-
114
- def ecs_describe_tasks
115
- @ecs_client.describe_tasks(
116
- {
117
- cluster: @cluster,
118
- tasks: [@task_arn] # required
119
- }
120
- )
121
- end
122
-
123
- def completed?
124
- resp = ecs_describe_tasks
125
- status = last_task_status(resp)
126
- @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
127
- if status == 'STOPPED'
128
- @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
129
- # parse exit_code(s) and return completion
130
- @success = determine_success(resp)
131
- return true
132
- end
133
- false
134
- rescue => e
135
- @logger.error('SEQUENCER_ERROR')
136
- @logger.error(e.message)
137
- e.backtrace.each { |r| @logger.error(r) }
138
- end
139
-
140
- # parses last status from aws API response
141
- def last_task_status(resp)
142
- resp.tasks[0].last_status
143
- end
144
-
145
- # success is determined by all containers having zero exit code
146
- def determine_success(resp)
147
- success = true
148
- resp.tasks[0].containers.each do |c|
149
- @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
150
- if c.exit_code != 0
151
- # we had a problem!
152
- success = false
153
- end
154
- end
155
- success
156
- end
157
- end
158
- end
159
- end
1
+ require "full360_sequencer"
@@ -0,0 +1,103 @@
1
+ require "logger"
2
+ require "aws-sdk-ecs"
3
+
4
+ module Full360
5
+ module Sequencer
6
+ class RunECSTask < RunTaskBase
7
+ attr_accessor :task_name
8
+ attr_accessor :params
9
+ attr_accessor :ecs_client
10
+ attr_accessor :logger
11
+
12
+ attr_reader :cluster
13
+ attr_reader :task_arn
14
+ attr_reader :start_time
15
+
16
+ def initialize(task_name, params, ecs_client = nil, logger = nil)
17
+ @logger = logger ||= Logger.new(STDOUT)
18
+ @ecs_client = ecs_client ||= Aws::ECS::Client.new
19
+ @task_name = task_name
20
+ @params = params[:parameters]
21
+ @cluster = self.params[:cluster]
22
+ end
23
+
24
+ def run_task
25
+ logger.info("starting ECS task #{task_name}")
26
+
27
+ resp = ecs_run_task
28
+ @task_arn = resp.tasks.first.task_arn
29
+
30
+ logger.info("#{task_name} task created #{task_arn} on cluster #{cluster}")
31
+ end
32
+
33
+ def ecs_run_task
34
+ logger.debug("running ECS task #{task_name}...")
35
+ @start_time = Time.new.utc
36
+
37
+ resp = ecs_client.run_task(params)
38
+ resp
39
+ rescue => e
40
+ logger.error("SEQUENCER_ERROR: response from ECS #{resp}")
41
+ raise e
42
+ end
43
+
44
+ def ecs_describe_tasks
45
+ ecs_client.describe_tasks(
46
+ {
47
+ cluster: cluster,
48
+ tasks: [task_arn],
49
+ }
50
+ )
51
+ end
52
+
53
+ def completed?
54
+ retries ||= 0
55
+
56
+ resp = ecs_describe_tasks
57
+ status = last_task_status(resp)
58
+
59
+ logger.info("#{task_name}: #{task_arn} current status: #{status}")
60
+
61
+ completed = false
62
+
63
+ if status == "STOPPED"
64
+ logger.info("#{task_name} completed in #{Time.new.utc - start_time} seconds")
65
+ # parse exit_code(s) and return completion
66
+ @success = determine_success(resp)
67
+ completed = true
68
+ end
69
+
70
+ completed
71
+ rescue => e
72
+ logger.warn(e.message)
73
+ logger.warn("task completion check failed, trying again ##{retries}")
74
+
75
+ sleep 10*retries
76
+
77
+ retry if (retries += 1) < 3
78
+
79
+ logger.error("SEQUENCER_ERROR: #{e.message}")
80
+ e.backtrace.each { |r| logger.error(r) }
81
+ end
82
+
83
+ # parses last status from aws API response
84
+ def last_task_status(resp)
85
+ resp.tasks.first.last_status
86
+ end
87
+
88
+ # success is determined by all containers having zero exit code
89
+ def determine_success(resp)
90
+ success = true
91
+
92
+ resp.tasks.first.containers.each do |c|
93
+ logger.info("#{task_name}: container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
94
+
95
+ # we had a problem!
96
+ success = false if c.exit_code != 0
97
+ end
98
+
99
+ success
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,12 @@
1
+ module Full360
2
+ module Sequencer
3
+ class RunTaskBase
4
+ attr_reader :success
5
+ attr_reader :exit_code
6
+
7
+ def run_task; end
8
+ def completed?; end
9
+ def kill_task; end # will be used for timeout
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,79 @@
1
+ require "yaml"
2
+ require "logger"
3
+ require "full360_sequencer/run_ecs_task"
4
+
5
+ module Full360
6
+ module Sequencer
7
+ class Runner
8
+ attr_accessor :sleep_between_checks
9
+ attr_accessor :logger
10
+
11
+ attr_reader :config
12
+
13
+ # Initializes the class
14
+ # @params sleep_between_checks [Int]
15
+ # @params logger [Logger]
16
+ # @return [Full360::Sequencer::Runner]
17
+ def initialize(sleep_between_checks, logger = nil)
18
+ @sleep_between_checks = sleep_between_checks
19
+ @logger = logger ||= Logger.new(STDOUT)
20
+ end
21
+
22
+ def config_from_file(yaml_path)
23
+ @config = parse_config_file(yaml_path)
24
+ end
25
+
26
+ def run_task_class(task_type_string)
27
+ case task_type_string
28
+ when "ecs_task"
29
+ Full360::Sequencer::RunECSTask
30
+ else
31
+ nil
32
+ end
33
+ end
34
+
35
+ def run
36
+ config.each do |params|
37
+ this_task_name = task_name(params)
38
+
39
+ this_task = run_task_class(params[this_task_name][:type]).new(
40
+ this_task_name,
41
+ params[this_task_name]
42
+ )
43
+
44
+ this_task.run_task
45
+
46
+ until this_task.completed?
47
+ sleep sleep_between_checks
48
+ end
49
+
50
+ raise "task failed error" unless this_task.success
51
+ end
52
+ rescue => e
53
+ logger.error("SEQUENCER_ERROR: #{e.message}")
54
+
55
+ e.backtrace.each { |r| logger.error(r) }
56
+ raise e
57
+ end
58
+
59
+ def task_name(params)
60
+ params.keys.first
61
+ end
62
+
63
+ def config_valid?(config)
64
+ return false unless config.is_a? Array
65
+ true
66
+ end
67
+
68
+ private
69
+
70
+ # parse_config_file reads and parses the configuration file and returns
71
+ # an array of hashes with symbolize keys ready to consume.
72
+ # @params yaml_path [String]
73
+ # @return [Array]
74
+ def parse_config_file(yaml_path)
75
+ YAML.safe_load(File.read(yaml_path), symbolize_names: true)
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,5 @@
1
+ module Full360
2
+ module Sequencer
3
+ VERSION = "0.2.5".freeze
4
+ end
5
+ end
@@ -0,0 +1,11 @@
1
+ require "logger"
2
+
3
+ # Autoload all gem classes
4
+ module Full360
5
+ module Sequencer
6
+ autoload :VERSION, "full360_sequencer/version"
7
+ autoload :Runner, "full360_sequencer/runner"
8
+ autoload :RunECSTask, "full360_sequencer/run_ecs_task"
9
+ autoload :RunTaskBase, "full360_sequencer/run_task_base"
10
+ end
11
+ end
metadata CHANGED
@@ -1,72 +1,105 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: full360-sequencer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.2.5
5
5
  platform: ruby
6
6
  authors:
7
- - pankaj batra
8
- - jeremy winters
9
- autorequire:
7
+ - Full 360 Group
8
+ autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
11
  date: 2017-10-17 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-ecs
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.2'
20
- name: logger
21
- prerelease: false
19
+ version: '1.85'
22
20
  type: :runtime
21
+ prerelease: false
23
22
  version_requirements: !ruby/object:Gem::Requirement
24
23
  requirements:
25
24
  - - "~>"
26
25
  - !ruby/object:Gem::Version
27
- version: '1.2'
26
+ version: '1.85'
28
27
  - !ruby/object:Gem::Dependency
28
+ name: minitest
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '2.9'
34
- name: aws-sdk
33
+ version: '5.9'
34
+ type: :development
35
35
  prerelease: false
36
- type: :runtime
37
36
  version_requirements: !ruby/object:Gem::Requirement
38
37
  requirements:
39
38
  - - "~>"
40
39
  - !ruby/object:Gem::Version
41
- version: '2.9'
40
+ version: '5.9'
42
41
  - !ruby/object:Gem::Dependency
42
+ name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '5.9'
48
- name: minitest
47
+ version: '12'
48
+ type: :development
49
49
  prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '12'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.14'
50
62
  type: :development
63
+ prerelease: false
51
64
  version_requirements: !ruby/object:Gem::Requirement
52
65
  requirements:
53
66
  - - "~>"
54
67
  - !ruby/object:Gem::Version
55
- version: '5.9'
56
- description: automation for simple batch jobs run in AWS
57
- email: pankaj.batra@full360.com
68
+ version: '0.14'
69
+ description: Automation for simple batch jobs run in AWS
70
+ email: support@full360.com
58
71
  executables:
72
+ - console
59
73
  - sequencer
60
74
  extensions: []
61
75
  extra_rdoc_files: []
62
76
  files:
77
+ - ".github/workflows/release.yml"
78
+ - ".github/workflows/test.yml"
79
+ - ".gitignore"
80
+ - CHANGELOG.md
81
+ - Dockerfile.jruby
82
+ - Dockerfile.ruby
83
+ - Gemfile
84
+ - Gemfile.lock
85
+ - LICENSE
86
+ - README.md
87
+ - Rakefile
88
+ - bin/console
63
89
  - bin/sequencer
90
+ - entrypoint.sh
91
+ - full360-sequencer.gemspec
64
92
  - lib/full360-sequencer.rb
65
- homepage: https://www.full360.com
93
+ - lib/full360_sequencer.rb
94
+ - lib/full360_sequencer/run_ecs_task.rb
95
+ - lib/full360_sequencer/run_task_base.rb
96
+ - lib/full360_sequencer/runner.rb
97
+ - lib/full360_sequencer/version.rb
98
+ homepage: https://full360.com
66
99
  licenses:
67
100
  - MIT
68
101
  metadata: {}
69
- post_install_message:
102
+ post_install_message:
70
103
  rdoc_options: []
71
104
  require_paths:
72
105
  - lib
@@ -81,9 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
81
114
  - !ruby/object:Gem::Version
82
115
  version: '0'
83
116
  requirements: []
84
- rubyforge_project:
85
- rubygems_version: 2.6.11
86
- signing_key:
117
+ rubygems_version: 3.1.6
118
+ signing_key:
87
119
  specification_version: 4
88
- summary: full360 sequencer utility
120
+ summary: Full 360 sequencer utility
89
121
  test_files: []