full360-sequencer 0.0.4 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: abbbd848e9b1781d94ad645dd9cc22f633fdf3f614e9933e2bc644f6062681cb
4
- data.tar.gz: e193689050a930b1a39a5007eb325c314ea4f895052cd9bef00bc7b2f780337c
3
+ metadata.gz: 54bca4472a18632b312e84aff4c3ca5958053645dca8860f579c27e257689337
4
+ data.tar.gz: 024a6bad17c794fe021cfd31df13a03ada093aa4c0c12c3dfd7432e38b43a518
5
5
  SHA512:
6
- metadata.gz: 8c4614eda5a674ce7a6b112fc3832223f92ad04c52b051649259563b5dca4a86b7344b7edf3ecafc0a2f1c4aebbd675689dc6be59593bd82eef7c21e1f8e0039
7
- data.tar.gz: 5caef9605f16951eb9c6bfc33a0153928707e96aa99cd922d5c96f73f59f560b2a906fc33a8ba4e0cd1c86336df8748f56cc59bbcac573fee3374c3a83f2f411
6
+ metadata.gz: 04e30b01ccbb05f4622d1596c37cbff99ad15cabd7ab8e6bd8bc1906a46482a35e7f1cc5fc51588e66b281a71ea3e31b4ab945b01abe5650cbf30eb777117c16
7
+ data.tar.gz: 510b7447b7182e6a746d6584a48ea4949aeeeb320a222bc0605e4ebcae93729f56928dd301d2ae3fa01a54843364a9f5672c1f2526197374b4425f8a0fd57015
@@ -0,0 +1,35 @@
1
+ name: Release Gem
2
+
3
+ on:
4
+ workflow_run:
5
+ workflows: ["CI Test"]
6
+ types: [completed]
7
+ branches:
8
+ - master
9
+ tags:
10
+ - v*
11
+
12
+ jobs:
13
+ release:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ # Checkout code if release was created
17
+ - uses: actions/checkout@v2
18
+
19
+ # Setup ruby if a release was created
20
+ - uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: 2.7
23
+ bundler-cache: true
24
+
25
+ # Publish
26
+ - name: publish gem
27
+ run: |
28
+ mkdir -p $HOME/.gem
29
+ touch $HOME/.gem/credentials
30
+ chmod 0600 $HOME/.gem/credentials
31
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
32
+ gem build *.gemspec
33
+ gem push *.gem
34
+ env:
35
+ RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
@@ -0,0 +1,26 @@
1
+ name: CI Test
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ strategy:
8
+ fail-fast: false
9
+ matrix:
10
+ os: [ubuntu-latest, macos-latest]
11
+ # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
12
+ ruby: ["2.7", "3.0", "jruby-9.2.19.0", "jruby-9.3.0.0"]
13
+ runs-on: ${{ matrix.os }}
14
+
15
+ if: "!contains(github.event.head_commit.message, '[ci skip]')"
16
+
17
+ steps:
18
+ - name: Checkout
19
+ uses: actions/checkout@v2
20
+
21
+ - name: Test ${{ matrix.ruby }}
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby }}
25
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
26
+ - run: bundle exec rake
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ .DS_Store
2
+ .ruby-version
3
+
4
+ /.bundle/
5
+ /.yardoc
6
+ /_yardoc/
7
+ /coverage/
8
+ /doc/
9
+ /pkg/
10
+ /spec/reports/
11
+ /tmp/
12
+
13
+ # rspec failure tracking
14
+ .rspec_status
15
+
16
+ # generated gems
17
+ *.gem
data/CHANGELOG.md ADDED
@@ -0,0 +1,34 @@
1
+ # Changelog
2
+ All notable changes to this project will be documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
5
+ and this project adheres to [Semantic
6
+ Versioning](http://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+ ### Added
10
+ ### Changed
11
+ ### Removed
12
+
13
+ ## 0.1.1
14
+ ### Added
15
+ - Workflows dependable.
16
+ ### Changed
17
+ ### Removed
18
+
19
+ ## 0.1.0
20
+ ### Added
21
+ - Working GitHub Actions workflow for tests and releases.
22
+ - Add Gemfile to the Gem.
23
+ ### Changed
24
+ - Update the .gemspec file and dependencies.
25
+ - Update the gem file format.
26
+ - Change from single quote to double quotes.
27
+ - Fix deprecation warnings in tests
28
+ ### Removed
29
+
30
+ ## 0.0.7
31
+ ### Added
32
+ - Working version.
33
+ ### Changed
34
+ ### Removed
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,50 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ full360-sequencer (0.1.3)
5
+ aws-sdk (~> 2.9)
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ aws-eventstream (1.2.0)
11
+ aws-sdk (2.11.632)
12
+ aws-sdk-resources (= 2.11.632)
13
+ aws-sdk-core (2.11.632)
14
+ aws-sigv4 (~> 1.0)
15
+ jmespath (~> 1.0)
16
+ aws-sdk-resources (2.11.632)
17
+ aws-sdk-core (= 2.11.632)
18
+ aws-sigv4 (1.4.0)
19
+ aws-eventstream (~> 1, >= 1.0.2)
20
+ coderay (1.1.3)
21
+ ffi (1.15.4-java)
22
+ jmespath (1.4.0)
23
+ method_source (1.0.0)
24
+ minitest (5.14.4)
25
+ pry (0.14.1)
26
+ coderay (~> 1.1)
27
+ method_source (~> 1.0)
28
+ pry (0.14.1-java)
29
+ coderay (~> 1.1)
30
+ method_source (~> 1.0)
31
+ spoon (~> 0.0)
32
+ rake (12.3.3)
33
+ spoon (0.0.6)
34
+ ffi
35
+
36
+ PLATFORMS
37
+ universal-java-11
38
+ universal-java-8
39
+ x86_64-darwin-18
40
+ x86_64-darwin-19
41
+ x86_64-linux
42
+
43
+ DEPENDENCIES
44
+ full360-sequencer!
45
+ minitest (~> 5.9)
46
+ pry (~> 0.14)
47
+ rake (~> 12)
48
+
49
+ BUNDLED WITH
50
+ 2.2.27
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2017 Full 360 Inc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,97 @@
1
+ # Full 360 Sequencer
2
+
3
+ Sequencer is a tool that supports the synchronous execution of docker containers
4
+ running in Amazon ECS for the purpose of batch processing.
5
+
6
+ While there are many solutions for container orchestration, most of them are
7
+ focused on managing and scaling long running microservices, where a group of
8
+ containers need to kept alive and able to talk to each other. This model is
9
+ quite different from the execution of a nightly ETL batch where a container
10
+ performs a series of actions such as:
11
+
12
+ * collect the data from source system
13
+ * transform the data into the data lake
14
+ * ingest the data into the database
15
+ * transform the data inside the database with SQL
16
+ * export data to downstream consumers such as third parties or BI tools
17
+
18
+ AWS Batch service could do the above but it is currently only available in a
19
+ single region.
20
+
21
+ At one end of the spectrum your other options fall into the realm of shell
22
+ scripts.. while the other end brings you powerful (and cool!) DAG based tools
23
+ such as Nomad and Airflow.
24
+
25
+ We consider the DAG based tools to be way overkill for a simple batch of
26
+ synchronous jobs, so we opted to expand on the idea of a shell script.
27
+
28
+ Putting a shell script into a container is easy enough, but it requires that you
29
+ build and deploy a new container every time you make a change.
30
+
31
+ Sequencer allows you to define a list of ECS tasks in a yaml file, the location
32
+ of which is passed into the container at run time. This yaml file can sit in an
33
+ S3 repository (git support forthcoming). Sequencer will synchronously run each
34
+ task, passing in your override parameters as specified.
35
+
36
+ ## Installing Sequencer
37
+
38
+ Sequencer is installed as a ruby gem:
39
+
40
+ gem install full360-sequencer
41
+
42
+ Once installed... you can run the sequencer command from anywhere on your
43
+ system. Simply pass the path to your YAML file as a parameter. Note that you
44
+ will need to export some environment variables in order to use the AWS API.
45
+
46
+ $ export AWS_ACCESS_KEY_ID=aaaaaaaaayourkey
47
+ $ export AWS_SECRET_ACCESS_KEY=bbbbbbbbbbbbbbbbbbbbbbbbbbbbyoursecret
48
+ $ export AWS_REGION=us-west-2
49
+ $ sequencer mybatch.yml
50
+
51
+ Sequencer will utilize AWS instance or container roles if appropriate, though
52
+ you will have to provide `AWS_REGION` if you are not in us-east-1.
53
+
54
+ While using the command line sequencer is just fine, we suggest that you use the
55
+ full360/sequencer container available on Dockerhub.
56
+
57
+ ## YAML File Specification
58
+
59
+ Sequencer requires a yaml file with the following structure:
60
+
61
+ ```yaml
62
+ - first_task:
63
+ type: ecs_task
64
+ parameters:
65
+ cluster: yourcluster
66
+ task_definition: test-task-def:1
67
+ count: 1
68
+ - first_task:
69
+ type: ecs_task
70
+ parameters:
71
+ cluster: yourcluster2
72
+ task_definition: test-task-def:2
73
+ count: 1
74
+ ```
75
+
76
+ The top level structure is an array of tasks to be executed in the order shown.
77
+ Each task can be provided a name. Below each task you must specify the task
78
+ `type`. At this time, only ecs_task is supported as a task type.
79
+
80
+ The `parameters` element correlates to the request provided to the ECS run_task
81
+ API call. All of the elements are supported as long as you provide a YAML
82
+ structure. Refer to the API [reference][ref] for more details.
83
+
84
+ ## Releasing
85
+
86
+ Releasing a new version of the Gem requires a few steps:
87
+ - Update the `CHANGELOG.md` file
88
+ - Add a new section for the [UNRELEASED] code
89
+ - Check that all changes are reflected for the current release version
90
+ - Update the `version.rb` file
91
+ - Do a bundle install to ensure we are locking the latest version
92
+ - Commit all the changes
93
+ - Example: `git commit -m "Bump gem to version 1.0.0"`
94
+ - Create a Git tag that matches the version number in `version.rb`
95
+ - Example: `git tag -m "Version 1.0.0" v1.0.0`
96
+
97
+ [ref]: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html
data/Rakefile ADDED
@@ -0,0 +1,7 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task :test do
4
+ Dir["./test/*_test.rb"].each { |file| require file}
5
+ end
6
+
7
+ task default: %w(test)
data/bin/console ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pry"
5
+ require "full360-sequencer"
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ Pry.start
data/bin/sequencer CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'full360-sequencer'
3
+ require "full360-sequencer"
4
4
 
5
5
  config_file = ARGV[0]
6
6
 
@@ -14,27 +14,29 @@ def logger
14
14
  end
15
15
 
16
16
  def sequencer_version
17
- Gem.loaded_specs['full360-sequencer'].version
17
+ Gem.loaded_specs["full360-sequencer"].version
18
18
  end
19
19
 
20
20
  begin
21
21
  logger.info("sequencer version #{sequencer_version}")
22
- logger.level = ENV['SEQUENCER_LOG_DEBUG'] ? Logger::DEBUG : Logger::INFO
22
+ logger.level = ENV["SEQUENCER_LOG_DEBUG"] ? Logger::DEBUG : Logger::INFO
23
23
 
24
24
  if config_file == nil
25
- logger.error('YAML file not provided... exiting with error code 1')
25
+ logger.error("SEQUENCER_ERROR")
26
+ logger.error("YAML file not provided... exiting with error code 1")
26
27
  exit 1
27
28
  else
28
29
  r = Full360::Sequencer::Runner.new(logger)
29
30
  r.config_from_file(config_file)
30
- r.sleep_between_checks = ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS'].to_i if ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS']
31
+ r.sleep_between_checks = ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"].to_i if ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"]
31
32
  r.run
32
33
  end
33
- logger.info('all steps succeeded... exiting with code 0')
34
+ logger.info("all steps succeeded... exiting with code 0")
34
35
  exit 0
35
36
  rescue => e
37
+ logger.error("SEQUENCER_ERROR")
36
38
  logger.error(e.message)
37
39
  e.backtrace.each { |r| logger.error(r) }
38
- logger.error('failure... exiting with code 1')
40
+ logger.error("failure... exiting with code 1")
39
41
  exit 1
40
- end
42
+ end
@@ -0,0 +1,34 @@
1
+ lib = File.expand_path("../lib", __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require "full360_sequencer/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "full360-sequencer"
7
+ spec.version = Full360::Sequencer::VERSION
8
+ spec.date = "2017-10-17"
9
+ spec.summary = "Full 360 sequencer utility"
10
+ spec.description = "Automation for simple batch jobs run in AWS"
11
+ spec.authors = ["Full 360 Group"]
12
+ spec.email = "support@full360.com"
13
+ spec.files = ["lib/full360-sequencer.rb"]
14
+ spec.homepage = "https://full360.com"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ end
20
+
21
+ spec.bindir = "bin"
22
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
+ spec.require_paths = ["lib"]
24
+
25
+ # Make it Ruby 2+ only
26
+ spec.required_ruby_version = ">= 2.0"
27
+
28
+ spec.add_runtime_dependency "aws-sdk", "~> 2.9"
29
+
30
+ # development dependencies
31
+ spec.add_development_dependency "minitest", "~> 5.9"
32
+ spec.add_development_dependency "rake", "~> 12"
33
+ spec.add_development_dependency "pry", "~> 0.14"
34
+ end
@@ -1,155 +1 @@
1
- #!/usr/bin/ruby
2
- require 'aws-sdk'
3
- require 'yaml'
4
- require 'logger'
5
- require 'pp'
6
-
7
- module Full360
8
- module Sequencer
9
- class Runner
10
- attr_accessor :sleep_between_checks
11
- attr_accessor :config
12
-
13
- def initialize(logger = nil)
14
- @logger = logger ? logger : Logger.new(STDOUT)
15
-
16
- # default 5 seconds between completed? checks
17
- @sleep_between_checks = 5
18
- end
19
-
20
- def config_from_file(yaml_path)
21
- @config = parse_config_file(yaml_path)
22
- end
23
-
24
- def run_task_class(task_type_string)
25
- case task_type_string
26
- when 'ecs_task' then Full360::Sequencer::RunECSTask
27
- else nil
28
- end
29
- end
30
-
31
- def run
32
- @config.each do |params|
33
- this_task_name = task_name(params)
34
- this_task = run_task_class(params[this_task_name]['type']).new(
35
- this_task_name,
36
- params[this_task_name]
37
- )
38
- this_task.run_task
39
- until this_task.completed?
40
- sleep @sleep_between_checks
41
- end
42
- raise "task failed error" unless this_task.success
43
- end
44
- rescue => e
45
- @logger.error(e.message)
46
- e.backtrace.each { |r| @logger.error(r) }
47
- end
48
-
49
- def task_name(params)
50
- params.keys.first
51
- end
52
-
53
- def parse_config_file(yaml_path)
54
- YAML.load_file(yaml_path)
55
- end
56
-
57
- def config_valid?(config)
58
- return false unless config.is_a? Array
59
- true
60
- end
61
- end
62
-
63
- class RunTaskBase
64
- attr_reader :success
65
- attr_reader :exit_code
66
-
67
- def run_task; end
68
- def completed?; end
69
- def kill_task; end #will be used for timeout
70
- end
71
-
72
- class RunECSTask < RunTaskBase
73
- def initialize(task_name, params, logger = nil)
74
- @logger = logger ? logger : Logger.new(STDOUT)
75
- @task_name = task_name
76
- @params = params['parameters']
77
- @params = keys_to_symbol(@params)
78
- @cluster = @params[:cluster]
79
- end
80
-
81
- def keys_to_symbol(params)
82
- # replaces string keys with symbol keys
83
- # required by AWS SDK
84
- if params.is_a?(Hash)
85
- params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
86
- else
87
- nil
88
- end
89
- end
90
-
91
- def run_task
92
- @logger.info("starting ECS task #{@task_name}")
93
- resp = ecs_run_task
94
- @task_arn = resp.tasks[0].task_arn
95
- @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
96
- end
97
-
98
- def ecs_run_task
99
- @logger.debug("creating AWS client for ECS task #{@task_name}...")
100
- @ecs_client = ::Aws::ECS::Client.new
101
- @logger.debug("running ECS task #{@task_name}...")
102
- @start_time = Time.new
103
- resp = @ecs_client.run_task(@params)
104
- return resp
105
- rescue => e
106
- @logger.error("error creating ECS task...")
107
- @logger.error("response from ECS: #{resp}")
108
- raise e
109
- end
110
-
111
- def ecs_describe_tasks
112
- @ecs_client.describe_tasks(
113
- {
114
- cluster: @cluster,
115
- tasks: [@task_arn] # required
116
- }
117
- )
118
- end
119
-
120
- def completed?
121
- resp = ecs_describe_tasks
122
- status = last_task_status(resp)
123
- @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
124
- if status == 'STOPPED'
125
- @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
126
- # parse exit_code(s) and return completion
127
- @success = determine_success(resp)
128
- return true
129
- end
130
- false
131
- rescue => e
132
- @logger.error(e.message)
133
- e.backtrace.each { |r| @logger.error(r) }
134
- end
135
-
136
- # parses last status from aws API response
137
- def last_task_status(resp)
138
- resp.tasks[0].last_status
139
- end
140
-
141
- # success is determined by all containers having zero exit code
142
- def determine_success(resp)
143
- success = true
144
- resp.tasks[0].containers.each do |c|
145
- @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
146
- if c.exit_code != 0
147
- # we had a problem!
148
- success = false
149
- end
150
- end
151
- success
152
- end
153
- end
154
- end
155
- end
1
+ require "full360_sequencer"
@@ -0,0 +1,97 @@
1
+ require "aws-sdk"
2
+ require "logger"
3
+
4
+ module Full360
5
+ module Sequencer
6
+ class RunECSTask < RunTaskBase
7
+ def initialize(task_name, params, logger = nil)
8
+ @logger = logger ? logger : Logger.new(STDOUT)
9
+ @task_name = task_name
10
+ @params = params["parameters"]
11
+ @params = keys_to_symbol(@params)
12
+ @cluster = @params[:cluster]
13
+ end
14
+
15
+ def keys_to_symbol(params)
16
+ # replaces string keys with symbol keys
17
+ # required by AWS SDK
18
+ if params.is_a?(Hash)
19
+ params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
20
+ else
21
+ nil
22
+ end
23
+ end
24
+
25
+ def run_task
26
+ @logger.info("starting ECS task #{@task_name}")
27
+ resp = ecs_run_task
28
+ @task_arn = resp.tasks[0].task_arn
29
+ @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
30
+ end
31
+
32
+ def ecs_run_task
33
+ @logger.debug("creating AWS client for ECS task #{@task_name}...")
34
+ @ecs_client = ::Aws::ECS::Client.new
35
+ @logger.debug("running ECS task #{@task_name}...")
36
+ @start_time = Time.new
37
+ resp = @ecs_client.run_task(@params)
38
+ return resp
39
+ rescue => e
40
+ @logger.error("SEQUENCER_ERROR")
41
+ @logger.error("error creating ECS task...")
42
+ @logger.error("response from ECS: #{resp}")
43
+ raise e
44
+ end
45
+
46
+ def ecs_describe_tasks
47
+ @ecs_client.describe_tasks(
48
+ {
49
+ cluster: @cluster,
50
+ tasks: [@task_arn] # required
51
+ }
52
+ )
53
+ end
54
+
55
+ def completed?
56
+ retries ||= 0
57
+ resp = ecs_describe_tasks
58
+ status = last_task_status(resp)
59
+ @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
60
+ if status == "STOPPED"
61
+ @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
62
+ # parse exit_code(s) and return completion
63
+ @success = determine_success(resp)
64
+ return true
65
+ end
66
+ false
67
+ rescue => e
68
+ @logger.warn(e.message)
69
+ @logger.warn("task completion check failed, trying again ##{ retries }")
70
+ sleep 10*retries
71
+ retry if (retries += 1) < 3
72
+
73
+ @logger.error("SEQUENCER_ERROR")
74
+ @logger.error(e.message)
75
+ e.backtrace.each { |r| @logger.error(r) }
76
+ end
77
+
78
+ # parses last status from aws API response
79
+ def last_task_status(resp)
80
+ resp.tasks[0].last_status
81
+ end
82
+
83
+ # success is determined by all containers having zero exit code
84
+ def determine_success(resp)
85
+ success = true
86
+ resp.tasks[0].containers.each do |c|
87
+ @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
88
+ if c.exit_code != 0
89
+ # we had a problem!
90
+ success = false
91
+ end
92
+ end
93
+ success
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,12 @@
1
+ module Full360
2
+ module Sequencer
3
+ class RunTaskBase
4
+ attr_reader :success
5
+ attr_reader :exit_code
6
+
7
+ def run_task; end
8
+ def completed?; end
9
+ def kill_task; end # will be used for timeout
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,62 @@
1
+ require "yaml"
2
+ require "logger"
3
+
4
+ module Full360
5
+ module Sequencer
6
+ class Runner
7
+ attr_accessor :sleep_between_checks
8
+ attr_accessor :config
9
+
10
+ def initialize(logger = nil)
11
+ @logger = logger ? logger : Logger.new(STDOUT)
12
+
13
+ # default 5 seconds between completed? checks
14
+ @sleep_between_checks = 5
15
+ end
16
+
17
+ def config_from_file(yaml_path)
18
+ @config = parse_config_file(yaml_path)
19
+ end
20
+
21
+ def run_task_class(task_type_string)
22
+ case task_type_string
23
+ when "ecs_task" then Full360::Sequencer::RunECSTask
24
+ else nil
25
+ end
26
+ end
27
+
28
+ def run
29
+ @config.each do |params|
30
+ this_task_name = task_name(params)
31
+ this_task = run_task_class(params[this_task_name]["type"]).new(
32
+ this_task_name,
33
+ params[this_task_name]
34
+ )
35
+ this_task.run_task
36
+ until this_task.completed?
37
+ sleep @sleep_between_checks
38
+ end
39
+ raise "task failed error" unless this_task.success
40
+ end
41
+ rescue => e
42
+ @logger.error("SEQUENCER_ERROR")
43
+ @logger.error(e.message)
44
+ e.backtrace.each { |r| @logger.error(r) }
45
+ raise e
46
+ end
47
+
48
+ def task_name(params)
49
+ params.keys.first
50
+ end
51
+
52
+ def parse_config_file(yaml_path)
53
+ YAML.load_file(yaml_path)
54
+ end
55
+
56
+ def config_valid?(config)
57
+ return false unless config.is_a? Array
58
+ true
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,5 @@
1
+ module Full360
2
+ module Sequencer
3
+ VERSION = "0.1.3".freeze
4
+ end
5
+ end
@@ -0,0 +1,9 @@
1
+ # Autoload all gem classes
2
+ module Full360
3
+ module Sequencer
4
+ autoload :VERSION, "full360_sequencer/version"
5
+ autoload :Runner, "full360_sequencer/runner"
6
+ autoload :RunECSTask, "full360_sequencer/run_ecs_task"
7
+ autoload :RunTaskBase, "full360_sequencer/run_task_base"
8
+ end
9
+ end
metadata CHANGED
@@ -1,72 +1,102 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: full360-sequencer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
- - pankaj batra
8
- - jeremy winters
9
- autorequire:
7
+ - Full 360 Group
8
+ autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2017-07-25 00:00:00.000000000 Z
11
+ date: 2017-10-17 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
14
+ name: aws-sdk
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.2'
20
- name: logger
21
- prerelease: false
19
+ version: '2.9'
22
20
  type: :runtime
21
+ prerelease: false
23
22
  version_requirements: !ruby/object:Gem::Requirement
24
23
  requirements:
25
24
  - - "~>"
26
25
  - !ruby/object:Gem::Version
27
- version: '1.2'
26
+ version: '2.9'
28
27
  - !ruby/object:Gem::Dependency
28
+ name: minitest
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '2.9'
34
- name: aws-sdk
33
+ version: '5.9'
34
+ type: :development
35
35
  prerelease: false
36
- type: :runtime
37
36
  version_requirements: !ruby/object:Gem::Requirement
38
37
  requirements:
39
38
  - - "~>"
40
39
  - !ruby/object:Gem::Version
41
- version: '2.9'
40
+ version: '5.9'
42
41
  - !ruby/object:Gem::Dependency
42
+ name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '5.9'
48
- name: minitest
47
+ version: '12'
48
+ type: :development
49
49
  prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '12'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.14'
50
62
  type: :development
63
+ prerelease: false
51
64
  version_requirements: !ruby/object:Gem::Requirement
52
65
  requirements:
53
66
  - - "~>"
54
67
  - !ruby/object:Gem::Version
55
- version: '5.9'
56
- description: automation for simple batch jobs run in AWS
57
- email: pankaj.batra@full360.com
68
+ version: '0.14'
69
+ description: Automation for simple batch jobs run in AWS
70
+ email: support@full360.com
58
71
  executables:
72
+ - console
59
73
  - sequencer
60
74
  extensions: []
61
75
  extra_rdoc_files: []
62
76
  files:
77
+ - ".github/workflows/release.yml"
78
+ - ".github/workflows/test.yml"
79
+ - ".gitignore"
80
+ - CHANGELOG.md
81
+ - Gemfile
82
+ - Gemfile.lock
83
+ - LICENSE
84
+ - README.md
85
+ - Rakefile
86
+ - bin/console
63
87
  - bin/sequencer
88
+ - full360-sequencer.gemspec
64
89
  - lib/full360-sequencer.rb
65
- homepage: https://www.full360.com
90
+ - lib/full360_sequencer.rb
91
+ - lib/full360_sequencer/run_ecs_task.rb
92
+ - lib/full360_sequencer/run_task_base.rb
93
+ - lib/full360_sequencer/runner.rb
94
+ - lib/full360_sequencer/version.rb
95
+ homepage: https://full360.com
66
96
  licenses:
67
97
  - MIT
68
98
  metadata: {}
69
- post_install_message:
99
+ post_install_message:
70
100
  rdoc_options: []
71
101
  require_paths:
72
102
  - lib
@@ -81,9 +111,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
81
111
  - !ruby/object:Gem::Version
82
112
  version: '0'
83
113
  requirements: []
84
- rubyforge_project:
85
- rubygems_version: 2.6.11
86
- signing_key:
114
+ rubygems_version: 3.1.6
115
+ signing_key:
87
116
  specification_version: 4
88
- summary: full360 sequencer utility
117
+ summary: Full 360 sequencer utility
89
118
  test_files: []