full360-sequencer 0.0.7 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 11689f8562fd17688883eca48d0e173a573d3823
4
- data.tar.gz: 2328c3194fc48a4333bfeb4b7300dc839a9539a5
2
+ SHA256:
3
+ metadata.gz: 54bca4472a18632b312e84aff4c3ca5958053645dca8860f579c27e257689337
4
+ data.tar.gz: 024a6bad17c794fe021cfd31df13a03ada093aa4c0c12c3dfd7432e38b43a518
5
5
  SHA512:
6
- metadata.gz: 163c12b36137fd5a09c96851fe43315f50f191afd793e14b32817fd5dc6fd260b1afdf05e2a694ca4c942883fcf6bf7bf902332552e168cf97c5c86986661cfd
7
- data.tar.gz: e5ef1c88f6cad5d4c4c37f435b7e3be01b1460bdeb7c8cf1b84ad3b7f98eba5ec25968aff1af19162235f92faf972b82fd456be82a24ca7d60e21a78a7228f18
6
+ metadata.gz: 04e30b01ccbb05f4622d1596c37cbff99ad15cabd7ab8e6bd8bc1906a46482a35e7f1cc5fc51588e66b281a71ea3e31b4ab945b01abe5650cbf30eb777117c16
7
+ data.tar.gz: 510b7447b7182e6a746d6584a48ea4949aeeeb320a222bc0605e4ebcae93729f56928dd301d2ae3fa01a54843364a9f5672c1f2526197374b4425f8a0fd57015
@@ -0,0 +1,35 @@
1
+ name: Release Gem
2
+
3
+ on:
4
+ workflow_run:
5
+ workflows: ["CI Test"]
6
+ types: [completed]
7
+ branches:
8
+ - master
9
+ tags:
10
+ - v*
11
+
12
+ jobs:
13
+ release:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ # Checkout code if release was created
17
+ - uses: actions/checkout@v2
18
+
19
+ # Setup ruby if a release was created
20
+ - uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: 2.7
23
+ bundler-cache: true
24
+
25
+ # Publish
26
+ - name: publish gem
27
+ run: |
28
+ mkdir -p $HOME/.gem
29
+ touch $HOME/.gem/credentials
30
+ chmod 0600 $HOME/.gem/credentials
31
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
32
+ gem build *.gemspec
33
+ gem push *.gem
34
+ env:
35
+ RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
@@ -0,0 +1,26 @@
1
+ name: CI Test
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ strategy:
8
+ fail-fast: false
9
+ matrix:
10
+ os: [ubuntu-latest, macos-latest]
11
+ # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
12
+ ruby: ["2.7", "3.0", "jruby-9.2.19.0", "jruby-9.3.0.0"]
13
+ runs-on: ${{ matrix.os }}
14
+
15
+ if: "!contains(github.event.head_commit.message, '[ci skip]')"
16
+
17
+ steps:
18
+ - name: Checkout
19
+ uses: actions/checkout@v2
20
+
21
+ - name: Test ${{ matrix.ruby }}
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby }}
25
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
26
+ - run: bundle exec rake
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ .DS_Store
2
+ .ruby-version
3
+
4
+ /.bundle/
5
+ /.yardoc
6
+ /_yardoc/
7
+ /coverage/
8
+ /doc/
9
+ /pkg/
10
+ /spec/reports/
11
+ /tmp/
12
+
13
+ # rspec failure tracking
14
+ .rspec_status
15
+
16
+ # generated gems
17
+ *.gem
data/CHANGELOG.md ADDED
@@ -0,0 +1,34 @@
1
+ # Changelog
2
+ All notable changes to this project will be documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
5
+ and this project adheres to [Semantic
6
+ Versioning](http://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+ ### Added
10
+ ### Changed
11
+ ### Removed
12
+
13
+ ## 0.1.1
14
+ ### Added
15
+ - Workflows dependable.
16
+ ### Changed
17
+ ### Removed
18
+
19
+ ## 0.1.0
20
+ ### Added
21
+ - Working GitHub Actions workflow for tests and releases.
22
+ - Add Gemfile to the Gem.
23
+ ### Changed
24
+ - Update the .gemspec file and dependencies.
25
+ - Update the gem file format.
26
+ - Change from single quote to double quotes.
27
+ - Fix deprecation warnings in tests
28
+ ### Removed
29
+
30
+ ## 0.0.7
31
+ ### Added
32
+ - Working version.
33
+ ### Changed
34
+ ### Removed
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,50 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ full360-sequencer (0.1.3)
5
+ aws-sdk (~> 2.9)
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ aws-eventstream (1.2.0)
11
+ aws-sdk (2.11.632)
12
+ aws-sdk-resources (= 2.11.632)
13
+ aws-sdk-core (2.11.632)
14
+ aws-sigv4 (~> 1.0)
15
+ jmespath (~> 1.0)
16
+ aws-sdk-resources (2.11.632)
17
+ aws-sdk-core (= 2.11.632)
18
+ aws-sigv4 (1.4.0)
19
+ aws-eventstream (~> 1, >= 1.0.2)
20
+ coderay (1.1.3)
21
+ ffi (1.15.4-java)
22
+ jmespath (1.4.0)
23
+ method_source (1.0.0)
24
+ minitest (5.14.4)
25
+ pry (0.14.1)
26
+ coderay (~> 1.1)
27
+ method_source (~> 1.0)
28
+ pry (0.14.1-java)
29
+ coderay (~> 1.1)
30
+ method_source (~> 1.0)
31
+ spoon (~> 0.0)
32
+ rake (12.3.3)
33
+ spoon (0.0.6)
34
+ ffi
35
+
36
+ PLATFORMS
37
+ universal-java-11
38
+ universal-java-8
39
+ x86_64-darwin-18
40
+ x86_64-darwin-19
41
+ x86_64-linux
42
+
43
+ DEPENDENCIES
44
+ full360-sequencer!
45
+ minitest (~> 5.9)
46
+ pry (~> 0.14)
47
+ rake (~> 12)
48
+
49
+ BUNDLED WITH
50
+ 2.2.27
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2017 Full 360 Inc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,97 @@
1
+ # Full 360 Sequencer
2
+
3
+ Sequencer is a tool that supports the synchronous execution of docker containers
4
+ running in Amazon ECS for the purpose of batch processing.
5
+
6
+ While there are many solutions for container orchestration, most of them are
7
+ focused on managing and scaling long running microservices, where a group of
8
+ containers need to kept alive and able to talk to each other. This model is
9
+ quite different from the execution of a nightly ETL batch where a container
10
+ performs a series of actions such as:
11
+
12
+ * collect the data from source system
13
+ * transform the data into the data lake
14
+ * ingest the data into the database
15
+ * transform the data inside the database with SQL
16
+ * export data to downstream consumers such as third parties or BI tools
17
+
18
+ AWS Batch service could do the above but it is currently only available in a
19
+ single region.
20
+
21
+ At one end of the spectrum your other options fall into the realm of shell
22
+ scripts.. while the other end brings you powerful (and cool!) DAG based tools
23
+ such as Nomad and Airflow.
24
+
25
+ We consider the DAG based tools to be way overkill for a simple batch of
26
+ synchronous jobs, so we opted to expand on the idea of a shell script.
27
+
28
+ Putting a shell script into a container is easy enough, but it requires that you
29
+ build and deploy a new container every time you make a change.
30
+
31
+ Sequencer allows you to define a list of ECS tasks in a yaml file, the location
32
+ of which is passed into the container at run time. This yaml file can sit in an
33
+ S3 repository (git support forthcoming). Sequencer will synchronously run each
34
+ task, passing in your override parameters as specified.
35
+
36
+ ## Installing Sequencer
37
+
38
+ Sequencer is installed as a ruby gem:
39
+
40
+ gem install full360-sequencer
41
+
42
+ Once installed... you can run the sequencer command from anywhere on your
43
+ system. Simply pass the path to your YAML file as a parameter. Note that you
44
+ will need to export some environment variables in order to use the AWS API.
45
+
46
+ $ export AWS_ACCESS_KEY_ID=aaaaaaaaayourkey
47
+ $ export AWS_SECRET_ACCESS_KEY=bbbbbbbbbbbbbbbbbbbbbbbbbbbbyoursecret
48
+ $ export AWS_REGION=us-west-2
49
+ $ sequencer mybatch.yml
50
+
51
+ Sequencer will utilize AWS instance or container roles if appropriate, though
52
+ you will have to provide `AWS_REGION` if you are not in us-east-1.
53
+
54
+ While using the command line sequencer is just fine, we suggest that you use the
55
+ full360/sequencer container available on Dockerhub.
56
+
57
+ ## YAML File Specification
58
+
59
+ Sequencer requires a yaml file with the following structure:
60
+
61
+ ```yaml
62
+ - first_task:
63
+ type: ecs_task
64
+ parameters:
65
+ cluster: yourcluster
66
+ task_definition: test-task-def:1
67
+ count: 1
68
+ - first_task:
69
+ type: ecs_task
70
+ parameters:
71
+ cluster: yourcluster2
72
+ task_definition: test-task-def:2
73
+ count: 1
74
+ ```
75
+
76
+ The top level structure is an array of tasks to be executed in the order shown.
77
+ Each task can be provided a name. Below each task you must specify the task
78
+ `type`. At this time, only ecs_task is supported as a task type.
79
+
80
+ The `parameters` element correlates to the request provided to the ECS run_task
81
+ API call. All of the elements are supported as long as you provide a YAML
82
+ structure. Refer to the API [reference][ref] for more details.
83
+
84
+ ## Releasing
85
+
86
+ Releasing a new version of the Gem requires a few steps:
87
+ - Update the `CHANGELOG.md` file
88
+ - Add a new section for the [UNRELEASED] code
89
+ - Check that all changes are reflected for the current release version
90
+ - Update the `version.rb` file
91
+ - Do a bundle install to ensure we are locking the latest version
92
+ - Commit all the changes
93
+ - Example: `git commit -m "Bump gem to version 1.0.0"`
94
+ - Create a Git tag that matches the version number in `version.rb`
95
+ - Example: `git tag -m "Version 1.0.0" v1.0.0`
96
+
97
+ [ref]: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html
data/Rakefile ADDED
@@ -0,0 +1,7 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task :test do
4
+ Dir["./test/*_test.rb"].each { |file| require file}
5
+ end
6
+
7
+ task default: %w(test)
data/bin/console ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pry"
5
+ require "full360-sequencer"
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ Pry.start
data/bin/sequencer CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'full360-sequencer'
3
+ require "full360-sequencer"
4
4
 
5
5
  config_file = ARGV[0]
6
6
 
@@ -14,29 +14,29 @@ def logger
14
14
  end
15
15
 
16
16
  def sequencer_version
17
- Gem.loaded_specs['full360-sequencer'].version
17
+ Gem.loaded_specs["full360-sequencer"].version
18
18
  end
19
19
 
20
20
  begin
21
21
  logger.info("sequencer version #{sequencer_version}")
22
- logger.level = ENV['SEQUENCER_LOG_DEBUG'] ? Logger::DEBUG : Logger::INFO
22
+ logger.level = ENV["SEQUENCER_LOG_DEBUG"] ? Logger::DEBUG : Logger::INFO
23
23
 
24
24
  if config_file == nil
25
- logger.error('SEQUENCER_ERROR')
26
- logger.error('YAML file not provided... exiting with error code 1')
25
+ logger.error("SEQUENCER_ERROR")
26
+ logger.error("YAML file not provided... exiting with error code 1")
27
27
  exit 1
28
28
  else
29
29
  r = Full360::Sequencer::Runner.new(logger)
30
30
  r.config_from_file(config_file)
31
- r.sleep_between_checks = ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS'].to_i if ENV['SEQUENCER_SLEEP_BETWEEN_CHECKS']
31
+ r.sleep_between_checks = ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"].to_i if ENV["SEQUENCER_SLEEP_BETWEEN_CHECKS"]
32
32
  r.run
33
33
  end
34
- logger.info('all steps succeeded... exiting with code 0')
34
+ logger.info("all steps succeeded... exiting with code 0")
35
35
  exit 0
36
36
  rescue => e
37
- logger.error('SEQUENCER_ERROR')
37
+ logger.error("SEQUENCER_ERROR")
38
38
  logger.error(e.message)
39
39
  e.backtrace.each { |r| logger.error(r) }
40
- logger.error('failure... exiting with code 1')
40
+ logger.error("failure... exiting with code 1")
41
41
  exit 1
42
- end
42
+ end
@@ -0,0 +1,34 @@
1
+ lib = File.expand_path("../lib", __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require "full360_sequencer/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "full360-sequencer"
7
+ spec.version = Full360::Sequencer::VERSION
8
+ spec.date = "2017-10-17"
9
+ spec.summary = "Full 360 sequencer utility"
10
+ spec.description = "Automation for simple batch jobs run in AWS"
11
+ spec.authors = ["Full 360 Group"]
12
+ spec.email = "support@full360.com"
13
+ spec.files = ["lib/full360-sequencer.rb"]
14
+ spec.homepage = "https://full360.com"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ end
20
+
21
+ spec.bindir = "bin"
22
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
+ spec.require_paths = ["lib"]
24
+
25
+ # Make it Ruby 2+ only
26
+ spec.required_ruby_version = ">= 2.0"
27
+
28
+ spec.add_runtime_dependency "aws-sdk", "~> 2.9"
29
+
30
+ # development dependencies
31
+ spec.add_development_dependency "minitest", "~> 5.9"
32
+ spec.add_development_dependency "rake", "~> 12"
33
+ spec.add_development_dependency "pry", "~> 0.14"
34
+ end
@@ -1,165 +1 @@
1
- #!/usr/bin/ruby
2
- require 'aws-sdk'
3
- require 'yaml'
4
- require 'logger'
5
- require 'pp'
6
-
7
- module Full360
8
- module Sequencer
9
- class Runner
10
- attr_accessor :sleep_between_checks
11
- attr_accessor :config
12
-
13
- def initialize(logger = nil)
14
- @logger = logger ? logger : Logger.new(STDOUT)
15
-
16
- # default 5 seconds between completed? checks
17
- @sleep_between_checks = 5
18
- end
19
-
20
- def config_from_file(yaml_path)
21
- @config = parse_config_file(yaml_path)
22
- end
23
-
24
- def run_task_class(task_type_string)
25
- case task_type_string
26
- when 'ecs_task' then Full360::Sequencer::RunECSTask
27
- else nil
28
- end
29
- end
30
-
31
- def run
32
- @config.each do |params|
33
- this_task_name = task_name(params)
34
- this_task = run_task_class(params[this_task_name]['type']).new(
35
- this_task_name,
36
- params[this_task_name]
37
- )
38
- this_task.run_task
39
- until this_task.completed?
40
- sleep @sleep_between_checks
41
- end
42
- raise "task failed error" unless this_task.success
43
- end
44
- rescue => e
45
- @logger.error('SEQUENCER_ERROR')
46
- @logger.error(e.message)
47
- e.backtrace.each { |r| @logger.error(r) }
48
- raise e
49
- end
50
-
51
- def task_name(params)
52
- params.keys.first
53
- end
54
-
55
- def parse_config_file(yaml_path)
56
- YAML.load_file(yaml_path)
57
- end
58
-
59
- def config_valid?(config)
60
- return false unless config.is_a? Array
61
- true
62
- end
63
- end
64
-
65
- class RunTaskBase
66
- attr_reader :success
67
- attr_reader :exit_code
68
-
69
- def run_task; end
70
- def completed?; end
71
- def kill_task; end #will be used for timeout
72
- end
73
-
74
- class RunECSTask < RunTaskBase
75
- def initialize(task_name, params, logger = nil)
76
- @logger = logger ? logger : Logger.new(STDOUT)
77
- @task_name = task_name
78
- @params = params['parameters']
79
- @params = keys_to_symbol(@params)
80
- @cluster = @params[:cluster]
81
- end
82
-
83
- def keys_to_symbol(params)
84
- # replaces string keys with symbol keys
85
- # required by AWS SDK
86
- if params.is_a?(Hash)
87
- params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
88
- else
89
- nil
90
- end
91
- end
92
-
93
- def run_task
94
- @logger.info("starting ECS task #{@task_name}")
95
- resp = ecs_run_task
96
- @task_arn = resp.tasks[0].task_arn
97
- @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
98
- end
99
-
100
- def ecs_run_task
101
- @logger.debug("creating AWS client for ECS task #{@task_name}...")
102
- @ecs_client = ::Aws::ECS::Client.new
103
- @logger.debug("running ECS task #{@task_name}...")
104
- @start_time = Time.new
105
- resp = @ecs_client.run_task(@params)
106
- return resp
107
- rescue => e
108
- @logger.error('SEQUENCER_ERROR')
109
- @logger.error("error creating ECS task...")
110
- @logger.error("response from ECS: #{resp}")
111
- raise e
112
- end
113
-
114
- def ecs_describe_tasks
115
- @ecs_client.describe_tasks(
116
- {
117
- cluster: @cluster,
118
- tasks: [@task_arn] # required
119
- }
120
- )
121
- end
122
-
123
- def completed?
124
- retries ||= 0
125
- resp = ecs_describe_tasks
126
- status = last_task_status(resp)
127
- @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
128
- if status == 'STOPPED'
129
- @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
130
- # parse exit_code(s) and return completion
131
- @success = determine_success(resp)
132
- return true
133
- end
134
- false
135
- rescue => e
136
- @logger.warn(e.message)
137
- @logger.warn("task completion check failed, trying again ##{ retries }")
138
- sleep 10*retries
139
- retry if (retries += 1) < 3
140
-
141
- @logger.error('SEQUENCER_ERROR')
142
- @logger.error(e.message)
143
- e.backtrace.each { |r| @logger.error(r) }
144
- end
145
-
146
- # parses last status from aws API response
147
- def last_task_status(resp)
148
- resp.tasks[0].last_status
149
- end
150
-
151
- # success is determined by all containers having zero exit code
152
- def determine_success(resp)
153
- success = true
154
- resp.tasks[0].containers.each do |c|
155
- @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
156
- if c.exit_code != 0
157
- # we had a problem!
158
- success = false
159
- end
160
- end
161
- success
162
- end
163
- end
164
- end
165
- end
1
+ require "full360_sequencer"
@@ -0,0 +1,97 @@
1
+ require "aws-sdk"
2
+ require "logger"
3
+
4
+ module Full360
5
+ module Sequencer
6
+ class RunECSTask < RunTaskBase
7
+ def initialize(task_name, params, logger = nil)
8
+ @logger = logger ? logger : Logger.new(STDOUT)
9
+ @task_name = task_name
10
+ @params = params["parameters"]
11
+ @params = keys_to_symbol(@params)
12
+ @cluster = @params[:cluster]
13
+ end
14
+
15
+ def keys_to_symbol(params)
16
+ # replaces string keys with symbol keys
17
+ # required by AWS SDK
18
+ if params.is_a?(Hash)
19
+ params.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
20
+ else
21
+ nil
22
+ end
23
+ end
24
+
25
+ def run_task
26
+ @logger.info("starting ECS task #{@task_name}")
27
+ resp = ecs_run_task
28
+ @task_arn = resp.tasks[0].task_arn
29
+ @logger.info("#{@task_name} task created #{@task_arn} on cluster #{@cluster}")
30
+ end
31
+
32
+ def ecs_run_task
33
+ @logger.debug("creating AWS client for ECS task #{@task_name}...")
34
+ @ecs_client = ::Aws::ECS::Client.new
35
+ @logger.debug("running ECS task #{@task_name}...")
36
+ @start_time = Time.new
37
+ resp = @ecs_client.run_task(@params)
38
+ return resp
39
+ rescue => e
40
+ @logger.error("SEQUENCER_ERROR")
41
+ @logger.error("error creating ECS task...")
42
+ @logger.error("response from ECS: #{resp}")
43
+ raise e
44
+ end
45
+
46
+ def ecs_describe_tasks
47
+ @ecs_client.describe_tasks(
48
+ {
49
+ cluster: @cluster,
50
+ tasks: [@task_arn] # required
51
+ }
52
+ )
53
+ end
54
+
55
+ def completed?
56
+ retries ||= 0
57
+ resp = ecs_describe_tasks
58
+ status = last_task_status(resp)
59
+ @logger.info("#{@task_name} : #{@task_arn} current status: #{status}")
60
+ if status == "STOPPED"
61
+ @logger.info("#{@task_name} completed in #{Time.new - @start_time} seconds")
62
+ # parse exit_code(s) and return completion
63
+ @success = determine_success(resp)
64
+ return true
65
+ end
66
+ false
67
+ rescue => e
68
+ @logger.warn(e.message)
69
+ @logger.warn("task completion check failed, trying again ##{ retries }")
70
+ sleep 10*retries
71
+ retry if (retries += 1) < 3
72
+
73
+ @logger.error("SEQUENCER_ERROR")
74
+ @logger.error(e.message)
75
+ e.backtrace.each { |r| @logger.error(r) }
76
+ end
77
+
78
+ # parses last status from aws API response
79
+ def last_task_status(resp)
80
+ resp.tasks[0].last_status
81
+ end
82
+
83
+ # success is determined by all containers having zero exit code
84
+ def determine_success(resp)
85
+ success = true
86
+ resp.tasks[0].containers.each do |c|
87
+ @logger.info("#{@task_name} : container #{c.name} #{c.container_arn} completed with exit_code #{c.exit_code}")
88
+ if c.exit_code != 0
89
+ # we had a problem!
90
+ success = false
91
+ end
92
+ end
93
+ success
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,12 @@
1
+ module Full360
2
+ module Sequencer
3
+ class RunTaskBase
4
+ attr_reader :success
5
+ attr_reader :exit_code
6
+
7
+ def run_task; end
8
+ def completed?; end
9
+ def kill_task; end # will be used for timeout
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,62 @@
1
+ require "yaml"
2
+ require "logger"
3
+
4
+ module Full360
5
+ module Sequencer
6
+ class Runner
7
+ attr_accessor :sleep_between_checks
8
+ attr_accessor :config
9
+
10
+ def initialize(logger = nil)
11
+ @logger = logger ? logger : Logger.new(STDOUT)
12
+
13
+ # default 5 seconds between completed? checks
14
+ @sleep_between_checks = 5
15
+ end
16
+
17
+ def config_from_file(yaml_path)
18
+ @config = parse_config_file(yaml_path)
19
+ end
20
+
21
+ def run_task_class(task_type_string)
22
+ case task_type_string
23
+ when "ecs_task" then Full360::Sequencer::RunECSTask
24
+ else nil
25
+ end
26
+ end
27
+
28
+ def run
29
+ @config.each do |params|
30
+ this_task_name = task_name(params)
31
+ this_task = run_task_class(params[this_task_name]["type"]).new(
32
+ this_task_name,
33
+ params[this_task_name]
34
+ )
35
+ this_task.run_task
36
+ until this_task.completed?
37
+ sleep @sleep_between_checks
38
+ end
39
+ raise "task failed error" unless this_task.success
40
+ end
41
+ rescue => e
42
+ @logger.error("SEQUENCER_ERROR")
43
+ @logger.error(e.message)
44
+ e.backtrace.each { |r| @logger.error(r) }
45
+ raise e
46
+ end
47
+
48
+ def task_name(params)
49
+ params.keys.first
50
+ end
51
+
52
+ def parse_config_file(yaml_path)
53
+ YAML.load_file(yaml_path)
54
+ end
55
+
56
+ def config_valid?(config)
57
+ return false unless config.is_a? Array
58
+ true
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,5 @@
1
+ module Full360
2
+ module Sequencer
3
+ VERSION = "0.1.3".freeze
4
+ end
5
+ end
@@ -0,0 +1,9 @@
1
+ # Autoload all gem classes
2
+ module Full360
3
+ module Sequencer
4
+ autoload :VERSION, "full360_sequencer/version"
5
+ autoload :Runner, "full360_sequencer/runner"
6
+ autoload :RunECSTask, "full360_sequencer/run_ecs_task"
7
+ autoload :RunTaskBase, "full360_sequencer/run_task_base"
8
+ end
9
+ end
metadata CHANGED
@@ -1,68 +1,98 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: full360-sequencer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
- - pankaj batra
8
- - jeremy winters
7
+ - Full 360 Group
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
11
  date: 2017-10-17 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
- name: logger
14
+ name: aws-sdk
16
15
  requirement: !ruby/object:Gem::Requirement
17
16
  requirements:
18
17
  - - "~>"
19
18
  - !ruby/object:Gem::Version
20
- version: '1.2'
19
+ version: '2.9'
21
20
  type: :runtime
22
21
  prerelease: false
23
22
  version_requirements: !ruby/object:Gem::Requirement
24
23
  requirements:
25
24
  - - "~>"
26
25
  - !ruby/object:Gem::Version
27
- version: '1.2'
26
+ version: '2.9'
28
27
  - !ruby/object:Gem::Dependency
29
- name: aws-sdk
28
+ name: minitest
30
29
  requirement: !ruby/object:Gem::Requirement
31
30
  requirements:
32
31
  - - "~>"
33
32
  - !ruby/object:Gem::Version
34
- version: '2.9'
35
- type: :runtime
33
+ version: '5.9'
34
+ type: :development
36
35
  prerelease: false
37
36
  version_requirements: !ruby/object:Gem::Requirement
38
37
  requirements:
39
38
  - - "~>"
40
39
  - !ruby/object:Gem::Version
41
- version: '2.9'
40
+ version: '5.9'
42
41
  - !ruby/object:Gem::Dependency
43
- name: minitest
42
+ name: rake
44
43
  requirement: !ruby/object:Gem::Requirement
45
44
  requirements:
46
45
  - - "~>"
47
46
  - !ruby/object:Gem::Version
48
- version: '5.9'
47
+ version: '12'
49
48
  type: :development
50
49
  prerelease: false
51
50
  version_requirements: !ruby/object:Gem::Requirement
52
51
  requirements:
53
52
  - - "~>"
54
53
  - !ruby/object:Gem::Version
55
- version: '5.9'
56
- description: automation for simple batch jobs run in AWS
57
- email: pankaj.batra@full360.com
54
+ version: '12'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.14'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.14'
69
+ description: Automation for simple batch jobs run in AWS
70
+ email: support@full360.com
58
71
  executables:
72
+ - console
59
73
  - sequencer
60
74
  extensions: []
61
75
  extra_rdoc_files: []
62
76
  files:
77
+ - ".github/workflows/release.yml"
78
+ - ".github/workflows/test.yml"
79
+ - ".gitignore"
80
+ - CHANGELOG.md
81
+ - Gemfile
82
+ - Gemfile.lock
83
+ - LICENSE
84
+ - README.md
85
+ - Rakefile
86
+ - bin/console
63
87
  - bin/sequencer
88
+ - full360-sequencer.gemspec
64
89
  - lib/full360-sequencer.rb
65
- homepage: https://www.full360.com
90
+ - lib/full360_sequencer.rb
91
+ - lib/full360_sequencer/run_ecs_task.rb
92
+ - lib/full360_sequencer/run_task_base.rb
93
+ - lib/full360_sequencer/runner.rb
94
+ - lib/full360_sequencer/version.rb
95
+ homepage: https://full360.com
66
96
  licenses:
67
97
  - MIT
68
98
  metadata: {}
@@ -81,9 +111,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
81
111
  - !ruby/object:Gem::Version
82
112
  version: '0'
83
113
  requirements: []
84
- rubyforge_project:
85
- rubygems_version: 2.6.11
114
+ rubygems_version: 3.1.6
86
115
  signing_key:
87
116
  specification_version: 4
88
- summary: full360 sequencer utility
117
+ summary: Full 360 sequencer utility
89
118
  test_files: []