elastic_whenever 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 448f6e4c937fdc568cf15f2bd0c488f89c2d65fc83396f5cec02f7b8cdb791a7
-  data.tar.gz: 0d48b483b1dfc7991398ce6b499426b21657466c62588235ed155ab3acf7fecd
+  metadata.gz: faaf28462435852a35046ac97c388f60eaaa3348f9b386251960d83d4351f3a5
+  data.tar.gz: c35bf7a5281b25667c3a96dda86920a6b304ba35a147d2421105233f5843fb69
 SHA512:
-  metadata.gz: 05a39822bd08957051d973ab2ca2fe1f39a976962bcb73e4381831b2c18660c9880f0034ba491749618601429494bb53ba1877bc07501d495f2f3304c3b1c139
-  data.tar.gz: 8a3ca786d459492a724a9cfe65e9616d009ad5ffd7982e8517a2e44ddbed3d8a6892f89aef879fc8a8f6369d18c3d44c69a8d606135eedeb7cc2e2f9b141d27c
+  metadata.gz: f5639992267344f3360d8115ae4ad4beef874bea395d666c0840f80bc04a736c098a781f94b8f29cf4c0f0174ec875caf92c2a3ce0d311657efd91427313b18b
+  data.tar.gz: 4165f7c3d71f02f991e4d509f3a6493c2ff36ee11d1733bda7def83825da07bf468796fe83e079603ca974cc9981d63c0e53055fac00339ccc327bee17cdba3a

data/.github/workflows/build.yml

@@ -16,15 +16,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby_version: ['2.5.x', '2.6.x', '2.7.x']
+        ruby_version: ['2.6', '2.7', '3.0']
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby ${{ matrix.ruby_version }}
-      uses: actions/setup-ruby@v1
+    - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby_version }}
-    - name: Run spec
-      run: |
-        gem install bundler
-        bundle install --jobs 4 --retry 3
-        bundle exec rake
+        bundler-cache: true
+    - run: bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## v0.7.0 (2021-09-25)
+
+This release contains a major change to the behavior when updating tasks. In most cases, this change has no effect, but be aware of the change in behavior when omitting a revision of the task definition. In particular, if you are building a deployment workflow where the update timing of task definitions and the update timing of scheduled tasks are different, the revisions that are executed may be different.
+
+### Breaking Changes
+
+- [#57](https://github.com/wata727/elastic_whenever/pull/57): Selective Updates ([@HistoireDeBabar](https://github.com/HistoireDeBabar))
+  - Previously, Elastic Whenever recreates all scheduled tasks after deleting all tasks when updating tasks. However, in this case, there is a risk that frequently invoked tasks will not be executed, so this change now creates/deletes only those tasks that have changed.
+  - Due to this change, the naming convention for scheduled tasks has changed. When updating from v0.6, all rules will be deleted and recreated due to different naming conventions, and the behavior will be the same as before. After that, the task will not change if the names are the same.
+  - The breaking change is that the behavior when omitting a revision of a task definition has changed. Previously, when you created a task, you resolved the latest revision at that time, so even if the task definition was updated, the revision that was executed was always the same. In v0.7 and later, revisions are not resolved when you created a task, so the latest revision is always adopted.
+
+### Chores
+
+- [#55](https://github.com/wata727/elastic_whenever/pull/55): CI against Ruby 3.0
+- [#56](https://github.com/wata727/elastic_whenever/pull/56): Add the rexml dependency explictly
+- [#58](https://github.com/wata727/elastic_whenever/pull/58): Fix typo
+
 ## v0.6.1 (2020-11-08)
 
 ### BugFixes

data/README.md

@@ -86,12 +86,13 @@ end
 Especially, `@environment` defaults to `"production"`.
 
 ## How it works
-Elastic Whenever creates CloudWatch Events as many as `every` block. Each event has as many targets as there are commands in the block.
+Elastic Whenever creates CloudWatch Events for every command. Each rule has a one to one mapping to a target.
+for example, the following input will generate two Rules each with one Target.
 
 ```ruby
-every '0 0 * * *' do # scheduled task (identifier_68237a3b152a6c44359e1cf5cd8e7cf0def303d7)
-  rake "hoge:run"    # target for `identifier_68237a3b152a6c44359e1cf5cd8e7cf0def303d7`
-  command "awesome"  # target for `identifier_68237a3b152a6c44359e1cf5cd8e7cf0def303d7`
+every '0 0 * * *' do
+  rake "hoge:run"
+  command "awesome"
 end
 ```
 
@@ -99,7 +100,7 @@ The scheduled task's name is a digest value calculated from an identifier, comma
 
 NOTE: You should not use the same identifier across different clusters because CloudWatch Events rule names are unique across all clusters.
 
-## Compatible with Whenever
+## Compatibility with Whenever
 ### `job_type`
 Whenever supports custom job type with `job_type` method, but Elastic Whenever doesn't support it.
 

data/elastic_whenever.gemspec

@@ -29,4 +29,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk-iam", "~> 1.0"
   spec.add_dependency "chronic", "~> 0.10"
   spec.add_dependency "retryable", "~> 3.0"
+  spec.add_dependency "rexml", ">= 0"
 end

data/exe/elastic_whenever

@@ -2,4 +2,4 @@
 
 require "elastic_whenever"
 
-exit ElasticWhenever::CLI.run(ARGV)
+exit ElasticWhenever::CLI.new(ARGV).run

data/lib/elastic_whenever/cli.rb

@@ -3,123 +3,149 @@ module ElasticWhenever
     SUCCESS_EXIT_CODE = 0
     ERROR_EXIT_CODE = 1
 
-    class << self
-      def run(args)
-        option = Option.new(args)
-        case option.mode
-        when Option::DRYRUN_MODE
-          option.validate!
-          update_tasks(option, dry_run: true)
-          Logger.instance.message("Above is your schedule file converted to scheduled tasks; your scheduled tasks was not updated.")
-          Logger.instance.message("Run `elastic_whenever --help' for more options.")
-        when Option::UPDATE_MODE
-          option.validate!
-          with_concurrent_modification_handling do
-            update_tasks(option, dry_run: false)
-          end
-          Logger.instance.log("write", "scheduled tasks updated")
-        when Option::CLEAR_MODE
-          with_concurrent_modification_handling do
-            clear_tasks(option)
-          end
-          Logger.instance.log("write", "scheduled tasks cleared")
-        when Option::LIST_MODE
-          list_tasks(option)
-          Logger.instance.message("Above is your scheduled tasks.")
-          Logger.instance.message("Run `elastic_whenever --help` for more options.")
-        when Option::PRINT_VERSION_MODE
-          print_version
-        end
+    attr_reader :args, :option
+
+    def initialize(args)
+      @args = args
+      @option = Option.new(args)
+    end
 
-        SUCCESS_EXIT_CODE
-      rescue Aws::Errors::MissingRegionError
-        Logger.instance.fail("missing region error occurred; please use `--region` option or export `AWS_REGION` environment variable.")
-        ERROR_EXIT_CODE
-      rescue Aws::Errors::MissingCredentialsError
-        Logger.instance.fail("missing credential error occurred; please specify it with arguments, use shared credentials, or export `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variable.")
-        ERROR_EXIT_CODE
-      rescue OptionParser::MissingArgument,
-        Option::InvalidOptionException,
-        Task::Target::InvalidContainerException => exn
-
-        Logger.instance.fail(exn.message)
-        ERROR_EXIT_CODE
+    def run
+      case option.mode
+      when Option::DRYRUN_MODE
+        option.validate!
+        update_tasks(dry_run: true)
+        Logger.instance.message("Above is your schedule file converted to scheduled tasks; your scheduled tasks was not updated.")
+        Logger.instance.message("Run `elastic_whenever --help' for more options.")
+      when Option::UPDATE_MODE
+        option.validate!
+        with_concurrent_modification_handling do
+          update_tasks(dry_run: false)
+        end
+        Logger.instance.log("write", "scheduled tasks updated")
+      when Option::CLEAR_MODE
+        with_concurrent_modification_handling do
+          clear_tasks
+        end
+        Logger.instance.log("write", "scheduled tasks cleared")
+      when Option::LIST_MODE
+        list_tasks
+        Logger.instance.message("Above is your scheduled tasks.")
+        Logger.instance.message("Run `elastic_whenever --help` for more options.")
+      when Option::PRINT_VERSION_MODE
+        print_version
       end
 
-      private
+      SUCCESS_EXIT_CODE
+    rescue Aws::Errors::MissingRegionError
+      Logger.instance.fail("missing region error occurred; please use `--region` option or export `AWS_REGION` environment variable.")
+      ERROR_EXIT_CODE
+    rescue Aws::Errors::MissingCredentialsError => e
+      Logger.instance.fail("missing credential error occurred; please specify it with arguments, use shared credentials, or export `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variable")
+      ERROR_EXIT_CODE
+    rescue OptionParser::MissingArgument,
+      Option::InvalidOptionException,
+      Task::Target::InvalidContainerException => exn
 
-      def update_tasks(option, dry_run:)
-        schedule = Schedule.new(option.schedule_file, option.verbose, option.variables)
+      Logger.instance.fail(exn.message)
+      ERROR_EXIT_CODE
+    end
 
-        cluster = Task::Cluster.new(option, option.cluster)
-        definition = Task::Definition.new(option, option.task_definition)
-        role = Task::Role.new(option)
-        if !role.exists? && !dry_run
-          role.create
-        end
+    private
 
-        clear_tasks(option) unless dry_run
-        schedule.tasks.each do |task|
-          rule = Task::Rule.convert(option, task)
-          targets = task.commands.map do |command|
-            Task::Target.new(
-              option,
-              cluster: cluster,
-              definition: definition,
-              container: option.container,
-              commands: command,
-              rule: rule,
-              role: role,
-            )
-          end
-
-          if dry_run
-            print_task(rule, targets)
-          else
-            begin
-              rule.create
-            rescue Aws::CloudWatchEvents::Errors::ValidationException => exn
-              Logger.instance.warn("#{exn.message} Ignore this task: name=#{rule.name} expression=#{rule.expression}")
-              next
-            end
-            targets.each(&:create)
-          end
+    def update_tasks(dry_run:)
+      schedule = Schedule.new(option.schedule_file, option.verbose, option.variables)
+
+      cluster = Task::Cluster.new(option, option.cluster)
+      definition = Task::Definition.new(option, option.task_definition)
+      role = Task::Role.new(option)
+      if !role.exists? && !dry_run
+        role.create
+      end
+
+      targets = schedule.tasks.map do |task|
+        task.commands.map do |command|
+          Task::Target.new(
+            option,
+            cluster: cluster,
+            definition: definition,
+            container: option.container,
+            commands: command,
+            rule: Task::Rule.convert(option, task.expression, command),
+            role: role,
+          )
         end
+      end.flatten
+
+      if dry_run
+        print_task(targets)
+      else
+        create_missing_rules_from_targets(targets)
+        delete_unused_rules_from_targets(targets)
       end
+    end
+
+    def remote_rules
+      Task::Rule.fetch(option)
+    end
 
-      def clear_tasks(option)
-        Task::Rule.fetch(option).each(&:delete)
+    # Creates a rule but only persists the rule remotely if it does not exist
+    def create_missing_rules_from_targets(targets)
+      cached_remote_rules = remote_rules
+      targets.each do |target|
+        exists = cached_remote_rules.any? do |remote_rule|
+          target.rule.name == remote_rule.name
+        end
+
+        unless exists
+          target.rule.create
+          target.create
+        end
       end
+    end
 
-      def list_tasks(option)
-        Task::Rule.fetch(option).each do |rule|
-          targets = Task::Target.fetch(option, rule)
-          print_task(rule, targets)
+    def delete_unused_rules_from_targets(targets)
+      remote_rules.each do |remote_rule|
+        rule_exists_in_schedule = targets.any? do |target|
+          target.rule.name == remote_rule.name
         end
+
+        remote_rule.delete unless rule_exists_in_schedule
       end
+    end
 
-      def print_version
-        puts "Elastic Whenever v#{ElasticWhenever::VERSION}"
+    def clear_tasks
+      Task::Rule.fetch(option).each(&:delete)
+    end
+
+    def list_tasks
+      Task::Rule.fetch(option).each do |rule|
+        targets = Task::Target.fetch(option, rule)
+        print_task(targets)
       end
+    end
 
-      def print_task(rule, targets)
-        targets.each do |target|
-          puts "#{rule.expression} #{target.cluster.name} #{target.definition.name} #{target.container} #{target.commands.join(" ")}"
-          puts
-        end
+    def print_version
+      puts "Elastic Whenever v#{ElasticWhenever::VERSION}"
+    end
+
+    def print_task(targets)
+      targets.each do |target|
+        puts "#{target.rule.expression} #{target.cluster.name} #{target.definition.name} #{target.container} #{target.commands.join(" ")}"
+        puts
       end
+    end
 
-      def with_concurrent_modification_handling
-        Retryable.retryable(
-          tries: 5,
-          on: Aws::CloudWatchEvents::Errors::ConcurrentModificationException,
-          sleep: lambda { |_n| rand(1..10) },
-        ) do |retries, exn|
-          if retries > 0
-            Logger.instance.warn("concurrent modification detected; Retrying...")
-          end
-          yield
+    def with_concurrent_modification_handling
+      Retryable.retryable(
+        tries: 5,
+        on: Aws::CloudWatchEvents::Errors::ConcurrentModificationException,
+        sleep: lambda { |_n| rand(1..10) },
+      ) do |retries, exn|
+        if retries > 0
+          Logger.instance.warn("concurrent modification detected; Retrying...")
         end
+        yield
       end
     end
   end

data/lib/elastic_whenever/option.rb

@@ -23,6 +23,10 @@ module ElasticWhenever
     attr_reader :schedule_file
     attr_reader :iam_role
     attr_reader :rule_state
+    attr_reader :aws_config
+    attr_reader :ecs_client
+    attr_reader :iam_client
+    attr_reader :cloudwatch_events_client
 
     class InvalidOptionException < StandardError; end
 
@@ -48,7 +52,7 @@ module ElasticWhenever
       @region = nil
 
       OptionParser.new do |opts|
-        opts.on('-i', '--update identifier', 'Clear and create scheduled tasks by schedule file') do |identifier|
+        opts.on('-i', '--update identifier', 'Creates and deletes tasks as needed by schedule file') do |identifier|
           @identifier = identifier
           @mode = UPDATE_MODE
         end
@@ -132,7 +136,19 @@ module ElasticWhenever
     end
 
     def aws_config
-      { credentials: credentials, region: region }.delete_if { |_k, v| v.nil? }
+      @aws_config ||= { credentials: credentials, region: region }.delete_if { |_k, v| v.nil? }
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(aws_config)
+    end
+
+    def iam_client
+      @iam_client ||= Aws::IAM::Client.new(aws_config)
+    end
+
+    def cloudwatch_events_client
+      @cloudwatch_events_client ||= Aws::CloudWatchEvents::Client.new(aws_config)
     end
 
     def validate!
@@ -143,6 +159,25 @@ module ElasticWhenever
       raise InvalidOptionException.new("Invalid rule state. Possible values are #{POSSIBLE_RULE_STATES.join(", ")}") unless POSSIBLE_RULE_STATES.include?(rule_state)
     end
 
+    def key
+      Digest::SHA1.hexdigest(
+        [
+          identifier,
+          variables,
+          cluster,
+          task_definition,
+          container,
+          assign_public_ip,
+          launch_type,
+          platform_version,
+          security_groups,
+          subnets,
+          iam_role,
+          rule_state,
+        ].join
+      )
+    end
+
     private
 
     attr_reader :profile

data/lib/elastic_whenever/task/cluster.rb

@@ -4,7 +4,7 @@ module ElasticWhenever
       class InvalidInputException < StandardError; end
 
       def initialize(option, name)
-        @client = Aws::ECS::Client.new(option.aws_config)
+        @client = option.ecs_client
         @cluster = client.describe_clusters(
           clusters: [name]
         ).clusters.first

data/lib/elastic_whenever/task/definition.rb

@@ -2,7 +2,8 @@ module ElasticWhenever
   class Task
     class Definition
       def initialize(option, family)
-        @client = Aws::ECS::Client.new(option.aws_config)
+        @client = option.ecs_client
+        @family = family
         @definition = client.describe_task_definition(
           task_definition: family
         ).task_definition
@@ -13,7 +14,12 @@ module ElasticWhenever
       end
 
       def arn
-        definition&.task_definition_arn
+        arn = definition&.task_definition_arn
+        if family_with_revision?
+          arn
+        else
+          remove_revision(arn)
+        end
       end
 
       def containers
@@ -24,6 +30,14 @@ module ElasticWhenever
 
       attr_reader :client
       attr_reader :definition
+
+      def family_with_revision?
+        @family.include?(":")
+      end
+
+      def remove_revision(arn)
+        arn.split(":")[0...-1].join(":")
+      end
     end
   end
 end

data/lib/elastic_whenever/task/role.rb

@@ -2,7 +2,7 @@ module ElasticWhenever
   class Task
     class Role
       def initialize(option)
-        client = Aws::IAM::Client.new(option.aws_config)
+        client = option.iam_client
         @resource = Aws::IAM::Resource.new(client: client)
         @role_name = option.iam_role
         @role = resource.role(@role_name)

data/lib/elastic_whenever/task/rule.rb

@@ -8,10 +8,13 @@ module ElasticWhenever
 
       class UnsupportedOptionException < StandardError; end
 
-      def self.fetch(option)
-        client = Aws::CloudWatchEvents::Client.new(option.aws_config)
-        client.list_rules(name_prefix: option.identifier).rules.map do |rule|
-          self.new(
+      def self.fetch(option, rules: [], next_token: nil)
+        client = option.cloudwatch_events_client
+        prefix = option.identifier
+
+        response = client.list_rules(name_prefix: prefix, next_token: next_token)
+        response.rules.each do |rule|
+          rules << self.new(
             option,
             name: rule.name,
             expression: rule.schedule_expression,
@@ -19,14 +22,19 @@ module ElasticWhenever
             client: client
           )
         end
+        if response.next_token.nil?
+          rules
+        else
+          fetch(option, rules: rules, next_token: response.next_token)
+        end
       end
 
-      def self.convert(option, task)
+      def self.convert(option, expression, command)
         self.new(
           option,
-          name: rule_name(option.identifier, task.expression, task.commands),
-          expression: task.expression,
-          description: rule_description(option.identifier, task.expression, task.commands)
+          name: rule_name(option, expression, command),
+          expression: expression,
+          description: rule_description(option.identifier, expression, command)
         )
       end
 
@@ -38,12 +46,13 @@ module ElasticWhenever
         if client != nil
           @client = client
         else
-          @client = Aws::CloudWatchEvents::Client.new(option.aws_config)
+          @client = option.cloudwatch_events_client
         end
       end
 
       def create
         # See https://docs.aws.amazon.com/eventbridge/latest/APIReference/API_PutRule.html#API_PutRule_RequestSyntax
+        Logger.instance.message("Creating Rule: #{name} #{expression}")
         client.put_rule(
           name: name,
           schedule_expression: expression,
@@ -55,17 +64,18 @@ module ElasticWhenever
       def delete
         targets = client.list_targets_by_rule(rule: name).targets
         client.remove_targets(rule: name, ids: targets.map(&:id)) unless targets.empty?
+        Logger.instance.message("Removing Rule: #{name}")
         client.delete_rule(name: name)
       end
 
       private
 
-      def self.rule_name(identifier, expression, commands)
-        "#{identifier}_#{Digest::SHA1.hexdigest([expression, commands.map { |command| command.join("-") }.join("-")].join("-"))}"
+      def self.rule_name(option, expression, command)
+        "#{option.identifier}_#{Digest::SHA1.hexdigest([option.key, expression, command.join("-")].join("-"))}"
       end
 
-      def self.rule_description(identifier, expression, commands)
-        "#{identifier} - #{expression} - #{commands.map { |command| command.join(" ") }.join(" - ")}"
+      def self.rule_description(identifier, expression, command)
+        "#{identifier} - #{expression} - #{command.join(" ")}"
       end
 
       def truncate(string, max)

data/lib/elastic_whenever/task/target.rb

@@ -10,11 +10,12 @@ module ElasticWhenever
       attr_reader :platform_version
       attr_reader :security_groups
       attr_reader :subnets
+      attr_reader :rule
 
       class InvalidContainerException < StandardError; end
 
       def self.fetch(option, rule)
-        client = Aws::CloudWatchEvents::Client.new(option.aws_config)
+        client = option.cloudwatch_events_client
         targets = client.list_targets_by_rule(rule: rule.name).targets
         targets.map do |target|
           input = JSON.parse(target.input, symbolize_names: true)
@@ -26,7 +27,7 @@ module ElasticWhenever
             container: input[:containerOverrides].first[:name],
             commands: input[:containerOverrides].first[:command],
             rule: rule,
-            role: Role.new(option)
+            role: Role.new(option),
           )
         end
       end
@@ -47,7 +48,7 @@ module ElasticWhenever
         @platform_version = option.platform_version
         @security_groups = option.security_groups
         @subnets = option.subnets
-        @client = Aws::CloudWatchEvents::Client.new(option.aws_config)
+        @client = option.cloudwatch_events_client
       end
 
       def create
@@ -102,7 +103,6 @@ module ElasticWhenever
         end
       end
 
-      attr_reader :rule
       attr_reader :role
       attr_reader :client
     end

data/lib/elastic_whenever/version.rb

@@ -1,3 +1,3 @@
 module ElasticWhenever
-  VERSION = "0.6.1"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic_whenever
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Kazuma Watanabe
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-08 00:00:00.000000000 Z
+date: 2021-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Manage ECS Scheduled Tasks like Whenever
 email:
 - watassbass@gmail.com