xronor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9da6e45fffe237ba7dc775a577a5b13aaccf6ed
+  data.tar.gz: 1fb3ef2dfdd321ee7462ca6c9120ee749d9e8e88
+SHA512:
+  metadata.gz: dbe0036b34de528d5cccbd770243333eb9bf91ae8441f88023eea7611636680ef273c75e96df45624e443e1f0c70fa3981c09291cdfbe04f697cb036d3f700e2
+  data.tar.gz: 062ca0625de294f93a11ffc572a09d0be09e50e98fa43521157f37625a137add6819c653243a9249cb8ae394e82a5c7c74dde8b7e6953277494226414b65617b

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.6
+  - 2.3.3
+  - 2.4.0
+before_install: gem install bundler -v 1.14.6

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# [v0.1.0](https://github.com/dtan4/xronor/releases/tag/v0.1.0) (2017-03-24)
+
+Initial release.

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in xronor.gemspec
+gemspec
+
+group :development do
+  gem "guard", "~> 2.14", require: false
+  gem "guard-rspec", "~> 4.7", require: false
+end
+
+group :test do
+  gem "codecov", "~> 0.1", require: false
+end

data/Guardfile

@@ -0,0 +1,16 @@
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Daisuke Fujita
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,140 @@
+# Xronor
+
+[![Build Status](https://travis-ci.org/dtan4/xronor.svg?branch=master)](https://travis-ci.org/dtan4/xronor)
+[![codecov](https://codecov.io/gh/dtan4/xronor/branch/master/graph/badge.svg)](https://codecov.io/gh/dtan4/xronor)
+
+Timezone-aware Job Scheduler DSL and Converter
+
+```ruby
+job_template "/bin/bash -l -c ':job'"
+
+job_type :rake, "bundle exec rake :task RAILS_ENV=production"
+
+default do
+  timezone "Asia/Tokyo" # UTC+9
+end
+
+every 1.hour, at: 15 do
+  name "Send awesome mails"
+  rake "send_awesome_mail"
+end
+
+every :day, at: '0:00 am' do
+  name "Send greeting notifications"
+  description "Send greeting notifications for all users"
+  rake "send_greeting_notification"
+end
+
+every :day, at: '0:00 am', timezone: "Europe/Berlin" do # UTC+1
+  name "Send notifications for Berlin"
+  description "Send notifications for Berlin"
+  rake "send_notification[Europe/Berlin]"
+end
+
+every :wednesday, at: '0:10 am' do
+  name "Create new companies"
+  rake "create_new_companies"
+end
+
+every "0 10 10,20 * *" do
+  name "Healthcheck"
+  rake "ping"
+end
+```
+
+## Table of contents
+
+- [Why](#why)
+  * [Scheduled job execution system](#scheduled-job-execution-system)
+  * [Scheduler DSL](#scheduler-dsl)
+  * [:point_right:](#point_right)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Xronor DSL](#xronor-dsl)
+- [Development](#development)
+- [Contributing](#contributing)
+- [Author](#author)
+- [License](#license)
+
+## Why
+
+### Scheduled job execution system
+
+As you know, Cron is commonly used for scheduled jobs.
+However, Cron has some difficulties:
+
+- Does not consider timezone. It depends on machine environment where cron daemon runs.
+- Does not contain job metadata (name, description, ...).
+- Cron daemon cannot be distributed. Machine where cron daemon runs can be SPOF.
+
+Recently there are solutions for the last point, e.g. [Azure Scheduler](https://azure.microsoft.com/en-us/services/scheduler/), [CloudWatch Events](http://docs.aws.amazon.com/AmazonCloudWatch/latest/events/WhatIsCloudWatchEvents.html) and [Kubernetes Cron Job](https://kubernetes.io/docs/user-guide/cron-jobs/), but those services still have fixed timezone to UTC.
+
+### Scheduler DSL
+
+[Whenever gem](https://github.com/javan/whenever) is very useful to describe scheduled jobs in human-friendly format.
+However, Whenever cannot treat timezone and metadata so that it is just a wrapper of Cron expression.
+
+### :point_right:
+
+To resolve above problems, we need:
+
+- _timezone-aware_ job scheduler DSL
+  - Just like an enhance of Whenever
+- a DSL converter which is easy to register CloudWatch Events rule
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'xronor'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install xronor
+
+## Usage
+
+```
+Commands:
+  xronor crontab SCHEDULEFILE
+  xronor cwa SCHEDULEFILE --cluster=CLUSTER --container=CONTAINER --function=FUNCTION --table=TABLE --task-definition=TASK_DEFINITION
+  xronor template SCHEDULEFILE --template=TEMPLATE
+  xronor template_per_job SCHEDULERFILE --outdir=OUTDIR --template=TEMPLATE
+```
+
+Xronor CLI converts DSL file to:
+
+- `xronor cwa` CloudWatch Events Rule (requires `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION` environment variables)
+- `xronor crontab` crontab file
+- file(s) from ERB template
+  - `xronor template` write all jobs in one file
+  - `xronor template_per_job` generate files per job
+
+## Xronor DSL
+
+:point_right: [docs/dsl](docs/dsl.md)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/dtan4/xronor.
+
+## Author
+
+Daisuke Fujita ([@dtan4](https://github.com/dtan4))
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "xronor"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/docs/README.md

@@ -0,0 +1,3 @@
+# Xronor documents
+
+- [Xronor DSL](dsl.md)

data/docs/dsl.md

@@ -0,0 +1,179 @@
+# Xronor DSL
+
+Xronor DSL is heavily inspired by [Whenever](https://github.com/javan/whenever) DSL.
+
+For example,
+
+```ruby
+job_template "/bin/bash -l -c ':job'"
+
+job_type :rake, "bundle exec rake :task RAILS_ENV=production"
+
+default do
+  timezone "Asia/Tokyo" # UTC+9
+end
+
+every 1.hour, at: 15 do
+  name "Send awesome mails"
+  rake "send_awesome_mail"
+end
+
+every :day, at: '0:00 am' do
+  name "Send greeting notifications"
+  description "Send greeting notifications for all users"
+  rake "send_greeting_notification"
+end
+
+every :day, at: '0:00 am', timezone: "Europe/Berlin" do # UTC+1
+  name "Send notifications for Berlin"
+  description "Send notifications for Berlin"
+  rake "send_notification[Europe/Berlin]"
+end
+
+every :wednesday, at: '0:10 am' do
+  name "Create new companies"
+  rake "create_new_companies"
+end
+
+every "0 10 10,20 * *" do
+  name "Healthcheck"
+  rake "ping"
+end
+```
+
+will be converted to the following crontab:
+
+```
+# Send awesome mails - Send awesome mails
+15 * * * * /bin/bash -l -c 'bundle exec rake send_awesome_mail RAILS_ENV=production'
+
+# Update Elasticsearch indices - Update Elasticsearch indices
+10 * * * * /bin/bash -l -c 'bundle exec rake update_elasticsearch RAILS_ENV=production'
+
+# Send greeting notifications - Send greeting notifications for all users
+0 15 * * * /bin/bash -l -c 'bundle exec rake send_greeting_notification RAILS_ENV=production'
+
+# Send notifications for Berlin - Send notifications for Berlin
+0 23 * * * /bin/bash -l -c 'bundle exec rake send_notification[Europe/Berlin] RAILS_ENV=production'
+
+# Create new companies - Create new companies
+10 15 * * 2 /bin/bash -l -c 'bundle exec rake create_new_companies RAILS_ENV=production'
+
+# Healthcheck - Healthcheck
+0 10 10,20 * * /bin/bash -l -c 'bundle exec rake ping RAILS_ENV=production'
+```
+
+## Table of contents
+
+- [Configuration](#configuration)
+  * [`job_template`](#job_template)
+  * [`job_type` (Required at least one)](#job_type-required-at-least-one)
+  * [`default`](#default)
+- [Job definition](#job-definition)
+  * [`every do ... end`](#every---do--end)
+  * [`name` (Required)](#name-required)
+  * [`description`](#description)
+  * [`job_type` (e.g. `rake`)](#job_type-eg-rake)
+
+## Configuration
+
+```ruby
+job_template "/bin/bash -l -c ':job'"
+
+job_type :rake, "bundle exec rake :task RAILS_ENV=production"
+
+default do
+  timezone "Asia/Tokyo" # UTC+9
+end
+```
+
+### `job_template`
+
+Define common job command template.
+`:job` will be replaced to per-job command.
+
+Default: `:job`
+
+### `job_type` (Required at least one)
+
+Define job type and per-job command template.
+`:task` will be replaced to the first argument of job.
+
+For example, the following configuration will generate `/bin/bash -l -c 'bundle exec rake update_elasticsearch'`.
+
+```ruby
+job_template "/bin/bash -l -c ':job'"
+
+job_type :rake, "bundle exec rake :task"
+
+every 1.minute do
+  rake "update_elasticsearch"
+end
+```
+
+### `default`
+
+Set default timezone.
+Timezone format follows [tz database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (e.g. `UTC`, `Asia/Tokyo`).
+
+|key|description|default|
+|---|---|---|
+|`timezone`|Timezone of described time in DSL|`UTC`|
+|`cron_timezone`|Timezone of the machine where schedule engine runs|`UTC`|
+
+For example, the following configuration will parse `10:30 am` as `10:30 am UTC+9` then convert to `30 1 * * *` (`1:30 am UTC`).
+
+```ruby
+default do
+  timezone "Asia/Tokyo" # UTC+9
+  cron_timezone "UTC"
+end
+```
+
+## Job definition
+
+```ruby
+every :day, at: '0:00 am', timezone: "Europe/Berlin" do # UTC+1
+  name "Send notifications for Berlin"
+  description "Send notifications for Berlin"
+  rake "send_notification[Europe/Berlin]"
+end
+```
+
+### `every <frequency> <options> do ... end`
+
+Define job schedule.
+
+Available `<frequency>`:
+
+|key|description|
+|---|---|
+|`:minute`|Invoke at every minute|
+|`:hour`|Invoke at every hour|
+|`:day`|Invoke at every day|
+|`:sunday`, `:monday`, ..., `:saturday`|Invoke at every weekday|
+|`N.minutes` (N = 1,2,3,...)|Invoke at every N minutes|
+|`N.hours` (N = 1,2,3,...)|Invoke at every N hours|
+|`N.days` (N = 1,2,3,...)|Invoke at every N days|
+|`0 10 10,20 * *`|Cron expression in `cron_timezone`|
+
+Available `<options>`:
+
+|key|description|
+|---|---|
+|`at`|Invocation time|
+|`timezone`|Timezone of described time in DSL|
+|`cron_timezone`|Timezone of the machine where schedule engine runs|
+
+### `name` (Required)
+
+Define job name
+
+### `description`
+
+Define job description.
+If `description` is not specified, job name will be used as description.
+
+### `job_type` (e.g. `rake`)
+
+Define job command.

data/exe/xronor

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "xronor"
+
+Xronor::CLI.start(ARGV)

data/lib/xronor/aws/cloud_watch_events.rb

@@ -0,0 +1,70 @@
+module Xronor
+  module AWS
+    class CloudWatchEvents
+      def initialize(client: Aws::CloudWatchEvents::Client.new)
+        @client = client
+      end
+
+      def deregister_job(job_name)
+        targets = @client.list_targets_by_rule(rule: job_name).targets
+        @client.remove_targets(rule: job_name, ids: targets.map(&:id))
+        @client.delete_rule(name: job_name)
+      end
+
+      def list_jobs(prefix = "")
+        @client.list_rules.rules.select { |rule| rule.name.start_with?(prefix) }.map(&:name)
+      end
+
+      def register_job(job, prefix, cluster, task_definition, container, target_function_arn)
+        rule_name = job.cloud_watch_rule_name(prefix)
+        rule_arn = put_rule(rule_name, job.description, job.cloud_watch_schedule)
+        put_target(rule_name, cluster, task_definition, container, job.command, target_function_arn)
+        rule_arn
+      end
+
+      private
+
+      def put_rule(name, description, schedule)
+        @client.put_rule({
+          name: name,
+          description: description,
+          schedule_expression: schedule,
+        }).rule_arn
+      end
+
+      def put_target(name, cluster, task_definition, container, command, target_function_arn)
+        @client.put_targets({
+          rule: name,
+          targets: [
+            {
+              id: generate_id,
+              arn: target_function_arn,
+              input_transformer: {
+                input_paths_map: {
+                  "resources" => "$.resources",
+                  "time" => "$.time",
+                },
+                input_template: generate_input_template(cluster, task_definition, container, command),
+              },
+            },
+          ],
+        })
+      end
+
+      def generate_input_template(cluster, task_definition, container, command)
+        JSON.generate({
+          "resources" => '###$.resources###',
+          "time" => '###$.time###',
+          "cluster" => cluster,
+          "task_definition" => task_definition,
+          "container" => container,
+          "command" => Shellwords.split(command),
+        }).sub('"###$.resources###"', "<resources>").sub('"###$.time###"', "<time>")
+      end
+
+      def generate_id
+        SecureRandom.uuid
+      end
+    end
+  end
+end

data/lib/xronor/aws/dynamo_db.rb

@@ -0,0 +1,44 @@
+module Xronor
+  module AWS
+    class DynamoDB
+      BATCH_WRITE_ITEM_MAX = 25
+
+      def initialize(client: Aws::DynamoDB::Client.new)
+        @client = client
+      end
+
+      def sync_rule_arns(table, add_rule_arns, delete_rule_arns)
+        put_requests = add_rule_arns.map do |arn|
+          {
+            put_request: {
+              item: {
+                "ARN" => arn,
+                "InvokedAt" => "0"
+              },
+            },
+          }
+        end
+
+        delete_requests = delete_rule_arns.map do |arn|
+          {
+            delete_request: {
+              key: {
+                "ARN" => arn,
+              },
+            },
+          }
+        end
+
+        requests = put_requests + delete_requests
+
+        requests.each_slice(BATCH_WRITE_ITEM_MAX) do |reqs|
+          @client.batch_write_item({
+            request_items: {
+              table => reqs,
+            },
+          })
+        end
+      end
+    end
+  end
+end

data/lib/xronor/aws/lambda.rb

@@ -0,0 +1,14 @@
+module Xronor
+  module AWS
+    class Lambda
+      def initialize(client: Aws::Lambda::Client.new)
+        @client = client
+      end
+
+      def retrieve_function_arn(name)
+        function = @client.list_functions.functions.find { |fn| fn.function_name == name }
+        function ? function.function_arn : ""
+      end
+    end
+  end
+end

data/lib/xronor/cli.rb

@@ -0,0 +1,43 @@
+module Xronor
+  class CLI < Thor
+    DEFAULT_JOB_PREFIX = "scheduler-"
+
+    desc "crontab SCHEDULEFILE", "Generate crontab file"
+    def crontab(filename)
+      body = Xronor::Generator::Crontab.generate(filename, options)
+      puts body
+    end
+
+    desc "cwa SCHEDULEFILE", "Register CloudWatch Events - Scheduler & ECS job runner"
+    option :prefix, default: DEFAULT_JOB_PREFIX
+    option :function, required: true
+    option :cluster, required: true
+    option :task_definition, required: true
+    option :container, required: true
+    option :table, required: true
+    option :dry_run, type: :boolean, default: false
+    def cwa(filename)
+      Xronor::Generator::CloudWatchEvents.generate(filename, options)
+    end
+
+    desc "template SCHEDULEFILE", "Generate single file"
+    option :template, required: true
+    def template(filename)
+      body = Xronor::Generator::ERB.generate_all_in_one(filename, options)
+      puts body
+    end
+
+    desc "template_per_job SCHEDULERFILE", "Generate files per job"
+    option :ext, default: nil
+    option :outdir, required: true
+    option :template, required: true
+    def template_per_job(filename)
+      contents = Xronor::Generator::ERB.generate_per_job(filename, options)
+
+      contents.each do |name, body|
+        path = File.join(options[:outdir], options[:ext] ? "#{name}.#{options[:ext]}" : name)
+        open(path, "w+") { |f| f.puts body }
+      end
+    end
+  end
+end

data/lib/xronor/core_ext/numeric.rb

@@ -0,0 +1,13 @@
+class Numeric
+  def respond_to?(method, include_private = false)
+    super || Xronor::DSL::NumericSeconds.public_method_defined?(method)
+  end
+
+  def method_missing(method, *args, &block)
+    if Xronor::DSL::NumericSeconds.public_method_defined?(method)
+      Xronor::DSL::NumericSeconds.new(self).send(method)
+    else
+      super
+    end
+  end
+end

data/lib/xronor/dsl/checker.rb

@@ -0,0 +1,25 @@
+module Xronor
+  class DSL
+    module Checker
+      class ValidationError < StandardError
+      end
+
+      def required(name, value)
+        invalid = false
+
+        if value
+          case value
+          when String
+            invalid = value.strip.empty?
+          when Array, Hash
+            invalid = value.empty?
+          end
+        else
+          invalid = true
+        end
+
+        raise ValidationError.new("'#{name}' is required") if invalid
+      end
+    end
+  end
+end