pendulum 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +92 -0
- data/Rakefile +10 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/exe/pendulum +7 -0
- data/lib/pendulum.rb +19 -0
- data/lib/pendulum/client.rb +50 -0
- data/lib/pendulum/command/apply.rb +35 -0
- data/lib/pendulum/command/apply/result_url.rb +57 -0
- data/lib/pendulum/command/apply/schedule.rb +165 -0
- data/lib/pendulum/configuration.rb +18 -0
- data/lib/pendulum/dsl/converter.rb +51 -0
- data/lib/pendulum/dsl/helper.rb +17 -0
- data/lib/pendulum/dsl/output/base.rb +46 -0
- data/lib/pendulum/dsl/output/postgresql.rb +11 -0
- data/lib/pendulum/dsl/output/result.rb +14 -0
- data/lib/pendulum/dsl/output/treasure_data.rb +9 -0
- data/lib/pendulum/dsl/result.rb +28 -0
- data/lib/pendulum/dsl/schedule.rb +56 -0
- data/lib/pendulum/runner.rb +67 -0
- data/lib/pendulum/settings.rb +23 -0
- data/lib/pendulum/version.rb +3 -0
- data/pendulum.gemspec +31 -0
- metadata +184 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 39768125e6cbdf710a729beac0133f6c3e53d7b9
|
4
|
+
data.tar.gz: ef90d7cedb538420b6cdae2dcea602a2b5bafb5b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b9e1c70253542545c488fd040d7640f907343920783feae932def202a27f274f23781251f060fb3deeeff382b49910eb2b723a681f47a3750c9df53cd3535556
|
7
|
+
data.tar.gz: da9651c21dc64f7b674b94b2f66edd0f88c869744191c149493bf2a3918f2b42faad57ac958ca842e549a6c02ee4075e86313e73d42631dd3f300a103e993f98
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 monochromegane
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
# Pendulum
|
2
|
+
|
3
|
+
Pendulum is a tool to manage Treasure Data scheduled jobs.
|
4
|
+
|
5
|
+
It defines the state of Treasure Data scheduled jobs using DSL, and updates the jobs according as DSL.
|
6
|
+
|
7
|
+
## Usage
|
8
|
+
|
9
|
+
```sh
|
10
|
+
# Export from Treasure Data
|
11
|
+
$ pendulum --apikey='...' -e -o Schedfile
|
12
|
+
|
13
|
+
# Update Schedfile
|
14
|
+
$ vi Schedfile
|
15
|
+
|
16
|
+
# Apply scheduled jobs
|
17
|
+
$ pendulum --apikey='...' -a --dry-run
|
18
|
+
$ pendulum --apikey='...' -a
|
19
|
+
```
|
20
|
+
|
21
|
+
## Schedfile
|
22
|
+
|
23
|
+
```rb
|
24
|
+
schedule 'test-scheduled-job' do
|
25
|
+
database 'db_name'
|
26
|
+
query 'select time from access;'
|
27
|
+
retry_limit 0
|
28
|
+
priority :normal
|
29
|
+
cron '30 0 * * *'
|
30
|
+
timezone 'Asia/Tokyo'
|
31
|
+
delay 0
|
32
|
+
result_url 'td://@/db_name/table_name'
|
33
|
+
end
|
34
|
+
```
|
35
|
+
|
36
|
+
#### query_file
|
37
|
+
|
38
|
+
If your query is long, you can specify `query_file`.
|
39
|
+
|
40
|
+
```rb
|
41
|
+
query_file 'queries/test-scheduled-job.hql'
|
42
|
+
```
|
43
|
+
|
44
|
+
#### result
|
45
|
+
|
46
|
+
You can use `result` DSL instead of `result_url`.
|
47
|
+
|
48
|
+
```rb
|
49
|
+
schedule 'test-scheduled-job' do
|
50
|
+
database 'db_name'
|
51
|
+
...
|
52
|
+
result :td do
|
53
|
+
database 'db_name'
|
54
|
+
table 'table_name'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
```
|
58
|
+
|
59
|
+
Now, Pendulum supports `td` and `postgresql` result export.
|
60
|
+
If you want to use other result export, please send pull request :smile:
|
61
|
+
|
62
|
+
## Installation
|
63
|
+
|
64
|
+
Add this line to your application's Gemfile:
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
gem 'pendulum'
|
68
|
+
```
|
69
|
+
|
70
|
+
And then execute:
|
71
|
+
|
72
|
+
$ bundle
|
73
|
+
|
74
|
+
Or install it yourself as:
|
75
|
+
|
76
|
+
$ gem install pendulum
|
77
|
+
|
78
|
+
## Development
|
79
|
+
|
80
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
81
|
+
|
82
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
83
|
+
|
84
|
+
## Contributing
|
85
|
+
|
86
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/monochromegane/pendulum.
|
87
|
+
|
88
|
+
|
89
|
+
## License
|
90
|
+
|
91
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
92
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
data/bin/setup
ADDED
data/exe/pendulum
ADDED
data/lib/pendulum.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require "pendulum/version"
|
2
|
+
require "pendulum/client"
|
3
|
+
require "pendulum/settings"
|
4
|
+
require "pendulum/configuration"
|
5
|
+
require "pendulum/dsl/helper"
|
6
|
+
require "pendulum/dsl/schedule"
|
7
|
+
require "pendulum/dsl/result"
|
8
|
+
require "pendulum/dsl/output/base"
|
9
|
+
require "pendulum/dsl/output/treasure_data"
|
10
|
+
require "pendulum/dsl/output/postgresql"
|
11
|
+
require "pendulum/dsl/output/result"
|
12
|
+
require "pendulum/dsl/converter"
|
13
|
+
require "pendulum/command/apply"
|
14
|
+
require "pendulum/command/apply/result_url"
|
15
|
+
require "pendulum/command/apply/schedule"
|
16
|
+
require "pendulum/runner"
|
17
|
+
|
18
|
+
module Pendulum
|
19
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'td'
|
2
|
+
require 'td-client'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Pendulum
|
6
|
+
class Client
|
7
|
+
def initialize(api_key='', options={}, &block)
|
8
|
+
@api_key = api_key
|
9
|
+
@config = Configuration.new(options)
|
10
|
+
@config.instance_eval(&block) if block_given?
|
11
|
+
end
|
12
|
+
|
13
|
+
def apply(dry_run: false, force: false, color: false)
|
14
|
+
Pendulum::Command::Apply.new(
|
15
|
+
td_client,
|
16
|
+
current_schedules,
|
17
|
+
@config.schedules,
|
18
|
+
dry_run,
|
19
|
+
force,
|
20
|
+
color,
|
21
|
+
).execute
|
22
|
+
end
|
23
|
+
|
24
|
+
def export(output)
|
25
|
+
result = DSL::Converter.new(td_client.schedules).convert
|
26
|
+
# schedule
|
27
|
+
File.write(output, result[:schedule])
|
28
|
+
# queries
|
29
|
+
query_dir = File.join(File.dirname(output), 'queries')
|
30
|
+
make_dir(query_dir)
|
31
|
+
result[:queries].each do |query|
|
32
|
+
File.write(File.join(query_dir, query[:name]), query[:query])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def current_schedules
|
39
|
+
td_client.schedules
|
40
|
+
end
|
41
|
+
|
42
|
+
def td_client
|
43
|
+
@td_client ||= TreasureData::Client.new(@api_key, {ssl: true})
|
44
|
+
end
|
45
|
+
|
46
|
+
def make_dir(dir)
|
47
|
+
FileUtils.mkdir(dir) unless File.exist?(dir)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Pendulum::Command
|
2
|
+
class Apply
|
3
|
+
attr_accessor :client, :dry_run, :force
|
4
|
+
|
5
|
+
def initialize(client, from, to, dry_run=false, force=false, color=false)
|
6
|
+
@schedules = matched_schedules(client, from, to, dry_run, force, color)
|
7
|
+
end
|
8
|
+
|
9
|
+
def execute
|
10
|
+
@schedules.each{|s| s.apply }
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def matched_schedules(client, from, to, dry_run, force, color)
|
16
|
+
# create or update
|
17
|
+
schedules = to.map do |schedule|
|
18
|
+
Schedule.new(
|
19
|
+
client,
|
20
|
+
from.find{|f| f.name == schedule.name},
|
21
|
+
schedule,
|
22
|
+
dry_run,
|
23
|
+
force,
|
24
|
+
color
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
# delete
|
29
|
+
from.reject{|f| to.any?{|t| t.name == f.name}}.each do |schedule|
|
30
|
+
schedules << Schedule.new(client, schedule, nil, dry_run, force, color)
|
31
|
+
end
|
32
|
+
schedules
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Pendulum::Command
|
2
|
+
class Apply
|
3
|
+
class ResultURL
|
4
|
+
attr_accessor :client, :from, :to
|
5
|
+
def initialize(client, from, to)
|
6
|
+
self.client = client
|
7
|
+
self.from = from
|
8
|
+
self.to = to
|
9
|
+
end
|
10
|
+
|
11
|
+
def changed?
|
12
|
+
from_uri = to_uri(from)
|
13
|
+
to_uri = mask(to_uri(to))
|
14
|
+
|
15
|
+
uri_without_query(from_uri) != uri_without_query(to_uri) ||
|
16
|
+
query_hash(from_uri) != query_hash(to_uri)
|
17
|
+
end
|
18
|
+
|
19
|
+
def mask(uri)
|
20
|
+
uri.password = '***' if uri.user
|
21
|
+
uri
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def uri_without_query(uri)
|
27
|
+
uri.to_s.split('?').first
|
28
|
+
end
|
29
|
+
|
30
|
+
def query_hash(uri)
|
31
|
+
Hash[URI::decode_www_form(uri.query || '')]
|
32
|
+
end
|
33
|
+
|
34
|
+
def to_uri(url)
|
35
|
+
return URI.parse(url) if url.include?('://')
|
36
|
+
|
37
|
+
# use result
|
38
|
+
name, table = url.split(':', 2)
|
39
|
+
|
40
|
+
result = result_by(name)
|
41
|
+
return URI.parse(url) unless result
|
42
|
+
|
43
|
+
uri = URI.parse(result.url)
|
44
|
+
uri.path += "/#{table}"
|
45
|
+
uri
|
46
|
+
end
|
47
|
+
|
48
|
+
def results
|
49
|
+
@results ||= client.results
|
50
|
+
end
|
51
|
+
|
52
|
+
def result_by(name)
|
53
|
+
results.find{|r| name == r.name}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
require 'highline'
|
2
|
+
|
3
|
+
module Pendulum::Command
|
4
|
+
class Apply
|
5
|
+
class Schedule
|
6
|
+
attr_accessor :client, :from, :to, :dry_run, :force, :color
|
7
|
+
def initialize(client, from, to, dry_run=false, force=false, color=false)
|
8
|
+
self.client = client
|
9
|
+
self.from = from
|
10
|
+
self.to = to
|
11
|
+
self.dry_run = dry_run
|
12
|
+
self.force = force
|
13
|
+
self.color = color
|
14
|
+
end
|
15
|
+
|
16
|
+
def apply
|
17
|
+
case
|
18
|
+
when will_create? then create
|
19
|
+
when will_update? then update
|
20
|
+
when will_delete? then delete
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def create
|
25
|
+
puts message_for_create
|
26
|
+
client.create_schedule(to.name, to.to_params) unless dry_run?
|
27
|
+
end
|
28
|
+
|
29
|
+
def update
|
30
|
+
puts message_for_update
|
31
|
+
puts message_for_diff if has_diff?
|
32
|
+
if force? || has_diff?
|
33
|
+
client.update_schedule(to.name, to.to_params) unless dry_run?
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete
|
38
|
+
puts message_for_delete
|
39
|
+
client.delete_schedule(from.name) if force? && !dry_run?
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def will_create?
|
45
|
+
!from && to
|
46
|
+
end
|
47
|
+
|
48
|
+
def will_update?
|
49
|
+
from && to
|
50
|
+
end
|
51
|
+
|
52
|
+
def will_delete?
|
53
|
+
from && !to
|
54
|
+
end
|
55
|
+
|
56
|
+
def has_diff?
|
57
|
+
!diff.empty?
|
58
|
+
end
|
59
|
+
|
60
|
+
def diff
|
61
|
+
return {} unless will_update?
|
62
|
+
|
63
|
+
@diff ||= begin
|
64
|
+
default_params.merge(to.to_params).select do |k, v|
|
65
|
+
if k == :result
|
66
|
+
result_url_changed?(from.result_url, v)
|
67
|
+
else
|
68
|
+
v != from.send(k)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def masked_diff
|
75
|
+
return diff unless diff.key?(:result)
|
76
|
+
|
77
|
+
masked = diff.dup
|
78
|
+
uri = URI.parse(masked[:result])
|
79
|
+
uri.password = '***' if uri.user
|
80
|
+
masked[:result] = uri.to_s
|
81
|
+
|
82
|
+
masked
|
83
|
+
end
|
84
|
+
|
85
|
+
def message_for_create
|
86
|
+
colorize message_for(:create), :cyan
|
87
|
+
end
|
88
|
+
|
89
|
+
def message_for_update
|
90
|
+
if force? || has_diff?
|
91
|
+
colorize message_for(:update), :green
|
92
|
+
else
|
93
|
+
colorize message_with_dry_run("No change schedule: #{name}"), :blue
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def message_for_delete
|
98
|
+
if force?
|
99
|
+
colorize message_for(:delete), :red
|
100
|
+
else
|
101
|
+
colorize message_with_dry_run("Undefined schedule (pass `--force` if you want to remove): #{name}"), :yellow
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def message_with_dry_run(message)
|
106
|
+
message += ' (dry-run)' if dry_run?
|
107
|
+
message
|
108
|
+
end
|
109
|
+
|
110
|
+
def message_for(action)
|
111
|
+
message_with_dry_run "#{action.to_s.capitalize} schedule: #{name}"
|
112
|
+
end
|
113
|
+
|
114
|
+
def message_for_diff
|
115
|
+
message = masked_diff.map do |name, value|
|
116
|
+
" set #{name}=#{value}"
|
117
|
+
end.join("\n")
|
118
|
+
colorize message, :green
|
119
|
+
end
|
120
|
+
|
121
|
+
def name
|
122
|
+
(from && from.name) || (to && to.name)
|
123
|
+
end
|
124
|
+
|
125
|
+
def dry_run?
|
126
|
+
dry_run
|
127
|
+
end
|
128
|
+
|
129
|
+
def force?
|
130
|
+
force
|
131
|
+
end
|
132
|
+
|
133
|
+
def color?
|
134
|
+
color
|
135
|
+
end
|
136
|
+
|
137
|
+
def default_params
|
138
|
+
{
|
139
|
+
database: '',
|
140
|
+
query: nil,
|
141
|
+
retry_limit: 0,
|
142
|
+
priority: 0,
|
143
|
+
cron: nil,
|
144
|
+
timezone: 'Asia/Tokyo', # TODO: require timezone.
|
145
|
+
delay: 0,
|
146
|
+
result: ''
|
147
|
+
}
|
148
|
+
end
|
149
|
+
|
150
|
+
def result_url_changed?(from_url, to_url)
|
151
|
+
Apply::ResultURL.new(client, from_url, to_url).changed?
|
152
|
+
end
|
153
|
+
|
154
|
+
def colorize(message, color)
|
155
|
+
return message unless color?
|
156
|
+
h.color message, color
|
157
|
+
end
|
158
|
+
|
159
|
+
def h
|
160
|
+
@h ||= HighLine.new
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Pendulum
|
2
|
+
class Configuration
|
3
|
+
def initialize(options={})
|
4
|
+
settings = Pendulum::Settings.load(options[:env])
|
5
|
+
if file = options[:file]
|
6
|
+
self.instance_eval(File.read(file), file)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def schedule(name, &block)
|
11
|
+
schedules << DSL::Schedule.new(name, &block)
|
12
|
+
end
|
13
|
+
|
14
|
+
def schedules
|
15
|
+
@schedules ||= []
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'erb'
|
2
|
+
|
3
|
+
module Pendulum::DSL
|
4
|
+
class Converter
|
5
|
+
def initialize(schedules)
|
6
|
+
@schedules = schedules
|
7
|
+
end
|
8
|
+
|
9
|
+
def convert
|
10
|
+
result = {}
|
11
|
+
result[:schedule] = @schedules.map do |schedule|
|
12
|
+
to_dsl(schedule)
|
13
|
+
end.join("\n")
|
14
|
+
|
15
|
+
result[:queries] = @schedules.map do |schedule|
|
16
|
+
to_query(schedule)
|
17
|
+
end.compact
|
18
|
+
|
19
|
+
result
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def to_dsl(schedule)
|
25
|
+
ERB.new(<<-EOS, nil, '-').result(binding)
|
26
|
+
schedule '<%= schedule.name %>' do
|
27
|
+
database '<%= schedule.database %>'
|
28
|
+
<% if schedule.query -%>
|
29
|
+
query_file 'queries/<%= schedule.name %>.hql'
|
30
|
+
# type :hive # FIXME: Treasure Data schedule api dosen't contain type result.
|
31
|
+
retry_limit <%= schedule.retry_limit %>
|
32
|
+
priority <%= schedule.priority %>
|
33
|
+
<% end -%>
|
34
|
+
<% if schedule.cron -%>
|
35
|
+
cron '<%= schedule.cron %>'
|
36
|
+
timezone '<%= schedule.timezone %>'
|
37
|
+
delay <%= schedule.delay %>
|
38
|
+
<% end -%>
|
39
|
+
<% if schedule.result_url != '' -%>
|
40
|
+
result_url '<%= schedule.result_url %>'
|
41
|
+
<% end -%>
|
42
|
+
end
|
43
|
+
EOS
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_query(schedule)
|
47
|
+
return nil unless schedule.query
|
48
|
+
{name: "#{schedule.name}.hql", query: schedule.query}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Pendulum::DSL
|
2
|
+
module Helper
|
3
|
+
def self.included(base)
|
4
|
+
base.extend(ClassMethods)
|
5
|
+
end
|
6
|
+
|
7
|
+
module ClassMethods
|
8
|
+
def define_setter(*names)
|
9
|
+
names.each do |name|
|
10
|
+
define_method(name) do |value|
|
11
|
+
instance_variable_set("@#{name}", value)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Pendulum::DSL::Output
|
2
|
+
class Base
|
3
|
+
include Pendulum::DSL::Helper
|
4
|
+
|
5
|
+
def initialize(&block)
|
6
|
+
self.instance_eval(&block) if block_given?
|
7
|
+
end
|
8
|
+
|
9
|
+
def to_url
|
10
|
+
raise NotImplementedError, "You must implement #{self.class}##{__method__}"
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def with_options(url, *options)
|
16
|
+
params = (options || []).select do |option|
|
17
|
+
instance_variable_defined?("@#{option}")
|
18
|
+
end.map do |option|
|
19
|
+
"#{option}=#{instance_variable_get("@#{option}")}"
|
20
|
+
end.join('&')
|
21
|
+
url + (params.empty? ? '' : "?#{params}")
|
22
|
+
end
|
23
|
+
|
24
|
+
def username_and_password
|
25
|
+
case
|
26
|
+
when @username && @password
|
27
|
+
"#{@username}:#{@password}"
|
28
|
+
when @username
|
29
|
+
@username
|
30
|
+
when @password
|
31
|
+
":#{@password}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def hostname_and_port
|
36
|
+
case
|
37
|
+
when @hostname && @port
|
38
|
+
"#{@hostname}:#{@port}"
|
39
|
+
when @hostname
|
40
|
+
@hostname
|
41
|
+
when @port
|
42
|
+
":#{@port}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
module Pendulum::DSL::Output
|
2
|
+
class Postgresql < Base
|
3
|
+
define_setter :username, :password, :hostname, :port,
|
4
|
+
:database, :table, :ssl, :schema, :mode, :method
|
5
|
+
|
6
|
+
def to_url
|
7
|
+
url = "postgresql://#{username_and_password}@#{hostname_and_port}/#{@database}/#{@table}"
|
8
|
+
with_options(url, :ssl, :schema, :mode, :method)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Pendulum::DSL
|
2
|
+
class Result
|
3
|
+
attr_accessor :type, :output
|
4
|
+
|
5
|
+
def initialize(type, &block)
|
6
|
+
self.type = type
|
7
|
+
self.output = output_by(type)
|
8
|
+
self.output.instance_eval(&block) if block_given?
|
9
|
+
end
|
10
|
+
|
11
|
+
def to_url
|
12
|
+
output.to_url
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def output_by(type)
|
18
|
+
case type.to_sym
|
19
|
+
when :treasure_data, :td
|
20
|
+
Output::TreasureData.new
|
21
|
+
when :postgresql
|
22
|
+
Output::Postgresql.new
|
23
|
+
else
|
24
|
+
Output::Result.new(type)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Pendulum::DSL
|
2
|
+
class Schedule
|
3
|
+
include Helper
|
4
|
+
|
5
|
+
attr_accessor :name
|
6
|
+
|
7
|
+
def initialize(name, &block)
|
8
|
+
self.name = name
|
9
|
+
self.instance_eval(&block) if block_given?
|
10
|
+
end
|
11
|
+
|
12
|
+
define_setter :database, :query, :timezone,
|
13
|
+
:delay, :retry_limit, :type
|
14
|
+
|
15
|
+
def query_file(path)
|
16
|
+
query(File.read(path))
|
17
|
+
end
|
18
|
+
|
19
|
+
def cron(cron)
|
20
|
+
@cron = %i(hourly daily monthly).include?(cron) ? "@#{cron}" : cron
|
21
|
+
end
|
22
|
+
|
23
|
+
def priority(priority)
|
24
|
+
@priority = priority.is_a?(Integer) ? priority : priority_id_of(priority)
|
25
|
+
end
|
26
|
+
|
27
|
+
def result_url(url)
|
28
|
+
@result = url
|
29
|
+
end
|
30
|
+
|
31
|
+
def result(type, &block)
|
32
|
+
result = Result.new(type, &block)
|
33
|
+
@result = result.to_url
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_params
|
37
|
+
instance_variables.inject({}) do |params, v|
|
38
|
+
params[v.to_s.delete('@').to_sym] = instance_variable_get(v)
|
39
|
+
params
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def priority_id_of(name)
|
46
|
+
case name.to_sym
|
47
|
+
when :very_low then -2
|
48
|
+
when :low then -1
|
49
|
+
when :normal then 0
|
50
|
+
when :high then 1
|
51
|
+
when :very_high then 2
|
52
|
+
else 0
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
|
4
|
+
module Pendulum
|
5
|
+
DEFAULT_SCHEDFILE = 'Schedfile'
|
6
|
+
class Runner
|
7
|
+
def run(argv=ARGV)
|
8
|
+
api_key = nil
|
9
|
+
mode = nil
|
10
|
+
dry_run = false
|
11
|
+
force = false
|
12
|
+
color = true
|
13
|
+
options = {
|
14
|
+
file: DEFAULT_SCHEDFILE,
|
15
|
+
env: :development,
|
16
|
+
}
|
17
|
+
output = DEFAULT_SCHEDFILE
|
18
|
+
|
19
|
+
opt.on('-k', '--apikey=KEY') {|v| api_key = v }
|
20
|
+
|
21
|
+
# apply
|
22
|
+
opt.on('-a', '--apply') { mode = :apply }
|
23
|
+
opt.on('-E', '--environment=ENV') {|v| options[:env] = v }
|
24
|
+
opt.on('-f', '--file=FILE') {|v| options[:file] = v }
|
25
|
+
opt.on('', '--dry-run') { dry_run = true }
|
26
|
+
opt.on('', '--force') { force = true }
|
27
|
+
opt.on('', '--no-color') { color = false }
|
28
|
+
|
29
|
+
# export
|
30
|
+
opt.on('-e', '--export') do
|
31
|
+
mode = :export
|
32
|
+
options.delete(:file)
|
33
|
+
end
|
34
|
+
opt.on('-o', '--output=FILE') {|v| output = v }
|
35
|
+
|
36
|
+
opt.parse!(argv) rescue return usage $!
|
37
|
+
return usage if (api_key.nil? || mode.nil?)
|
38
|
+
|
39
|
+
begin
|
40
|
+
client = Client.new(api_key, options)
|
41
|
+
case mode
|
42
|
+
when :apply
|
43
|
+
client.apply(dry_run: dry_run, force: force, color: color)
|
44
|
+
when :export
|
45
|
+
client.export(output)
|
46
|
+
end
|
47
|
+
rescue
|
48
|
+
$stderr.puts $!
|
49
|
+
return 1
|
50
|
+
end
|
51
|
+
|
52
|
+
return 0
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def usage(err=nil)
|
58
|
+
puts err if err
|
59
|
+
puts opt.help
|
60
|
+
err ? 1 : 0
|
61
|
+
end
|
62
|
+
|
63
|
+
def opt
|
64
|
+
@opt ||= OptionParser.new
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'hashie'
|
2
|
+
|
3
|
+
module Pendulum
|
4
|
+
class Settings
|
5
|
+
class << self
|
6
|
+
def load(env)
|
7
|
+
merge(load_from(:default), load_from(env))
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def load_from(env)
|
13
|
+
path = File.join('environments', "#{env}.yml")
|
14
|
+
return Hashie::Mash.new unless File.file?(path)
|
15
|
+
Hashie::Mash.load(path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def merge(org, new)
|
19
|
+
org.deep_merge(new)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/pendulum.gemspec
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'pendulum/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "pendulum"
|
8
|
+
spec.version = Pendulum::VERSION
|
9
|
+
spec.authors = ["monochromegane"]
|
10
|
+
spec.email = ["dev.kuro.obi@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Pendulum is a tool to manage Treasure Data scheduled jobs.}
|
13
|
+
spec.description = %q{Pendulum is a tool to manage Treasure Data scheduled jobs.}
|
14
|
+
spec.homepage = "https://github.com/monochromegane/pendulum"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
+
spec.bindir = "exe"
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_dependency "td"
|
23
|
+
spec.add_dependency "td-client"
|
24
|
+
spec.add_dependency "hashie"
|
25
|
+
spec.add_dependency "highline"
|
26
|
+
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.11"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
spec.add_development_dependency "minitest", "~> 5.0"
|
30
|
+
spec.add_development_dependency "pry"
|
31
|
+
end
|
metadata
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pendulum
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- monochromegane
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-03-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: td
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: td-client
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: hashie
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: highline
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.11'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.11'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '10.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '10.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: minitest
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '5.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '5.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pry
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: Pendulum is a tool to manage Treasure Data scheduled jobs.
|
126
|
+
email:
|
127
|
+
- dev.kuro.obi@gmail.com
|
128
|
+
executables:
|
129
|
+
- pendulum
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files: []
|
132
|
+
files:
|
133
|
+
- ".gitignore"
|
134
|
+
- ".travis.yml"
|
135
|
+
- Gemfile
|
136
|
+
- LICENSE.txt
|
137
|
+
- README.md
|
138
|
+
- Rakefile
|
139
|
+
- bin/console
|
140
|
+
- bin/setup
|
141
|
+
- exe/pendulum
|
142
|
+
- lib/pendulum.rb
|
143
|
+
- lib/pendulum/client.rb
|
144
|
+
- lib/pendulum/command/apply.rb
|
145
|
+
- lib/pendulum/command/apply/result_url.rb
|
146
|
+
- lib/pendulum/command/apply/schedule.rb
|
147
|
+
- lib/pendulum/configuration.rb
|
148
|
+
- lib/pendulum/dsl/converter.rb
|
149
|
+
- lib/pendulum/dsl/helper.rb
|
150
|
+
- lib/pendulum/dsl/output/base.rb
|
151
|
+
- lib/pendulum/dsl/output/postgresql.rb
|
152
|
+
- lib/pendulum/dsl/output/result.rb
|
153
|
+
- lib/pendulum/dsl/output/treasure_data.rb
|
154
|
+
- lib/pendulum/dsl/result.rb
|
155
|
+
- lib/pendulum/dsl/schedule.rb
|
156
|
+
- lib/pendulum/runner.rb
|
157
|
+
- lib/pendulum/settings.rb
|
158
|
+
- lib/pendulum/version.rb
|
159
|
+
- pendulum.gemspec
|
160
|
+
homepage: https://github.com/monochromegane/pendulum
|
161
|
+
licenses:
|
162
|
+
- MIT
|
163
|
+
metadata: {}
|
164
|
+
post_install_message:
|
165
|
+
rdoc_options: []
|
166
|
+
require_paths:
|
167
|
+
- lib
|
168
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
169
|
+
requirements:
|
170
|
+
- - ">="
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '0'
|
173
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
174
|
+
requirements:
|
175
|
+
- - ">="
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '0'
|
178
|
+
requirements: []
|
179
|
+
rubyforge_project:
|
180
|
+
rubygems_version: 2.4.5.1
|
181
|
+
signing_key:
|
182
|
+
specification_version: 4
|
183
|
+
summary: Pendulum is a tool to manage Treasure Data scheduled jobs.
|
184
|
+
test_files: []
|