bricolage-td 5.23.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +18 -0
- data/jobclass/td-delete.rb +20 -0
- data/jobclass/td-export.rb +30 -0
- data/lib/bricolage-td.rb +5 -0
- data/lib/bricolage/tddatasource.rb +116 -0
- metadata +76 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 82f1cd2fc01908e53ac39cfe202686c8822b65bb
|
4
|
+
data.tar.gz: 9a2bc72fdd603edcd2b03db6692fc487f3a8ae4e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c3637b49c5662ab676a148e4c079d78f61c4c527b5bb41b4cd81858335ea70937bbd3d7e7bc07d52986a6edb4fc8ec0c6764cd5f6d35d6cd0589daa4943d6131
|
7
|
+
data.tar.gz: b622838892b8cf9bcc204134ffcd3f42360b050491570f5d840892e28646198d80c1f7a302c948f3892e18d5233834c28dcd64e94b539cc8acc6732bc8104e7f
|
data/README.md
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# bricolage-td
|
2
|
+
|
3
|
+
Treasure Data add-on for Bricolage.
|
4
|
+
|
5
|
+
## Home Page
|
6
|
+
|
7
|
+
https://github.com/aamine/bricolage-td
|
8
|
+
|
9
|
+
## License
|
10
|
+
|
11
|
+
MIT license.
|
12
|
+
See LICENSES file for details.
|
13
|
+
|
14
|
+
## Credit
|
15
|
+
|
16
|
+
Author: Minero Aoki
|
17
|
+
|
18
|
+
This software is written in working time in Cookpad, Inc.
|
@@ -0,0 +1,20 @@
|
|
1
|
+
JobClass.define('td-delete') {
|
2
|
+
parameters {|params|
|
3
|
+
params.add DestTableParam.new(optional: false)
|
4
|
+
params.add DataSourceParam.new('td')
|
5
|
+
params.add DateParam.new('from', 'DATE', 'Start date of logs to delete (%Y-%m-%d).')
|
6
|
+
params.add DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
|
7
|
+
}
|
8
|
+
|
9
|
+
declarations {|params|
|
10
|
+
Declarations.new("dest_table" => nil)
|
11
|
+
}
|
12
|
+
|
13
|
+
script {|params, script|
|
14
|
+
script.task(params['data-source']) {|task|
|
15
|
+
task.delete params['dest-table'],
|
16
|
+
from: params['from'],
|
17
|
+
to: params['to']
|
18
|
+
}
|
19
|
+
}
|
20
|
+
}
|
@@ -0,0 +1,30 @@
|
|
1
|
+
JobClass.define('td-export') {
|
2
|
+
parameters {|params|
|
3
|
+
params.add SQLFileParam.new
|
4
|
+
params.add DestFileParam.new
|
5
|
+
params.add SrcTableParam.new
|
6
|
+
params.add StringParam.new('src-database', 'NAME', 'Source TD database name.', optional: true)
|
7
|
+
params.add EnumParam.new('format', %w(msgpack tsv csv json), 'Target file format.', default: 'msgpack')
|
8
|
+
params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
|
9
|
+
params.add OptionalBoolParam.new('override', 'If true, clears target file. Otherwise causes error.')
|
10
|
+
params.add DataSourceParam.new('td')
|
11
|
+
}
|
12
|
+
|
13
|
+
parameters_filter {|job|
|
14
|
+
job.provide_sql_file_by_job_id
|
15
|
+
}
|
16
|
+
|
17
|
+
declarations {|params|
|
18
|
+
params['sql-file'].declarations
|
19
|
+
}
|
20
|
+
|
21
|
+
script {|params, script|
|
22
|
+
script.task(params['data-source']) {|task|
|
23
|
+
task.export params['sql-file'],
|
24
|
+
path: params['dest-file'],
|
25
|
+
format: params['format'],
|
26
|
+
gzip: params['gzip'],
|
27
|
+
override: params['override']
|
28
|
+
}
|
29
|
+
}
|
30
|
+
}
|
data/lib/bricolage-td.rb
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'bricolage/datasource'
|
2
|
+
require 'bricolage/commandutils'
|
3
|
+
require 'stringio'
|
4
|
+
require 'date'
|
5
|
+
|
6
|
+
module Bricolage
|
7
|
+
|
8
|
+
class TDDataSource < DataSource
|
9
|
+
declare_type 'td'
|
10
|
+
|
11
|
+
include CommandUtils
|
12
|
+
|
13
|
+
def initialize(database: nil, username: nil, apikey: nil, td: 'td', priority: -2)
|
14
|
+
@database = database
|
15
|
+
@apikey = apikey
|
16
|
+
@td = td
|
17
|
+
@priority = priority
|
18
|
+
end
|
19
|
+
|
20
|
+
def new_task
|
21
|
+
TDTask.new(self)
|
22
|
+
end
|
23
|
+
|
24
|
+
def query_command_args(*args)
|
25
|
+
[@td, "--apikey=#{@apikey}", "query", "--database=#{@database}", "--wait"] + args
|
26
|
+
end
|
27
|
+
|
28
|
+
def delete_command_args(*args)
|
29
|
+
[@td, "--apikey=#{@apikey}", "table:partial_delete", @database, *args, "--wait"]
|
30
|
+
end
|
31
|
+
|
32
|
+
def exec(*args)
|
33
|
+
JobResult.for_process_status(command(*args))
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
class TDTask < DataSourceTask
|
38
|
+
def export(stmt, path: nil, **opts)
|
39
|
+
add TDTask::Export.new(stmt, path: path, **opts)
|
40
|
+
end
|
41
|
+
|
42
|
+
class Export < Action
|
43
|
+
include CommandUtils
|
44
|
+
|
45
|
+
def initialize(stmt, path: nil, format: nil, gzip: false, override: false)
|
46
|
+
@statement = stmt
|
47
|
+
@path = path
|
48
|
+
@format = format
|
49
|
+
@gzip = gzip
|
50
|
+
@override = override
|
51
|
+
end
|
52
|
+
|
53
|
+
def_delegator '@statement', :bind
|
54
|
+
|
55
|
+
def source
|
56
|
+
buf = StringIO.new
|
57
|
+
buf.puts command_args(new_tmpfile_path).join(' ') + " <<EndTDSQL"
|
58
|
+
buf.puts @statement.stripped_source
|
59
|
+
buf.puts 'EndTDSQL'
|
60
|
+
buf.string
|
61
|
+
end
|
62
|
+
|
63
|
+
GZIP_COMMAND = 'gzip' # FIXME: parameterize
|
64
|
+
|
65
|
+
def run
|
66
|
+
if File.exist?(@path) and not @override
|
67
|
+
raise JobFailure, "target file exists: #{@path.inspect}"
|
68
|
+
end
|
69
|
+
puts @statement.source
|
70
|
+
td_result = make_tmpfile(@statement.stripped_source) {|query_path|
|
71
|
+
ds.exec(*command_args(query_path))
|
72
|
+
}
|
73
|
+
if @gzip
|
74
|
+
gzip_result = ds.command(GZIP_COMMAND, export_path)
|
75
|
+
raise JobFailure, "gzip failed" unless gzip_result.success?
|
76
|
+
end
|
77
|
+
td_result
|
78
|
+
end
|
79
|
+
|
80
|
+
def command_args(query_path)
|
81
|
+
ds.query_command_args("--query=#{query_path}", "--output=#{export_path}", "--format=#{@format}")
|
82
|
+
end
|
83
|
+
|
84
|
+
def export_path
|
85
|
+
@gzip ? @path.sub(/\.gz\z/, '') : @path
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def delete(table_name, **opts)
|
90
|
+
add TDTask::Delete.new(table_name, **opts)
|
91
|
+
end
|
92
|
+
|
93
|
+
class Delete < Action
|
94
|
+
def initialize(table_name, from:, to:)
|
95
|
+
@table = table_name
|
96
|
+
@from = from.to_time.to_i
|
97
|
+
@to = to.to_time.to_i
|
98
|
+
end
|
99
|
+
|
100
|
+
def run
|
101
|
+
td_result = ds.exec(source)
|
102
|
+
end
|
103
|
+
|
104
|
+
def command_args
|
105
|
+
ds.delete_command_args(@table, "--from #{@from}", "--to #{@to}")
|
106
|
+
end
|
107
|
+
|
108
|
+
def source
|
109
|
+
buf = StringIO.new
|
110
|
+
buf.puts command_args.join(' ')
|
111
|
+
buf.string
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
metadata
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: bricolage-td
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 5.23.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Minero Aoki
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-02-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bricolage
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 5.23.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 5.23.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: td
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description: This gem adds Treasure Data support for Bricolage batch framework.
|
42
|
+
email: aamine@loveruby.net
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- README.md
|
48
|
+
- jobclass/td-delete.rb
|
49
|
+
- jobclass/td-export.rb
|
50
|
+
- lib/bricolage-td.rb
|
51
|
+
- lib/bricolage/tddatasource.rb
|
52
|
+
homepage: https://github.com/aamine/bricolage-td
|
53
|
+
licenses:
|
54
|
+
- MIT
|
55
|
+
metadata: {}
|
56
|
+
post_install_message:
|
57
|
+
rdoc_options: []
|
58
|
+
require_paths:
|
59
|
+
- lib
|
60
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: 2.0.0
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
requirements: []
|
71
|
+
rubyforge_project:
|
72
|
+
rubygems_version: 2.6.8
|
73
|
+
signing_key:
|
74
|
+
specification_version: 4
|
75
|
+
summary: Treasure Data add-on for Bricolage
|
76
|
+
test_files: []
|