bricolage-td 5.23.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 82f1cd2fc01908e53ac39cfe202686c8822b65bb
4
+ data.tar.gz: 9a2bc72fdd603edcd2b03db6692fc487f3a8ae4e
5
+ SHA512:
6
+ metadata.gz: c3637b49c5662ab676a148e4c079d78f61c4c527b5bb41b4cd81858335ea70937bbd3d7e7bc07d52986a6edb4fc8ec0c6764cd5f6d35d6cd0589daa4943d6131
7
+ data.tar.gz: b622838892b8cf9bcc204134ffcd3f42360b050491570f5d840892e28646198d80c1f7a302c948f3892e18d5233834c28dcd64e94b539cc8acc6732bc8104e7f
@@ -0,0 +1,18 @@
1
+ # bricolage-td
2
+
3
+ Treasure Data add-on for Bricolage.
4
+
5
+ ## Home Page
6
+
7
+ https://github.com/aamine/bricolage-td
8
+
9
+ ## License
10
+
11
+ MIT license.
12
+ See LICENSES file for details.
13
+
14
+ ## Credit
15
+
16
+ Author: Minero Aoki
17
+
18
+ This software is written in working time in Cookpad, Inc.
@@ -0,0 +1,20 @@
1
+ JobClass.define('td-delete') {
2
+ parameters {|params|
3
+ params.add DestTableParam.new(optional: false)
4
+ params.add DataSourceParam.new('td')
5
+ params.add DateParam.new('from', 'DATE', 'Start date of logs to delete (%Y-%m-%d).')
6
+ params.add DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
7
+ }
8
+
9
+ declarations {|params|
10
+ Declarations.new("dest_table" => nil)
11
+ }
12
+
13
+ script {|params, script|
14
+ script.task(params['data-source']) {|task|
15
+ task.delete params['dest-table'],
16
+ from: params['from'],
17
+ to: params['to']
18
+ }
19
+ }
20
+ }
@@ -0,0 +1,30 @@
1
+ JobClass.define('td-export') {
2
+ parameters {|params|
3
+ params.add SQLFileParam.new
4
+ params.add DestFileParam.new
5
+ params.add SrcTableParam.new
6
+ params.add StringParam.new('src-database', 'NAME', 'Source TD database name.', optional: true)
7
+ params.add EnumParam.new('format', %w(msgpack tsv csv json), 'Target file format.', default: 'msgpack')
8
+ params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
9
+ params.add OptionalBoolParam.new('override', 'If true, clears target file. Otherwise causes error.')
10
+ params.add DataSourceParam.new('td')
11
+ }
12
+
13
+ parameters_filter {|job|
14
+ job.provide_sql_file_by_job_id
15
+ }
16
+
17
+ declarations {|params|
18
+ params['sql-file'].declarations
19
+ }
20
+
21
+ script {|params, script|
22
+ script.task(params['data-source']) {|task|
23
+ task.export params['sql-file'],
24
+ path: params['dest-file'],
25
+ format: params['format'],
26
+ gzip: params['gzip'],
27
+ override: params['override']
28
+ }
29
+ }
30
+ }
@@ -0,0 +1,5 @@
1
+ require 'bricolage/jobclass'
2
+ require 'pathname'
3
+
4
+ jobclass_path = Pathname(__dir__).realpath.parent.cleanpath + 'jobclass'
5
+ Bricolage::JobClass.add_load_path jobclass_path
@@ -0,0 +1,116 @@
1
+ require 'bricolage/datasource'
2
+ require 'bricolage/commandutils'
3
+ require 'stringio'
4
+ require 'date'
5
+
6
+ module Bricolage
7
+
8
+ class TDDataSource < DataSource
9
+ declare_type 'td'
10
+
11
+ include CommandUtils
12
+
13
+ def initialize(database: nil, username: nil, apikey: nil, td: 'td', priority: -2)
14
+ @database = database
15
+ @apikey = apikey
16
+ @td = td
17
+ @priority = priority
18
+ end
19
+
20
+ def new_task
21
+ TDTask.new(self)
22
+ end
23
+
24
+ def query_command_args(*args)
25
+ [@td, "--apikey=#{@apikey}", "query", "--database=#{@database}", "--wait"] + args
26
+ end
27
+
28
+ def delete_command_args(*args)
29
+ [@td, "--apikey=#{@apikey}", "table:partial_delete", @database, *args, "--wait"]
30
+ end
31
+
32
+ def exec(*args)
33
+ JobResult.for_process_status(command(*args))
34
+ end
35
+ end
36
+
37
+ class TDTask < DataSourceTask
38
+ def export(stmt, path: nil, **opts)
39
+ add TDTask::Export.new(stmt, path: path, **opts)
40
+ end
41
+
42
+ class Export < Action
43
+ include CommandUtils
44
+
45
+ def initialize(stmt, path: nil, format: nil, gzip: false, override: false)
46
+ @statement = stmt
47
+ @path = path
48
+ @format = format
49
+ @gzip = gzip
50
+ @override = override
51
+ end
52
+
53
+ def_delegator '@statement', :bind
54
+
55
+ def source
56
+ buf = StringIO.new
57
+ buf.puts command_args(new_tmpfile_path).join(' ') + " <<EndTDSQL"
58
+ buf.puts @statement.stripped_source
59
+ buf.puts 'EndTDSQL'
60
+ buf.string
61
+ end
62
+
63
+ GZIP_COMMAND = 'gzip' # FIXME: parameterize
64
+
65
+ def run
66
+ if File.exist?(@path) and not @override
67
+ raise JobFailure, "target file exists: #{@path.inspect}"
68
+ end
69
+ puts @statement.source
70
+ td_result = make_tmpfile(@statement.stripped_source) {|query_path|
71
+ ds.exec(*command_args(query_path))
72
+ }
73
+ if @gzip
74
+ gzip_result = ds.command(GZIP_COMMAND, export_path)
75
+ raise JobFailure, "gzip failed" unless gzip_result.success?
76
+ end
77
+ td_result
78
+ end
79
+
80
+ def command_args(query_path)
81
+ ds.query_command_args("--query=#{query_path}", "--output=#{export_path}", "--format=#{@format}")
82
+ end
83
+
84
+ def export_path
85
+ @gzip ? @path.sub(/\.gz\z/, '') : @path
86
+ end
87
+ end
88
+
89
+ def delete(table_name, **opts)
90
+ add TDTask::Delete.new(table_name, **opts)
91
+ end
92
+
93
+ class Delete < Action
94
+ def initialize(table_name, from:, to:)
95
+ @table = table_name
96
+ @from = from.to_time.to_i
97
+ @to = to.to_time.to_i
98
+ end
99
+
100
+ def run
101
+ td_result = ds.exec(source)
102
+ end
103
+
104
+ def command_args
105
+ ds.delete_command_args(@table, "--from #{@from}", "--to #{@to}")
106
+ end
107
+
108
+ def source
109
+ buf = StringIO.new
110
+ buf.puts command_args.join(' ')
111
+ buf.string
112
+ end
113
+ end
114
+ end
115
+
116
+ end
metadata ADDED
@@ -0,0 +1,76 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: bricolage-td
3
+ version: !ruby/object:Gem::Version
4
+ version: 5.23.0
5
+ platform: ruby
6
+ authors:
7
+ - Minero Aoki
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-02-10 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bricolage
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 5.23.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 5.23.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: td
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ description: This gem adds Treasure Data support for Bricolage batch framework.
42
+ email: aamine@loveruby.net
43
+ executables: []
44
+ extensions: []
45
+ extra_rdoc_files: []
46
+ files:
47
+ - README.md
48
+ - jobclass/td-delete.rb
49
+ - jobclass/td-export.rb
50
+ - lib/bricolage-td.rb
51
+ - lib/bricolage/tddatasource.rb
52
+ homepage: https://github.com/aamine/bricolage-td
53
+ licenses:
54
+ - MIT
55
+ metadata: {}
56
+ post_install_message:
57
+ rdoc_options: []
58
+ require_paths:
59
+ - lib
60
+ required_ruby_version: !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ">="
63
+ - !ruby/object:Gem::Version
64
+ version: 2.0.0
65
+ required_rubygems_version: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ requirements: []
71
+ rubyforge_project:
72
+ rubygems_version: 2.6.8
73
+ signing_key:
74
+ specification_version: 4
75
+ summary: Treasure Data add-on for Bricolage
76
+ test_files: []