bricolage-td 5.23.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82f1cd2fc01908e53ac39cfe202686c8822b65bb
+  data.tar.gz: 9a2bc72fdd603edcd2b03db6692fc487f3a8ae4e
+SHA512:
+  metadata.gz: c3637b49c5662ab676a148e4c079d78f61c4c527b5bb41b4cd81858335ea70937bbd3d7e7bc07d52986a6edb4fc8ec0c6764cd5f6d35d6cd0589daa4943d6131
+  data.tar.gz: b622838892b8cf9bcc204134ffcd3f42360b050491570f5d840892e28646198d80c1f7a302c948f3892e18d5233834c28dcd64e94b539cc8acc6732bc8104e7f

data/README.md

@@ -0,0 +1,18 @@
+# bricolage-td
+
+Treasure Data add-on for Bricolage.
+
+## Home Page
+
+https://github.com/aamine/bricolage-td
+
+## License
+
+MIT license.
+See LICENSES file for details.
+
+## Credit
+
+Author: Minero Aoki
+
+This software is written in working time in Cookpad, Inc.

data/jobclass/td-delete.rb

@@ -0,0 +1,20 @@
+JobClass.define('td-delete') {
+  parameters {|params|
+    params.add DestTableParam.new(optional: false)
+    params.add DataSourceParam.new('td')
+    params.add DateParam.new('from', 'DATE', 'Start date of logs to delete (%Y-%m-%d).')
+    params.add DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
+  }
+
+  declarations {|params|
+    Declarations.new("dest_table" => nil)
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.delete params['dest-table'],
+        from: params['from'],
+        to: params['to']
+    }
+  }
+}

data/jobclass/td-export.rb

@@ -0,0 +1,30 @@
+JobClass.define('td-export') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestFileParam.new
+    params.add SrcTableParam.new
+    params.add StringParam.new('src-database', 'NAME', 'Source TD database name.', optional: true)
+    params.add EnumParam.new('format', %w(msgpack tsv csv json), 'Target file format.', default: 'msgpack')
+    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
+    params.add OptionalBoolParam.new('override', 'If true, clears target file.  Otherwise causes error.')
+    params.add DataSourceParam.new('td')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.export params['sql-file'],
+        path: params['dest-file'],
+        format: params['format'],
+        gzip: params['gzip'],
+        override: params['override']
+    }
+  }
+}

data/lib/bricolage-td.rb

@@ -0,0 +1,5 @@
+require 'bricolage/jobclass'
+require 'pathname'
+
+jobclass_path = Pathname(__dir__).realpath.parent.cleanpath + 'jobclass'
+Bricolage::JobClass.add_load_path jobclass_path

data/lib/bricolage/tddatasource.rb

@@ -0,0 +1,116 @@
+require 'bricolage/datasource'
+require 'bricolage/commandutils'
+require 'stringio'
+require 'date'
+
+module Bricolage
+
+  class TDDataSource < DataSource
+    declare_type 'td'
+
+    include CommandUtils
+
+    def initialize(database: nil, username: nil, apikey: nil, td: 'td', priority: -2)
+      @database = database
+      @apikey = apikey
+      @td = td
+      @priority = priority
+    end
+
+    def new_task
+      TDTask.new(self)
+    end
+
+    def query_command_args(*args)
+      [@td, "--apikey=#{@apikey}", "query", "--database=#{@database}", "--wait"] + args
+    end
+
+    def delete_command_args(*args)
+      [@td, "--apikey=#{@apikey}", "table:partial_delete", @database, *args, "--wait"]
+    end
+
+    def exec(*args)
+      JobResult.for_process_status(command(*args))
+    end
+  end
+
+  class TDTask < DataSourceTask
+    def export(stmt, path: nil, **opts)
+      add TDTask::Export.new(stmt, path: path, **opts)
+    end
+
+    class Export < Action
+      include CommandUtils
+
+      def initialize(stmt, path: nil, format: nil, gzip: false, override: false)
+        @statement = stmt
+        @path = path
+        @format = format
+        @gzip = gzip
+        @override = override
+      end
+
+      def_delegator '@statement', :bind
+
+      def source
+        buf = StringIO.new
+        buf.puts command_args(new_tmpfile_path).join(' ') + " <<EndTDSQL"
+        buf.puts @statement.stripped_source
+        buf.puts 'EndTDSQL'
+        buf.string
+      end
+
+      GZIP_COMMAND = 'gzip'   # FIXME: parameterize
+
+      def run
+        if File.exist?(@path) and not @override
+          raise JobFailure, "target file exists: #{@path.inspect}"
+        end
+        puts @statement.source
+        td_result = make_tmpfile(@statement.stripped_source) {|query_path|
+          ds.exec(*command_args(query_path))
+        }
+        if @gzip
+          gzip_result = ds.command(GZIP_COMMAND, export_path)
+          raise JobFailure, "gzip failed" unless gzip_result.success?
+        end
+        td_result
+      end
+
+      def command_args(query_path)
+        ds.query_command_args("--query=#{query_path}", "--output=#{export_path}", "--format=#{@format}")
+      end
+
+      def export_path
+        @gzip ? @path.sub(/\.gz\z/, '') : @path
+      end
+    end
+
+    def delete(table_name, **opts)
+      add TDTask::Delete.new(table_name, **opts)
+    end
+
+    class Delete < Action
+      def initialize(table_name, from:, to:)
+        @table = table_name
+        @from = from.to_time.to_i
+        @to = to.to_time.to_i
+      end
+
+      def run
+        td_result = ds.exec(source)
+      end
+
+      def command_args
+        ds.delete_command_args(@table, "--from #{@from}", "--to #{@to}")
+      end
+
+      def source
+        buf = StringIO.new
+        buf.puts command_args.join(' ')
+        buf.string
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: bricolage-td
+version: !ruby/object:Gem::Version
+  version: 5.23.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bricolage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.23.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.23.0
+- !ruby/object:Gem::Dependency
+  name: td
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem adds Treasure Data support for Bricolage batch framework.
+email: aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- jobclass/td-delete.rb
+- jobclass/td-export.rb
+- lib/bricolage-td.rb
+- lib/bricolage/tddatasource.rb
+homepage: https://github.com/aamine/bricolage-td
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Treasure Data add-on for Bricolage
+test_files: []