bricolage 5.22.3 → 5.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 888f08c5389eb2cd139787f0d1e5953dffb4ba73
-  data.tar.gz: 4a6ef9137c4e5ac55976e9a4620a8b407294563b
+  metadata.gz: 68aff8b010487891f214258a2c0b3e4c175e9195
+  data.tar.gz: c9cf9a4ee6d938e7b05456a0643a687094cc379a
 SHA512:
-  metadata.gz: c7467cfe78a919f20a07555bd67c7088faf3bfa9b40e2fa3240644bbf7d69fb87fb6a65c41152ea3ffa978a54625efca08b1b829522ed117c85368303a2148c8
-  data.tar.gz: 90fdf06402e01edee7f53302c97ac43ebe47f8684b806372d818188fb6c4f8d64378b8025f3b1b74af70f85ed7bc6a35c4d153accd609306f529af0ef73037b0
+  metadata.gz: 3c923cd19ed97f203a12e090efafd55edfbe79f2807cd739e92704d1788caa45bebe074c56ce156276ee0af69fbf9cbfcbb7b0a25bdcb73068c93d9df8b5aed3
+  data.tar.gz: 25d863da9ce8fb1f0d380f4fe9ce80cbe8f9201fc82ca1ad3fe49259453f6a12325723254884ac87b77b0a22da5c8856c0cb10332539e7832fa5ebffc1b01c92

data/lib/bricolage/jobclass.rb

@@ -17,13 +17,27 @@ module Bricolage
       CLASSES[id] = c
     end
 
-    srcdir = Pathname(__FILE__).realpath.parent.parent.parent.cleanpath
-    LOAD_PATH = [srcdir + 'jobclass']
+    LOAD_PATHES = []
+
+    def JobClass.add_load_path(path)
+      LOAD_PATHES.push path
+    end
+
+    def JobClass.primary_load_path
+      load_path_for_lib_file(__FILE__)
+    end
+
+    def JobClass.load_path_for_lib_file(path)
+      srcdir = Pathname(path).realpath.parent.parent.parent.cleanpath
+      srcdir + 'jobclass'
+    end
+
+    add_load_path primary_load_path
 
     def JobClass.get(id)
       unless CLASSES[id.to_s]
         begin
-          path = LOAD_PATH.map {|prefix| prefix + "#{id}.rb" }.detect(&:exist?)
+          path = LOAD_PATHES.map {|prefix| prefix + "#{id}.rb" }.detect(&:exist?)
           raise ParameterError, "no such job class: #{id}" unless path
           ::Bricolage.module_eval File.read(path), path.to_path, 1
         rescue SystemCallError => err
@@ -35,7 +49,7 @@ module Bricolage
     end
 
     def JobClass.list
-      LOAD_PATH.map {|dir|
+      LOAD_PATHES.map {|dir|
         Dir.glob("#{dir}/*.rb").map {|path| File.basename(path, '.rb') }
       }.flatten.uniq.sort
     end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.22.3'
+  VERSION = '5.23.0'
 end

data/test/home/Gemfile.lock

@@ -1,65 +1,34 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.22.3)
+    bricolage (5.23.0)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)
       redis (>= 3.0.0)
-      td
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.6.36)
-      aws-sdk-resources (= 2.6.36)
-    aws-sdk-core (2.6.36)
+    aws-sdk (2.7.5)
+      aws-sdk-resources (= 2.7.5)
+    aws-sdk-core (2.7.5)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.36)
-      aws-sdk-core (= 2.6.36)
+    aws-sdk-resources (2.7.5)
+      aws-sdk-core (= 2.7.5)
     aws-sigv4 (1.0.0)
-    coderay (1.1.0)
-    fluent-logger (0.6.1)
-      msgpack (>= 0.5.6, < 2)
-    hirb (0.7.3)
-    httpclient (2.8.3)
+    coderay (1.1.1)
     jmespath (1.3.1)
-    json (2.0.2)
     method_source (0.8.2)
-    msgpack (1.0.2)
     mysql2 (0.4.5)
-    parallel (1.8.0)
     pg (0.18.4)
-    pry (0.10.3)
+    pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    redis (3.3.2)
-    ruby-progressbar (1.8.1)
-    rubyzip (1.1.7)
+    redis (3.3.3)
     slop (3.6.0)
-    td (0.15.2)
-      hirb (>= 0.4.5)
-      msgpack
-      parallel (~> 1.8.0)
-      ruby-progressbar (~> 1.7)
-      rubyzip (~> 1.1.7)
-      td-client (~> 0.8.85)
-      td-logger (>= 0.3.21, < 2)
-      yajl-ruby (~> 1.1)
-      zip-zip (~> 0.3)
-    td-client (0.8.85)
-      httpclient (>= 2.7)
-      json (>= 1.7.6)
-      msgpack (>= 0.5.6, < 2)
-    td-logger (0.3.26)
-      fluent-logger (>= 0.5.0, < 2.0)
-      msgpack (>= 0.5.6, < 2.0)
-      td-client (~> 0.8.66)
-    yajl-ruby (1.3.0)
-    zip-zip (0.3)
-      rubyzip (>= 1.0.0)
 
 PLATFORMS
   ruby

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.22.3
+  version: 5.23.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: td
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -164,8 +150,6 @@ files:
 - jobclass/s3-put.rb
 - jobclass/sql.rb
 - jobclass/streaming_load.rb
-- jobclass/td-delete.rb
-- jobclass/td-export.rb
 - jobclass/unload.rb
 - jobclass/wait-file.rb
 - lib/bricolage.rb
@@ -202,7 +186,6 @@ files:
 - lib/bricolage/sqlstatement.rb
 - lib/bricolage/sqlutils.rb
 - lib/bricolage/taskqueue.rb
-- lib/bricolage/tddatasource.rb
 - lib/bricolage/vacuumlock.rb
 - lib/bricolage/variables.rb
 - lib/bricolage/version.rb

data/jobclass/td-delete.rb

@@ -1,20 +0,0 @@
-JobClass.define('td-delete') {
-  parameters {|params|
-    params.add DestTableParam.new(optional: false)
-    params.add DataSourceParam.new('td')
-    params.add DateParam.new('from', 'DATE', 'Start date of logs to delete (%Y-%m-%d).')
-    params.add DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
-  }
-
-  declarations {|params|
-    Declarations.new("dest_table" => nil)
-  }
-
-  script {|params, script|
-    script.task(params['data-source']) {|task|
-      task.delete params['dest-table'],
-        from: params['from'],
-        to: params['to']
-    }
-  }
-}

data/jobclass/td-export.rb

@@ -1,30 +0,0 @@
-JobClass.define('td-export') {
-  parameters {|params|
-    params.add SQLFileParam.new
-    params.add DestFileParam.new
-    params.add SrcTableParam.new
-    params.add StringParam.new('src-database', 'NAME', 'Source TD database name.', optional: true)
-    params.add EnumParam.new('format', %w(msgpack tsv csv json), 'Target file format.', default: 'msgpack')
-    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
-    params.add OptionalBoolParam.new('override', 'If true, clears target file.  Otherwise causes error.')
-    params.add DataSourceParam.new('td')
-  }
-
-  parameters_filter {|job|
-    job.provide_sql_file_by_job_id
-  }
-
-  declarations {|params|
-    params['sql-file'].declarations
-  }
-
-  script {|params, script|
-    script.task(params['data-source']) {|task|
-      task.export params['sql-file'],
-        path: params['dest-file'],
-        format: params['format'],
-        gzip: params['gzip'],
-        override: params['override']
-    }
-  }
-}

data/lib/bricolage/tddatasource.rb

@@ -1,116 +0,0 @@
-require 'bricolage/datasource'
-require 'bricolage/commandutils'
-require 'stringio'
-require 'date'
-
-module Bricolage
-
-  class TDDataSource < DataSource
-    declare_type 'td'
-
-    include CommandUtils
-
-    def initialize(database: nil, username: nil, apikey: nil, td: 'td', priority: -2)
-      @database = database
-      @apikey = apikey
-      @td = td
-      @priority = priority
-    end
-
-    def new_task
-      TDTask.new(self)
-    end
-
-    def query_command_args(*args)
-      [@td, "--apikey=#{@apikey}", "query", "--database=#{@database}", "--wait"] + args
-    end
-
-    def delete_command_args(*args)
-      [@td, "--apikey=#{@apikey}", "table:partial_delete", @database, *args, "--wait"]
-    end
-
-    def exec(*args)
-      JobResult.for_process_status(command(*args))
-    end
-  end
-
-  class TDTask < DataSourceTask
-    def export(stmt, path: nil, **opts)
-      add TDTask::Export.new(stmt, path: path, **opts)
-    end
-
-    class Export < Action
-      include CommandUtils
-
-      def initialize(stmt, path: nil, format: nil, gzip: false, override: false)
-        @statement = stmt
-        @path = path
-        @format = format
-        @gzip = gzip
-        @override = override
-      end
-
-      def_delegator '@statement', :bind
-
-      def source
-        buf = StringIO.new
-        buf.puts command_args(new_tmpfile_path).join(' ') + " <<EndTDSQL"
-        buf.puts @statement.stripped_source
-        buf.puts 'EndTDSQL'
-        buf.string
-      end
-
-      GZIP_COMMAND = 'gzip'   # FIXME: parameterize
-
-      def run
-        if File.exist?(@path) and not @override
-          raise JobFailure, "target file exists: #{@path.inspect}"
-        end
-        puts @statement.source
-        td_result = make_tmpfile(@statement.stripped_source) {|query_path|
-          ds.exec(*command_args(query_path))
-        }
-        if @gzip
-          gzip_result = ds.command(GZIP_COMMAND, export_path)
-          raise JobFailure, "gzip failed" unless gzip_result.success?
-        end
-        td_result
-      end
-
-      def command_args(query_path)
-        ds.query_command_args("--query=#{query_path}", "--output=#{export_path}", "--format=#{@format}")
-      end
-
-      def export_path
-        @gzip ? @path.sub(/\.gz\z/, '') : @path
-      end
-    end
-
-    def delete(table_name, **opts)
-      add TDTask::Delete.new(table_name, **opts)
-    end
-
-    class Delete < Action
-      def initialize(table_name, from:, to:)
-        @table = table_name
-        @from = from.to_time.to_i
-        @to = to.to_time.to_i
-      end
-
-      def run
-        td_result = ds.exec(source)
-      end
-
-      def command_args
-        ds.delete_command_args(@table, "--from #{@from}", "--to #{@to}")
-      end
-
-      def source
-        buf = StringIO.new
-        buf.puts command_args.join(' ')
-        buf.string
-      end
-    end
-  end
-
-end