bricolage 5.25.1 → 5.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b26f9e8d81a62e099e26e9c7f0bf3394c10ff754
-  data.tar.gz: d2201749ca5fc7fee832116ab88071e5a65502f0
+  metadata.gz: 7331a52e0d7ce7f6e37a033455c4bbd0cd20ec98
+  data.tar.gz: 8c794e3df428209e2467e8edd7dca371d109eb17
 SHA512:
-  metadata.gz: 40f1006c34882e1718445aaa26eb1bb0e3695eb9763b325c116132636a0c1b4aba57a7bb8112162c5a02bf643c56e3cc4eb6622a876717118b7d05e2a4e8c77e
-  data.tar.gz: 8e62c714a483670c1307e4b7eda9bd3bfaf746c44232aa4d6437034166e32ffa6d3ae5349d981a9a280587875fb1cdbe5d68c4ee97943287f27da5c31570153f
+  metadata.gz: 8f58e1983098afd0dace568e5f32e415e12e50e39be6a04502ef9cab9347c1325370c4304de4859d481d2b1ad5246aeaf816bc7971ca7f5adb0747f1af47ad19
+  data.tar.gz: c9831a9e1c8f5694654e69ed695f8886db98b3fe46e9a8ef7384a9f120bd8890e2557a14f90318665f0ad713cb087b6ce1ae103e9943c55aef98eb71ac684ff8

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.25.1'
+  VERSION = '5.26.0'
 end

data/test/home/Gemfile.lock

@@ -1,12 +1,24 @@
+PATH
+  remote: ../../../bricolage-mysql
+  specs:
+    bricolage-mysql (5.26.0)
+      bricolage (>= 5.26.0)
+      mysql2
+
+PATH
+  remote: ../../../bricolage-redis
+  specs:
+    bricolage-redis (5.26.0)
+      bricolage (>= 5.26.0)
+      redis (~> 3)
+
 PATH
   remote: ../..
   specs:
-    bricolage (5.25.0)
+    bricolage (5.26.0)
       aws-sdk-s3 (~> 1)
       aws-sdk-sns (~> 1)
-      mysql2
       pg (~> 0.18.0)
-      redis (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -35,13 +47,15 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    redis (4.0.1)
+    redis (3.3.3)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bricolage!
+  bricolage-mysql!
+  bricolage-redis!
   pry
 
 BUNDLED WITH

data/test/home/config/development/database.yml

@@ -79,29 +79,16 @@ mysql_summary:
   password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
 
-sqs_preproc:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"
-  max_number_of_messages: 10
-  visibility_timeout: 60
-
-sqs_dispatch:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
-  max_number_of_messages: 10
-  visibility_timeout: 60
-
-sqs_task:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
-  max_number_of_messages: 1
-  visibility_timeout: 180
-
 sns:
   type: sns
   region: "ap-northeast-1"
   topic_arn: "arn:aws:sns:ap-northeast-1:789035092620:dwh-service-notification"
 
+redis:
+  type: redis
+  host: localhost
+  port: 6379
+
 td:
   type: td
   database: logs

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.25.1
+  version: 5.26.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -52,34 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
-- !ruby/object:Gem::Dependency
-  name: mysql2
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -154,14 +126,9 @@ files:
 - jobclass/insert-delta.rb
 - jobclass/insert.rb
 - jobclass/load.rb
-- jobclass/my-export.rb
-- jobclass/my-import-delta.rb
-- jobclass/my-import.rb
-- jobclass/my-migrate.rb
 - jobclass/noop.rb
 - jobclass/rebuild-drop.rb
 - jobclass/rebuild-rename.rb
-- jobclass/redis-export.rb
 - jobclass/s3-put.rb
 - jobclass/sql.rb
 - jobclass/streaming_load.rb
@@ -190,11 +157,9 @@ files:
 - lib/bricolage/logger.rb
 - lib/bricolage/loglocator.rb
 - lib/bricolage/loglocatorbuilder.rb
-- lib/bricolage/mysqldatasource.rb
 - lib/bricolage/parameters.rb
 - lib/bricolage/postgresconnection.rb
 - lib/bricolage/psqldatasource.rb
-- lib/bricolage/redisdatasource.rb
 - lib/bricolage/resource.rb
 - lib/bricolage/rubyjobclass.rb
 - lib/bricolage/s3datasource.rb
@@ -208,10 +173,6 @@ files:
 - lib/bricolage/variables.rb
 - lib/bricolage/version.rb
 - libexec/create-lockfile
-- libexec/mys3dump.jar
-- libexec/sqldump
-- libexec/sqldump.Darwin
-- libexec/sqldump.Linux
 - test/all.rb
 - test/home/Gemfile
 - test/home/Gemfile.lock
@@ -219,7 +180,6 @@ files:
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
 - test/home/config/development/variable.yml
-- test/home/config/streamingload.yml
 - test/home/data/20141002-1355_00.txt
 - test/home/data/20141002-1355_01.txt
 - test/home/data/20141002-1355_02.txt

data/jobclass/my-export.rb

@@ -1,40 +0,0 @@
-JobClass.define('my-export') {
-  parameters {|params|
-    params.add SQLFileParam.new(optional: true)
-    params.add DestFileParam.new
-    params.add SrcTableParam.new
-    params.add EnumParam.new('format', %w(json tsv csv), 'Target file format.', default: 'json')
-    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
-    params.add OptionalBoolParam.new('override', 'If true, clears target file.  Otherwise causes error.')
-    params.add OptionalBoolParam.new('sqldump', 'If true, clears use sqldump command to dump, only wheen usable.')
-    params.add DataSourceParam.new('mysql')
-  }
-
-  declarations {|params|
-    sql_statement(params).declarations
-  }
-
-  script {|params, script|
-    script.task(params['data-source']) {|task|
-      task.export sql_statement(params),
-        path: params['dest-file'],
-        format: params['format'],
-        override: params['override'],
-        gzip: params['gzip'],
-        sqldump: params['sqldump']
-    }
-  }
-
-  def sql_statement(params)
-    if sql = params['sql-file']
-      sql
-    else
-      srcs = params['src-tables']
-      raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
-      src_table_var = srcs.keys.first
-      stmt = SQLStatement.for_string("select * from $#{src_table_var};")
-      stmt.declarations = Declarations.new({src_table_var => src_table_var})
-      stmt
-    end
-  end
-}

data/jobclass/my-import-delta.rb

@@ -1,66 +0,0 @@
-require 'bricolage/psqldatasource'
-require 'bricolage/mysqldatasource'
-
-JobClass.define('my-import-delta') {
-  parameters {|params|
-    # S3Export
-    params.add SrcTableParam.new(optional: false)
-    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
-    params.add SQLFileParam.new(optional: true)
-    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
-    params.add DestFileParam.new('s3-prefix', 'PREFIX', 'Temporary S3 prefix.')
-    params.add KeyValuePairsParam.new('dump-options', 'KEY:VALUE', 'dump options.', optional: true)
-
-    # Delete, Load
-    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
-    params.add StringParam.new('delete-cond', 'SQL_EXPR', 'DELETE condition.')
-    params.add DestTableParam.new(optional: false)
-    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
-        optional: true, default: PSQLLoadOptions.new,
-        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
-
-    # Misc
-    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
-    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
-    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-
-    # All
-    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
-    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
-    params.add OptionalBoolParam.new('gzip', 'Compress Temporary files.')
-  }
-
-  script {|params, script|
-    run_all = !params['export'] && !params['load']
-
-    # S3Export
-    if params['export'] || run_all
-      script.task(params['src-ds']) {|task|
-        task.s3export params['src-tables'].values.first.to_s,
-                      params['sql-file'],
-                      params['s3-ds'],
-                      params['s3-prefix'],
-                      params['gzip'],
-                      dump_options: params['dump-options']
-      }
-    end
-
-    # Load
-    if params['load'] || run_all
-      script.task(params['dest-ds']) {|task|
-        task.transaction {
-          # DELETE
-          task.exec SQLStatement.delete_where(params['delete-cond']) if params['delete-cond']
-
-          # COPY
-          task.load params['s3-ds'], params['s3-prefix'], params['dest-table'],
-              'json', nil, params['options'].merge('gzip' => params['gzip'])
-        }
-
-        # VACUUM, ANALYZE
-        task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
-        task.analyze_if params['analyze'], params['dest-table']
-      }
-    end
-  }
-}

data/jobclass/my-import.rb

@@ -1,84 +0,0 @@
-require 'bricolage/psqldatasource'
-require 'bricolage/mysqldatasource'
-
-JobClass.define('my-import') {
-  parameters {|params|
-    # S3Export
-    params.add SrcTableParam.new(optional: false)
-    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
-    params.add SQLFileParam.new(optional: true)
-    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
-    params.add DestFileParam.new('s3-prefix', 'PREFIX', 'Temporary S3 prefix.')
-    params.add KeyValuePairsParam.new('dump-options', 'KEY:VALUE', 'dump options.', optional: true)
-
-    # Load
-    params.add DestTableParam.new(optional: false)
-    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
-    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
-        optional: true, default: PSQLLoadOptions.new,
-        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
-    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
-    params.add OptionalBoolParam.new('no-backup', 'Do not backup current table with suffix "_old".', default: false)
-
-    # Misc
-    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
-    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
-    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
-
-    # All
-    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
-    params.add OptionalBoolParam.new('put', 'Runs PUT task.')
-    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
-    params.add OptionalBoolParam.new('gzip', 'Compress Temporary files.')
-  }
-
-  script {|params, script|
-    run_all = !params['export'] && !params['put'] && !params['load']
-
-    # S3Export
-    if params['export'] || run_all
-      script.task(params['src-ds']) {|task|
-        task.s3export params['src-tables'].keys.first,
-                      params['sql-file'],
-                      params['s3-ds'],
-                      params['s3-prefix'],
-                      params['gzip'],
-                      dump_options: params['dump-options']
-      }
-    end
-
-    # Load
-    if params['load'] || run_all
-      script.task(params['dest-ds']) {|task|
-        prev_table = '${dest_table}_old'
-        work_table = '${dest_table}_wk'
-
-        task.transaction {
-          # CREATE
-          task.drop_force prev_table
-          task.drop_force work_table
-          task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
-
-          # COPY
-          task.load params['s3-ds'], params['s3-prefix'], work_table,
-              'json', nil, params['options'].merge('gzip' => params['gzip'])
-
-          # GRANT, ANALYZE
-          task.grant_if params['grant'], work_table
-          task.analyze_if params['analyze'], work_table
-
-          # RENAME
-          task.create_dummy_table '${dest_table}'
-          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
-          task.rename_table work_table, params['dest-table'].name
-        }
-
-        task.drop_force prev_table if params['no-backup']
-
-        # VACUUM: vacuum is needless for newly created table, applying vacuum after exposure is not a problem.
-        task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table'].to_s
-      }
-    end
-  }
-}

data/jobclass/my-migrate.rb

@@ -1,116 +0,0 @@
-require 'bricolage/psqldatasource'
-
-JobClass.define('my-migrate') {
-  parameters {|params|
-    # Export
-    params.add SrcTableParam.new(optional: false)
-    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
-    params.add DestFileParam.new('tmp-file', 'PATH', 'Temporary local file path.')
-    params.add OptionalBoolParam.new('sqldump', 'If true, use sqldump command to dump, only on available.', default: true)
-    params.add SQLFileParam.new(optional: true)
-
-    # Put
-    params.add DestFileParam.new('s3-file', 'PATH', 'Temporary S3 file path.')
-    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
-    params.add OptionalBoolParam.new('override', 'If true, overwrite s3 target file.  Otherwise causes error.')
-    params.add OptionalBoolParam.new('remove-tmp', 'Removes temporary local files after S3-PUT is succeeded.')
-
-    # Load
-    params.add DestTableParam.new(optional: false)
-    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
-    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
-        optional: true, default: PSQLLoadOptions.new,
-        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
-    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
-    params.add OptionalBoolParam.new('no-backup', 'Do not backup current table with suffix "_old".', default: false)
-
-    # Misc
-    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
-    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
-    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
-
-    # All
-    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
-    params.add OptionalBoolParam.new('put', 'Runs PUT task.')
-    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
-    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.', default: true)
-  }
-
-  declarations {|params|
-    decls = sql_statement(params).declarations
-    decls.declare 'dest-table', nil
-    decls
-  }
-
-  script {|params, script|
-    run_all = !params['export'] && !params['put'] && !params['load']
-
-    # Export
-    if params['export'] || run_all
-      script.task(params['src-ds']) {|task|
-        task.export sql_statement(params),
-          path: params['tmp-file'],
-          format: 'json',
-          override: true,
-          gzip: params['gzip'],
-          sqldump: params['sqldump']
-      }
-    end
-
-    # Put
-    if params['put'] || run_all
-      script.task(params['s3-ds']) {|task|
-        task.put params['tmp-file'], params['s3-file'], check_args: false
-      }
-      if params['remove-tmp']
-        script.task(params.file_ds) {|task|
-          task.remove params['tmp-file']
-        }
-      end
-    end
-
-    # Load
-    if params['load'] || run_all
-      script.task(params['dest-ds']) {|task|
-        prev_table = '${dest_table}_old'
-        work_table = '${dest_table}_wk'
-
-        task.transaction {
-          # CREATE
-          task.drop_force prev_table
-          task.drop_force work_table
-          task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
-
-          # COPY
-          task.load params['s3-ds'], params['s3-file'], work_table,
-              'json', nil, params['options'].merge('gzip' => params['gzip'])
-
-          # GRANT, ANALYZE
-          task.grant_if params['grant'], work_table
-          task.analyze_if params['analyze'], work_table
-
-          # RENAME
-          task.create_dummy_table '${dest_table}'
-          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
-          task.rename_table work_table, params['dest-table'].name
-        }
-
-        task.drop_force prev_table if params['no-backup']
-
-        # VACUUM: vacuum is needless for newly created table, applying vacuum after exposure is not a problem.
-        task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table'].to_s
-      }
-    end
-  }
-
-  def sql_statement(params)
-    return params['sql-file'] if params['sql-file']
-    srcs = params['src-tables']
-    raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
-    src_table_var = srcs.keys.first
-    stmt = SQLStatement.for_string("select * from $#{src_table_var};")
-    stmt.declarations = Declarations.new({src_table_var => src_table_var})
-    stmt
-  end
-}

data/jobclass/redis-export.rb

@@ -1,42 +0,0 @@
-require 'bricolage/psqldatasource'
-require 'bricolage/redisdatasource'
-require 'redis'
-
-JobClass.define('redis-export') {
-  parameters {|params|
-    # Export
-    params.add DataSourceParam.new('psql', 'src-ds', 'Source data source.')
-    params.add SrcTableParam.new(optional: false)
-    params.add SQLFileParam.new(optional: true)
-
-    # Redis import
-    params.add DataSourceParam.new('redis', 'dest-ds', 'Redis cluster')
-    params.add StringParam.new('key-column', 'REDIS_KEY', 'Redis object key. default: id', optional: true)
-    params.add StringParam.new('prefix', 'REDIS_PREFIX', 'Redis object key prefix', optional: true)
-    params.add StringParam.new('encode', 'REDIS_ENCODE', 'Redis object encoding. default: hash', optional: true)
-    params.add StringParam.new('expire', 'REDIS_TTL', 'Redis object TTL. default: none', optional: true)
-  }
-
-  script {|params, script|
-    # Export
-    script.task(params['dest-ds']) {|task|
-      task.import params['src-ds'],
-        params['src-tables'].first,
-        sql_statement(params),
-        params['key-column'] || "id",
-        params['prefix'],
-        params['encode'] || "hash",
-        expire: params['expire'].to_i
-    }
-  }
-
-  def sql_statement(params)
-    return params['sql-file'] if params['sql-file']
-    srcs = params['src-tables']
-    raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
-    src_table_var = srcs.keys.first
-    stmt = SQLStatement.for_string("select * from $#{src_table_var};")
-    stmt.declarations = Declarations.new({src_table_var => src_table_var})
-    stmt
-  end
-}

data/lib/bricolage/mysqldatasource.rb

@@ -1,363 +0,0 @@
-require 'bricolage/datasource'
-require 'mysql2'
-require 'json'
-require 'csv'
-require 'stringio'
-require 'open3'
-
-module Bricolage
-
-  class MySQLDataSource < DataSource
-    declare_type 'mysql'
-
-    def initialize(**mysql_options)
-      @mysql_options = mysql_options
-      @client = nil
-    end
-
-    attr_reader :mysql_options
-
-    def host
-      @mysql_options[:host]
-    end
-
-    def port
-      @mysql_options[:port]
-    end
-
-    def username
-      @mysql_options[:username]
-    end
-
-    def password
-      @mysql_options[:password]
-    end
-
-    def database
-      @mysql_options[:database]
-    end
-
-    def new_task
-      MySQLTask.new(self)
-    end
-
-    def open
-      @client = Mysql2::Client.new(**@mysql_options)
-      begin
-        yield self
-      ensure
-        c = @client
-        @client = nil
-        c.close
-      end
-    end
-
-    def query(sql, **opts)
-      logger.info "[SQL] #{sql}"
-      connection_check
-      @client.query(sql, **opts)
-    end
-
-    private
-
-    def connection_check
-      unless @client
-        raise FatalError, "#{self.class} used outside of \#open block"
-      end
-    end
-  end
-
-  class MySQLTask < DataSourceTask
-    def export(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: false)
-      add Export.new(stmt, path: path, format: format, override: override, gzip: gzip, sqldump: sqldump)
-    end
-
-    class Export < Action
-      def initialize(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: false)
-        @statement = stmt
-        @path = path
-        @format = format
-        @override = override
-        @gzip = gzip
-        @sqldump = sqldump
-      end
-
-      def bind(*args)
-        @statement.bind(*args)
-      end
-
-      def source
-        @statement.stripped_source
-      end
-
-      def run
-        if @sqldump and sqldump_available? and sqldump_usable?
-          export_by_sqldump
-        else
-          export_by_ruby
-        end
-        JobResult.success
-      end
-
-      def export_by_sqldump
-        cmds = [[{"SQLDUMP_PASSWORD" => ds.password}, sqldump_path.to_s, "--#{@format}", ds.host, ds.port.to_s, ds.username, ds.database, @statement.stripped_source]]
-        cmds.push [GZIP_COMMAND] if @gzip
-        cmds.last.push({out: @path.to_s})
-        ds.logger.info '[CMD] ' + format_pipeline(cmds)
-        statuses = Open3.pipeline(*cmds)
-        statuses.each_with_index do |st, idx|
-          unless st.success?
-            cmd = cmds[idx].first
-            raise JobFailure, "sqldump failed (status #{st.to_i})"
-          end
-        end
-      end
-
-      def format_pipeline(cmds)
-        cmds = cmds.map {|args| args[0].kind_of?(Hash) ? args[1..-1] : args.dup }   # do not show env
-        cmds.map {|args| %Q("#{args.join('" "')}") }.join(' | ')
-      end
-
-      def sqldump_available?
-        sqldump_real_path.executable?
-      end
-
-      def sqldump_path
-        Pathname(__dir__).parent.parent + "libexec/sqldump"
-      end
-
-      def sqldump_real_path
-        Pathname("#{sqldump_path}.#{platform_name}")
-      end
-
-      def platform_name
-        @platform_name ||= `uname -s`.strip
-      end
-
-      def sqldump_usable?
-        %w[json tsv].include?(@format)
-      end
-
-      def export_by_ruby
-        ds.logger.info "exporting table into #{@path} ..."
-        count = 0
-        open_target_file(@path) {|f|
-          writer_class = WRITER_CLASSES[@format] or raise ArgumentError, "unknown export format: #{@format.inspect}"
-          writer = writer_class.new(f)
-          rs = ds.query(@statement.stripped_source, as: writer_class.record_format, stream: true, cache_rows: false)
-          ds.logger.info "got result set, writing..."
-          rs.each do |values|
-            writer.write_record values
-            count += 1
-            ds.logger.info "#{count} records exported..." if count % 10_0000 == 0
-          end
-        }
-        ds.logger.info "#{count} records exported; export finished"
-      end
-
-      private
-
-      # FIXME: parameterize
-      GZIP_COMMAND = 'gzip'
-
-      def open_target_file(path, &block)
-        unless @override
-          raise JobFailure, "destination file already exists: #{path}" if File.exist?(path)
-        end
-        if @gzip
-          ds.logger.info "enable compression: gzip"
-          IO.popen(%Q(#{GZIP_COMMAND} > "#{path}"), 'w', &block)
-        else
-          File.open(path, 'w', &block)
-        end
-      end
-    end
-
-    def s3export(table, stmt, s3ds, prefix, gzip, dump_options)
-      options = dump_options.nil? ? {} : dump_options[:dump_options]
-      add S3Export.new(table, stmt, s3ds, prefix, gzip: gzip,
-        format: options['format'],
-        partition_column: options['partition_column'],
-        partition_number: options['partition_number'],
-        write_concurrency: options['write_concurrency'],
-        rotation_size: options['rotation_size'],
-        delete_objects: options['delete_objects'],
-        object_key_delimiter: options['object_key_delimiter'],
-        src_zone_offset: options['src_zone_offset'],
-        dst_zone_offset: options['dst_zone_offset'])
-    end
-
-    class S3Export < Action
-
-      def initialize(table, stmt, s3ds, prefix, gzip: true,
-         format: "json",
-         partition_column: nil,
-         partition_number: 4,
-         write_concurrency: 4,
-         rotation_size: nil,
-         delete_objects: false,
-         object_key_delimiter: nil,
-         src_zone_offset: nil,
-         dst_zone_offset: nil)
-        @table = table
-        @statement = stmt
-        @s3ds = s3ds
-        @prefix = build_prefix @s3ds.prefix, prefix
-        @format = format
-        @gzip = gzip
-        @partition_column = partition_column
-        @partition_number = partition_number
-        @write_concurrency = write_concurrency
-        @rotation_size = rotation_size
-        @delete_objects = delete_objects
-        @object_key_delimiter = object_key_delimiter
-        @src_zone_offset = src_zone_offset
-        @dst_zone_offset = dst_zone_offset
-      end
-
-      def run
-        s3export
-        JobResult.success
-      end
-
-      def bind(*args)
-        @statement.bind(*args) if @statement
-      end
-
-      def source
-        "-- myexport #{@table} -> #{@s3ds.bucket_name}/#{@prefix}" +
-          (@statement ? "\n#{@statement.stripped_source}" : "")
-      end
-
-      def s3export
-        cmd = build_cmd(command_parameters)
-        ds.logger.info "[CMD] #{cmd}"
-        out, st = Open3.capture2e(environment_variables, cmd)
-        ds.logger.info "[CMDOUT] #{out}"
-        unless st.success?
-          msg = extract_exception_message(out)
-          raise JobFailure, "mys3dump failed (status: #{st.to_i}): #{msg}"
-        end
-      end
-
-      def environment_variables
-        {
-          'AWS_ACCESS_KEY_ID' => @s3ds.access_key,
-          'AWS_SECRET_ACCESS_KEY' => @s3ds.secret_key,
-          'MYS3DUMP_PASSWORD' => ds.password
-        }
-      end
-
-      def command_parameters
-        params = {
-          jar: mys3dump_path.to_s,
-          h: ds.host,
-          P: ds.port.to_s,
-          D: ds.database,
-          u: ds.username,
-          #p: ds.password,
-          o: connection_property,
-          t: @table,
-          b: @s3ds.bucket.name,
-          x: @prefix
-        }
-        params[:q] = @statement.stripped_source.chomp(';') if @statement
-        params[:f] = @format if @format
-        params[:C] = nil if @gzip
-        params[:c] = @partition_column if @partition_column
-        params[:n] = @partition_number if @partition_number
-        params[:w] = @write_concurrency if @write_concurrency
-        params[:r] = @rotation_size if @rotation_size
-        params[:d] = nil if @delete_objects
-        params[:k] = @object_key_delimiter if @object_key_delimiter
-        if src_zone_offset = @src_zone_offset || ds.mysql_options[:src_zone_offset]
-          params[:S] = src_zone_offset
-        end
-        if dst_zone_offset = @dst_zone_offset || ds.mysql_options[:dst_zone_offset]
-          params[:T] = dst_zone_offset
-        end
-        params
-      end
-
-      OPTION_MAP = {
-        encoding: 'useUnicode=true&characterEncoding',
-        read_timeout: 'netTimeoutForStreamingResults',
-        connect_timeout: 'connectTimeout',
-        reconnect: 'autoReconnect',
-        collation: 'connectionCollation'
-      }
-
-      def connection_property
-        ds.mysql_options.map {|k, v| opt = OPTION_MAP[k] ; opt ? "#{opt}=#{v}" : nil }.compact.join('&')
-      end
-
-      def build_prefix(ds_prefix, pm_prefix)
-        ((ds_prefix || "") + "//" +  (pm_prefix.to_s || "")).gsub(%r<\A/>, '').gsub(%r<//>, '/')
-      end
-
-      def mys3dump_path
-        Pathname(__dir__).parent.parent + "libexec/mys3dump.jar"
-      end
-
-      def build_cmd(options)
-        (['java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }.map {|o| %Q("#{o}") }).join(" ")
-      end
-
-      def extract_exception_message(out)
-        out.lines do |line|
-          if /^.*Exception: (?<msg>.*)$/ =~ line
-            return msg
-          end
-        end
-      end
-    end
-
-    WRITER_CLASSES = {}
-
-    class JSONWriter
-      def JSONWriter.record_format
-        :hash
-      end
-
-      def initialize(f)
-        @f = f
-      end
-
-      def write_record(values)
-        @f.puts JSON.dump(values)
-      end
-    end
-    WRITER_CLASSES['json'] = JSONWriter
-
-    class TSVWriter
-      def TSVWriter.record_format
-        :array
-      end
-
-      def initialize(f)
-        @f = f
-      end
-
-      def write_record(values)
-        @f.puts values.join("\t")
-      end
-    end
-    WRITER_CLASSES['tsv'] = TSVWriter
-
-    class CSVWriter
-      def CSVWriter.record_format
-        :array
-      end
-
-      def initialize(f)
-        @csv = CSV.new(f)
-      end
-
-      def write_record(values)
-        @csv.add_row values
-      end
-    end
-    WRITER_CLASSES['csv'] = CSVWriter
-  end
-
-end

data/lib/bricolage/redisdatasource.rb

@@ -1,153 +0,0 @@
-require 'bricolage/datasource'
-require 'bricolage/commandutils'
-require 'redis'
-require 'json'
-
-module Bricolage
-
-  class RedisDataSource < DataSource
-    declare_type 'redis'
-
-    def initialize(host: 'localhost', port: 6380, **options)
-      @host = host
-      @port = port
-      @options = options
-    end
-
-    attr_reader :host
-    attr_reader :port
-    attr_reader :options
-
-    def new_task
-      RedisTask.new(self)
-    end
-
-    def open
-      client = Redis.new(host: @host, port: @port, **@options)
-      yield client
-    end
-  end
-
-  class RedisTask < DataSourceTask
-    def import(src, table, query, key_column, prefix, encode, expire: nil)
-      add Import.new(src, table, query, key_column, prefix, encode, expire)
-    end
-
-    class Import < Action
-      def initialize(src, table, query, key_column, prefix, encode, expire)
-        @src = src
-        @table = table
-        @query = query
-        @key_columns = key_column.split(',').map(&:strip)
-        @prefix = prefix || "#{@table.last.schema}_#{@table.last.name}_"
-        @encode = encode
-        @expire = expire
-      end
-
-      def bind(*args)
-        @query.bind(*args)
-      end
-
-      def source
-        @query.stripped_source
-      end
-
-      def run
-        logger = ds.logger
-        begin
-          logger.info "Key Pattern: #{@prefix}<#{@key_columns.join('_')}>"
-          logger.info "Encode: #{@encode}"
-          logger.info "Expire: #{@expire}"
-          ds.open {|client|
-            writer = RedisRowWriter.for_encode(@encode).new(client, @prefix, @key_columns)
-            import writer
-          }
-        rescue => ex
-          logger.exception ex
-          raise JobFailure, ex.message
-        end
-        JobResult.success
-      end
-
-      BATCH_SIZE = 5000
-
-      def import(writer)
-        count = 0
-        @src.query_batch(source, BATCH_SIZE) do |rs|
-          writer.pipelined {
-            rs.each do |row|
-              writer.write(row)
-              count += 1
-              ds.logger.info "transfered: #{count} rows" if count % 100_0000 == 0
-            end
-          }
-        end
-        ds.logger.info "all rows written: #{count} rows"
-      end
-    end
-  end
-
-  class RedisRowWriter
-    def RedisRowWriter.for_encode(encode)
-      case encode
-      when 'hash' then RedisHashRowWriter
-      when 'json' then RedisJSONRowWriter
-      else
-        raise ParameterError, "unsupported Redis encode: #{encode.inspect}"
-      end
-    end
-
-    def initialize(client, prefix, key_columns)
-      @client = client
-      @prefix = prefix
-      @key_columns = key_columns
-    end
-
-    attr_reader :prefix
-    attr_reader :write_count
-
-    def key(row)
-      @prefix + @key_columns.map {|k| row[k] }.join('_')
-    end
-
-    def value_columns(row)
-      r = row.dup
-      @key_columns.each do |key|
-        r.delete(key)
-      end
-      r.empty? ? {1 => 1} : r
-    end
-
-    def pipelined(&block)
-      @client.pipelined(&block)
-    end
-
-    def write(row)
-      key = key(row)
-      futures = do_write(key, value_columns(row))
-      futures.push @client.expire(key, @expire) if @expire
-      futures
-    end
-
-    def expire
-      @client.expire(key, @expire)
-    end
-  end
-
-  class RedisHashRowWriter < RedisRowWriter
-    def do_write(key, values)
-      # set a value for each key:field pair
-      values.map {|field, value|
-        @client.hset(key, field, value)
-      }
-    end
-  end
-
-  class RedisJSONRowWriter < RedisRowWriter
-    def do_write(key, values)
-      future = @client.set(key, JSON.generate(values))
-      [future]
-    end
-  end
-
-end

data/libexec/sqldump

@@ -1,9 +0,0 @@
-#!/bin/sh
-
-binary="$0.$(uname -s)"
-if ! [[ -x $binary ]]
-then
-    echo "$0: error: sqldump does not support $(uname -s)" 1>&2
-    exit 1
-fi
-exec "$binary" "$@"

data/test/home/config/streamingload.yml

@@ -1,8 +0,0 @@
-redshift-ds: sql
-event-queue-ds: sqs_strload_event
-task-queue-ds: sqs_strload_task
-url_patterns:
-  -
-    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
-    schema: "aamine"
-    table: "pv_log"