bricolage 5.15.2 → 5.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5acaeef75f7cfc4d74814c7f036d7e7a187f728f
-  data.tar.gz: 4ef05fb1c2b2e292a51f2c6331ceaa9a12d445e9
+  metadata.gz: 2660120a7623731fb14dac13b2efa0582e535265
+  data.tar.gz: 859704e487ea718d76bc4567d4e46451acad39db
 SHA512:
-  metadata.gz: 4569b393900821b004f84b341e9e5ce87298c6e83d8fb49719960448bac8ad6d8d59cb29ce49acc7e5e5047a0a05324170e76957928ed7ea102944835a733c09
-  data.tar.gz: c17dddd3e8a46e246250e746a02e9f8eaa8f335a3cbfe245ab709aa35c8e59caf1629f0636b0928bfc285f60a84968456ae7556132386a71854634e546b09054
+  metadata.gz: a68f420cbfbc54ad9596200c3dc00204e4c0105b68051e06eeb256c7096bc2334cd9dceb678c3b997efe09dca22103ea6545845b76a8824c6eea69fb4779d9cc
+  data.tar.gz: a39835ca04114f75c778c9c129b8ab7e08ea8d9601dcd45504c2fbac84bb5ac286914cc55db308ad49cef94d55de61d58682e23537a0d595a8916e003ded9891

data/jobclass/my-import.rb

@@ -0,0 +1,79 @@
+require 'bricolage/psqldatasource'
+require 'bricolage/mysqldatasource'
+
+JobClass.define('my-import') {
+  parameters {|params|
+    # S3Export
+    params.add SrcTableParam.new(optional: false)
+    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
+    params.add SQLFileParam.new(optional: true)
+    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
+    params.add DestFileParam.new('s3-prefix', 'PREFIX', 'Temporary S3 prefix.')
+    params.add KeyValuePairsParam.new('dump-options', 'KEY:VALUE', 'dump options.', optional: true)
+
+    # Load
+    params.add DestTableParam.new(optional: false)
+    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
+    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+        optional: true, default: PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+
+    # Misc
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
+
+    # All
+    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
+    params.add OptionalBoolParam.new('put', 'Runs PUT task.')
+    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
+    params.add OptionalBoolParam.new('gzip', 'Compress Temporary files.')
+  }
+
+  script {|params, script|
+    run_all = !params['export'] && !params['put'] && !params['load']
+
+    # S3Export
+    if params['export'] || run_all
+      script.task(params['src-ds']) {|task|
+        task.s3export params['src-tables'].keys.first,
+                      params['sql-file'],
+                      params['s3-ds'],
+                      params['s3-prefix'],
+                      params['gzip'],
+                      dump_options: params['dump-options']
+      }
+    end
+
+    # Load
+    if params['load'] || run_all
+      script.task(params['dest-ds']) {|task|
+        prev_table = '${dest_table}_old'
+        work_table = '${dest_table}_wk'
+
+        # CREATE
+        task.drop_force prev_table
+        task.drop_force work_table
+        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+        # COPY
+        task.load params['s3-ds'], params['s3-prefix'], work_table,
+            'json', nil, params['options'].merge('gzip' => params['gzip'])
+
+        # VACUUM, ANALYZE, GRANT
+        task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
+        task.analyze_if params['analyze'], work_table
+        task.grant_if params['grant'], work_table
+
+        # RENAME
+        task.create_dummy_table '${dest_table}'
+        task.transaction {
+          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          task.rename_table work_table, params['dest-table'].name
+        }
+      }
+    end
+  }
+}

data/jobclass/streaming_load.rb

@@ -261,21 +261,13 @@ class StreamingLoadJobClass < RubyJobClass
 
     def create_tmp_log_table(conn, log_url)
       target_table = log_table_wk
-      execute_update conn, "create table #{target_table} (like #{@log_table});"
+      execute_update conn, "truncate #{target_table};"
       execute_update conn, load_log_copy_stmt(target_table, log_url, @src.credential_string)
-      begin
-        yield target_table
-      ensure
-        begin
-          execute_update conn, "drop table #{target_table}"
-        rescue PostgreSQLException => ex
-          @logger.error ex.message + " (ignored)"
-        end
-      end
+      yield target_table
     end
 
     def log_table_wk
-      "#{@log_table}_tmp#{Process.pid}"
+      "#{@log_table}_wk"
     end
 
     def load_log_copy_stmt(target_table, log_url, credential_string)

data/lib/bricolage/mysqldatasource.rb

@@ -15,6 +15,8 @@ module Bricolage
       @client = nil
     end
 
+    attr_reader :mysql_options
+
     def host
       @mysql_options[:host]
     end
@@ -172,6 +174,104 @@ module Bricolage
       end
     end
 
+    def s3export(table, stmt, s3ds, prefix, gzip, dump_options)
+      options = dump_options.nil? ? {} : dump_options[:dump_options]
+      add S3Export.new(table, stmt, s3ds, prefix, gzip: gzip,
+        format: options['format'],
+        partition_column: options['partition_column'],
+        partition_number: options['partition_number'],
+        write_concurrency: options['write_concurrency'],
+        rotation_size: options['rotation_size'],
+        delete_objects: options['delete_objects'],
+        object_key_delimiter: options['object_key_delimiter'])
+    end
+
+    class S3Export < Action
+
+      def initialize(table, stmt, s3ds, prefix, gzip: true,
+         format: "json",
+         partition_column: nil,
+         partition_number: 4,
+         write_concurrency: 4,
+         rotation_size: nil,
+         delete_objects: false,
+         object_key_delimiter: nil)
+        @table = table
+        @statement = stmt
+        @s3ds = s3ds
+        @prefix = build_prefix @s3ds.prefix, prefix
+        @format = format
+        @gzip = gzip
+        @partition_column = partition_column
+        @partition_number = partition_number
+        @write_concurrency = write_concurrency
+        @rotation_size = rotation_size
+        @delete_objects = delete_objects
+        @object_key_delimiter = object_key_delimiter
+      end
+
+      def run
+        s3export
+        JobResult.success
+      end
+
+      def bind(*args)
+        @statement.bind(*args) if @statement
+      end
+
+      def s3export
+        cmd = build_cmd(command_parameters)
+        ds.logger.info '[CMD] ' + cmd.join(' ')
+        Open3.popen2e(*cmd) do |input, output, thread|
+          input.close
+          output.each do |line|
+            puts line
+          end
+          unless thread.value.success?
+            raise JobFailure, "#{cmd.join(' ')} failed (status #{thread.value.to_i})"
+          end
+        end
+      end
+
+      def command_parameters
+        params = {jar: mys3dump_path.to_s, h: ds.host, P: ds.port.to_s, D: ds.database, u: ds.username, p: ds.password, o: connection_property, t: @table,
+                   'Daws.accessKeyId': @s3ds.access_key, 'Daws.secretKey': @s3ds.secret_key, b: @s3ds.bucket.name, x: @prefix}
+        params[:q] = @statement.stripped_source.chop if @statement
+        params[:f] = @format if @format
+        params[:C] = nil if @gzip
+        params[:c] = @partition_column if @partition_column
+        params[:n] = @partition_number if @partition_number
+        params[:w] = @write_concurrency if @write_concurrency
+        params[:r] = @rotation_size if @rotation_size
+        params[:d] = nil if @delete_objects
+        params[:k] = @object_key_delimiter if @object_key_delimiter
+        params
+      end
+
+      OPTION_MAP = {
+        encoding: 'useUnicode=true&characterEncoding',
+        read_timeout: 'netTimeoutForStreamingResults',
+        connect_timeout: 'connectTimeout',
+        reconnect: 'autoReconnect'
+      }
+
+      def connection_property
+        ds.mysql_options.map {|k, v| opt = OPTION_MAP[k] ; opt ? "#{opt}=#{v}" : nil }.compact.join('&')
+      end
+
+      def build_prefix(ds_prefix, pm_prefix)
+        ((ds_prefix || "") + "//" +  (pm_prefix.to_s || "")).gsub(%r<\A/>, '').gsub(%r<//>, '/')
+      end
+
+      def mys3dump_path
+        Pathname(__dir__).parent.parent + "libexec/mys3dump.jar"
+      end
+
+      def build_cmd(options)
+        ['java'] + options.flat_map {|k, v| v ? ["-#{k}", v.to_s] : ["-#{k}"] }
+      end
+    end
+
     WRITER_CLASSES = {}
 
     class JSONWriter

data/lib/bricolage/postgresconnection.rb

@@ -1,5 +1,4 @@
 require 'bricolage/exception'
-require 'securerandom'
 require 'pg'
 
 module Bricolage
@@ -11,14 +10,13 @@ module Bricolage
       @connection = connection
       @ds = ds
       @logger = logger
-      @cursor = nil
     end
 
     def source
       @connection
     end
 
-    def execute(query)
+    def execute_update(query)
       @logger.info "[#{@ds.name}] #{query}"
       log_elapsed_time {
         rs = @connection.exec(query)
@@ -30,42 +28,97 @@ module Bricolage
       raise PostgreSQLException.wrap(ex)
     end
 
+    alias execute execute_update
+    alias update execute_update
+
+    def select(table, &block)
+      execute_query("select * from #{table}", &block)
+    end
+
     def execute_query(query, &block)
       @logger.info "[#{@ds.name}] #{query}"
-      exec(query, &block)
-    end
-
-    def execute_query_with_cursor(query, fetch_size, cursor, &block)
-      raise "Begin transaction before invoking this method" unless in_transaction?
-      if @cursor.nil?
-        @cursor = cursor || (0...32).map { alphabets[rand(alphabets.length)] }.join
-        declare_cursor = "declare #{@cursor} cursor for #{query}"
-        @logger.info "[#{@ds.name}] #{declare_cursor}"
-        @connection.exec(declare_cursor)
-      elsif !@cursor.nil? && cursor.nil?
-        raise "Cannot declare new cursor. Cursor in use: #{@cursor}"
-      elsif @cursor != cursor
-        raise "Specified cursor not exists. Specified: #{cursor}, Current: #{@cursor}"
-      end
-      fetch = "fetch #{fetch_size} in #{@cursor}"
-      @logger.info "[#{@ds.name}] #{fetch}" if cursor.nil?
-      yield @connection.exec(fetch)
-      return @cursor
+      rs = log_elapsed_time {
+        @connection.exec(query)
+      }
+      result = yield rs
+      rs.clear
+      result
+    rescue PG::Error => ex
+      raise PostgreSQLException.wrap(ex)
     end
 
-    def clear_cursor
-      @cursor = nil
+    alias query execute_query
+
+    def query_batch(query, batch_size = 5000, &block)
+      open_cursor(query) {|cur|
+        cur.each_result_set(batch_size, &block)
+      }
     end
 
-    def alphabets
-      @alphabets = @alphabets || [('a'...'z'), ('A'...'Z')].map { |i| i.to_a }.flatten
+    def streaming_execute_query(query, &block)
+      @logger.info "[#{@ds.name}] #{query}"
+      log_elapsed_time {
+        @connection.send_query(query)
+      }
+      @connection.set_single_row_mode
+      while rs = @connection.get_result
+        begin
+          rs.check
+          yield rs
+        ensure
+          rs.clear
+        end
+      end
+    rescue PG::Error => ex
+      raise PostgreSQLException.wrap(ex)
     end
 
     def in_transaction?
       @connection.transaction_status == PG::Constants::PQTRANS_INTRANS
     end
 
-    alias update execute
+    def transaction
+      execute 'begin transaction'
+      yield
+      execute 'commit'
+    end
+
+    def open_cursor(query, name = nil, &block)
+      unless in_transaction?
+        transaction {
+          return open_cursor(query, &block)
+        }
+      end
+      name ||= make_unique_cursor_name
+      execute "declare #{name} cursor for #{query}"
+      yield Cursor.new(name, self, @logger)
+    end
+
+    Thread.current['bricolage_cursor_seq'] = 0
+
+    def make_unique_cursor_name
+      seq = (Thread.current['bricolage_cursor_seq'] += 1)
+      "cur_bric_#{$$}_#{'%X' % Thread.current.object_id}_#{seq}"
+    end
+
+    class Cursor
+      def initialize(name, conn, logger)
+        @name = name
+        @conn = conn
+        @logger = logger
+      end
+
+      attr_reader :name
+
+      def each_result_set(fetch_size = 5000)
+        while true
+          @conn.execute_query("fetch #{fetch_size} in #{@name}") {|rs|
+            return if rs.values.empty?
+            yield rs
+          }
+        end
+      end
+    end
 
     def drop_table(name)
       execute "drop table #{name} cascade;"
@@ -77,12 +130,6 @@ module Bricolage
       @logger.error err.message
     end
 
-    def select(table, &block)
-      query = "select * from #{table}"
-      @logger.info "[#{@ds.name}] #{query}"
-      exec(query, &block)
-    end
-
     def vacuum(table)
       execute "vacuum #{table};"
     end
@@ -105,19 +152,6 @@ module Bricolage
       t = e - b
       @logger.info "#{'%.1f' % t} secs"
     end
-    
-    def exec(query, &block)
-      @connection.send_query(query)
-      @connection.set_single_row_mode
-      loop do
-        rs = @connection.get_result or break
-        begin
-          rs.check
-          yield rs
-        ensure
-          rs.clear
-        end
-      end
-    end
   end
+
 end

data/lib/bricolage/psqldatasource.rb

@@ -102,34 +102,8 @@ module Bricolage
       conn.close if conn
     end
 
-    def query(query)
-      open {|conn| conn.execute(query) }
-    end
-    
-    def execute_query(query, &block)
-      open {|conn| conn.execute_query(query, &block) }
-    end
-
-    def cursor_transaction(&block)
-      raise "Cursor in use" if cursor_in_transaction?
-      conn = PG::Connection.open(host: @host, port: @port, dbname: @database, user: @user, password: password)
-      @cur_conn = PostgresConnection.new(conn, self, logger)
-      @cur_conn.execute("begin transaction")
-      yield
-    ensure
-      @cur_conn.execute("commit") if cursor_in_transaction?
-      @cur_conn.clear_cursor if @cur_conn
-      @cur_conn = nil
-      conn.close if conn
-    end
-
-    def cursor_execute_query(query, fetch_size: 10000, cursor: nil, &block)
-      raise "Begin transaction before invoking this method" unless cursor_in_transaction?
-      @cur_conn.execute_query_with_cursor(query, fetch_size, cursor, &block)
-    end
-
-    def cursor_in_transaction?
-      @cur_conn && @cur_conn.in_transaction?
+    def query_batch(query, batch_size = 5000, &block)
+      open {|conn| conn.query_batch(query, batch_size, &block) }
     end
 
     def drop_table(name)

data/lib/bricolage/redisdatasource.rb

@@ -22,8 +22,9 @@ module Bricolage
       RedisTask.new(self)
     end
 
-    def client
-      @client = @client || Redis.new(:host => @host, :port => @port, **@options)
+    def open
+      client = Redis.new(host: @host, port: @port, **@options)
+      yield client
     end
   end
 
@@ -38,11 +39,9 @@ module Bricolage
         @table = table
         @query = query
         @key_columns = key_column.split(',').map(&:strip)
-        @prefix = prefix
+        @prefix = prefix || "#{@table.last.schema}_#{@table.last.name}_"
         @encode = encode
         @expire = expire
-        @read_count = 0
-        @write_count = 0
       end
 
       def bind(*args)
@@ -53,84 +52,102 @@ module Bricolage
         @query.stripped_source
       end
 
-      def prefix
-        @prefix = @prefix || "#{@table.last.schema}_#{@table.last.name}_"
+      def run
+        logger = ds.logger
+        begin
+          logger.info "Key Pattern: #{@prefix}<#{@key_columns.join('_')}>"
+          logger.info "Encode: #{@encode}"
+          logger.info "Expire: #{@expire}"
+          ds.open {|client|
+            writer = RedisRowWriter.for_encode(@encode).new(client, @prefix, @key_columns)
+            import writer
+          }
+        rescue => ex
+          logger.exception ex
+          raise JobFailure, ex.message
+        end
+        JobResult.success
       end
 
-      def import
-        @src.cursor_transaction {
-          read_count = 0
-          loop do
-            ds.client.pipelined do 
-              read_count = read_row do |row|
-                write_row row
-              end
-            end
-            break if read_count == 0
-          end
-        }
-        @cursor = nil
-      end
+      BATCH_SIZE = 5000
 
-      def read_row
-        rs_size = 0
-        @cursor = @src.cursor_execute_query(source, cursor: @cursor) do |rs|
-          rs_size = rs.values.size
-          break if rs_size == 0
-          rs.each do |row|
-            yield row
-            @read_count += 1
-            ds.logger.info "Rows read: #{@read_count}" if @read_count % 100000 == 0
-          end
+      def import(writer)
+        count = 0
+        @src.query_batch(source, BATCH_SIZE) do |rs|
+          writer.pipelined {
+            rs.each do |row|
+              writer.write(row)
+              count += 1
+              ds.logger.info "transfered: #{count} rows" if count % 100_0000 == 0
+            end
+          }
         end
-        rs_size
+        ds.logger.info "all rows written: #{count} rows"
       end
+    end
+  end
 
-      def write_row(row)
-        key = key(row)
-        data = delete_key_columns(row)
-        f = []
-        case @encode
-        when 'hash'
-          # set a value for each key:field pair
-          data.each do |field,value|
-            f.push ds.client.hset(key, field, value)
-          end
-        when 'json'
-          f.push ds.client.set(key, JSON.generate(data))
-        else
-          raise %Q("encode: #{type}" is not supported)
-        end
-        f.push ds.client.expire(key, @expire) if @expire
-        @write_count += 1
-        return f
+  class RedisRowWriter
+    def RedisRowWriter.for_encode(encode)
+      case encode
+      when 'hash' then RedisHashRowWriter
+      when 'json' then RedisJSONRowWriter
+      else
+        raise ParameterError, "unsupported Redis encode: #{encode.inspect}"
       end
+    end
 
-      def delete_key_columns(row)
-        r = row.dup
-        @key_columns.each do |key|
-          r.delete(key)
-        end
-        r.empty? ? {1 => 1} : r
-      end
+    def initialize(client, prefix, key_columns)
+      @client = client
+      @prefix = prefix
+      @key_columns = key_columns
+    end
 
-      def key(row)
-        prefix + @key_columns.map {|k| row[k]}.join('_')
-      end
+    attr_reader :prefix
+    attr_reader :write_count
 
-      def run
-        begin
-          ds.logger.info "Key Pattern: #{prefix}<#{@key_columns.join('_')}>"
-          ds.logger.info "Encode: #{@encode}"
-          ds.logger.info "Expire: #{@expire}"
-          import
-        rescue => ex
-          ds.logger.error ex.backtrace.join("\n")
-          raise JobFailure, ex.message
-        end
-        ds.logger.info "Rows written: #{@write_count}"
-        JobResult.success
+    def key(row)
+      @prefix + @key_columns.map {|k| row[k] }.join('_')
+    end
+
+    def value_columns(row)
+      r = row.dup
+      @key_columns.each do |key|
+        r.delete(key)
       end
+      r.empty? ? {1 => 1} : r
+    end
+
+    def pipelined(&block)
+      @client.pipelined(&block)
+    end
+
+    def write(row)
+      key = key(row)
+      futures = do_write(key, value_columns(row))
+      futures.push @client.expire(key, @expire) if @expire
+      futures
+    end
+
+    def expire
+      @client.expire(key, @expire)
+    end
+  end
+
+  class RedisHashRowWriter < RedisRowWriter
+    def do_write(key, values)
+      # set a value for each key:field pair
+      values.map {|field, value|
+        @client.hset(key, field, value)
+      }
     end
   end
+
+  class RedisJSONRowWriter < RedisRowWriter
+    def do_write(key, values)
+      future = @client.set(key, JSON.generate(values))
+      [future]
+    end
+  end
+
 end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.15.2'
+  VERSION = '5.16.0'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.15.2)
+    bricolage (5.16.0)
       aws-sdk (~> 2)
       mysql2
       pg

data/test/home/config/development/database.yml

@@ -1,7 +1,7 @@
 sql: &redshift_parameters
   type: psql
-  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
-  port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
+  host: <%= ENV['RSHOST'] || 'dwh.ckpd.co' %>
+  port: <%= ENV['RSPORT'] || 5439 %>
   database: production
   username: <%= ENV['RSUSER'] || 'tabemirudev' %>
   pgpass: <%= user_home_relative_path '.pgpass' %>

data/test/home/config.yml

@@ -0,0 +1,13 @@
+event_queue:
+  sqs_url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
+  visibility_timeout: 60
+
+load_queue:
+  sqs_url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/XXXXXXXXXXXXXX"
+  visibility_timeout: 60
+
+url_patterns:
+  -
+    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
+    schema: "aamine"
+    table: "pv_log"

data/test/home/subsys/my-import.job

@@ -0,0 +1,32 @@
+class: my-import
+src-ds: mysql
+src-tables:
+    users: main.users
+s3-ds: s3
+s3-prefix: shimpeko/test-abc-
+gzip: true
+dump-options:
+    partition_column: id
+    partition_number: 8
+    write_concurrency: 16
+    rotation_size: 16000000
+    delete_objects: true
+dest-ds: sql
+dest-table: $test_schema.users
+table-def: users.ct
+options:
+    statupdate: false
+    compupdate: false
+    maxerror: 3
+    acceptinvchars: " "
+    #trimblanks: true
+    #truncatecolumns: true
+    ## datetime
+    #acceptanydate: true
+    #dateformat: "auto"
+    #timeformat: "auto"
+vacuum-sort: true
+analyze: true
+grant:
+    privilege: select
+    to: "$test_group"

data/test/home/subsys/users.ct

@@ -0,0 +1,13 @@
+--dest-table: users
+
+create table $dest_table
+( id int encode delta
+, user_name varchar(1000) encode lzo
+, birthday date encode lzo
+, zip varchar(255) encode lzo
+, created_at timestamp encode lzo
+, updated_at timestamp encode lzo
+)
+distkey (id)
+sortkey (id)
+;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.15.2
+  version: 5.16.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2016-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -126,6 +126,7 @@ files:
 - jobclass/insert.rb
 - jobclass/load.rb
 - jobclass/my-export.rb
+- jobclass/my-import.rb
 - jobclass/my-migrate.rb
 - jobclass/noop.rb
 - jobclass/rebuild-drop.rb
@@ -173,12 +174,14 @@ files:
 - lib/bricolage/variables.rb
 - lib/bricolage/version.rb
 - libexec/create-lockfile
+- libexec/mys3dump.jar
 - libexec/sqldump
 - libexec/sqldump.Darwin
 - libexec/sqldump.Linux
 - test/all.rb
 - test/home/Gemfile
 - test/home/Gemfile.lock
+- test/home/config.yml
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
 - test/home/config/development/variable.yml
@@ -202,6 +205,7 @@ files:
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
 - test/home/subsys/migrate.job
+- test/home/subsys/my-import.job
 - test/home/subsys/net1.jobnet
 - test/home/subsys/net2.jobnet
 - test/home/subsys/put.job
@@ -216,6 +220,7 @@ files:
 - test/home/subsys/some_view-cv.sql.job
 - test/home/subsys/unified.jobnet
 - test/home/subsys/unified.sql.job
+- test/home/subsys/users.ct
 - test/home/subsys/variable.yml
 - test/test_c_streaming_load.rb
 - test/test_filesystem.rb