bricolage 5.8.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: afb5228b706d506b2b2f7b468f216f76cdd82dc9
+  data.tar.gz: 9c947de18235c29e7db2b3a7b54fa095f8bf2335
+SHA512:
+  metadata.gz: 3759c24eda01c02d78d0f3724c14254dc8dd8b687fc0bdd82869ec3db4ef9120b6cff2a5312533a04ef79d912fc9be96191edde3a81e0f79477783fc563245e9
+  data.tar.gz: 3873fafbb3f9dbbec2b2a38afbcbe4f2230ecd1f4467c421ceb37717a6be6723fa111a456c568adf03353471e9c3f5a89ecee9d1d649b02248b95d24858b36d0

data/README.md

@@ -0,0 +1,4 @@
+bricolage
+=========
+
+SQL batch framework

data/bin/bricolage

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+Bundler.require(:default) if defined?(Bundler)
+require 'bricolage/application'
+
+Bricolage::Application.main

data/bin/bricolage-jobnet

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+Bundler.require(:default) if defined?(Bundler)
+require 'bricolage/jobnetrunner'
+
+Bricolage::JobNetRunner.main

data/jobclass/create.rb

@@ -0,0 +1,21 @@
+JobClass.define('create') {
+  parameters {|params|
+    params.add SQLFileParam.new('table-def', 'PATH', 'CREATE TABLE file.')
+    params.add DestTableParam.new
+    params.add OptionalBoolParam.new('drop', 'DROP table before CREATE.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add DataSourceParam.new('sql')
+  }
+
+  declarations {|params|
+    params['table-def'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.drop_force_if params['drop']
+      task.exec params['table-def']
+      task.analyze_if params['analyze']
+    }
+  }
+}

data/jobclass/exec.rb

@@ -0,0 +1,17 @@
+JobClass.define('exec') {
+  parameters {|params|
+    params.add StringListParam.new('args', 'ARG', 'Command line arguments.')
+  }
+
+  script {|params, script|
+    script.task(params.generic_ds) {|task|
+      task.action(params['args'].join(' ')) {|ds|
+        ds.logger.info '[CMD] ' + params['args'].join(' ')
+        system(*params['args'])
+        st = $?
+        ds.logger.info "status=#{st.exitstatus}"
+        JobResult.for_bool(st.exitstatus == 0)
+      }
+    }
+  }
+}

data/jobclass/insert-delta.rb

@@ -0,0 +1,31 @@
+JobClass.define('insert-delta') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add StringParam.new('delete-cond', 'SQL_EXPR', 'DELETE condition.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add DataSourceParam.new('sql')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.transaction {
+        task.exec SQLStatement.delete_where(params['delete-cond'])
+        task.exec params['sql-file']
+      }
+      task.vacuum_if params['vacuum'], params['vacuum-sort']
+      task.analyze_if params['analyze']
+    }
+  }
+}

data/jobclass/insert.rb

@@ -0,0 +1,33 @@
+JobClass.define('insert') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.', optional: true)
+    params.add OptionalBoolParam.new('drop', 'DROP table before CREATE.')
+    params.add OptionalBoolParam.new('truncate', 'TRUNCATE table before SQL is executed.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add DataSourceParam.new('sql')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.truncate_if params['truncate']
+      task.drop_force_if params['drop']
+      task.exec params['table-def'] if params['table-def']
+      task.exec params['sql-file']
+      task.vacuum_if params['vacuum'], params['vacuum-sort']
+      task.analyze_if params['analyze']
+    }
+  }
+}

data/jobclass/load.rb

@@ -0,0 +1,39 @@
+require 'bricolage/psqldatasource'
+
+JobClass.define('load') {
+  parameters {|params|
+    params.add DataSourceParam.new('s3', 'src-ds', 'Input data source.')
+    params.add SrcFileParam.new
+    params.add DataSourceParam.new('sql', 'dest-ds', 'Target data source.')
+    params.add DestTableParam.new(optional: false)
+    params.add EnumParam.new('format', %w(tsv json), 'Data file format.', default: 'tsv')
+    params.add StringParam.new('jsonpath', 'PATH', 'jsonpath to specify columns of json format records', optional: true)
+    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+        optional: true, default: PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.', optional: true)
+    params.add OptionalBoolParam.new('drop', 'DROP table before CREATE.')
+    params.add OptionalBoolParam.new('truncate', 'TRUNCATE table before SQL is executed.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
+  }
+
+  declarations {|params|
+    Declarations.new("dest_table" => nil)
+  }
+
+  script {|params, script|
+    script.task(params['dest-ds']) {|task|
+      task.drop_force_if params['drop']
+      task.exec params['table-def'] if params['table-def']
+      task.truncate_if params['truncate']
+      task.load params['src-ds'], params['src-file'], params['dest-table'],
+          params['format'], params['jsonpath'], params['options']
+      task.vacuum_if params['vacuum'], params['vacuum-sort']
+      task.analyze_if params['analyze']
+      task.grant_if params['grant'], params['dest-table']
+    }
+  }
+}

data/jobclass/my-export.rb

@@ -0,0 +1,40 @@
+JobClass.define('my-export') {
+  parameters {|params|
+    params.add SQLFileParam.new(optional: true)
+    params.add DestFileParam.new
+    params.add SrcTableParam.new
+    params.add EnumParam.new('format', %w(json tsv csv), 'Target file format.', default: 'json')
+    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
+    params.add OptionalBoolParam.new('override', 'If true, clears target file.  Otherwise causes error.')
+    params.add OptionalBoolParam.new('sqldump', 'If true, clears use sqldump command to dump, only wheen usable.')
+    params.add DataSourceParam.new('mysql')
+  }
+
+  declarations {|params|
+    sql_statement(params).declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.export sql_statement(params),
+        path: params['dest-file'],
+        format: params['format'],
+        override: params['override'],
+        gzip: params['gzip'],
+        sqldump: params['sqldump']
+    }
+  }
+
+  def sql_statement(params)
+    if sql = params['sql-file']
+      sql
+    else
+      srcs = params['src-tables']
+      raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
+      src_table_var = srcs.keys.first
+      stmt = SQLStatement.for_string("select * from $#{src_table_var};")
+      stmt.declarations = Declarations.new({src_table_var => src_table_var})
+      stmt
+    end
+  end
+}

data/jobclass/my-migrate.rb

@@ -0,0 +1,103 @@
+require 'bricolage/psqldatasource'
+
+JobClass.define('my-migrate') {
+  parameters {|params|
+    # Export
+    params.add SrcTableParam.new(optional: false)
+    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
+    params.add DestFileParam.new('tmp-file', 'PATH', 'Temporary local file path.')
+    params.add OptionalBoolParam.new('sqldump', 'If true, use sqldump command to dump, only on available.', default: true)
+
+    # Put
+    params.add DestFileParam.new('s3-file', 'PATH', 'Temporary S3 file path.')
+    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
+    params.add OptionalBoolParam.new('override', 'If true, overwrite s3 target file.  Otherwise causes error.')
+
+    # Load
+    params.add DestTableParam.new(optional: false)
+    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
+    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+        optional: true, default: PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+
+    # Misc
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
+
+    # All
+    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
+    params.add OptionalBoolParam.new('put', 'Runs PUT task.')
+    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
+    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.', default: true)
+  }
+
+  declarations {|params|
+    decls = sql_statement(params).declarations
+    decls.declare 'dest-table', nil
+    decls
+  }
+
+  script {|params, script|
+    run_all = !params['export'] && !params['put'] && !params['load']
+
+    # Export
+    if params['export'] || run_all
+      script.task(params['src-ds']) {|task|
+        task.export sql_statement(params),
+          path: params['tmp-file'],
+          format: 'json',
+          override: true,
+          gzip: params['gzip'],
+          sqldump: params['sqldump']
+      }
+    end
+
+    # Put
+    if params['put'] || run_all
+      script.task(params['s3-ds']) {|task|
+        task.put params['tmp-file'], params['s3-file'], check_args: false
+      }
+    end
+
+    # Load
+    if params['load'] || run_all
+      script.task(params['dest-ds']) {|task|
+        prev_table = '${dest_table}_old'
+        work_table = '${dest_table}_wk'
+
+        # CREATE
+        task.drop_force prev_table
+        task.drop_force work_table
+        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+        # COPY
+        task.load params['s3-ds'], params['s3-file'], work_table,
+            'json', nil, params['options'].merge('gzip' => params['gzip'])
+
+        # VACUUM, ANALYZE, GRANT
+        task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
+        task.analyze_if params['analyze'], work_table
+        task.grant_if params['grant'], work_table
+
+        # RENAME
+        task.create_dummy_table '${dest_table}'
+        task.transaction {
+          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          task.rename_table work_table, params['dest-table'].name
+        }
+      }
+    end
+  }
+
+  def sql_statement(params)
+    srcs = params['src-tables']
+    raise ParameterError, "src-tables must be singleton when no sql-file is given" unless srcs.size == 1
+    src_table_var = srcs.keys.first
+    stmt = SQLStatement.for_string("select * from $#{src_table_var};")
+    stmt.declarations = Declarations.new({src_table_var => src_table_var})
+    stmt
+  end
+}

data/jobclass/noop.rb

@@ -0,0 +1,13 @@
+JobClass.define('noop') {
+  parameters {|params|
+    params.add OptionalBoolParam.new('failure', 'Finish with failure or not.')
+  }
+
+  script {|params, script|
+    script.task(params.generic_ds) {|task|
+      task.action('return !$failure') {
+        JobResult.for_bool(!params['failure'])
+      }
+    }
+  }
+}

data/jobclass/rebuild-drop.rb

@@ -0,0 +1,37 @@
+JobClass.define('rebuild-drop') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: on, to)')
+    params.add DataSourceParam.new('sql')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      # CREATE
+      task.drop_force params['dest-table']
+      task.exec params['table-def']
+
+      # INSERT
+      task.exec params['sql-file']
+
+      # VACUUM, ANALYZE, GRANT
+      task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
+      task.analyze_if params['analyze'], params['dest-table']
+      task.grant_if params['grant'], params['dest-table']
+    }
+  }
+}

data/jobclass/rebuild-rename.rb

@@ -0,0 +1,49 @@
+JobClass.define('rebuild-rename') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
+    params.add DataSourceParam.new('sql')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      prev_table = '${dest_table}_old'
+      work_table = '${dest_table}_wk'
+
+      # CREATE
+      task.drop_force prev_table
+      task.drop_force work_table
+      task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+      # INSERT
+      task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+      # VACUUM, ANALYZE, GRANT
+      task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
+      task.analyze_if params['analyze'], work_table
+      task.grant_if params['grant'], work_table
+
+      # RENAME
+      task.create_dummy_table '$dest_table'
+      task.transaction {
+        dest_table = params['dest-table']
+        task.rename_table dest_table.to_s, "#{dest_table.name}_old"
+        task.rename_table "#{dest_table}_wk", dest_table.name
+      }
+    }
+  }
+}

data/jobclass/s3-put.rb

@@ -0,0 +1,19 @@
+JobClass.define('s3-put') {
+  parameters {|params|
+    params.add DestFileParam.new
+    params.add SrcFileParam.new
+    params.add OptionalBoolParam.new('remove', 'Removes source files after PUT is succeeded.')
+    params.add DataSourceParam.new('s3')
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.put params['src-file'], params['dest-file']
+    }
+    if params['remove']
+      script.task(params.file_ds) {|task|
+        task.remove params['src-file']
+      }
+    end
+  }
+}

data/jobclass/sql.rb

@@ -0,0 +1,29 @@
+JobClass.define('sql') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add OptionalBoolParam.new('truncate', 'TRUNCATE table before SQL is executed.')
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add DataSourceParam.new('sql')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.truncate_if params['truncate']
+      task.exec params['sql-file']
+      task.vacuum_if params['vacuum'], params['vacuum-sort']
+      task.analyze_if params['analyze']
+    }
+  }
+}

data/jobclass/td-delete.rb

@@ -0,0 +1,20 @@
+JobClass.define('td-delete') {
+  parameters {|params|
+    params.add DestTableParam.new(optional: false)
+    params.add DataSourceParam.new('td')
+    params.add DateParam.new('from', 'DATE', 'Start date of logs to delete (%Y-%m-%d).')
+    params.add DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
+  }
+
+  declarations {|params|
+    Declarations.new("dest_table" => nil)
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.delete params['dest-table'],
+        from: params['from'],
+        to: params['to']
+    }
+  }
+}

data/jobclass/td-export.rb

@@ -0,0 +1,30 @@
+JobClass.define('td-export') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DestFileParam.new
+    params.add SrcTableParam.new
+    params.add StringParam.new('src-database', 'NAME', 'Source TD database name.', optional: true)
+    params.add EnumParam.new('format', %w(msgpack tsv csv json), 'Target file format.', default: 'msgpack')
+    params.add OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.')
+    params.add OptionalBoolParam.new('override', 'If true, clears target file.  Otherwise causes error.')
+    params.add DataSourceParam.new('td')
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.export params['sql-file'],
+        path: params['dest-file'],
+        format: params['format'],
+        gzip: params['gzip'],
+        override: params['override']
+    }
+  }
+}

data/jobclass/unload.rb

@@ -0,0 +1,30 @@
+require 'bricolage/psqldatasource'
+
+JobClass.define('unload') {
+  parameters {|params|
+    params.add SQLFileParam.new
+    params.add DataSourceParam.new('sql', 'src-ds', 'Input data source (sql).')
+    params.add SrcTableParam.new(optional: false)
+    params.add DataSourceParam.new('s3', 'dest-ds', 'Target data source (s3).')
+    params.add DestFileParam.new
+    params.add EnumParam.new('format', %w(tsv), 'Data file format.', default: 'tsv')
+    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+        optional: true, default: PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+  }
+
+  parameters_filter {|job|
+    job.provide_sql_file_by_job_id
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['src-ds']) {|task|
+      task.unload params['sql-file'], params['dest-ds'], params['dest-file'],
+          params['format'], params['options']
+    }
+  }
+}

data/jobclass/wait-file.rb

@@ -0,0 +1,48 @@
+JobClass.define('wait-file') {
+  parameters {|params|
+    params.add DestFileParam.new
+    params.add StringParam.new('condition-expr', 'RUBY_EXPR', 'Wait condition expression (in Ruby).  (default: check file existance)', optional: true)
+    params.add StringParam.new('max-wait-minutes', 'NUM', 'Max waiting minutes.')
+  }
+
+  script {|params, script|
+    script.task(params.generic_ds) {|task|
+      dest_file = params['dest-file']
+      max_wait_minutes = params['max-wait-minutes'].to_i
+
+      default_expr = "File.exist?('#{dest_file}')"
+      condition_expr = params['condition-expr'] || default_expr
+
+      task.action("wait #{dest_file} (condition: #{condition_expr})") {|ds|
+        wait_file_condition(dest_file, condition_expr, max_wait_minutes, ds.logger)
+      }
+    }
+  }
+
+  REPORT_INTERVAL = 90   # 15 min
+  WAIT_INTERVAL = 5
+
+  def wait_file_condition(dest_file, condition_expr, max_wait_minutes, logger)
+    logger.info "waiting file by { #{condition_expr} }"
+
+    # DO NOT CHANGE these variables name; they are condition expression interface.
+    start_time = Time.now
+    wait_limit = start_time + max_wait_minutes * 60
+    last_log_time = start_time
+
+    until eval(condition_expr)
+      now = Time.now
+      if now > wait_limit
+        logger.error "exceeded wait limit (#{max_wait_minutes} minutes); abort"
+        return JobResult.for_bool(false)
+      end
+      if now - last_log_time > REPORT_INTERVAL
+        logger.info "waiting..."
+        last_log_time = Time.now
+      end
+      sleep WAIT_INTERVAL
+    end
+    logger.info "condition fullfilled"
+    JobResult.for_bool(true)
+  end
+}