bricolage 5.27.2 → 5.28.0

data/test/home/subsys/my-import-delta.sql.job

@@ -1,40 +0,0 @@
-/*
-class: my-import-delta
-src-ds: mysql_summary
-src-tables:
-    item_pv: item_pv
-s3-ds: s3
-s3-prefix: bricolage/my-import-delta/$data_date/data.
-gzip: true
-dump-options:
-    partition_column: id
-    delete_objects: true
-dest-ds: sql
-dest-table: $test_schema.item_pv_acc
-delete-cond: "data_date = '$data_date'"
-options:
-    statupdate: false
-    compupdate: false
-    maxerror: 0
-    #acceptinvchars: " "
-    #trimblanks: true
-    #truncatecolumns: true
-    ## datetime
-    #acceptanydate: true
-    #dateformat: "auto"
-    #timeformat: "auto"
-#vacuum-sort: true
-analyze: true
-*/
-
-select
-    data_date
-    , item_id
-    , pv
-    , uu
-from
-    $item_pv
-where
-    date = '$data_date'
-    and (@PARTITION_CONDITION@)
-;

data/test/home/subsys/my-import.job

@@ -1,31 +0,0 @@
-class: my-import
-src-ds: mysql
-src-tables:
-    users: main.users
-s3-ds: s3
-s3-prefix: $test_schema/test-abc-
-gzip: true
-dump-options:
-    src_zone_offset: "+00:00"
-    dst_zone_offset: "+09:00"
-    delete_objects: true
-dest-ds: sql
-dest-table: $test_schema.users
-table-def: users.ct
-#no-backup: true
-options:
-    statupdate: false
-    compupdate: false
-    maxerror: 3
-    acceptinvchars: " "
-    #trimblanks: true
-    #truncatecolumns: true
-    ## datetime
-    #acceptanydate: true
-    #dateformat: "auto"
-    #timeformat: "auto"
-#vacuum-sort: true
-analyze: true
-grant:
-    privilege: select
-    to: "$test_group"

data/test/home/subsys/net1.jobnet

@@ -1,3 +0,0 @@
-job1
--> *net2
--> job4

data/test/home/subsys/net2.jobnet

@@ -1,2 +0,0 @@
-job2
--> job3

data/test/home/subsys/put.job

@@ -1,3 +0,0 @@
-class: s3-put
-dest-file: test-dest.txt
-src-file: $bricolage_home/data/test.txt

data/test/home/subsys/raw-vacuum.jobnet

@@ -1 +0,0 @@
-raw-vacuum

data/test/home/subsys/raw-vacuum.sql.job

@@ -1,7 +0,0 @@
-/*
-class: adhoc
-*/
-
-<%= psql_serialize_vacuum_begin %>
-vacuum s;
-<%= psql_serialize_vacuum_end %>

data/test/home/subsys/rebuild.sql.job

@@ -1,14 +0,0 @@
-/*
-    class: rebuild-drop
-    table-def: d.ct
-    dest-table: d
-    src-tables:
-        s: s
-    analyze: true
-    vacuum-sort: true
-    #grant: false
-*/
-
-insert into $dest_table
-select * from $s
-;

data/test/home/subsys/redis_export.job

@@ -1,9 +0,0 @@
-class: redis-export
-src-ds: sql
-src-tables:
-    user_cook_recipes: $test_schema.user_cook_recipes
-dest-ds: redis
-prefix: ha_ucp_
-key-column: user_id, recipe_id
-encode: hash
-expire: 604800

data/test/home/subsys/redis_export2.job

@@ -1,8 +0,0 @@
-class: redis-export
-src-ds: sql
-src-tables:
-    user_cook_recipes: $test_schema.user_cook_recipes
-dest-ds: redis
-key-column: user_id
-encode: json
-expire: 60

data/test/home/subsys/search_backends.ct

@@ -1,13 +0,0 @@
---dest-table: search_backends
-
-create table $dest_table
-( id int
-, role varchar(10)
-, group_id smallint
-, index_type varchar(32)
-, config_key varchar(32)
-, updated_at timestamp
-)
-distkey (id)
-sortkey (id)
-;

data/test/home/subsys/separated.job

@@ -1 +0,0 @@
-class: sql

data/test/home/subsys/separated.sql

@@ -1 +0,0 @@
-select 1;

data/test/home/subsys/some_view-cv.sql.job

@@ -1,11 +0,0 @@
-/*
-class: createview
-drop: true
-dest-table: some_view
-src-tables:
-    d: d
-*/
-
-create view $dest_table as
-select * from $d
-;

data/test/home/subsys/unified.jobnet

@@ -1 +0,0 @@
-unified

data/test/home/subsys/unified.sql.job

@@ -1,5 +0,0 @@
-/*
-    class: sql
-*/
-
-select 1;

data/test/home/subsys/users.ct

@@ -1,13 +0,0 @@
---dest-table: users
-
-create table $dest_table
-( id int encode delta
-, user_name varchar(1000) encode lzo
-, birthday date encode lzo
-, zip varchar(255) encode lzo
-, created_at timestamp encode lzo
-, updated_at timestamp encode lzo
-)
-distkey (id)
-sortkey (id)
-;

data/test/home/subsys/variable.yml

@@ -1,6 +0,0 @@
-data_date: "<%= (Date.today - 1).strftime('%Y-%m-%d') %>"
-defaults:
-    analyze: true
-    grant:
-        privilege: "select"
-        to: "$test_group"

data/test/test_c_streaming_load.rb

@@ -1,13 +0,0 @@
-require 'test/unit'
-require 'bricolage/jobclass'
-Bricolage::JobClass.get('streaming_load')
-
-module Bricolage
-  class TestStreamingLoadJobClass_S3Queue < Test::Unit::TestCase
-    def test_compile_name_pattern
-      q = StreamingLoadJobClass::S3Queue.new(data_source: nil, queue_path: nil, ctl_prefix: nil, persistent_path: nil, file_name: nil, logger: nil)
-      re = q.compile_name_pattern("%*%Y%m%d-%H%M_%Q.gz")
-      assert_equal /\A[^\/]*(?<year>\d{4})(?<month>\d{2})(?<day>\d{2})\-(?<hour>\d{2})(?<minute>\d{2})_(?<seq>\d+)\.gz\z/, re
-    end
-  end
-end

data/test/test_filesystem.rb

@@ -1,19 +0,0 @@
-require 'test/unit'
-require 'bricolage/filesystem'
-require 'pathname'
-require 'pp'
-
-module Bricolage
-  class TestFileSystem < Test::Unit::TestCase
-    fs = FileSystem.new("#{__dir__}/home", "test")
-    subfs = fs.subsystem('subsys')
-
-    test "FileSystem.job_file" do
-      path = subfs.job_file('unified')
-      assert_instance_of Pathname, path
-      assert_equal subfs.relative('unified.sql.job'), path
-
-      assert_equal subfs.relative('separated.job'), subfs.job_file('separated')
-    end
-  end
-end

data/test/test_logfilepath.rb

@@ -1,19 +0,0 @@
-require 'test/unit'
-require 'bricolage/logfilepath'
-require 'bricolage/jobnet'
-
-module Bricolage
-  class TestLogFilePath < Test::Unit::TestCase
-    test '#format' do
-      path = LogFilePath.new('./log/%{std}.log')
-      ref = JobNet::JobRef.new('subsys', 'somejob', '-')
-      start = Time.new(2012,3,4,12,34,56)
-      assert_equal './log/20120304/subsys::rebuild/20120304_123456000/subsys-somejob.log', path.format(
-        job_ref: ref,
-        jobnet_id: 'subsys/rebuild',
-        job_start_time: start,
-        jobnet_start_time: start
-      )
-    end
-  end
-end

data/test/test_parameters.rb

@@ -1,401 +0,0 @@
-require 'test/unit'
-require 'bricolage/parameters'
-require 'optparse'
-require 'pp'
-
-module Bricolage
-  class TestParameters < Test::Unit::TestCase
-    def apply_values(decls, values)
-      parser = Parameters::DirectValueHandler.new(decls)
-      parser.parse(values)
-    end
-
-    def apply_options(decls, argv)
-      parser = Parameters::CommandLineOptionHandler.new(decls)
-      opts = OptionParser.new
-      parser.define_options(opts)
-      opts.parse!(argv)
-      parser.values
-    end
-
-    def wrap_decl(decl)
-      Parameters::Declarations.new.tap {|params|
-        params.add decl
-      }
-    end
-
-    default_context = nil
-    default_variables = ResolvedVariables.new
-
-    # StringParam (8)
-
-    test "StringParam (*.job)" do
-      decls = wrap_decl StringParam.new('options', 'OPTIONS', 'Loader options.', optional: true)
-      pvals = apply_values(decls, {'options' => 'gzip, maxerror=3'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal 'gzip, maxerror=3', params['options']
-      assert_false params.variables.bound?('options')
-    end
-
-    test "StringParam (--opt)" do
-      decls = wrap_decl StringParam.new('options', 'OPTIONS', 'Loader options.', optional: true)
-      pvals = apply_options(decls, ['--options=gzip, maxerror=3'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal 'gzip, maxerror=3', params['options']
-      assert_false params.variables.bound?('options')
-    end
-
-    test "StringParam (default value)" do
-      decls = wrap_decl StringParam.new('options', 'OPTIONS', 'Loader options.', optional: true)
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_nil params['options']
-    end
-
-    test "StringParam (missing value)" do
-      decls = wrap_decl StringParam.new('delete-cond', 'SQL_EXPR', 'DELETE condition.')
-      pvals = apply_values(decls, {})
-      assert_raise(ParameterError) {
-        pvals.resolve(default_context, default_variables)
-      }
-    end
-
-    # BoolParam (0)
-
-    # OptionalBoolParam (41)
-
-    test "OptionalBoolParam (*.job)" do
-      decls = wrap_decl OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-      pvals = apply_values(decls, {'vacuum-sort' => true})
-      params = pvals.resolve(default_context, default_variables)
-      assert_true params['vacuum-sort']
-      assert_false params.variables.bound?('vacuum_sort')
-    end
-
-    test "OptionalBoolParam (--opt)" do
-      decls = wrap_decl OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.', publish: true)
-      pvals = apply_options(decls, ['--vacuum-sort'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_true params['vacuum-sort']
-      assert_equal 'true', params.variables['vacuum_sort']
-    end
-
-    test "OptionalBoolParam (default value #1)" do
-      decls = wrap_decl OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_false params['vacuum']
-      assert_false params.variables.bound?('vacuum')
-    end
-
-    test "OptionalBoolParam (default value #2)" do
-      decls = wrap_decl OptionalBoolParam.new('gzip', 'If true, compresses target file by gzip.', default: true)
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_true params['gzip']
-      assert_false params.variables.bound?('gzip')
-    end
-
-    # DateParam (2)
-
-    test "DateParam (*.job)" do
-      decls = wrap_decl DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).')
-      pvals = apply_values(decls, {'to' => '2014-01-23'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Date.new(2014, 1, 23), params['to']
-      assert_false params.variables.bound?('to')
-    end
-
-    test "DateParam (--opt)" do
-      decls = wrap_decl DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).', publish: true)
-      pvals = apply_options(decls, ['--to=2014-01-23'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Date.new(2014, 1, 23), params['to']
-      assert_equal '2014-01-23', params.variables['to']
-    end
-
-    test "DateParam (default value)" do
-      decls = wrap_decl DateParam.new('to', 'DATE', 'End date of logs to delete (%Y-%m-%d).', optional: true)
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_nil params['to']
-      assert_false params.variables.bound?('to')
-    end
-
-    # EnumParam (4)
-
-    test "EnumParam (*.job)" do
-      decls = wrap_decl EnumParam.new('format', %w(tsv json), 'Data file format.', default: 'tsv')
-      pvals = apply_values(decls, {'format' => 'json'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal 'json', params['format']
-      assert_false params.variables.bound?('format')
-    end
-
-    test "EnumParam (--opt)" do
-      decls = wrap_decl EnumParam.new('format', %w(tsv json), 'Data file format.', default: nil, publish: true)
-      pvals = apply_options(decls, ['--format=tsv'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal 'tsv', params['format']
-      assert_equal 'tsv', params.variables['format']
-    end
-
-    test "EnumParam (default value)" do
-      decls = wrap_decl EnumParam.new('format', %w(tsv json), 'Data file format.', default: 'tsv')
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal 'tsv', params['format']
-    end
-
-    # DataSourceParam (17)
-
-    class DummyContext_ds
-      def initialize(kind, name, result)
-        @kind = kind
-        @name = name
-        @result = result
-      end
-
-      def get_data_source(kind, name)
-        if kind == @kind and name == @name
-          @result
-        else
-          raise ParameterError, "wrong ds argument: #{kind}, #{name}"
-        end
-      end
-    end
-
-    DummyDataSource = Struct.new(:name)
-    app_ds = DummyDataSource.new('app')
-    sql_ds = DummyDataSource.new('sql')
-
-    test "DataSourceParam (*.job)" do
-      decls = wrap_decl DataSourceParam.new('sql')
-      pvals = apply_values(decls, {'data-source' => 'app'})
-      ctx = DummyContext_ds.new('sql', app_ds.name, app_ds)
-      params = pvals.resolve(ctx, default_variables)
-      assert_equal app_ds, params['data-source']
-      assert_false params.variables.bound?('data-source')
-    end
-
-    test "DataSourceParam (--opt)" do
-      decls = wrap_decl DataSourceParam.new('sql')
-      pvals = apply_options(decls, ['--data-source=app'])
-      ctx = DummyContext_ds.new('sql', app_ds.name, app_ds)
-      params = pvals.resolve(ctx, default_variables)
-      assert_equal app_ds, params['data-source']
-      assert_false params.variables.bound?('data-source')
-    end
-
-    test "DataSourceParam (default value)" do
-      decls = wrap_decl DataSourceParam.new('sql')
-      pvals = apply_values(decls, {})
-      ctx = DummyContext_ds.new('sql', nil, sql_ds)
-      params = pvals.resolve(ctx, default_variables)
-      assert_equal sql_ds, params['data-source']
-      assert_false params.variables.bound?('data-source')
-    end
-
-    # SQLFileParam (14)
-
-    DummyContext_sqlfile = Struct.new(:name, :ext, :result)
-    class DummyContext_sqlfile
-      def parameter_file(_name, _ext)
-        raise ParameterError, "bad argument: #{_name}, #{_ext}" unless [name, ext] == [_name, _ext]
-        result
-      end
-    end
-
-    dummy_sql_resource = StringResource.new("select * from t;")
-    dummy_sql_file = SQLStatement.new(dummy_sql_resource)
-
-    test "SQLFileParam (*.job)" do
-      decls = wrap_decl SQLFileParam.new
-      pvals = apply_values(decls, {'sql-file' => 'some_path.sql'})
-      ctx = DummyContext_sqlfile.new('some_path.sql', 'sql', dummy_sql_resource)
-      params = pvals.resolve(ctx, default_variables)
-      assert_equal dummy_sql_file, params['sql-file']
-      assert_false params.variables.bound?('sql_file')
-    end
-
-    test "SQLFileParam (--opt)" do
-      decls = wrap_decl SQLFileParam.new
-      pvals = apply_options(decls, ['--sql-file=some_path.sql'])
-      ctx = DummyContext_sqlfile.new('some_path.sql', 'sql', dummy_sql_resource)
-      params = pvals.resolve(ctx, default_variables)
-      assert_equal dummy_sql_file, params['sql-file']
-      assert_false params.variables.bound?('sql_file')
-    end
-
-    test "SQLFileParam (default value)" do
-      decls = wrap_decl SQLFileParam.new(optional: true)
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_nil params['sql-file']
-    end
-
-    # DestTableParam (9)
-
-    test "DestTableParam (*.job)" do
-      decls = wrap_decl DestTableParam.new
-      pvals = apply_values(decls, {'dest-table' => 'schemaA.tableA'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal TableSpec.new('schemaA', 'tableA'), params['dest-table']
-      assert_equal 'schemaA.tableA', params.variables['dest_table']
-    end
-
-    test "DestTableParam (--opt)" do
-      decls = wrap_decl DestTableParam.new(optional: false)
-      pvals = apply_options(decls, ['--dest-table=schemaA.tableA'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal TableSpec.new('schemaA', 'tableA'), params['dest-table']
-      assert_equal 'schemaA.tableA', params.variables['dest_table']
-    end
-
-    test "DestTableParam (default value)" do
-      decls = wrap_decl DestTableParam.new
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_nil params['dest-table']
-      assert_false params.variables.bound?('dest_table')
-    end
-
-    test "DestTableParam (variable expansion)" do
-      decls = wrap_decl DestTableParam.new
-      pvals = apply_values(decls, {'dest-table' => '$s.t'})
-      vars = ResolvedVariables.new
-      vars.add ResolvedVariable.new('s', 'SCH')
-      params = pvals.resolve(default_context, vars)
-      assert_equal TableSpec.new('SCH', 't'), params['dest-table']
-      assert_equal 'SCH.t', params.variables['dest_table']
-    end
-
-    test "DestTableParam (no such variable)" do
-      decls = wrap_decl DestTableParam.new
-      pvals = apply_values(decls, {'dest-table' => '$s.t'})
-      assert_raise(ParameterError) {
-        pvals.resolve(default_context, default_variables)
-      }
-    end
-
-    # SrcTableParam (9)
-
-    test "SrcTableParam (*.job)" do
-      decls = wrap_decl SrcTableParam.new
-      pvals = apply_values(decls, {'src-tables' => {'a' => '$s.A', 'b' => 'B'}})
-      vars = ResolvedVariables.new
-      vars.add ResolvedVariable.new('s', 'SCH')
-      params = pvals.resolve(default_context, vars)
-      srcs = {'a' => TableSpec.new('SCH', 'A'), 'b' => TableSpec.new(nil, 'B')}
-      assert_equal srcs, params['src-tables']
-      assert_equal 'SCH.A', params.variables['a']
-      assert_equal 'B', params.variables['b']
-    end
-
-    test "SrcTableParam (--opt)" do
-      decls = wrap_decl SrcTableParam.new
-      pvals = apply_options(decls, ['--src-table=a:A', '--src-table=b:B'])
-      params = pvals.resolve(default_context, default_variables)
-      srcs = {'a' => TableSpec.new(nil, 'A'), 'b' => TableSpec.new(nil, 'B')}
-      assert_equal srcs, params['src-tables']
-      assert_equal 'A', params.variables['a']
-      assert_equal 'B', params.variables['b']
-    end
-
-    test "SrcTableParam (default value)" do
-      decls = wrap_decl SrcTableParam.new
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal({}, params['src-tables'])
-      assert_false params.variables.bound?('a')
-      assert_false params.variables.bound?('b')
-    end
-
-    # DestFileParam (7)
-
-    test "DestFileParam (*.job)" do
-      decls = wrap_decl DestFileParam.new
-      pvals = apply_values(decls, {'dest-file' => '/some/path.txt'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Pathname.new('/some/path.txt'), params['dest-file']
-    end
-
-    test "DestFileParam (--opt)" do
-      decls = wrap_decl DestFileParam.new
-      pvals = apply_options(decls, ['--dest-file=/some/path.txt'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Pathname.new('/some/path.txt'), params['dest-file']
-    end
-
-    test "DestFileParam (no value error)" do
-      decls = wrap_decl DestFileParam.new
-      pvals = apply_values(decls, {})
-      assert_raise(ParameterError) {
-        pvals.resolve(default_context, default_variables)
-      }
-    end
-
-    # SrcFileParam (2)
-
-    test "SrcFileParam (*.job)" do
-      decls = wrap_decl SrcFileParam.new
-      pvals = apply_values(decls, {'src-file' => '/some/path.txt'})
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Pathname.new('/some/path.txt'), params['src-file']
-    end
-
-    test "SrcFileParam (--opt)" do
-      decls = wrap_decl SrcFileParam.new
-      pvals = apply_options(decls, ['--src-file=/some/path.txt'])
-      params = pvals.resolve(default_context, default_variables)
-      assert_equal Pathname.new('/some/path.txt'), params['src-file']
-    end
-
-    test "SrcFileParam (no value error)" do
-      decls = wrap_decl SrcFileParam.new
-      pvals = apply_values(decls, {})
-      assert_raise(ParameterError) {
-        pvals.resolve(default_context, default_variables)
-      }
-    end
-
-    # KeyValuePairsParam (3)
-
-    test "KeyValuePairsParam (*.job)" do
-      decls = wrap_decl KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
-      pvals = apply_values(decls, {'grant' => {'on'=>'tbl', 'to'=>'$user'}})
-      vars = ResolvedVariables.new
-      vars.add ResolvedVariable.new('user', 'group gg')
-      params = pvals.resolve(default_context, vars)
-      assert_equal({'on'=>'tbl', 'to'=>'group gg'}, params['grant'])
-    end
-
-    test "KeyValuePairsParam (default value)" do
-      decls = wrap_decl KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
-      pvals = apply_values(decls, {})
-      params = pvals.resolve(default_context, default_variables)
-      assert_nil params['grant']
-    end
-
-    # StringListParam (1)
-
-    test "StringListParam (*.job)" do
-      decls = wrap_decl StringListParam.new('args', 'ARG', 'Command line arguments.', publish: true)
-      pvals = apply_values(decls, {'args' => ['a', '$basedir', 'c']})
-      vars = ResolvedVariables.new
-      vars.add ResolvedVariable.new('basedir', '/base/dir')
-      params = pvals.resolve(default_context, vars)
-      assert_equal ['a', '/base/dir', 'c'], params['args']
-      assert_equal 'a /base/dir c', params.variables['args']
-    end
-
-    test "StringListParam (missing value)" do
-      decls = wrap_decl StringListParam.new('args', 'ARG', 'Command line arguments.')
-      pvals = apply_values(decls, {})
-      assert_raise(ParameterError) {
-        pvals.resolve(default_context, default_variables)
-      }
-    end
-
-  end
-end

data/test/test_psqldatasource.rb

@@ -1,12 +0,0 @@
-require 'test/unit'
-require 'bricolage/psqldatasource'
-
-module Bricolage
-  class TestPSQLLoadOptions < Test::Unit::TestCase
-    test "load option is correctly formatted" do
-      assert_equal 'gzip', PSQLLoadOptions::Option.new('gzip', true).to_s
-      assert_equal "json 'auto'", PSQLLoadOptions::Option.new('json', 'auto').to_s
-      assert_equal 'encoding utf16le', PSQLLoadOptions::Option.new('encoding', 'utf16le').to_s
-    end
-  end
-end