bricolage 5.27.2 → 5.28.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +22 -0
- data/Gemfile +2 -0
- data/LICENSES +21 -0
- data/RELEASE.md +271 -0
- data/Rakefile +3 -0
- data/bricolage.gemspec +26 -0
- data/jobclass/exec.rb +5 -2
- data/lib/bricolage/configloader.rb +12 -10
- data/lib/bricolage/datasource.rb +16 -11
- data/lib/bricolage/job.rb +3 -0
- data/lib/bricolage/jobfile.rb +42 -6
- data/lib/bricolage/parameters.rb +6 -2
- data/lib/bricolage/version.rb +1 -1
- data/streaming_load_log.ct +9 -0
- metadata +11 -143
- data/test/all.rb +0 -3
- data/test/home/Gemfile +0 -3
- data/test/home/Gemfile.lock +0 -44
- data/test/home/config/bricolage.yml +0 -1
- data/test/home/config/development/database.yml +0 -100
- data/test/home/config/development/password.yml +0 -6
- data/test/home/config/development/variable.yml +0 -7
- data/test/home/data/20141002-1355_00.txt +0 -2
- data/test/home/data/20141002-1355_01.txt +0 -2
- data/test/home/data/20141002-1355_02.txt +0 -2
- data/test/home/data/test.txt +0 -1
- data/test/home/jobnet-test.rb +0 -61
- data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log +0 -80
- data/test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log +0 -79
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log +0 -3
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log +0 -3
- data/test/home/log/20170629/subsys::job1/20170629_155543985/subsys-job1.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job1.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job2.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job3.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job4.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job1.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job2.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job3.log +0 -3
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job4.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job1.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job2.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job3.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job4.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job1.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job2.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job3.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job4.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job1.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job2.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job3.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job4.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_153437221/subsys-job1.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job1.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job2.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job3.log +0 -3
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job4.log +0 -3
- data/test/home/log/20170704/subsys::insert/20170704_164335210/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert/20170704_164344251/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert/20170704_164723299/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert.sql/20170704_164143661/subsys-insert.sql.log +0 -14
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_163033119/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_172410576/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_173130175/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_173201376/subsys-insert.log +0 -14
- data/test/home/log/20170704/subsys::net1/20170704_162457694/subsys-job1.log +0 -3
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job1.log +0 -3
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job2.log +0 -3
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job3.log +0 -3
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job4.log +0 -3
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job1.log +0 -3
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job2.log +0 -3
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job3.log +0 -3
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job4.log +0 -3
- data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log +0 -51
- data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log.status +0 -1
- data/test/home/log/20170707/subsys::job1/20170707_020542902/subsys-job1.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job1.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job2.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job3.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job4.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job1.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job2.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job3.log +0 -3
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job4.log +0 -3
- data/test/home/revert.sh +0 -7
- data/test/home/s +0 -1
- data/test/home/subsys/d.ct +0 -7
- data/test/home/subsys/insert.sql.job +0 -7
- data/test/home/subsys/item_pv_acc.ct +0 -8
- data/test/home/subsys/job1.job +0 -1
- data/test/home/subsys/job2.job +0 -1
- data/test/home/subsys/job3.job +0 -1
- data/test/home/subsys/job4.job +0 -1
- data/test/home/subsys/job5.job +0 -1
- data/test/home/subsys/job6.job +0 -1
- data/test/home/subsys/job7.job +0 -1
- data/test/home/subsys/load.job +0 -9
- data/test/home/subsys/load_test.ct +0 -6
- data/test/home/subsys/load_test.job +0 -14
- data/test/home/subsys/migrate.job +0 -33
- data/test/home/subsys/my-import-delta.sql.job +0 -40
- data/test/home/subsys/my-import.job +0 -31
- data/test/home/subsys/net1.jobnet +0 -3
- data/test/home/subsys/net2.jobnet +0 -2
- data/test/home/subsys/put.job +0 -3
- data/test/home/subsys/raw-vacuum.jobnet +0 -1
- data/test/home/subsys/raw-vacuum.sql.job +0 -7
- data/test/home/subsys/rebuild.sql.job +0 -14
- data/test/home/subsys/redis_export.job +0 -9
- data/test/home/subsys/redis_export2.job +0 -8
- data/test/home/subsys/search_backends.ct +0 -13
- data/test/home/subsys/separated.job +0 -1
- data/test/home/subsys/separated.sql +0 -1
- data/test/home/subsys/some_view-cv.sql.job +0 -11
- data/test/home/subsys/unified.jobnet +0 -1
- data/test/home/subsys/unified.sql.job +0 -5
- data/test/home/subsys/users.ct +0 -13
- data/test/home/subsys/variable.yml +0 -6
- data/test/test_c_streaming_load.rb +0 -13
- data/test/test_filesystem.rb +0 -19
- data/test/test_logfilepath.rb +0 -19
- data/test/test_parameters.rb +0 -401
- data/test/test_psqldatasource.rb +0 -12
- data/test/test_s3datasource.rb +0 -40
- data/test/test_variables.rb +0 -114
- data/test/vacuum-test.rb +0 -8
data/test/test_s3datasource.rb
DELETED
@@ -1,40 +0,0 @@
|
|
1
|
-
require 'test/unit'
|
2
|
-
require 'mocha/test_unit'
|
3
|
-
require 'bricolage/s3datasource'
|
4
|
-
require 'pp'
|
5
|
-
require 'aws-sdk'
|
6
|
-
|
7
|
-
module Bricolage
|
8
|
-
class TestS3datasouce < Test::Unit::TestCase
|
9
|
-
|
10
|
-
def setup
|
11
|
-
@ds = S3DataSource.new
|
12
|
-
@ds.stubs(:logger).returns(Logger.new)
|
13
|
-
end
|
14
|
-
|
15
|
-
test "traverse without exception" do
|
16
|
-
bucket = mock()
|
17
|
-
bucket.stubs(:objects).returns(true)
|
18
|
-
@ds.stubs(:bucket).returns(bucket)
|
19
|
-
assert @ds.traverse(nil)
|
20
|
-
end
|
21
|
-
|
22
|
-
test "traverse with 2 exception" do
|
23
|
-
bucket = mock()
|
24
|
-
bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
|
25
|
-
raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
|
26
|
-
@ds.stubs(:bucket).returns(bucket)
|
27
|
-
assert @ds.traverse(nil)
|
28
|
-
end
|
29
|
-
|
30
|
-
test "traverse with more than 3 exception" do
|
31
|
-
bucket = mock()
|
32
|
-
bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
|
33
|
-
raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
|
34
|
-
raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
|
35
|
-
raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
|
36
|
-
@ds.stubs(:bucket).returns(bucket)
|
37
|
-
assert_raise(Aws::Xml::Parser::ParsingError) {@ds.traverse(nil)}
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
data/test/test_variables.rb
DELETED
@@ -1,114 +0,0 @@
|
|
1
|
-
require 'test/unit'
|
2
|
-
require 'bricolage/job'
|
3
|
-
require 'bricolage/context'
|
4
|
-
require 'bricolage/variables'
|
5
|
-
require 'bricolage/parameters'
|
6
|
-
require 'optparse'
|
7
|
-
require 'pp'
|
8
|
-
|
9
|
-
module Bricolage
|
10
|
-
class TestVaribles < Test::Unit::TestCase
|
11
|
-
DummyFS = Struct.new(:home_path)
|
12
|
-
class DummyContextForGvarTest < Context
|
13
|
-
attr_accessor :variable_yml_vars
|
14
|
-
alias load_global_variables variable_yml_vars
|
15
|
-
end
|
16
|
-
|
17
|
-
test "global variable precedence" do
|
18
|
-
opt_gvars = Variables.new
|
19
|
-
opt_gvars.add Variable.new('var_global_opt', 'loc_global_opt')
|
20
|
-
fs = DummyFS.new('/home/path')
|
21
|
-
ctx = DummyContextForGvarTest.new(fs, 'development', global_variables: opt_gvars)
|
22
|
-
ctx.variable_yml_vars = Variables.define {|vars|
|
23
|
-
vars.add Variable.new('var_variable_yml', 'loc_variable_yml')
|
24
|
-
vars.add Variable.new('var_global_opt', 'loc_variable_yml')
|
25
|
-
}
|
26
|
-
result = ctx.global_variables
|
27
|
-
|
28
|
-
assert_equal 'loc_variable_yml', result['var_variable_yml']
|
29
|
-
assert_equal 'loc_global_opt', result['var_global_opt']
|
30
|
-
end
|
31
|
-
|
32
|
-
DummyContext = Struct.new(:global_variables)
|
33
|
-
class DummyContext
|
34
|
-
def job_dir
|
35
|
-
'/job/dir'
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
class DummyJobClass
|
40
|
-
def get_parameters
|
41
|
-
Parameters::Declarations.new
|
42
|
-
end
|
43
|
-
|
44
|
-
def invoke_parameters_filter(job)
|
45
|
-
end
|
46
|
-
|
47
|
-
def get_declarations(params)
|
48
|
-
Declarations.new
|
49
|
-
end
|
50
|
-
|
51
|
-
def get_script(params)
|
52
|
-
DummyScript.new
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
class DummyScript
|
57
|
-
def bind(ctx, vars)
|
58
|
-
vars
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
test "variable precedence (*.job)" do
|
63
|
-
gvars = Variables.new
|
64
|
-
gvars.add Variable.new('ow_global_variable', 'loc_global_variable')
|
65
|
-
gvars.add Variable.new('ow_rest_var', 'loc_global_variable')
|
66
|
-
gvars.add Variable.new('ow_job_opt', 'loc_global_variable')
|
67
|
-
ctx = DummyContext.new(gvars)
|
68
|
-
job_class = DummyJobClass.new
|
69
|
-
job = Job.new('varprec', job_class, ctx)
|
70
|
-
job.init_global_variables
|
71
|
-
job.bind_parameters({
|
72
|
-
'ow_rest_var' => 'loc_rest_var',
|
73
|
-
'ow_job_opt' => 'loc_rest_var'
|
74
|
-
})
|
75
|
-
job.parsing_options {|h|
|
76
|
-
opts = OptionParser.new
|
77
|
-
h.define_options(opts)
|
78
|
-
opts.parse!(['-v', 'ow_job_opt=loc_job_opt'])
|
79
|
-
}
|
80
|
-
job.compile
|
81
|
-
|
82
|
-
assert_equal 'loc_global_variable', job.variables['ow_global_variable']
|
83
|
-
assert_equal 'loc_rest_var', job.variables['ow_rest_var']
|
84
|
-
assert_equal 'loc_job_opt', job.variables['ow_job_opt']
|
85
|
-
end
|
86
|
-
|
87
|
-
test "lazy reference resolution" do
|
88
|
-
gvars = Variables.new
|
89
|
-
gvars.add Variable.new('gvar', 'GVAR')
|
90
|
-
gvars.add Variable.new('rest_var', '*global*')
|
91
|
-
gvars.add Variable.new('job_opt', '*global*')
|
92
|
-
gvars.add Variable.new('ref_gvar', '$gvar')
|
93
|
-
gvars.add Variable.new('ref_rest_var', '$rest_var')
|
94
|
-
gvars.add Variable.new('ref_job_opt', '$job_opt')
|
95
|
-
ctx = DummyContext.new(gvars)
|
96
|
-
job_class = DummyJobClass.new
|
97
|
-
job = Job.new('lazyres', job_class, ctx)
|
98
|
-
job.init_global_variables
|
99
|
-
job.bind_parameters({
|
100
|
-
'rest_var' => 'REST_VAR'
|
101
|
-
})
|
102
|
-
job.parsing_options {|h|
|
103
|
-
opts = OptionParser.new
|
104
|
-
h.define_options(opts)
|
105
|
-
opts.parse!(['-v', 'job_opt=JOB_OPT'])
|
106
|
-
}
|
107
|
-
job.compile
|
108
|
-
|
109
|
-
assert_equal 'GVAR', job.variables['ref_gvar']
|
110
|
-
assert_equal 'REST_VAR', job.variables['ref_rest_var']
|
111
|
-
assert_equal 'JOB_OPT', job.variables['ref_job_opt']
|
112
|
-
end
|
113
|
-
end
|
114
|
-
end
|
data/test/vacuum-test.rb
DELETED
@@ -1,8 +0,0 @@
|
|
1
|
-
$LOAD_PATH.unshift '../../lib'
|
2
|
-
require 'bricolage/psqldatasource'
|
3
|
-
|
4
|
-
ds = Bricolage::PSQLDataSource.new(port: 5444, database: 'production', username: 'aamine', pgpass: "#{ENV['HOME']}/.pgpass")
|
5
|
-
ds.__send__ :initialize_base, 'tmp', nil, Logger.new($stderr)
|
6
|
-
p ds
|
7
|
-
ds.vacuum 's'
|
8
|
-
puts 'OK'
|