bricolage 5.27.2 → 5.28.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +22 -0
  3. data/Gemfile +2 -0
  4. data/LICENSES +21 -0
  5. data/RELEASE.md +271 -0
  6. data/Rakefile +3 -0
  7. data/bricolage.gemspec +26 -0
  8. data/jobclass/exec.rb +5 -2
  9. data/lib/bricolage/configloader.rb +12 -10
  10. data/lib/bricolage/datasource.rb +16 -11
  11. data/lib/bricolage/job.rb +3 -0
  12. data/lib/bricolage/jobfile.rb +42 -6
  13. data/lib/bricolage/parameters.rb +6 -2
  14. data/lib/bricolage/version.rb +1 -1
  15. data/streaming_load_log.ct +9 -0
  16. metadata +11 -143
  17. data/test/all.rb +0 -3
  18. data/test/home/Gemfile +0 -3
  19. data/test/home/Gemfile.lock +0 -44
  20. data/test/home/config/bricolage.yml +0 -1
  21. data/test/home/config/development/database.yml +0 -100
  22. data/test/home/config/development/password.yml +0 -6
  23. data/test/home/config/development/variable.yml +0 -7
  24. data/test/home/data/20141002-1355_00.txt +0 -2
  25. data/test/home/data/20141002-1355_01.txt +0 -2
  26. data/test/home/data/20141002-1355_02.txt +0 -2
  27. data/test/home/data/test.txt +0 -1
  28. data/test/home/jobnet-test.rb +0 -61
  29. data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log +0 -3
  30. data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log +0 -3
  31. data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log +0 -3
  32. data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log +0 -80
  33. data/test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log +0 -79
  34. data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log +0 -3
  35. data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log +0 -3
  36. data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log +0 -3
  37. data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log +0 -3
  38. data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log +0 -3
  39. data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log +0 -3
  40. data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log +0 -3
  41. data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log +0 -3
  42. data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log +0 -3
  43. data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log +0 -3
  44. data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log +0 -3
  45. data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log +0 -3
  46. data/test/home/log/20170629/subsys::job1/20170629_155543985/subsys-job1.log +0 -3
  47. data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job1.log +0 -3
  48. data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job2.log +0 -3
  49. data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job3.log +0 -3
  50. data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job4.log +0 -3
  51. data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job1.log +0 -3
  52. data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job2.log +0 -3
  53. data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job3.log +0 -3
  54. data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job4.log +0 -3
  55. data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job1.log +0 -3
  56. data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job2.log +0 -3
  57. data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job3.log +0 -3
  58. data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job4.log +0 -3
  59. data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job1.log +0 -3
  60. data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job2.log +0 -3
  61. data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job3.log +0 -3
  62. data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job4.log +0 -3
  63. data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job1.log +0 -3
  64. data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job2.log +0 -3
  65. data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job3.log +0 -3
  66. data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job4.log +0 -3
  67. data/test/home/log/20170630/subsys::net1/20170630_153437221/subsys-job1.log +0 -3
  68. data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job1.log +0 -3
  69. data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job2.log +0 -3
  70. data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job3.log +0 -3
  71. data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job4.log +0 -3
  72. data/test/home/log/20170704/subsys::insert/20170704_164335210/subsys-insert.log +0 -14
  73. data/test/home/log/20170704/subsys::insert/20170704_164344251/subsys-insert.log +0 -14
  74. data/test/home/log/20170704/subsys::insert/20170704_164723299/subsys-insert.log +0 -14
  75. data/test/home/log/20170704/subsys::insert.sql/20170704_164143661/subsys-insert.sql.log +0 -14
  76. data/test/home/log/20170704/subsys::insert.sql.job/20170704_163033119/subsys-insert.log +0 -14
  77. data/test/home/log/20170704/subsys::insert.sql.job/20170704_172410576/subsys-insert.log +0 -14
  78. data/test/home/log/20170704/subsys::insert.sql.job/20170704_173130175/subsys-insert.log +0 -14
  79. data/test/home/log/20170704/subsys::insert.sql.job/20170704_173201376/subsys-insert.log +0 -14
  80. data/test/home/log/20170704/subsys::net1/20170704_162457694/subsys-job1.log +0 -3
  81. data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job1.log +0 -3
  82. data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job2.log +0 -3
  83. data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job3.log +0 -3
  84. data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job4.log +0 -3
  85. data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job1.log +0 -3
  86. data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job2.log +0 -3
  87. data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job3.log +0 -3
  88. data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job4.log +0 -3
  89. data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log +0 -51
  90. data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log.status +0 -1
  91. data/test/home/log/20170707/subsys::job1/20170707_020542902/subsys-job1.log +0 -3
  92. data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job1.log +0 -3
  93. data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job2.log +0 -3
  94. data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job3.log +0 -3
  95. data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job4.log +0 -3
  96. data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job1.log +0 -3
  97. data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job2.log +0 -3
  98. data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job3.log +0 -3
  99. data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job4.log +0 -3
  100. data/test/home/revert.sh +0 -7
  101. data/test/home/s +0 -1
  102. data/test/home/subsys/d.ct +0 -7
  103. data/test/home/subsys/insert.sql.job +0 -7
  104. data/test/home/subsys/item_pv_acc.ct +0 -8
  105. data/test/home/subsys/job1.job +0 -1
  106. data/test/home/subsys/job2.job +0 -1
  107. data/test/home/subsys/job3.job +0 -1
  108. data/test/home/subsys/job4.job +0 -1
  109. data/test/home/subsys/job5.job +0 -1
  110. data/test/home/subsys/job6.job +0 -1
  111. data/test/home/subsys/job7.job +0 -1
  112. data/test/home/subsys/load.job +0 -9
  113. data/test/home/subsys/load_test.ct +0 -6
  114. data/test/home/subsys/load_test.job +0 -14
  115. data/test/home/subsys/migrate.job +0 -33
  116. data/test/home/subsys/my-import-delta.sql.job +0 -40
  117. data/test/home/subsys/my-import.job +0 -31
  118. data/test/home/subsys/net1.jobnet +0 -3
  119. data/test/home/subsys/net2.jobnet +0 -2
  120. data/test/home/subsys/put.job +0 -3
  121. data/test/home/subsys/raw-vacuum.jobnet +0 -1
  122. data/test/home/subsys/raw-vacuum.sql.job +0 -7
  123. data/test/home/subsys/rebuild.sql.job +0 -14
  124. data/test/home/subsys/redis_export.job +0 -9
  125. data/test/home/subsys/redis_export2.job +0 -8
  126. data/test/home/subsys/search_backends.ct +0 -13
  127. data/test/home/subsys/separated.job +0 -1
  128. data/test/home/subsys/separated.sql +0 -1
  129. data/test/home/subsys/some_view-cv.sql.job +0 -11
  130. data/test/home/subsys/unified.jobnet +0 -1
  131. data/test/home/subsys/unified.sql.job +0 -5
  132. data/test/home/subsys/users.ct +0 -13
  133. data/test/home/subsys/variable.yml +0 -6
  134. data/test/test_c_streaming_load.rb +0 -13
  135. data/test/test_filesystem.rb +0 -19
  136. data/test/test_logfilepath.rb +0 -19
  137. data/test/test_parameters.rb +0 -401
  138. data/test/test_psqldatasource.rb +0 -12
  139. data/test/test_s3datasource.rb +0 -40
  140. data/test/test_variables.rb +0 -114
  141. data/test/vacuum-test.rb +0 -8
@@ -1,40 +0,0 @@
1
- require 'test/unit'
2
- require 'mocha/test_unit'
3
- require 'bricolage/s3datasource'
4
- require 'pp'
5
- require 'aws-sdk'
6
-
7
- module Bricolage
8
- class TestS3datasouce < Test::Unit::TestCase
9
-
10
- def setup
11
- @ds = S3DataSource.new
12
- @ds.stubs(:logger).returns(Logger.new)
13
- end
14
-
15
- test "traverse without exception" do
16
- bucket = mock()
17
- bucket.stubs(:objects).returns(true)
18
- @ds.stubs(:bucket).returns(bucket)
19
- assert @ds.traverse(nil)
20
- end
21
-
22
- test "traverse with 2 exception" do
23
- bucket = mock()
24
- bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
25
- raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
26
- @ds.stubs(:bucket).returns(bucket)
27
- assert @ds.traverse(nil)
28
- end
29
-
30
- test "traverse with more than 3 exception" do
31
- bucket = mock()
32
- bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
33
- raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
34
- raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
35
- raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
36
- @ds.stubs(:bucket).returns(bucket)
37
- assert_raise(Aws::Xml::Parser::ParsingError) {@ds.traverse(nil)}
38
- end
39
- end
40
- end
@@ -1,114 +0,0 @@
1
- require 'test/unit'
2
- require 'bricolage/job'
3
- require 'bricolage/context'
4
- require 'bricolage/variables'
5
- require 'bricolage/parameters'
6
- require 'optparse'
7
- require 'pp'
8
-
9
- module Bricolage
10
- class TestVaribles < Test::Unit::TestCase
11
- DummyFS = Struct.new(:home_path)
12
- class DummyContextForGvarTest < Context
13
- attr_accessor :variable_yml_vars
14
- alias load_global_variables variable_yml_vars
15
- end
16
-
17
- test "global variable precedence" do
18
- opt_gvars = Variables.new
19
- opt_gvars.add Variable.new('var_global_opt', 'loc_global_opt')
20
- fs = DummyFS.new('/home/path')
21
- ctx = DummyContextForGvarTest.new(fs, 'development', global_variables: opt_gvars)
22
- ctx.variable_yml_vars = Variables.define {|vars|
23
- vars.add Variable.new('var_variable_yml', 'loc_variable_yml')
24
- vars.add Variable.new('var_global_opt', 'loc_variable_yml')
25
- }
26
- result = ctx.global_variables
27
-
28
- assert_equal 'loc_variable_yml', result['var_variable_yml']
29
- assert_equal 'loc_global_opt', result['var_global_opt']
30
- end
31
-
32
- DummyContext = Struct.new(:global_variables)
33
- class DummyContext
34
- def job_dir
35
- '/job/dir'
36
- end
37
- end
38
-
39
- class DummyJobClass
40
- def get_parameters
41
- Parameters::Declarations.new
42
- end
43
-
44
- def invoke_parameters_filter(job)
45
- end
46
-
47
- def get_declarations(params)
48
- Declarations.new
49
- end
50
-
51
- def get_script(params)
52
- DummyScript.new
53
- end
54
- end
55
-
56
- class DummyScript
57
- def bind(ctx, vars)
58
- vars
59
- end
60
- end
61
-
62
- test "variable precedence (*.job)" do
63
- gvars = Variables.new
64
- gvars.add Variable.new('ow_global_variable', 'loc_global_variable')
65
- gvars.add Variable.new('ow_rest_var', 'loc_global_variable')
66
- gvars.add Variable.new('ow_job_opt', 'loc_global_variable')
67
- ctx = DummyContext.new(gvars)
68
- job_class = DummyJobClass.new
69
- job = Job.new('varprec', job_class, ctx)
70
- job.init_global_variables
71
- job.bind_parameters({
72
- 'ow_rest_var' => 'loc_rest_var',
73
- 'ow_job_opt' => 'loc_rest_var'
74
- })
75
- job.parsing_options {|h|
76
- opts = OptionParser.new
77
- h.define_options(opts)
78
- opts.parse!(['-v', 'ow_job_opt=loc_job_opt'])
79
- }
80
- job.compile
81
-
82
- assert_equal 'loc_global_variable', job.variables['ow_global_variable']
83
- assert_equal 'loc_rest_var', job.variables['ow_rest_var']
84
- assert_equal 'loc_job_opt', job.variables['ow_job_opt']
85
- end
86
-
87
- test "lazy reference resolution" do
88
- gvars = Variables.new
89
- gvars.add Variable.new('gvar', 'GVAR')
90
- gvars.add Variable.new('rest_var', '*global*')
91
- gvars.add Variable.new('job_opt', '*global*')
92
- gvars.add Variable.new('ref_gvar', '$gvar')
93
- gvars.add Variable.new('ref_rest_var', '$rest_var')
94
- gvars.add Variable.new('ref_job_opt', '$job_opt')
95
- ctx = DummyContext.new(gvars)
96
- job_class = DummyJobClass.new
97
- job = Job.new('lazyres', job_class, ctx)
98
- job.init_global_variables
99
- job.bind_parameters({
100
- 'rest_var' => 'REST_VAR'
101
- })
102
- job.parsing_options {|h|
103
- opts = OptionParser.new
104
- h.define_options(opts)
105
- opts.parse!(['-v', 'job_opt=JOB_OPT'])
106
- }
107
- job.compile
108
-
109
- assert_equal 'GVAR', job.variables['ref_gvar']
110
- assert_equal 'REST_VAR', job.variables['ref_rest_var']
111
- assert_equal 'JOB_OPT', job.variables['ref_job_opt']
112
- end
113
- end
114
- end
data/test/vacuum-test.rb DELETED
@@ -1,8 +0,0 @@
1
- $LOAD_PATH.unshift '../../lib'
2
- require 'bricolage/psqldatasource'
3
-
4
- ds = Bricolage::PSQLDataSource.new(port: 5444, database: 'production', username: 'aamine', pgpass: "#{ENV['HOME']}/.pgpass")
5
- ds.__send__ :initialize_base, 'tmp', nil, Logger.new($stderr)
6
- p ds
7
- ds.vacuum 's'
8
- puts 'OK'