bricolage 5.29.2 → 5.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4379a27ecfd37961cd70273944eb3a5a85eede39e4f9f45943fb61d6731e5a3c
-  data.tar.gz: d666c93b9bcb1cc1e41f26337b1de5b3e200f213ca987e92911b287a457f13d4
+  metadata.gz: cd34b3e85926b0f85f76d05415341066025a5117c77d1b3b644ac9ce3358f0d2
+  data.tar.gz: 067eed98758a6de824a0ab9133830a637f3cc3790e26974a946d39eaa54032fc
 SHA512:
-  metadata.gz: 3512422000c99775e53b0911fe5dbd9e604bcb49f6a9f1915051249b4005edc4ac888eb8efb469cc3a35da98048de2f4a4f3109149a74078469c40ddf93237dd
-  data.tar.gz: 346f571330a7aecd653f68216f9907f50b09d8f9da71237a356180bf6ba4e539a1772e9e367bff49cf453ea4557af55a5512aaa9cff69aa73dbb33d35e915c4d
+  metadata.gz: d4601475e0c9cbd70e2f54a68ddb06e417288fc52230b3ac553ff8a3f1ef79dafa6d83a8e543470d9c5cfd9d040a98df66217e2cd7a39aec6e1cd4b342640b0f
+  data.tar.gz: aa8d9bf658e8f36a84eacd18c3d680a18178d7612c995fbdbb8dfd695720331f953607614c1ad2d467c8a211e6fd6bf29f853257455b72700fe93172846375e3

data/README.md

@@ -13,6 +13,10 @@ See LICENSES file for details.
 
 % rake test
 
+## How to use
+
+See https://github.com/bricolages/bricolage/wiki
+
 ## Author
 
 Minero Aoki

data/RELEASE.md

@@ -1,5 +1,9 @@
 # Bricolage Release Note
 
+## version 5.30.0
+
+- [new] streaming_load: new option --ctl-ds to change S3 data source for metadata files.
+
 ## version 5.29.2
 
 - [new] load: Allows to set jsonpath by direct S3 URL, instead of relative S3 key.

data/jobclass/create.rb

@@ -18,7 +18,7 @@ JobClass.define('create') {
         task.drop_force_if params['drop']
         task.exec params['table-def']
         task.analyze_if params['analyze']
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
     }
   }

data/jobclass/createview.rb

@@ -17,7 +17,7 @@ JobClass.define('createview') {
       task.transaction {
         task.drop_view_force_if params['drop']
         task.exec params['sql-file']
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
     }
   }

data/jobclass/load.rb

@@ -35,9 +35,9 @@ JobClass.define('load') {
         task.transaction {
           task.drop_force '${dest_table}'
           task.exec params['table-def']
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
         }
         # ANALYZE, VACUUM is needless for newly loaded table, skip always.
 
@@ -49,9 +49,9 @@ JobClass.define('load') {
 
         task.truncate_if params['truncate']
         task.transaction {
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
         }
         # ANALYZE, VACUUM is needless for newly loaded table, skip always.
 
@@ -59,9 +59,9 @@ JobClass.define('load') {
         # load only pattern
 
         task.transaction {
-          task.load params['src-ds'], params['src-file'], params['dest-table'],
+          task.load params['src-ds'], params['src-file'], '$dest_table',
               params['format'], params['jsonpath'], params['options']
-          task.grant_if params['grant'], params['dest-table']
+          task.grant_if params['grant'], '$dest_table'
           task.analyze_if params['analyze']
         }
         # We cannot execute VACUUM in transaction

data/jobclass/rebuild-drop.rb

@@ -23,19 +23,19 @@ JobClass.define('rebuild-drop') {
     script.task(params['data-source']) {|task|
       task.transaction {
         # CREATE
-        task.drop_force params['dest-table']
+        task.drop_force '$dest_table'
         task.exec params['table-def']
 
         # INSERT
         task.exec params['sql-file']
 
         # GRANT
-        task.grant_if params['grant'], params['dest-table']
+        task.grant_if params['grant'], '$dest_table'
       }
 
       # VACUUM, ANALYZE
-      task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
-      task.analyze_if params['analyze'], params['dest-table']
+      task.vacuum_if params['vacuum'], params['vacuum-sort'], '$dest_table'
+      task.analyze_if params['analyze'], '$dest_table'
     }
   }
 }

data/jobclass/rebuild-rename.rb

@@ -21,6 +21,7 @@ JobClass.define('rebuild-rename') {
 
   script {|params, script|
     script.task(params['data-source']) {|task|
+      dest_table = '$dest_table'
       prev_table = '${dest_table}_old'
       work_table = '${dest_table}_wk'
 
@@ -43,10 +44,9 @@ JobClass.define('rebuild-rename') {
 
       # RENAME
       task.transaction {
-        task.create_dummy_table '$dest_table'
-        dest_table = params['dest-table']
-        task.rename_table dest_table.to_s, "#{dest_table.name}_old"
-        task.rename_table "#{dest_table}_wk", dest_table.name
+        task.create_dummy_table dest_table
+        task.rename_table dest_table, prev_table
+        task.rename_table work_table, dest_table
       }
     }
   }

data/jobclass/sql.rb

@@ -25,7 +25,7 @@ JobClass.define('sql') {
       task.exec params['sql-file']
       task.vacuum_if params['vacuum'], params['vacuum-sort']
       task.analyze_if params['analyze']
-      task.grant_if params['grant'], params['dest-table']
+      task.grant_if params['grant'], '$dest_table'
     }
   }
 }

data/jobclass/streaming_load.rb

@@ -18,6 +18,7 @@ class StreamingLoadJobClass < RubyJobClass
         optional: true, default: Bricolage::PSQLLoadOptions.new,
         value_handler: lambda {|value, ctx, vars| Bricolage::PSQLLoadOptions.parse(value) })
     params.add Bricolage::DataSourceParam.new('s3', 's3-ds', 'S3 data source.')
+    params.add Bricolage::DataSourceParam.new('s3', 'ctl-ds', 'S3 data source for control files. (default: $s3-ds)', optional: true)
     params.add Bricolage::StringParam.new('ctl-prefix', 'S3_PREFIX', 'S3 object key prefix for control files. (default: ${queue-path}/ctl)', optional: true)
     params.add Bricolage::OptionalBoolParam.new('keep-ctl', 'Does not delete control files if true.')
     params.add Bricolage::StringParam.new('queue-path', 'S3_PATH', 'S3 path for data file queue.')
@@ -59,8 +60,6 @@ class StreamingLoadJobClass < RubyJobClass
 
   def make_loader(params)
     ds = params['redshift-ds']
-    load_opts = params['load-options']
-    load_opts.provide_defaults(params['s3-ds'])
     RedshiftStreamingLoader.new(
       data_source: ds,
       queue: make_s3_queue(params),
@@ -68,7 +67,7 @@ class StreamingLoadJobClass < RubyJobClass
       table: string(params['dest-table']),
       work_table: string(params['work-table']),
       log_table: string(params['log-table']),
-      load_options: load_opts,
+      load_options: params['load-options'],
       sql: params['sql-file'],
       logger: ds.logger,
       noop: params['noop'],
@@ -81,6 +80,7 @@ class StreamingLoadJobClass < RubyJobClass
     ds = params['s3-ds']
     S3Queue.new(
       data_source: ds,
+      ctl_ds: (params['ctl-ds'] || params['s3-ds']),
       ctl_prefix: (params['ctl-prefix'] || "#{params['queue-path']}/ctl"),
       queue_path: params['queue-path'],
       persistent_path: params['persistent-path'],
@@ -362,8 +362,9 @@ class StreamingLoadJobClass < RubyJobClass
   class S3Queue
     extend Forwardable
 
-    def initialize(data_source:, ctl_prefix:, queue_path:, persistent_path:, file_name:, logger:)
+    def initialize(data_source:, ctl_ds:, ctl_prefix:, queue_path:, persistent_path:, file_name:, logger:)
       @ds = data_source
+      @ctl_ds = ctl_ds
       @ctl_prefix = ctl_prefix
       @queue_path = queue_path
       @persistent_path = persistent_path
@@ -388,18 +389,18 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def control_file_url(name)
-      @ds.url(control_file_path(name))
+      @ctl_ds.url(control_file_path(name))
     end
 
     def put_control_file(name, data, noop: false)
       @logger.info "s3 put: #{control_file_url(name)}"
-      @ds.object(control_file_path(name)).put(body: data) unless noop
+      @ctl_ds.object(control_file_path(name)).put(body: data) unless noop
       control_file_url(name)
     end
 
     def remove_control_file(name, noop: false)
       @logger.info "s3 delete: #{control_file_url(name)}"
-      @ds.object(control_file_path(name)).delete unless noop
+      @ctl_ds.object(control_file_path(name)).delete unless noop
     end
 
     def control_file_path(name)

data/lib/bricolage/psqldatasource.rb

@@ -328,7 +328,6 @@ module Bricolage
       unless src_ds.redshift_loader_source?
         raise ParameterError, "input data source does not support redshift as bulk loading source: #{src_ds.name}"
       end
-      opts.provide_defaults(src_ds)
       buf = StringIO.new
       buf.puts "copy #{dest_table}"
       buf.puts "from '#{src_ds.url(src_path)}'"

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.29.2'
+  VERSION = '5.30.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.29.2
+  version: 5.30.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-14 00:00:00.000000000 Z
+date: 2019-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -185,8 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: SQL Batch Framework