bricolage 5.9.7 → 5.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c19aa7e310f6f40ab0b9ce66b1782a81e82f5001
-  data.tar.gz: d5c3b3c822fb8f987ca49ef6f2b0e7091c194064
+  metadata.gz: 950cab4515b421cdfd9a29fba1fa7812d365647a
+  data.tar.gz: 93badbfd8528d66c97f2cdb5bda2217172386bd8
 SHA512:
-  metadata.gz: 28d0ffd735a81cbc9d48ea66f468567d7825809bf89c229c0310f7540feb6084157832d000fb0562ebe5d4ee3ec240156d2b3bedad7386904af3d1c7f6668ed6
-  data.tar.gz: 19cae6930b0717ac2efed5bc37644ed29b5c9b019abd7a04586b254457eb12d1b7267ca3f0b3887f6ac0f36dc90ffaf3b3ee370d054aad62272a41e097defa44
+  metadata.gz: 5bc1b110a47f6cf5f81a30595fe9b200b77a6b7918bf4b2f7a8064597b3af36e50cc73bf91d32e4def188c43dd2829a81a57184a87e3c82bcad054cdbf3da7a4
+  data.tar.gz: e9cba607a0436f613093ae654b347d69c04024a5ac97860fc113d61e6a34e3c7f9f55afa856c584442a07ed1a02781dc2634d1d9707df725ea9d0aa30c89d666

data/jobclass/streaming_load.rb

@@ -20,6 +20,7 @@ class StreamingLoadJobClass < RubyJobClass
     params.add Bricolage::StringParam.new('queue-path', 'S3_PATH', 'S3 path for data file queue.')
     params.add Bricolage::StringParam.new('persistent-path', 'S3_PATH', 'S3 path for persistent data file store.')
     params.add Bricolage::StringParam.new('file-name', 'PATTERN', 'name pattern of target data file.')
+    params.add Bricolage::SQLFileParam.new('sql-file', 'PATH', 'SQL to insert rows from the work table to the target table.', optional: true)
     params.add Bricolage::OptionalBoolParam.new('noop', 'Does not change any data.')
     params.add Bricolage::OptionalBoolParam.new('load-only', 'Just issues COPY statement to work table and quit. No INSERT, no dequeue, no load log.')
     params.add Bricolage::OptionalBoolParam.new('dequeue-only', 'Dequeues already loaded files.')
@@ -46,6 +47,10 @@ class StreamingLoadJobClass < RubyJobClass
     nil
   end
 
+  def bind(ctx, vars)
+    @loader.sql.bind(ctx, vars) if @loader.sql
+  end
+
   def make_loader(params)
     ds = params['redshift-ds']
     RedshiftStreamingLoader.new(
@@ -55,6 +60,7 @@ class StreamingLoadJobClass < RubyJobClass
       work_table: string(params['work-table']),
       log_table: string(params['log-table']),
       load_options: params['load-options'],
+      sql: params['sql-file'],
       logger: ds.logger,
       noop: params['noop'],
       load_only: params['load-only']
@@ -79,6 +85,7 @@ class StreamingLoadJobClass < RubyJobClass
   class RedshiftStreamingLoader
     def initialize(data_source:, queue:,
         table:, work_table: nil, log_table: nil, load_options: nil,
+        sql: nil,
         logger:, noop: false, load_only: false)
       @ds = data_source
       @src = queue
@@ -86,6 +93,7 @@ class StreamingLoadJobClass < RubyJobClass
       @work_table = work_table
       @log_table = log_table
       @load_options = load_options
+      @sql = sql
       @logger = logger
       @noop = noop
       @load_only = load_only
@@ -95,6 +103,8 @@ class StreamingLoadJobClass < RubyJobClass
       @job_process_id = "#{@start_time.strftime('%Y%m%d-%H%M%S')}.#{Socket.gethostname}.#{Process.pid}"
     end
 
+    attr_reader :sql
+
     def load
       load_in_parallel
     end
@@ -170,7 +180,8 @@ class StreamingLoadJobClass < RubyJobClass
 
     def commit_work_table(conn)
       return unless @work_table
-      execute_update conn, "insert into #{@table} select * from #{@work_table};"
+      insert_stmt = @sql ? @sql.source : "insert into #{@table} select * from #{@work_table};"
+      execute_update conn, insert_stmt
       # keep work table records for tracing
     end
 

data/lib/bricolage/datasource.rb

@@ -49,7 +49,7 @@ module Bricolage
       @logger = logger
     end
 
-    BUILTIN_TYPES = %w(generic file)
+    BUILTIN_TYPES = %w(generic file ruby)
 
     # For job classes
     def get(kind, name)

data/lib/bricolage/parameters.rb

@@ -222,6 +222,11 @@ module Bricolage
       @context.get_data_source('file', 'file')
     end
 
+    # FIXME: remove
+    def ruby_ds
+      @context.get_data_source('ruby', 'ruby')
+    end
+
   end   # class Parameters
 
   class Param

data/lib/bricolage/rubyjobclass.rb

@@ -1,6 +1,9 @@
 require 'bricolage/jobclass'
+require 'bricolage/datasource'
+require 'forwardable'
 
 module Bricolage
+
   class RubyJobClass
     def RubyJobClass.job_class_id(id)
       wrapper = self
@@ -24,19 +27,51 @@ module Bricolage
 
     def RubyJobClass.script(params, script)
       ruby_job = new(params)
-      script.task(params.generic_ds) {|task|
-        task.action('ruby job') {
-          ruby_job.run
-          nil   # job result
-        }
+      script.task(params.ruby_ds) {|task|
+        task.bind_ruby_job ruby_job
       }
     end
 
     def initialize(params, *args)
     end
 
+    def bind(ctx, vars)
+    end
+
+    def source
+      'ruby job'
+    end
+
     def run
       raise "bricolage: error: #{self.class}\#run is not overridden"
     end
   end
+
+  class RubyDataSource < DataSource
+    declare_type 'ruby'
+
+    # FIXME: keyword argument placeholder is required
+    def initialize(**)
+    end
+
+    def new_task
+      RubyTask.new(self)
+    end
+  end
+
+  class RubyTask < DataSourceTask
+    def bind_ruby_job(ruby_job)
+      add RubyAction.new(ruby_job)
+    end
+
+    class RubyAction < Action
+      def initialize(ruby_job)
+        @ruby_job = ruby_job
+      end
+
+      extend Forwardable
+      def_delegators '@ruby_job', :source, :bind, :run
+    end
+  end
+
 end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.9.7'
+  VERSION = '5.10.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.9.7
+  version: 5.10.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg