bricolage 5.8.7 → 5.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: afb5228b706d506b2b2f7b468f216f76cdd82dc9
-  data.tar.gz: 9c947de18235c29e7db2b3a7b54fa095f8bf2335
+  metadata.gz: 5d8fe450bc754d0074c361ebdec9b3f0119cf440
+  data.tar.gz: a12ba4bf06eb22f598e4d8e7df8b1028c84e0c52
 SHA512:
-  metadata.gz: 3759c24eda01c02d78d0f3724c14254dc8dd8b687fc0bdd82869ec3db4ef9120b6cff2a5312533a04ef79d912fc9be96191edde3a81e0f79477783fc563245e9
-  data.tar.gz: 3873fafbb3f9dbbec2b2a38afbcbe4f2230ecd1f4467c421ceb37717a6be6723fa111a456c568adf03353471e9c3f5a89ecee9d1d649b02248b95d24858b36d0
+  metadata.gz: 801a0ba6a167068141d446e2d8e598ef6bcafcfec591c9011dcedf8aa126afdfb2dd6b830dfc9786f530e2608a48d5c69faee6b510c1b0e833feca6ae661df6a
+  data.tar.gz: cbb2b228f85c74f5b1a4f83a9d55eddcccfbcf9d9ea91ab5b2e4564614a11ecb512222448bf2e71eebbb17237eb792368e780f5ca3104f63e3278543cf8bce28

data/README.md

@@ -1,4 +1,19 @@
-bricolage
+Bricolage
 =========
 
-SQL batch framework
+SQL batch framework for Redshift.
+
+This software is written in working time in Cookpad, Inc.
+
+
+License
+-------
+
+MIT license.
+See LICENSES file for details.
+
+
+Author
+------
+
+Minero Aoki

data/jobclass/streaming_load.rb

@@ -0,0 +1,451 @@
+require 'bricolage/rubyjobclass'
+require 'bricolage/psqldatasource'
+require 'bricolage/exception'
+require 'json'
+require 'socket'
+
+class StreamingLoadJobClass < RubyJobClass
+  job_class_id 'streaming_load'
+
+  def StreamingLoadJobClass.parameters(params)
+    super
+    params.add Bricolage::DataSourceParam.new('sql', 'redshift-ds', 'Redshift data source.')
+    params.add Bricolage::DestTableParam.new(optional: false)
+    params.add Bricolage::DestTableParam.new('work-table', optional: true)
+    params.add Bricolage::DestTableParam.new('log-table', optional: true)
+    params.add Bricolage::KeyValuePairsParam.new('load-options', 'OPTIONS', 'Loader options.',
+        optional: true, default: Bricolage::PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| Bricolage::PSQLLoadOptions.parse(value) })
+    params.add Bricolage::DataSourceParam.new('s3', 's3-ds', 'S3 data source.')
+    params.add Bricolage::StringParam.new('queue-path', 'S3_PATH', 'S3 path for data file queue.')
+    params.add Bricolage::StringParam.new('persistent-path', 'S3_PATH', 'S3 path for persistent data file store.')
+    params.add Bricolage::StringParam.new('file-name', 'PATTERN', 'name pattern of target data file.')
+    params.add Bricolage::OptionalBoolParam.new('noop', 'Does not change any data.')
+    params.add Bricolage::OptionalBoolParam.new('load-only', 'Just issues COPY statement to work table and quit. No INSERT, no dequeue, no load log.')
+    params.add Bricolage::OptionalBoolParam.new('dequeue-only', 'Dequeues already loaded files.')
+  end
+
+  def StreamingLoadJobClass.declarations(params)
+    Bricolage::Declarations.new(
+      'dest_table' => nil,
+      'work_table' => nil,
+      'log_table' => nil
+    )
+  end
+
+  def initialize(params)
+    @loader = make_loader(params)
+    @load_only = params['load-only']
+    @dequeue_only = params['dequeue-only']
+  end
+
+  def run
+    @loader.dequeue_loaded_files unless @load_only
+    return nil if @dequeue_only
+    @loader.load
+    nil
+  end
+
+  def make_loader(params)
+    ds = params['redshift-ds']
+    RedshiftStreamingLoader.new(
+      data_source: ds,
+      queue: make_s3_queue(params),
+      table: string(params['dest-table']),
+      work_table: string(params['work-table']),
+      log_table: string(params['log-table']),
+      load_options: params['load-options'],
+      logger: ds.logger,
+      noop: params['noop'],
+      load_only: params['load-only']
+    )
+  end
+
+  def make_s3_queue(params)
+    ds = params['s3-ds']
+    S3Queue.new(
+      data_source: ds,
+      queue_path: params['queue-path'],
+      persistent_path: params['persistent-path'],
+      file_name: params['file-name'],
+      logger: ds.logger
+    )
+  end
+
+  def string(obj)
+    obj ? obj.to_s : nil
+  end
+
+  class RedshiftStreamingLoader
+    def initialize(data_source:, queue:,
+        table:, work_table: nil, log_table: nil, load_options: nil,
+        logger:, noop: false, load_only: false)
+      @ds = data_source
+      @src = queue
+      @table = table
+      @work_table = work_table
+      @log_table = log_table
+      @load_options = load_options
+      @logger = logger
+      @noop = noop
+      @load_only = load_only
+
+      @start_time = Time.now
+      @end_time = nil
+      @job_process_id = "#{@start_time.strftime('%Y%m%d-%H%M%S')}.#{Socket.gethostname}.#{Process.pid}"
+    end
+
+    def load
+      load_in_parallel
+    end
+
+    def log_basic_info
+      @logger.info "start_time: #{@start_time}"
+      @logger.info "job_process_id: #{@job_process_id}"
+      @logger.info "queue: #{@src.queue_url}"
+    end
+
+    def load_in_parallel
+      log_basic_info
+      @logger.info 'load with manifest'
+      objects = @src.queued_objects
+      if objects.empty?
+        @logger.info 'no target data files; exit'
+        return
+      end
+      create_manifest_file(objects) {|manifest_url|
+        @ds.open {|conn|
+          init_work_table conn
+          execute_update conn, copy_manifest_statement(manifest_url, @src.credential_string)
+          @logger.info "load succeeded: #{manifest_url}" unless @noop
+          commit conn, objects
+        }
+        dequeue_all objects
+      }
+    end
+
+    def load_in_sequential
+      log_basic_info
+      @logger.info 'load each objects sequentially'
+      objects = @src.queued_objects
+      @ds.open {|conn|
+        init_work_table(conn)
+        objects.each do |obj|
+          @logger.info "load: #{obj.url}"
+          execute_update conn, copy_file_statement(obj)
+          @logger.info "load succeeded: #{obj.url}" unless @noop
+        end
+        commit conn, objects
+      }
+      dequeue_all objects
+    end
+
+    def commit(conn, objects)
+      @end_time = Time.now
+      return if @load_only
+      transaction(conn) {
+        commit_work_table conn
+        write_load_logs conn, objects
+      }
+    end
+
+    def dequeue_loaded_files
+      @logger.info "dequeue start"
+      objects = @src.queued_objects
+      @ds.open {|conn|
+        objects.each do |obj|
+          if loaded_object?(conn, obj)
+            obj.dequeue(@noop)
+          end
+        end
+      }
+    end
+
+    private
+
+    def init_work_table(conn)
+      return unless @work_table
+      execute_update conn, "truncate #{@work_table};"
+    end
+
+    def commit_work_table(conn)
+      return unless @work_table
+      execute_update conn, "insert into #{@table} select * from #{@work_table};"
+      # keep work table records for tracing
+    end
+
+    def copy_file_statement(obj)
+      %Q(
+        copy #{load_target_table} from '#{obj.url}'
+        credentials '#{obj.credential_string}'
+        #{@load_options}
+      ;).gsub(/\s+/, ' ').strip
+    end
+
+    def create_manifest_file(objects)
+      manifest_name = "manifest-#{@job_process_id}.json"
+      @logger.info "creating manifest: #{manifest_name}"
+      @logger.info "manifest:\n" + make_manifest_json(objects)
+      url = @src.put_control_file(manifest_name, make_manifest_json(objects), noop: @noop)
+      yield url
+      @src.remove_control_file(File.basename(url), noop: @noop)
+    end
+
+    def make_manifest_json(objects)
+      ents = objects.map {|obj|
+        { "url" => obj.url, "mandatory" => true }
+      }
+      JSON.pretty_generate({ "entries" => ents })
+    end
+
+    def copy_manifest_statement(manifest_url, credential_string)
+      %Q(
+        copy #{load_target_table}
+        from '#{manifest_url}'
+        credentials '#{credential_string}'
+        manifest
+        #{@load_options}
+      ;).gsub(/\s+/, ' ').strip
+    end
+
+    def load_target_table
+      @work_table || @table
+    end
+
+    def write_load_logs(conn, objects)
+      return unless @log_table
+      make_load_logs(objects).each do |record|
+        write_load_log conn, record
+      end
+    end
+
+    def make_load_logs(objects)
+      objects.map {|obj| make_load_log(obj) }
+    end
+
+    def make_load_log(obj)
+      LoadLogRecord.new(@job_process_id, @start_time, @end_time, @table, obj.url)
+    end
+
+    LoadLogRecord = Struct.new(:job_process_id, :start_time, :end_time, :target_table, :data_file)
+
+    def write_load_log(conn, record)
+      return unless @log_table
+      execute_update(conn, <<-EndSQL.gsub(/^\s+/, '').strip)
+        insert into #{@log_table}
+        ( job_process_id
+        , start_time
+        , end_time
+        , target_table
+        , data_file
+        )
+        values
+        ( #{sql_string record.job_process_id}
+        , #{sql_timestamp record.start_time}
+        , #{sql_timestamp record.end_time}
+        , #{sql_string record.target_table}
+        , #{sql_string record.data_file}
+        )
+        ;
+      EndSQL
+    end
+
+    def loaded_object?(conn, obj)
+      rs = conn.execute("select count(*) as c from #{@log_table} where data_file = #{sql_string obj.url}")
+      rs.first['c'].to_i > 0
+    end
+
+    def sql_timestamp(time)
+      %Q(timestamp '#{time.strftime('%Y-%m-%d %H:%M:%S')}')
+    end
+
+    def sql_string(str)
+      escaped = str.gsub("'", "''")
+      %Q('#{escaped}')
+    end
+
+    def transaction(conn)
+      execute_update conn, 'begin transaction'
+      yield
+      execute_update conn, 'commit'
+    end
+
+    def execute_update(conn, sql)
+      if @noop
+        log_query(sql)
+      else
+        conn.execute(sql)
+      end
+    end
+
+    def log_query(sql)
+      @logger.info "[#{@ds.name}] #{mask_secrets(sql)}"
+    end
+
+    def mask_secrets(log)
+      log.gsub(/\bcredentials\s+'.*?'/mi, "credentials '****'")
+    end
+
+    def dequeue_all(objects)
+      return if @load_only
+      objects.each do |obj|
+        obj.dequeue(@noop)
+      end
+    end
+  end
+
+  class S3Queue
+    def initialize(data_source:, queue_path:, persistent_path:, file_name:, logger:)
+      @ds = data_source
+      @queue_path = queue_path
+      @persistent_path = persistent_path
+      @file_name = file_name
+      @logger = logger
+    end
+
+    def credential_string
+      @ds.credential_string
+    end
+
+    attr_reader :queue_path
+
+    def queue_url
+      @ds.url(@queue_path)
+    end
+
+    def object_url(key)
+      @ds.url(key, no_prefix: true)
+    end
+
+    def control_file_url(name)
+      @ds.url(control_file_path(name))
+    end
+
+    def put_control_file(name, data, noop: false)
+      @logger.info "s3 put: #{control_file_url(name)}"
+      @ds.object(control_file_path(name)).write(data) unless noop
+      control_file_url(name)
+    end
+
+    def remove_control_file(name, noop: false)
+      @logger.info "s3 delete: #{control_file_url(name)}"
+      @ds.object(control_file_path(name)).delete unless noop
+    end
+
+    def control_file_path(name)
+      "#{queue_path}/#{name}"
+    end
+
+    def consume_each(noop: false, &block)
+      each do |obj|
+        yield obj and obj.save(noop: noop)
+      end
+    end
+
+    def each(&block)
+      queued_objects.each(&block)
+    end
+
+    def queue_directory
+      @ds.objects_with_prefix(queue_path)
+    end
+
+    def queued_file_nodes
+      queue_directory.as_tree.children.select {|node|
+        node.leaf? and
+          node.key[-1, 1] != '/' and
+          target_file_name?(File.basename(node.key))
+      }
+    end
+
+    def queued_objects
+      queued_file_nodes.map {|node| LoadableObject.new(self, node, @logger) }
+    end
+
+    def target_file_name?(name)
+      file_name_pattern =~ name
+    end
+
+    def persistent_path(name)
+      @ds.path("#{format_path(@persistent_path, name)}/#{name}")
+    end
+
+    def format_path(template, basename)
+      m = file_name_pattern.match(basename) or
+        raise ParameterError, "file name does not match the pattern: #{basename.inspect}"
+      template.gsub(/%./) {|op|
+        case op
+        when '%Y' then m[:year]
+        when '%m' then m[:month]
+        when '%d' then m[:day]
+        when '%H' then m[:hour]
+        when '%M' then m[:minute]
+        when '%S' then m[:second]
+        when '%N' then m[:nanosecond]
+        when '%Q' then m[:seq]
+        else raise ParameterError, "unknown time format in s3.file_name config: #{op}"
+        end
+      }
+    end
+
+    def file_name_pattern
+      @file_name_pattern ||= compile_name_pattern(@file_name)
+    end
+
+    def compile_name_pattern(template)
+      pattern = template.gsub(/[^%]+|%\d*./) {|op|
+        case op
+        when '%Y' then '(?<year>\\d{4})'
+        when '%m' then '(?<month>\\d{2})'
+        when '%d' then '(?<day>\\d{2})'
+        when '%H' then '(?<hour>\\d{2})'
+        when '%M' then '(?<minute>\\d{2})'
+        when '%S' then '(?<second>\\d{2})'
+        when /\A%(\d+)N\z/ then "(?<nanosecond>\\d{#{$1}})"
+        when '%Q' then '(?<seq>\\d+)'
+        when '%%' then '%'
+        when /\A%/ then raise ParameterError, "unknown time format in s3.file_name config: #{op}"
+        else Regexp.quote(op)
+        end
+      }
+      Regexp.compile("\\A#{pattern}\\z")
+    end
+  end
+
+  class LoadableObject
+    def initialize(s3queue, node, logger)
+      @s3queue = s3queue
+      @node = node
+      @logger = logger
+    end
+
+    def credential_string
+      @s3queue.credential_string
+    end
+
+    def path
+      @node.key
+    end
+
+    def basename
+      File.basename(path)
+    end
+
+    def url
+      @s3queue.object_url(path)
+    end
+
+    def save(noop = false)
+      @logger.info "s3 move: #{path} -> #{save_path}"
+      return if noop
+      @node.object.move_to save_path
+      @logger.info "file saved"
+    end
+
+    alias dequeue save
+
+    def save_path
+      @s3queue.persistent_path(basename)
+    end
+  end
+end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.8.7'
+  VERSION = '5.9.0'
 end

data/test/home/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gem 'bricolage', path: '../..'

data/test/home/Gemfile.lock

@@ -0,0 +1,54 @@
+PATH
+  remote: ../..
+  specs:
+    bricolage (5.8.7)
+      aws-sdk (< 2)
+      mysql2
+      pg
+      td
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (1.64.0)
+      aws-sdk-v1 (= 1.64.0)
+    aws-sdk-v1 (1.64.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    fluent-logger (0.4.10)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      yajl-ruby (~> 1.0)
+    hirb (0.7.3)
+    httpclient (2.5.3.3)
+    json (1.8.2)
+    mini_portile (0.6.2)
+    msgpack (0.5.12)
+    mysql2 (0.3.18)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    parallel (0.6.5)
+    pg (0.18.2)
+    rubyzip (0.9.9)
+    td (0.11.10)
+      hirb (>= 0.4.5)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      parallel (~> 0.6.1)
+      rubyzip (~> 0.9.9)
+      td-client (~> 0.8.70)
+      td-logger (~> 0.3.21)
+      yajl-ruby (~> 1.1)
+    td-client (0.8.70)
+      httpclient (>= 2.4.0, < 2.6.0)
+      json (>= 1.7.6)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+    td-logger (0.3.24)
+      fluent-logger (~> 0.4.9)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      td-client (~> 0.8.66)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bricolage!

data/test/home/config/development/database.yml

@@ -1,6 +1,6 @@
 sql: &redshift_parameters
   type: psql
-  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-201.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
+  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-301.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
   port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
   database: production
   username: tabemirudev
@@ -16,7 +16,7 @@ sql_prod:
 
 sql_app:
   type: psql
-  host: localhost   #cookpad-dwh-101.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
+  host: localhost   #cookpad-dwh-301.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
   port: 5445
   database: production
   username: tabemiru_batch
@@ -44,9 +44,12 @@ td_search_log:
 
 s3:
   type: s3
+  endpoint: "s3-ap-northeast-1.amazonaws.com"
   bucket: tabemiru-data.ap-northeast-1
   prefix: "/dev"
   s3cfg: <%= user_home_relative_path '.s3cfg' %>
+  access_key_id: "AKIAI5R5BL2W73PR3FXQ"
+  secret_access_key: "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"
 
 mysql:
   type: mysql

data/test/home/data/20141002-1355_00.txt

@@ -0,0 +1,2 @@
+1	aaa	AAA
+2	BBB	BBB

data/test/home/data/20141002-1355_01.txt

@@ -0,0 +1,2 @@
+3	ccc	CCC
+4	ddd	DDD

data/test/home/data/20141002-1355_02.txt

@@ -0,0 +1,2 @@
+5	eee	EEE
+6	fff	FFF

data/test/home/put.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+for name
+in \
+    20141002-1355_00.txt \
+    20141002-1355_01.txt \
+    20141002-1355_02.txt
+do
+    aws s3 cp \
+        $(dirname $0)/data/$name \
+        $S3_TABEMIRU/dev/queue/$name
+done

data/test/home/revert.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+for name
+in \
+    20141002-1355_00.txt \
+    20141002-1355_01.txt \
+    20141002-1355_02.txt
+do
+    aws s3 mv \
+        s3://tabemiru-data.ap-northeast-1/tmp/save/year=2014/month=10/day=02/hour=13/$name \
+        s3://tabemiru-data.ap-northeast-1/tmp/queue/$name
+done

data/test/home/subsys/load_test.ct

@@ -0,0 +1,6 @@
+create table load_test
+( x integer
+, y varchar(20)
+, z varchar(20)
+)
+;

data/test/home/subsys/load_test.job

@@ -0,0 +1,12 @@
+class: streaming_load
+
+s3-ds: s3
+queue-path: queue
+persistent-path: save/year=%Y/month=%m/day=%d/hour=%H
+file-name: "%Y%m%d-%H%M_%Q.txt"
+
+redshift-ds: sql
+dest-table: load_test
+work-table: load_test_wk
+log-table: load_test_l
+load-options: "delimiter '\\t'"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.8.7
+  version: 5.9.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -117,6 +117,7 @@ files:
 - jobclass/rebuild-rename.rb
 - jobclass/s3-put.rb
 - jobclass/sql.rb
+- jobclass/streaming_load.rb
 - jobclass/td-delete.rb
 - jobclass/td-export.rb
 - jobclass/unload.rb
@@ -156,8 +157,17 @@ files:
 - libexec/sqldump.Darwin
 - libexec/sqldump.Linux
 - test/all.rb
+- test/home/Gemfile
+- test/home/Gemfile.lock
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
+- test/home/data/20141002-1355_00.txt
+- test/home/data/20141002-1355_01.txt
+- test/home/data/20141002-1355_02.txt
+- test/home/put.sh
+- test/home/revert.sh
+- test/home/subsys/load_test.ct
+- test/home/subsys/load_test.job
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
 - test/home/subsys/unified.jobnet