bricolage 5.19.0 → 5.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e09f9be942c6431360ac3ec8afbd9cae5c03b06
-  data.tar.gz: b9c2dc9c5a91b42b0e60fb43c089b12f58670128
+  metadata.gz: 08ee4bdc2cc0df34636a9d370bd8dcc658f1fa5a
+  data.tar.gz: a26fcb9a3150e12572bacc4bd98168ebab68888f
 SHA512:
-  metadata.gz: be674ea50692704964d7360c35092e116078b5fb03ac0d2e2f620eb9abb083ffbc9a335d384e4f1c5a68594da4ff00e528097f707da6e03a2925e20a015fe7b5
-  data.tar.gz: 69241018dcfa1862b327bcc9d43404890b91bc38bcd57836b5a34187507318a3ba9907de33534fef15c606bc01a08e5f4cd6d70ba2c0e0215d9614f5005af7fb
+  metadata.gz: 5915c56686b6757f8a059aa3d7bd05ae14ff0d2b8d806cca59392698fcd1ffcdf8e574a3b1204208c9b4f7dbefe92a8984165920b4a400b1c282950e3cc7c649
+  data.tar.gz: 30c0701ff39c2e15c2723a0fb4aac188a655c0ceea04c73df7c3097594367e316a563b51a62958cd3e3747f7b0b2ac38caaa7dba0081baac7e3751f4337944c5

data/jobclass/streaming_load.rb

@@ -18,6 +18,8 @@ class StreamingLoadJobClass < RubyJobClass
         optional: true, default: Bricolage::PSQLLoadOptions.new,
         value_handler: lambda {|value, ctx, vars| Bricolage::PSQLLoadOptions.parse(value) })
     params.add Bricolage::DataSourceParam.new('s3', 's3-ds', 'S3 data source.')
+    params.add Bricolage::StringParam.new('ctl-prefix', 'S3_PREFIX', 'S3 object key prefix for control files. (default: ${queue-path}/ctl)', optional: true)
+    params.add Bricolage::OptionalBoolParam.new('keep-ctl', 'Does not delete control files if true.')
     params.add Bricolage::StringParam.new('queue-path', 'S3_PATH', 'S3 path for data file queue.')
     params.add Bricolage::StringParam.new('persistent-path', 'S3_PATH', 'S3 path for persistent data file store.')
     params.add Bricolage::StringParam.new('file-name', 'PATTERN', 'name pattern of target data file.')
@@ -61,6 +63,7 @@ class StreamingLoadJobClass < RubyJobClass
     RedshiftStreamingLoader.new(
       data_source: ds,
       queue: make_s3_queue(params),
+      keep_ctl: params['keep-ctl'],
       table: string(params['dest-table']),
       work_table: string(params['work-table']),
       log_table: string(params['log-table']),
@@ -76,6 +79,7 @@ class StreamingLoadJobClass < RubyJobClass
     ds = params['s3-ds']
     S3Queue.new(
       data_source: ds,
+      ctl_prefix: (params['ctl-prefix'] || "#{params['queue-path']}/ctl"),
       queue_path: params['queue-path'],
       persistent_path: params['persistent-path'],
       file_name: params['file-name'],
@@ -88,12 +92,13 @@ class StreamingLoadJobClass < RubyJobClass
   end
 
   class RedshiftStreamingLoader
-    def initialize(data_source:, queue:,
+    def initialize(data_source:, queue:, keep_ctl:,
         table:, work_table: nil, log_table: nil, load_options: nil,
         sql: nil,
         logger:, noop: false, load_only: false)
       @ds = data_source
       @src = queue
+      @keep_ctl = keep_ctl
       @table = table
       @work_table = work_table
       @log_table = log_table
@@ -200,13 +205,17 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def create_manifest_file(objects)
-      manifest_name = "manifest-#{@job_process_id}.json"
+      manifest_name = if @keep_ctl
+          "#{Time.now.strftime('%Y/%m/%d')}/#{@job_process_id}-#{@table}.json"
+        else
+          "manifest-#{@job_process_id}.json"
+        end
       @logger.info "creating manifest: #{manifest_name}"
       json = make_manifest_json(objects)
       @logger.info "manifest:\n" + json
       url = @src.put_control_file(manifest_name, json, noop: @noop)
       yield url
-      @src.remove_control_file(File.basename(url), noop: @noop)
+      @src.remove_control_file(File.basename(url), noop: @noop) unless @keep_ctl
     end
 
     def make_manifest_json(objects)
@@ -359,8 +368,9 @@ class StreamingLoadJobClass < RubyJobClass
   class S3Queue
     extend Forwardable
 
-    def initialize(data_source:, queue_path:, persistent_path:, file_name:, logger:)
+    def initialize(data_source:, ctl_prefix:, queue_path:, persistent_path:, file_name:, logger:)
       @ds = data_source
+      @ctl_prefix = ctl_prefix
       @queue_path = queue_path
       @persistent_path = persistent_path
       @file_name = file_name
@@ -399,7 +409,8 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def control_file_path(name)
-      "#{queue_path}/ctl/#{name}"
+      prefix = @ctl_prefix || "#{queue_path}/ctl"
+      "#{prefix}/#{name}"
     end
 
     def each(&block)

data/lib/bricolage/mysqldatasource.rb

@@ -183,7 +183,9 @@ module Bricolage
         write_concurrency: options['write_concurrency'],
         rotation_size: options['rotation_size'],
         delete_objects: options['delete_objects'],
-        object_key_delimiter: options['object_key_delimiter'])
+        object_key_delimiter: options['object_key_delimiter'],
+        src_zone_offset: options['src_zone_offset'],
+        dst_zone_offset: options['dst_zone_offset'])
     end
 
     class S3Export < Action
@@ -195,7 +197,9 @@ module Bricolage
          write_concurrency: 4,
          rotation_size: nil,
          delete_objects: false,
-         object_key_delimiter: nil)
+         object_key_delimiter: nil,
+         src_zone_offset: nil,
+         dst_zone_offset: nil)
         @table = table
         @statement = stmt
         @s3ds = s3ds
@@ -208,6 +212,8 @@ module Bricolage
         @rotation_size = rotation_size
         @delete_objects = delete_objects
         @object_key_delimiter = object_key_delimiter
+        @src_zone_offset = src_zone_offset
+        @dst_zone_offset = dst_zone_offset
       end
 
       def run
@@ -250,6 +256,12 @@ module Bricolage
         params[:r] = @rotation_size if @rotation_size
         params[:d] = nil if @delete_objects
         params[:k] = @object_key_delimiter if @object_key_delimiter
+        if src_zone_offset = @src_zone_offset || ds.mysql_options[:src_zone_offset]
+          params[:S] = src_zone_offset
+        end
+        if dst_zone_offset = @dst_zone_offset || ds.mysql_options[:dst_zone_offset]
+          params[:T] = dst_zone_offset
+        end
         params
       end
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.19.0'
+  VERSION = '5.19.1'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.19.0)
+    bricolage (5.19.1)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)
@@ -11,12 +11,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.6.7)
-      aws-sdk-resources (= 2.6.7)
-    aws-sdk-core (2.6.7)
+    aws-sdk (2.6.14)
+      aws-sdk-resources (= 2.6.14)
+    aws-sdk-core (2.6.14)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.7)
-      aws-sdk-core (= 2.6.7)
+    aws-sdk-resources (2.6.14)
+      aws-sdk-core (= 2.6.14)
     coderay (1.1.0)
     fluent-logger (0.5.1)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -47,10 +47,10 @@ GEM
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.83)
-      httpclient (~> 2.7)
+    td-client (0.8.84)
+      httpclient (>= 2.7)
       json (>= 1.7.6)
-      msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack (>= 0.5.6, < 2)
     td-logger (0.3.25)
       fluent-logger (~> 0.5.0)
       msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -67,4 +67,4 @@ DEPENDENCIES
   pry
 
 BUNDLED WITH
-   1.13.2
+   1.13.6

data/test/home/config/development/database.yml

@@ -42,16 +42,6 @@ postgres:
   encoding: utf8
   sql_log_level: DEBUG
 
-td:
-  type: td
-  database: logs
-  apikey: <%= password 'td_tabemiru' %>
-
-td_search_log:
-  type: td
-  database: search_log
-  apikey: <%= password 'td_tabemiru' %>
-
 s3:
   type: s3
   endpoint: "s3-ap-northeast-1.amazonaws.com"
@@ -74,27 +64,35 @@ mysql:
   password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
 
-s3_strload:
-  type: s3
-  region: "ap-northeast-1"
-  endpoint: "s3-ap-northeast-1.amazonaws.com"
-  bucket: tabemiru-data.ap-northeast-1
-  prefix: tmp/strload
-  access_key_id: "<%= password 'aws_access_key_id' %>"
-  secret_access_key: "<%= password 'aws_secret_access_key' %>"
+sqs_preproc:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"
+  max_number_of_messages: 10
+  visibility_timeout: 60
 
-sqs_strload_event:
+sqs_dispatch:
   type: sqs
   url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
   max_number_of_messages: 10
   visibility_timeout: 60
-  access_key_id: "<%= password 'aws_access_key_id' %>"
-  secret_access_key: "<%= password 'aws_secret_access_key' %>"
 
-sqs_strload_task:
+sqs_task:
   type: sqs
   url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
   max_number_of_messages: 1
   visibility_timeout: 180
-  access_key_id: "<%= password 'aws_access_key_id' %>"
-  secret_access_key: "<%= password 'aws_secret_access_key' %>"
+
+sns:
+  type: sns
+  region: "ap-northeast-1"
+  topic_arn: "arn:aws:sns:ap-northeast-1:789035092620:dwh-service-notification"
+
+td:
+  type: td
+  database: logs
+  apikey: <%= password 'td_tabemiru' %>
+
+td_search_log:
+  type: td
+  database: search_log
+  apikey: <%= password 'td_tabemiru' %>

data/test/home/revert.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+for i in data/2*.txt
+do
+    aws s3 cp $i s3://tabemiru-data.ap-northeast-1/dev/queue/
+done
+aws s3 rm --recursive s3://tabemiru-data.ap-northeast-1/dev/save/

data/test/home/subsys/load_test.job

@@ -1,12 +1,14 @@
 class: streaming_load
 
 s3-ds: s3
+ctl-prefix: ctl
+keep-ctl: true
 queue-path: queue
-persistent-path: save/year=%Y/month=%m/day=%d/hour=%H
+persistent-path: save/%Y/%m/%d
 file-name: "%Y%m%d-%H%M_%Q.txt"
 
 redshift-ds: sql
-dest-table: load_test
-work-table: load_test_wk
-log-table: load_test_l
+dest-table: $test_schema.load_test
+work-table: $test_schema.load_test_wk
+log-table: $test_schema.load_test_l
 load-options: "delimiter '\\t'"

data/test/home/subsys/my-import.job

@@ -6,10 +6,8 @@ s3-ds: s3
 s3-prefix: shimpeko/test-abc-
 gzip: true
 dump-options:
-    partition_column: id
-    partition_number: 8
-    write_concurrency: 16
-    rotation_size: 16000000
+    src_zone_offset: "+00:00"
+    dst_zone_offset: "+09:00"
     delete_objects: true
 dest-ds: sql
 dest-table: $test_schema.users

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.19.0
+  version: 5.19.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-17 00:00:00.000000000 Z
+date: 2016-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -207,7 +207,7 @@ files:
 - test/home/data/20141002-1355_02.txt
 - test/home/data/test.txt
 - test/home/jobnet-test.rb
-- test/home/sqltest.rb
+- test/home/revert.sh
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
 - test/home/subsys/job1.job

data/test/home/sqltest.rb

@@ -1,16 +0,0 @@
-require 'bricolage/commandlineapplication'
-require 'pp'
-
-app = Bricolage::CommandLineApplication.define {|opts|
-  opts.data_source_option('--ds', 'Target data source', short: '-D', kind: 'sql')
-}
-ds = app.data_source('--ds')
-ds.open {|conn|
-  #task_ids = conn.query_values('select task_id from strload_tasks')
-  #p [task_ids.class, task_ids.size]
-
-  conn.update('create table t (x int)')
-  pp conn.query_value('select count(*) from t')
-  conn.update('drop table t')
-  pp conn.query_value('select count(*) from t')
-}