bricolage 5.20.0 → 5.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb66877b6bc66492a6091a11eafbd9bb953f95e5
-  data.tar.gz: a31a4383dad5d6b8b1a319fa3a2d68900f57db55
+  metadata.gz: 952dae79a52b71032a1356582777888388e8727f
+  data.tar.gz: ec3a98ec8bfcdbd2fd26639555cd603794f28700
 SHA512:
-  metadata.gz: c2b402f7f2392a0dec1b4ed72753abed5a9683cf3860b3c6a906d86a1901abbccd9f88d5a4c7d6e52bd6e4ba235e4de0e8ce355971c3d91e064946224f8c91b7
-  data.tar.gz: c3cf0419ce2a8dc470ed15b9d9837bf8bf32cbfbc440d655813f167d101d2b9ebcb56d21099bd0653e79e44eba85b4062e189179cf4b4ff00e8b98ee1481c2fe
+  metadata.gz: 500bda6bef8694fa64285f9a76900f34fbcc14ea1c4169bc3b2c855c708b90837b78e293aa145225bed75a90847f0062e1b91ee8d905ae5dfa70012e152b8caf
+  data.tar.gz: a51c47307e485bf4042a745d1eb0d01b1d1fe1d7daf2e5a55d991fa594069975df2b4a98e2c8591c8204b04a33c8b7a27a8a91ebb3cdaaae0d5eb8b43d1604c9

data/jobclass/my-import-delta.rb

@@ -0,0 +1,66 @@
+require 'bricolage/psqldatasource'
+require 'bricolage/mysqldatasource'
+
+JobClass.define('my-import-delta') {
+  parameters {|params|
+    # S3Export
+    params.add SrcTableParam.new(optional: false)
+    params.add DataSourceParam.new('mysql', 'src-ds', 'Source data source.')
+    params.add SQLFileParam.new(optional: true)
+    params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
+    params.add DestFileParam.new('s3-prefix', 'PREFIX', 'Temporary S3 prefix.')
+    params.add KeyValuePairsParam.new('dump-options', 'KEY:VALUE', 'dump options.', optional: true)
+
+    # Delete, Load
+    params.add DataSourceParam.new('sql', 'dest-ds', 'Destination data source.')
+    params.add StringParam.new('delete-cond', 'SQL_EXPR', 'DELETE condition.')
+    params.add DestTableParam.new(optional: false)
+    params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+        optional: true, default: PSQLLoadOptions.new,
+        value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+
+    # Misc
+    params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+    params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
+    params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+
+    # All
+    params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')
+    params.add OptionalBoolParam.new('load', 'Runs LOAD task.')
+    params.add OptionalBoolParam.new('gzip', 'Compress Temporary files.')
+  }
+
+  script {|params, script|
+    run_all = !params['export'] && !params['load']
+
+    # S3Export
+    if params['export'] || run_all
+      script.task(params['src-ds']) {|task|
+        task.s3export params['src-tables'].values.first.to_s,
+                      params['sql-file'],
+                      params['s3-ds'],
+                      params['s3-prefix'],
+                      params['gzip'],
+                      dump_options: params['dump-options']
+      }
+    end
+
+    # Load
+    if params['load'] || run_all
+      script.task(params['dest-ds']) {|task|
+        task.transaction {
+          # DELETE
+          task.exec SQLStatement.delete_where(params['delete-cond']) if params['delete-cond']
+
+          # COPY
+          task.load params['s3-ds'], params['s3-prefix'], params['dest-table'],
+              'json', nil, params['options'].merge('gzip' => params['gzip'])
+        }
+
+        # VACUUM, ANALYZE
+        task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
+        task.analyze_if params['analyze'], params['dest-table']
+      }
+    end
+  }
+}

data/lib/bricolage/mysqldatasource.rb

@@ -247,7 +247,7 @@ module Bricolage
       def command_parameters
         params = {jar: mys3dump_path.to_s, h: ds.host, P: ds.port.to_s, D: ds.database, u: ds.username, p: ds.password, o: connection_property, t: @table,
                    'Daws.accessKeyId' => @s3ds.access_key, 'Daws.secretKey' => @s3ds.secret_key, b: @s3ds.bucket.name, x: @prefix}
-        params[:q] = @statement.stripped_source.chop if @statement
+        params[:q] = @statement.stripped_source.chomp(';') if @statement
         params[:f] = @format if @format
         params[:C] = nil if @gzip
         params[:c] = @partition_column if @partition_column

data/lib/bricolage/s3datasource.rb

@@ -125,7 +125,17 @@ module Bricolage
     end
 
     def traverse(rel, no_prefix: false)
-      bucket.objects(prefix: path(rel, no_prefix: no_prefix))
+      retries = client.config.retry_limit
+      begin
+        bucket.objects(prefix: path(rel, no_prefix: no_prefix))
+      rescue Aws::Xml::Parser::ParsingError => e
+        retries -= 1
+        if retries >= 0
+          logger.warn "Retry Bucket#objects() for XML parsing error: #{e.message}"
+          retry
+        end
+        raise
+      end
     end
   end
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.20.0'
+  VERSION = '5.20.1'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.20.0)
+    bricolage (5.20.1)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)
@@ -11,51 +11,53 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.6.14)
-      aws-sdk-resources (= 2.6.14)
-    aws-sdk-core (2.6.14)
+    aws-sdk (2.6.36)
+      aws-sdk-resources (= 2.6.36)
+    aws-sdk-core (2.6.36)
+      aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.14)
-      aws-sdk-core (= 2.6.14)
+    aws-sdk-resources (2.6.36)
+      aws-sdk-core (= 2.6.36)
+    aws-sigv4 (1.0.0)
     coderay (1.1.0)
-    fluent-logger (0.5.1)
-      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+    fluent-logger (0.6.1)
+      msgpack (>= 0.5.6, < 2)
     hirb (0.7.3)
-    httpclient (2.8.2.4)
+    httpclient (2.8.3)
     jmespath (1.3.1)
     json (2.0.2)
     method_source (0.8.2)
-    msgpack (0.5.12)
-    mysql2 (0.4.4)
+    msgpack (1.0.2)
+    mysql2 (0.4.5)
     parallel (1.8.0)
     pg (0.18.4)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    redis (3.3.1)
-    ruby-progressbar (1.7.5)
+    redis (3.3.2)
+    ruby-progressbar (1.8.1)
     rubyzip (1.1.7)
     slop (3.6.0)
-    td (0.15.0)
+    td (0.15.2)
       hirb (>= 0.4.5)
-      msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack
       parallel (~> 1.8.0)
-      ruby-progressbar (~> 1.7.5)
+      ruby-progressbar (~> 1.7)
       rubyzip (~> 1.1.7)
-      td-client (~> 0.8.82)
-      td-logger (~> 0.3.21)
+      td-client (~> 0.8.85)
+      td-logger (>= 0.3.21, < 2)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.84)
+    td-client (0.8.85)
       httpclient (>= 2.7)
       json (>= 1.7.6)
       msgpack (>= 0.5.6, < 2)
-    td-logger (0.3.25)
-      fluent-logger (~> 0.5.0)
-      msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+    td-logger (0.3.26)
+      fluent-logger (>= 0.5.0, < 2.0)
+      msgpack (>= 0.5.6, < 2.0)
       td-client (~> 0.8.66)
-    yajl-ruby (1.2.1)
+    yajl-ruby (1.3.0)
     zip-zip (0.3)
       rubyzip (>= 1.0.0)
 

data/test/home/config/development/database.yml

@@ -56,7 +56,6 @@ s3:
 
 mysql:
   type: mysql
-  #host: db-main-slave-free-001
   host: 127.0.0.1
   port: 9011
   database: main
@@ -64,6 +63,15 @@ mysql:
   password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
 
+mysql_summary:
+  type: mysql
+  host: 127.0.0.1
+  port: 40201
+  database: summarized_data
+  username: work_readonly
+  password: <%= password 'mysql_shared_work_readonly' %>
+  encoding: utf8
+
 sqs_preproc:
   type: sqs
   url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"

data/test/home/subsys/item_pv_acc.ct

@@ -0,0 +1,8 @@
+create table item_pv_acc
+( data_date date encode lzo
+, item_id integer encode lzo
+, pv bigint encode lzo
+, uu bigint encode lzo
+)
+sortkey (data_date, item_id)
+;

data/test/home/subsys/my-import-delta.sql.job

@@ -0,0 +1,40 @@
+/*
+class: my-import-delta
+src-ds: mysql_summary
+src-tables:
+    item_pv: summarized_data.daily_recipe_pvs
+s3-ds: s3
+s3-prefix: bricolage/my-import-delta/$data_date/data.
+gzip: true
+dump-options:
+    partition_column: id
+    delete_objects: true
+dest-ds: sql
+dest-table: $test_schema.item_pv_acc
+delete-cond: "data_date = '$data_date'"
+options:
+    statupdate: false
+    compupdate: false
+    maxerror: 0
+    #acceptinvchars: " "
+    #trimblanks: true
+    #truncatecolumns: true
+    ## datetime
+    #acceptanydate: true
+    #dateformat: "auto"
+    #timeformat: "auto"
+#vacuum-sort: true
+analyze: true
+*/
+
+select
+    coalesce(`date`) as data_date
+    , coalesce(recipe_id) as item_id
+    , coalesce(page_view) as pv
+    , coalesce(unique_ip) as uu
+from
+    $item_pv
+where
+    `date` = date '$data_date'
+    and (@PARTITION_CONDITION@)
+;

data/test/home/subsys/variable.yml

@@ -1,3 +1,4 @@
+data_date: "<%= (Date.today - 1).strftime('%Y-%m-%d') %>"
 defaults:
     analyze: true
     grant:

data/test/test_s3datasource.rb

@@ -0,0 +1,40 @@
+require 'test/unit'
+require 'mocha/test_unit'
+require 'bricolage/s3datasource'
+require 'pp'
+require 'aws-sdk'
+
+module Bricolage
+  class TestS3datasouce < Test::Unit::TestCase
+
+    def setup
+      @ds = S3DataSource.new
+      @ds.stubs(:logger).returns(Logger.new)
+    end
+
+    test "traverse without exception" do
+      bucket = mock()
+      bucket.stubs(:objects).returns(true)
+      @ds.stubs(:bucket).returns(bucket)
+      assert @ds.traverse(nil)
+    end
+
+    test "traverse with 2 exception" do
+      bucket = mock()
+      bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
+        raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
+      @ds.stubs(:bucket).returns(bucket)
+      assert @ds.traverse(nil)
+    end
+
+    test "traverse with more than 3 exception" do
+      bucket = mock()
+      bucket.stubs(:objects).raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
+        raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
+        raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.
+        raises(Aws::Xml::Parser::ParsingError.new("test message","0","test column")).then.returns(true)
+      @ds.stubs(:bucket).returns(bucket)
+      assert_raise(Aws::Xml::Parser::ParsingError) {@ds.traverse(nil)}
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.20.0
+  version: 5.20.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-11 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Redshift-oriented Data Warehouse Batch Framework
 email: aamine@loveruby.net
 executables:
@@ -140,6 +154,7 @@ files:
 - jobclass/insert.rb
 - jobclass/load.rb
 - jobclass/my-export.rb
+- jobclass/my-import-delta.rb
 - jobclass/my-import.rb
 - jobclass/my-migrate.rb
 - jobclass/noop.rb
@@ -210,6 +225,7 @@ files:
 - test/home/revert.sh
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
+- test/home/subsys/item_pv_acc.ct
 - test/home/subsys/job1.job
 - test/home/subsys/job2.job
 - test/home/subsys/job3.job
@@ -221,6 +237,7 @@ files:
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
 - test/home/subsys/migrate.job
+- test/home/subsys/my-import-delta.sql.job
 - test/home/subsys/my-import.job
 - test/home/subsys/net1.jobnet
 - test/home/subsys/net2.jobnet
@@ -241,6 +258,7 @@ files:
 - test/test_c_streaming_load.rb
 - test/test_filesystem.rb
 - test/test_parameters.rb
+- test/test_s3datasource.rb
 - test/test_variables.rb
 - test/vacuum-test.rb
 homepage: https://github.com/aamine/bricolage