bricolage-streamingload 0.14.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba544bbf6c3a133aeb0882e5193296dc734a25ec
-  data.tar.gz: 8706c5cd050e1083075d2890e422e3834680c2ea
+  metadata.gz: 01102c31f0c9f92fb7d6653c130f2a06633cf1fc
+  data.tar.gz: 4b160bb1a4c176faa0451319d94c8d6aa2404f85
 SHA512:
-  metadata.gz: d03f41030b3c7bc7175c1abf7b2f6e8e5c3a40c3cff5754e678c76338190b9c0293792dc4a8cc34fbf262419ac054467503c823ceafc2a55a6e59d9a2762f937
-  data.tar.gz: 8bd7c0cc4b826383d552ed196581b00a8d33010e24ada2cb9f3eb76ed381178b35e706585a2da449d019bf8bd4e43233afc84902695382df152925261d3fb848
+  metadata.gz: cd84fa57a717eda77e19710004df60ee780311e3ef89d8668f1d65ef0b67455911339194dae90c914e2c60f56eece109d2ca460998ad918a0d554a273aa5b950
+  data.tar.gz: 7e981e9c417716f0b5b51a90b442596849c906f62184222b4b2dd9347f3e4c510e2400748f93989ab60b46a95974b5842d5b557be721ac01362f3a410e25a678

data/lib/bricolage/streamingload/job.rb

@@ -69,7 +69,7 @@ module Bricolage
         return true
       end
 
-      MAX_RETRY = 5
+      MAX_RETRY = 2
 
       def execute_task
         @process_id = "#{Socket.gethostname}-#{$$}"
@@ -241,6 +241,12 @@ module Bricolage
               ;
           EndSQL
           @logger.info "load succeeded: #{manifest.url}"
+        rescue JobFailure => ex
+          if /stl_load_errors/ =~ ex.message
+            # We cannot resolve this load error by retry, give up now.
+            raise JobError, ex.message
+          end
+          raise
         end
 
         def write_load_log(log)

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.14.1'
+    VERSION = '0.14.2'
   end
 end

data/test/streamingload/test_job.rb

@@ -161,8 +161,7 @@ module Bricolage
             [1001, 's3://data-bucket/testschema.sql_fails/0001.json.gz', 1024, 'testschema.sql_fails', 'mmmm', current_timestamp, current_timestamp],
             [1002, 's3://data-bucket/testschema.sql_fails/0002.json.gz', 1024, 'testschema.sql_fails', 'mmmm', current_timestamp, current_timestamp]
           db.insert_into 'strload_jobs',
-            [101, 11, 'localhost-1234', 'failure', current_timestamp, current_timestamp, 'query failed'],
-            [102, 11, 'localhost-1234', 'failure', current_timestamp, current_timestamp, 'query failed']
+            [101, 11, 'localhost-1234', 'failure', current_timestamp, current_timestamp, 'query failed']
 
           job = new_job(task_id: 11, force: false)
           assert_raise(JobFailure) {
@@ -178,7 +177,7 @@ module Bricolage
           assert_equal 11, job_row['task_id'].to_i
           assert_equal job.process_id, job_row['process_id']
           assert_equal 'failure', job_row['status']
-          assert(/retry\#2/ =~ job_row['message'])
+          assert(/retry\#1/ =~ job_row['message'])
         }
       end
 
@@ -267,6 +266,33 @@ module Bricolage
         }
       end
 
+      test "execute_task (load error)" do
+        setup_context {|db|
+          db.insert_into 'strload_tables', [1, 'testschema.load_error', 'testschema', 'load_error', 100, 1800, false]
+          db.insert_into 'strload_tasks', [11, 'streaming_load_v3', 1, current_timestamp]
+          db.insert_into 'strload_task_objects', [11, 1001], [11, 1002]
+          db.insert_into 'strload_objects',
+            [1001, 's3://data-bucket/testschema.desttable/0001.json.gz', 1024, 'testschema.desttable', 'mmmm', current_timestamp, current_timestamp],
+            [1002, 's3://data-bucket/testschema.desttable/0002.json.gz', 1024, 'testschema.desttable', 'mmmm', current_timestamp, current_timestamp]
+
+          job = new_job(task_id: 11, force: false)
+          assert_raise(JobError) {
+            job.execute_task
+          }
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.load_error from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
+
+          job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
+          assert_equal 11, job_row['task_id'].to_i
+          assert_equal job.process_id, job_row['process_id']
+          assert_equal 'error', job_row['status']
+          assert(/stl_load_errors/ =~ job_row['message'])
+        }
+      end
+
       test "execute_task (unknown status, really=success)" do
         setup_context {|db|
           db.insert_into 'strload_tables', [1, 'testschema.desttable', 'testschema', 'desttable', 100, 1800, false]
@@ -430,11 +456,12 @@ module Bricolage
       class PSQLDataSourceMock < DataSource
         declare_type 'psql_mock'
 
-        def initialize(fail_pattern: nil, error_pattern: nil, exception_pattern: nil, **params)
+        def initialize(fail_pattern: nil, error_pattern: nil, exception_pattern: nil, load_error_pattern: nil, **params)
           @sql_list = []
           @fail_pattern = fail_pattern ? Regexp.compile(fail_pattern) : nil
           @error_pattern = error_pattern ? Regexp.compile(error_pattern) : nil
           @exception_pattern = exception_pattern ? Regexp.compile(exception_pattern) : nil
+          @load_error_pattern = load_error_pattern ? Regexp.compile(load_error_pattern) : nil
           @job_status = {}
         end
 
@@ -461,6 +488,9 @@ module Bricolage
           if @exception_pattern and @exception_pattern =~ sql
             raise ArgumentError, "unexpected exception"
           end
+          if @load_error_pattern and @load_error_pattern =~ sql
+            raise JobError, "Load into table 'xxxx_table' failed.  Check 'stl_load_errors' system table for details."
+          end
         end
 
         def provide_job_status(job_id, succeeded)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.2
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-09 00:00:00.000000000 Z
+date: 2018-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -99,13 +99,13 @@ description: Bricolage Streaming Load Daemon loads S3 data files to Redshift con
 email:
 - aamine@loveruby.net
 executables:
-- bricolage-streaming-dispatcher
+- send-flushtable-event
+- send-shutdown-event
+- send-load-task
 - bricolage-streaming-loader
 - send-checkpoint-event
 - send-data-event
-- send-flushtable-event
-- send-load-task
-- send-shutdown-event
+- bricolage-streaming-dispatcher
 extensions: []
 extra_rdoc_files: []
 files: