bricolage-streamingload 0.15.1 → 0.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e5d795da31bb8940f14d95be7ae2710ffdf026ab07bed08c619c20b25ce91e6
-  data.tar.gz: ce467fbb158a9a9a3a42eaf07a23c9741cf0f8721511a2cf66295d3f59f0c131
+  metadata.gz: dd7c6fd20043035518ecd176cbc35686794fb150dbc61ea8c1be40b7828fb8b7
+  data.tar.gz: 0bebfa811896f72308cabe2d27dafad5b2a07d31c6669c1bdc7a9499f634d1e9
 SHA512:
-  metadata.gz: 22008a5432ecf609084750804e39b5ac20e71f477c86cdb71066b4c5adea0a370ee15316bfba9972196ad431b0751afa4ea1f18b9c0d1524acd9f1a896994ad6
-  data.tar.gz: 84b45b8a683d971965ce2162ce5a2e9b47ffff015242290266dd64409e9c9890c7c52c5ed692a91010cbed076de79d1383d9f1c0d31619924f158dbb789ed76c
+  metadata.gz: b98c1709b7da75b91406b0ba670cdb78326635ab4d029bc01a737db6a2bd01004f27509fa1c92f6fac6d5ab93402155674e3bf3b66862970de20eeba4a053ccb
+  data.tar.gz: cc2b3659c714c711a1174872d9dfe09c3ee99f1806d262181e7949c47e65be9893a35f57ebd22436299cb5ff2cafe14116f4a4a50750422a240e630d8d9c6288

data/Gemfile.lock

@@ -1,44 +1,51 @@
 PATH
   remote: .
   specs:
-    bricolage-streamingload (0.15.1)
+    bricolage-streamingload (0.16.1)
       aws-sdk-s3 (~> 1.8)
       aws-sdk-sqs (~> 1.3)
-      bricolage (>= 5.29.2)
+      bricolage (~> 5.30)
       pg (~> 0.18.0)
+      sentry-raven (~> 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.0.1)
-    aws-partitions (1.100.0)
-    aws-sdk-core (3.24.1)
-      aws-eventstream (~> 1.0)
-      aws-partitions (~> 1.0)
-      aws-sigv4 (~> 1.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.351.0)
+    aws-sdk-core (3.104.3)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.7.0)
-      aws-sdk-core (~> 3)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-s3 (1.17.0)
-      aws-sdk-core (~> 3, >= 3.21.2)
+    aws-sdk-kms (1.36.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.75.0)
+      aws-sdk-core (~> 3, >= 3.104.1)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-sns (1.3.0)
-      aws-sdk-core (~> 3)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-sqs (1.4.0)
-      aws-sdk-core (~> 3)
-      aws-sigv4 (~> 1.0)
-    aws-sigv4 (1.0.3)
-    bricolage (5.29.2)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sns (1.28.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sqs (1.30.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+    bricolage (5.30.0)
       aws-sdk-s3 (~> 1)
       aws-sdk-sns (~> 1)
       pg (~> 0.18.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
     jmespath (1.4.0)
+    multipart-post (2.1.1)
     pg (0.18.4)
     power_assert (1.1.3)
-    rake (12.3.1)
+    rake (12.3.3)
+    sentry-raven (3.0.0)
+      faraday (>= 1.0)
     test-unit (3.2.8)
       power_assert
 
@@ -51,4 +58,4 @@ DEPENDENCIES
   test-unit
 
 BUNDLED WITH
-   1.16.1
+   1.17.2

data/RELEASE.md

@@ -1,5 +1,28 @@
 # Bricolage Streaming Load Release Note
 
+## version 0.17.1
+
+- [new][tmp] Sends a message to Sentry on start-up.
+
+## version 0.17.0
+
+- [new] Introduces Sentry error reporting.  Define SENTRY_DSN environment variable to enable it.
+
+## version 0.16.1
+
+- [fix] Stop retrying after 2 times retried (total 3 trial).
+
+## version 0.16.0
+
+- [new] Retry for also error tasks, not only failure tasks.
+  Error tasks are assumed as "non-retriable" e.g. DB login error, but some of them are really able to be resolved
+  by retrying on ECS environment. Retrying is not expensive (and human operation is relatively expensive),
+  we'll retry on all failures and errors.
+
+## version 0.15.2
+
+- [fix] send-data-event: Send objectSize=0 to indicate this message is fake.
+
 ## version 0.15.1
 
 - [new] Use the schema name in the parameters table by default (no more "no such variable: XXXX_schema" error).

data/bin/send-data-event

@@ -28,7 +28,7 @@ ARGF.each do |line|
       },
       object: {
         key: key,
-        size: 1
+        size: 0
       }
     },
     noDispatch: no_dispatch

data/bricolage-streamingload.gemspec

@@ -17,10 +17,11 @@ Gem::Specification.new do |s|
   s.require_path = 'lib'
 
   s.required_ruby_version = '>= 2.3.0'
-  s.add_dependency 'bricolage', '>= 5.29.2'
+  s.add_dependency 'bricolage', '~> 5.30'
   s.add_dependency 'pg', '~> 0.18.0'
   s.add_dependency 'aws-sdk-s3', '~> 1.8'
   s.add_dependency 'aws-sdk-sqs', '~> 1.3'
+  s.add_dependency 'sentry-raven', '~> 3.0'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'test-unit'
 end

data/lib/bricolage/streamingload/dispatcher.rb

@@ -10,8 +10,9 @@ require 'bricolage/streamingload/chunkbuffer'
 require 'bricolage/streamingload/loadtasklogger'
 require 'bricolage/streamingload/alertinglogger'
 require 'yaml'
-require 'optparse'
 require 'fileutils'
+require 'raven'
+require 'optparse'
 
 module Bricolage
 
@@ -20,6 +21,13 @@ module Bricolage
     class Dispatcher < SQSDataSource::MessageHandler
 
       def Dispatcher.main
+        Raven.capture_message("dispatcher start")
+        Raven.capture {
+          _main
+        }
+      end
+
+      def Dispatcher._main
         opts = DispatcherOptions.new(ARGV)
         opts.parse
         unless opts.rest_arguments.size == 1

data/lib/bricolage/streamingload/job.rb

@@ -3,6 +3,7 @@ require 'bricolage/streamingload/manifest'
 require 'bricolage/sqlutils'
 require 'socket'
 require 'json'
+require 'raven'
 
 module Bricolage
 
@@ -44,7 +45,8 @@ module Bricolage
       def execute(fail_fast: false)
         execute_task
         return true
-      rescue JobCancelled
+      rescue JobCancelled => ex
+        Raven.capture_exception(ex)
         return true
       rescue JobDuplicated
         return true
@@ -52,20 +54,25 @@ module Bricolage
         return false
       rescue ControlConnectionFailed => ex
         @logger.error ex.message
+        Raven.capture_exception(ex)
         wait_for_connection('ctl', @ctl_ds) unless fail_fast
         return false
       rescue DataConnectionFailed => ex
         @logger.error ex.message
+        Raven.capture_exception(ex)
         wait_for_connection('data', @data_ds) unless fail_fast
         return false
       rescue JobFailure => ex
         @logger.error ex.message
+        Raven.capture_exception(ex)
         return false
       rescue JobError => ex
         @logger.error ex.message
-        return true
+        Raven.capture_exception(ex)
+        return false
       rescue Exception => ex
         @logger.exception ex
+        Raven.capture_exception(ex)
         return true
       end
 
@@ -126,7 +133,11 @@ module Bricolage
           raise
         rescue JobError => ex
           ctl.open {
-            ctl.abort_job job_id, 'error', ex.message.lines.first.strip
+            fail_count = @task.failure_count
+            final_retry = (fail_count >= MAX_RETRY)
+            retry_msg = (fail_count > 0) ? "(retry\##{fail_count}#{final_retry ? ' FINAL' : ''}) " : ''
+            ctl.abort_job job_id, 'error', retry_msg + ex.message.lines.first.strip
+            raise JobCancelled, "retry count exceeds limit: task_id=#{@task_id}" if final_retry
           }
           raise
         rescue Exception => ex

data/lib/bricolage/streamingload/taskhandler.rb

@@ -7,6 +7,7 @@ require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
 require 'yaml'
+require 'raven'
 require 'optparse'
 
 module Bricolage
@@ -16,6 +17,8 @@ module Bricolage
     class TaskHandler < SQSDataSource::MessageHandler
 
       def TaskHandler.main
+        Raven.capture_message("loader start")
+
         opts = TaskHandlerOptions.new(ARGV)
         opts.parse
         unless opts.rest_arguments.size <= 1
@@ -149,12 +152,15 @@ module Bricolage
 
     class NoopJob
 
-      def initialize(context:, ctl_ds:, task_id:, force: false, logger:)
+      def initialize(context:, ctl_ds:, data_ds:, log_table:, task_id:, force: false, logger:)
         @ctx = context
         @ctl_ds = ctl_ds
+        @data_ds = data_ds
+        @log_table = log_table
         @task_id = task_id
         @force = force
         @logger = logger
+        @working_dir = Dir.getwd
       end
 
       def execute(fail_fast: false)
@@ -164,7 +170,7 @@ module Bricolage
       end
 
       def execute_task
-        @logger.info "execute_task: task_id=#{@task_id} force=#{@force} ctx=#{@ctx.home_path} ctl_ds=#{@ctl_ds.name} dir=#{@working_dir}"
+        @logger.info "execute_task: task_id=#{@task_id} force=#{@force} ctx=#{@ctx.home_path} ctl_ds=#{@ctl_ds.name} data_ds=#{@data_ds.name} dir=#{@working_dir}"
       end
 
     end

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.15.1'
+    VERSION = '0.17.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.17.1
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-21 00:00:00.000000000 Z
+date: 2020-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.29.2
+        version: '5.30'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.29.2
+        version: '5.30'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +67,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: sentry-raven
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -151,6 +165,7 @@ files:
 - sample/sqs-result.txt
 - strload_load_logs.ct
 - testschema/strload_test.ct
+- testschema/testlog.json.gz
 - testschema/with_work_table.job
 - testschema/with_work_table.sql
 - utils/init_strload_tables.sql
@@ -174,8 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Bricolage Streaming Load Daemon