bricolage-streamingload 0.15.0 → 0.15.1

data/testschema/strload_test.ct

@@ -0,0 +1,11 @@
+--dest-table: $test_schema.strload_test
+
+create table $dest_table
+( jst_time timestamp encode delta
+, full_controller varchar(128) encode lzo
+, action varchar(128) encode lzo
+, user_id integer encode lzo
+, unique_id varchar(80) encode lzo
+)
+sortkey (jst_time)
+;

data/testschema/with_work_table.job

@@ -0,0 +1,4 @@
+class: streaming_load_v3
+dest-table: $testschema_schema.with_work_table
+work-table: $testschema_schema.with_work_table_wk
+sql-file: with_work_table.sql

data/testschema/with_work_table.sql

@@ -0,0 +1 @@
+insert into $dest_table select * from $work_table;

data/utils/init_strload_tables.sql

@@ -0,0 +1,13 @@
+drop table if exists strload_objects cascade;
+drop table if exists strload_dup_objects cascade;
+drop table if exists strload_task_objects;
+drop table if exists strload_tasks cascade;
+drop table if exists strload_jobs cascade;
+
+\i schema/strload_objects.ct
+\i schema/strload_dup_objects.ct
+\i schema/strload_task_objects.ct
+\i schema/strload_tasks.ct
+\i schema/strload_jobs.ct
+\i schema/strload_stats.cv
+\i schema/strload_task_status.cv

data/utils/strload-stat.sql

@@ -0,0 +1,36 @@
+select
+    task_seq
+    , tbl.source_id
+    , tbl.schema_name
+    , tbl.table_name
+    , submit_time
+    , object_count
+    , total_object_size
+    , job_seq
+    , loader_id
+    , start_time
+    , finish_time
+    , status
+    , substring(message, 1, 30) as err_msg
+from
+    strload_tasks t
+    inner join (
+        select
+            task_seq
+            , count(*) as object_count
+            , sum(object_size) as total_object_size
+        from
+            strload_task_objects
+            inner join strload_objects
+                using (object_seq)
+        group by 1
+        ) o
+        using (task_seq)
+    left outer join strload_jobs j
+        using (task_seq)
+    left outer join strload_tables tbl
+        using (source_id)
+order by
+    task_seq
+    , job_seq
+;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.15.1
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-20 00:00:00.000000000 Z
+date: 2018-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.26'
+        version: 5.29.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.26'
+        version: 5.29.2
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -45,28 +45,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -95,35 +95,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Bricolage Streaming Load Daemon loads S3 data files to Redshift continuously.
 email:
 - aamine@loveruby.net
 executables:
-- send-flushtable-event
-- send-shutdown-event
-- send-load-task
+- bricolage-streaming-dispatcher
 - bricolage-streaming-loader
 - send-checkpoint-event
 - send-data-event
-- bricolage-streaming-dispatcher
+- send-flushtable-event
+- send-load-task
+- send-shutdown-event
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSES
 - README.md
+- RELEASE.md
+- Rakefile
 - bin/bricolage-streaming-dispatcher
 - bin/bricolage-streaming-loader
 - bin/send-checkpoint-event
@@ -131,6 +123,13 @@ files:
 - bin/send-flushtable-event
 - bin/send-load-task
 - bin/send-shutdown-event
+- bricolage-streamingload.gemspec
+- config/production/database.yml
+- config/production/password.yml
+- config/production/streamingload.yml
+- config/production/variable.yml
+- config/test/database.yml
+- config/test/variable.yml
 - lib/bricolage/sqsdatasource.rb
 - lib/bricolage/sqsmock.rb
 - lib/bricolage/streamingload/alertinglogger.rb
@@ -148,11 +147,14 @@ files:
 - lib/bricolage/streamingload/manifest.rb
 - lib/bricolage/streamingload/taskhandler.rb
 - lib/bricolage/streamingload/version.rb
-- test/all.rb
-- test/streamingload/test_dispatcher.rb
-- test/streamingload/test_dispatchermessage.rb
-- test/streamingload/test_job.rb
-- test/test_sqsdatasource.rb
+- sample/sqs-message.txt
+- sample/sqs-result.txt
+- strload_load_logs.ct
+- testschema/strload_test.ct
+- testschema/with_work_table.job
+- testschema/with_work_table.sql
+- utils/init_strload_tables.sql
+- utils/strload-stat.sql
 homepage: https://github.com/aamine/bricolage-streamingload
 licenses:
 - MIT
@@ -165,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -173,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Bricolage Streaming Load Daemon

data/test/all.rb

@@ -1,3 +0,0 @@
-Dir.glob("#{__dir__}/**/test_*.rb").each do |path|
-  load path
-end

data/test/streamingload/test_dispatcher.rb

@@ -1,241 +0,0 @@
-require 'test/unit'
-require 'bricolage/context'
-require 'bricolage/sqsdatasource'
-require 'bricolage/sqsmock'
-require 'bricolage/streamingload/dispatcher'
-require 'bricolage/streamingload/chunkrouter'
-require 'bricolage/streamingload/chunkbuffer'
-
-module Bricolage
-  module StreamingLoad
-
-    class TestDispatcher < Test::Unit::TestCase
-
-      test "checkpoint event" do
-        ctx = Context.for_application('.', environment: 'test', logger: NullLogger.new)
-        ctl_ds = ctx.get_data_source('sql', 'dwhctl')
-
-        event_queue = SQSDataSource.new_mock(queue: [
-          # 1st ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0003.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0004.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0005.json.gz')
-          ],
-          # 2nd ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0006.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0007.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0008.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0009.json.gz'),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'checkpoint'}),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0010.json.gz')
-          ]
-        ])
-
-        task_queue = SQSDataSource.new_mock
-
-        chunk_buffer = ChunkBuffer.new(
-          control_data_source: ctl_ds,
-          logger: ctx.logger
-        )
-
-        chunk_router = ChunkRouter.for_config([
-          {
-            "url" => %r<\As3://test-bucket/testschema\.desttable/datafile-\d{4}\.json\.gz>.source,
-            "schema" => 'testschema',
-            "table" => 'desttable'
-          }
-        ])
-
-        task_logger = DummyLoadTaskLogger.new
-
-        dispatcher = Dispatcher.new(
-          event_queue: event_queue,
-          task_queue: task_queue,
-          chunk_buffer: chunk_buffer,
-          chunk_router: chunk_router,
-          task_logger: task_logger,
-          dispatch_interval: 600,
-          logger: ctx.logger
-        )
-
-        # FIXME: database cleaner
-        ctl_ds.open {|conn|
-          conn.update("truncate strload_tables")
-          conn.update("truncate strload_objects")
-          conn.update("truncate strload_task_objects")
-          conn.update("truncate strload_tasks")
-          conn.update("insert into strload_tables values (1, 'testschema.desttable', 'testschema', 'desttable', 100, 1800, false)")
-        }
-        dispatcher.event_loop
-
-        # Event Queue Call Sequence
-        hst = event_queue.client.call_history
-        assert_equal 6, hst.size
-        assert_equal :send_message, hst[0].name      # start flush timer
-        assert_equal :receive_message, hst[1].name
-        assert_equal :delete_message_batch, hst[2].name
-        assert_equal :receive_message, hst[3].name
-        assert_equal :delete_message, hst[4].name    # delete checkpoint
-        assert_equal :delete_message_batch, hst[5].name
-
-        # Task Queue Call Sequence
-        hst = task_queue.client.call_history
-        assert_equal 1, hst.size
-        assert_equal :send_message, hst[0].name
-        assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
-        task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
-        assert_not_equal 0, task_id
-
-        # Object Buffer
-        assert_equal [], unassigned_objects(ctl_ds)
-        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
-        assert_equal 'testschema', task.schema_name
-        assert_equal 'desttable', task.table_name
-        assert_equal 10, task.object_urls.size
-
-        # Task Logger
-        assert_not_nil task_logger.last_task
-        assert_equal task_id, task_logger.last_task.id.to_i
-      end
-
-      def unassigned_objects(ctl_ds)
-        ctl_ds.open {|conn|
-          conn.query_values(<<-EndSQL)
-              select
-                  object_url
-              from
-                  strload_objects
-              where
-                  object_id not in (select object_id from strload_task_objects)
-              ;
-          EndSQL
-        }
-      end
-
-      test "flushtable event" do
-        ctx = Context.for_application('.', environment: 'test', logger: NullLogger.new)
-        ctl_ds = ctx.get_data_source('sql', 'dwhctl')
-
-        event_queue = SQSDataSource.new_mock(queue: [
-          # 1st ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0003.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0003.json.gz'),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'flushtable', tableName: 'testschema.bbb'}),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'shutdown'})
-          ]
-        ])
-
-        task_queue = SQSDataSource.new_mock
-
-        chunk_buffer = ChunkBuffer.new(
-          control_data_source: ctl_ds,
-          logger: ctx.logger
-        )
-
-        chunk_router = ChunkRouter.for_config([
-          {
-            "url" => %r<\As3://test-bucket/testschema\.(?<table>\w+)/datafile-\d{4}\.json\.gz>.source,
-            "schema" => 'testschema',
-            "table" => '%table'
-          }
-        ])
-
-        task_logger = DummyLoadTaskLogger.new
-
-        dispatcher = Dispatcher.new(
-          event_queue: event_queue,
-          task_queue: task_queue,
-          chunk_buffer: chunk_buffer,
-          chunk_router: chunk_router,
-          task_logger: task_logger,
-          dispatch_interval: 600,
-          logger: ctx.logger
-        )
-
-        # FIXME: database cleaner
-        ctl_ds.open {|conn|
-          conn.update("truncate strload_tables")
-          conn.update("truncate strload_objects")
-          conn.update("truncate strload_task_objects")
-          conn.update("truncate strload_tasks")
-          conn.update("insert into strload_tables values (1, 'testschema.aaa', 'testschema', 'aaa', 100, 1800, false)")
-          conn.update("insert into strload_tables values (2, 'testschema.bbb', 'testschema', 'bbb', 100, 1800, false)")
-          conn.update("insert into strload_tables values (3, 'testschema.ccc', 'testschema', 'ccc', 100, 1800, false)")
-        }
-        dispatcher.event_loop
-
-        # Event Queue Call Sequence
-        hst = event_queue.client.call_history
-        assert_equal 5, hst.size
-        assert_equal :send_message, hst[0].name      # start dispatch timer
-        assert_equal :receive_message, hst[1].name
-        assert_equal :delete_message, hst[2].name    # delete flushtable event
-        assert_equal :delete_message, hst[3].name    # delete shutdown event
-        assert_equal :delete_message_batch, hst[4].name
-
-        # Task Queue Call Sequence
-        hst = task_queue.client.call_history
-        assert_equal 1, hst.size
-        assert_equal :send_message, hst[0].name
-        assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
-        task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
-        assert_not_equal 0, task_id
-
-        # Object Buffer
-        assert_equal [], unassigned_table_objects(ctl_ds, 'testschema.bbb')
-        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
-        assert_equal 'testschema', task.schema_name
-        assert_equal 'bbb', task.table_name
-        assert_equal 2, task.object_urls.size
-
-        # Task Logger
-        assert_not_nil task_logger.last_task
-        assert_equal task_id, task_logger.last_task.id.to_i
-      end
-
-      def unassigned_table_objects(ctl_ds, table_name)
-        ctl_ds.open {|conn|
-          conn.query_values(<<-EndSQL)
-              select
-                  object_url
-              from
-                  strload_objects
-              where
-                  data_source_id = '#{table_name}'
-                  and object_id not in (select object_id from strload_task_objects)
-              ;
-          EndSQL
-        }
-      end
-
-    end
-
-
-    class DummyLoadTaskLogger
-
-      def initialize
-        @task = nil
-      end
-
-      def log(task)
-        @task = task
-        nil
-      end
-
-      def last_task
-        @task
-      end
-
-    end
-
-  end
-end