bricolage-streamingload 0.14.2 → 0.16.1

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.14.2'
+    VERSION = '0.16.1'
   end
 end

data/sample/sqs-message.txt

@@ -0,0 +1,38 @@
+{
+  "Records" => [
+    {
+      "eventVersion" => "2.0",
+      "eventSource" => "aws:s3",
+      "awsRegion" => "ap-northeast-1",
+      "eventTime" => "2016-02-13T11:40:08.001Z",
+      "eventName" => "ObjectCreated:Put",
+      "userIdentity" => {
+        "principalId" => "AWS:AIXXXXXXXXXXXXXXXXX6A"
+      },
+      "requestParameters" => {
+        "sourceIPAddress" => "111.222.111.90"
+      },
+      "responseElements" =>  {
+        "x-amz-request-id" => "1111AAAA9999AAAA",
+        "x-amz-id-2" => "6p9IZG+R+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxFyXiNMMB4="
+      },
+      "s3" => {
+        "s3SchemaVersion" => "1.0",
+        "configurationId" => "ObjectCreatedEvents",
+        "bucket" => {
+          "name" => "some-bucket",
+          "ownerIdentity" => {
+            "principalId" => "AAAAIIIIBBBB11"
+          },
+          "arn" => "arn:aws:s3:::some-bucket"
+        },
+        "object" => {
+          "key" => "development/logs/schema.table/20160125_0000_0_de37c5ad-d26a-42cc-a141-475676b65f69.gz",
+          "size" => 1302957,
+          "eTag" => "d704db7f9cb77b1ecb981c534526b542",
+          "sequencer" => "111122223333444499"
+        }
+      }
+    }
+  ]
+}

data/sample/sqs-result.txt

@@ -0,0 +1,18 @@
+#<struct Aws::SQS::Types::ReceiveMessageResult
+  messages = [
+    #<struct Aws::SQS::Types::Message
+      message_id = "11112222-b37f-4281-aee2-696408c482be",
+      receipt_handle = "AQEBErp4Vv159Hvx5oXSioWu7Ov1Jj1ht423LwTTUlgLohmKh02gAMAZ7kN1J4+aLzM54BRuMatnifWPH3cBh6rtdSWlllysBVmXDRKj83VCaseKPpq+Vdw/bLZrPEYRSGPuhKsHa2DFjo67KeND4AqasZO7lOCJX+YcsbqsqAtcODlDWlVbfedPkKIGM3Scra3uY9ysGjNx/zWuJ88fCtvRpu9tgk0nG7srHAm84Mxj5ArakJnKnJMbtkD/+lvm0Vqi2YCkXonS8+BP8gK4kSTbl1uv/Gp7ZfxkjKQJBB8xQiYo7e7qKLwhbImMZtO5tqDKzIKFVfMi1G2ODF+tEj4Ce+ryMRGA50GhON2ETCyofsl1T7Wdr61IEOV0NFlonGTWFf4q/1r3OPhFAchp+tYnyA==",
+      md5_of_body = "00006bf43abdff178ca0ffa96205aaaa",
+      body = "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-northeast-1\",\"eventTime\":\"2016-02-13T11:40:07.268Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AAAAJKS3A4VEF45XCAAAA\"},\"requestParameters\":{\"sourceIPAddress\":\"111.222.111.90\"},\"responseElements\":{\"x-amz-request-id\":\"10AAAA31A1EDCCCC\",\"x-amz-id-2\":\"YT+U/PxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxzfFGPjqP5AQg=\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"LogStreamDev\",\"bucket\":{\"name\":\"redshift-copy-buffer\",\"ownerIdentity\":{\"principalId\":\"AAAAFIDWR40000\"},\"arn\":\"arn:aws:s3:::redshift-copy-buffer\"},\"object\":{\"key\":\"development/logs/schema.table/20160125_0000_0_2d0b43ad-0db9-4655-bd98-b7464b123763.gz\",\"size\":1238953,\"eTag\":\"aaaa196c3935f4957c7bb645f9780000\",\"sequencer\":\"0000BF161697C1AAAA\"}}}]}",
+      attributes = {
+        "SenderId" => "FACE0VEO02BJMF37H2JKW",
+        "ApproximateFirstReceiveTimestamp" => "1455364193429",
+        "ApproximateReceiveCount" => "1",
+        "SentTimestamp" => "1455363607341"
+      },
+      md5_of_message_attributes = nil,
+      message_attributes = {}
+    >
+  ]
+>

data/strload_load_logs.ct

@@ -0,0 +1,13 @@
+--dest-table: bricolage.strload_load_logs
+
+/*
+    Redshift-side log table
+*/
+create table $dest_table
+( job_id bigint encode raw
+, task_id bigint encode zstd
+, finish_time timestamp encode delta
+)
+distkey (job_id)
+sortkey (job_id)
+;

data/testschema/strload_test.ct

@@ -0,0 +1,11 @@
+--dest-table: $test_schema.strload_test
+
+create table $dest_table
+( jst_time timestamp encode delta
+, full_controller varchar(128) encode lzo
+, action varchar(128) encode lzo
+, user_id integer encode lzo
+, unique_id varchar(80) encode lzo
+)
+sortkey (jst_time)
+;

data/testschema/with_work_table.job

@@ -0,0 +1,4 @@
+class: streaming_load_v3
+dest-table: $testschema_schema.with_work_table
+work-table: $testschema_schema.with_work_table_wk
+sql-file: with_work_table.sql

data/testschema/with_work_table.sql

@@ -0,0 +1 @@
+insert into $dest_table select * from $work_table;

data/utils/init_strload_tables.sql

@@ -0,0 +1,13 @@
+drop table if exists strload_objects cascade;
+drop table if exists strload_dup_objects cascade;
+drop table if exists strload_task_objects;
+drop table if exists strload_tasks cascade;
+drop table if exists strload_jobs cascade;
+
+\i schema/strload_objects.ct
+\i schema/strload_dup_objects.ct
+\i schema/strload_task_objects.ct
+\i schema/strload_tasks.ct
+\i schema/strload_jobs.ct
+\i schema/strload_stats.cv
+\i schema/strload_task_status.cv

data/utils/strload-stat.sql

@@ -0,0 +1,36 @@
+select
+    task_seq
+    , tbl.source_id
+    , tbl.schema_name
+    , tbl.table_name
+    , submit_time
+    , object_count
+    , total_object_size
+    , job_seq
+    , loader_id
+    , start_time
+    , finish_time
+    , status
+    , substring(message, 1, 30) as err_msg
+from
+    strload_tasks t
+    inner join (
+        select
+            task_seq
+            , count(*) as object_count
+            , sum(object_size) as total_object_size
+        from
+            strload_task_objects
+            inner join strload_objects
+                using (object_seq)
+        group by 1
+        ) o
+        using (task_seq)
+    left outer join strload_jobs j
+        using (task_seq)
+    left outer join strload_tables tbl
+        using (source_id)
+order by
+    task_seq
+    , job_seq
+;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.16.1
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-19 00:00:00.000000000 Z
+date: 2020-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.29.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.29.2
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -40,35 +40,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.18.0
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: aws-sdk-sqs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '1.3'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -82,7 +82,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -99,17 +99,23 @@ description: Bricolage Streaming Load Daemon loads S3 data files to Redshift con
 email:
 - aamine@loveruby.net
 executables:
-- send-flushtable-event
-- send-shutdown-event
-- send-load-task
+- bricolage-streaming-dispatcher
 - bricolage-streaming-loader
 - send-checkpoint-event
 - send-data-event
-- bricolage-streaming-dispatcher
+- send-flushtable-event
+- send-load-task
+- send-shutdown-event
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSES
 - README.md
+- RELEASE.md
+- Rakefile
 - bin/bricolage-streaming-dispatcher
 - bin/bricolage-streaming-loader
 - bin/send-checkpoint-event
@@ -117,6 +123,13 @@ files:
 - bin/send-flushtable-event
 - bin/send-load-task
 - bin/send-shutdown-event
+- bricolage-streamingload.gemspec
+- config/production/database.yml
+- config/production/password.yml
+- config/production/streamingload.yml
+- config/production/variable.yml
+- config/test/database.yml
+- config/test/variable.yml
 - lib/bricolage/sqsdatasource.rb
 - lib/bricolage/sqsmock.rb
 - lib/bricolage/streamingload/alertinglogger.rb
@@ -134,11 +147,15 @@ files:
 - lib/bricolage/streamingload/manifest.rb
 - lib/bricolage/streamingload/taskhandler.rb
 - lib/bricolage/streamingload/version.rb
-- test/all.rb
-- test/streamingload/test_dispatcher.rb
-- test/streamingload/test_dispatchermessage.rb
-- test/streamingload/test_job.rb
-- test/test_sqsdatasource.rb
+- sample/sqs-message.txt
+- sample/sqs-result.txt
+- strload_load_logs.ct
+- testschema/strload_test.ct
+- testschema/testlog.json.gz
+- testschema/with_work_table.job
+- testschema/with_work_table.sql
+- utils/init_strload_tables.sql
+- utils/strload-stat.sql
 homepage: https://github.com/aamine/bricolage-streamingload
 licenses:
 - MIT
@@ -151,15 +168,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Bricolage Streaming Load Daemon

data/test/all.rb

@@ -1,3 +0,0 @@
-Dir.glob("#{__dir__}/**/test_*.rb").each do |path|
-  load path
-end

data/test/streamingload/test_dispatcher.rb

@@ -1,241 +0,0 @@
-require 'test/unit'
-require 'bricolage/context'
-require 'bricolage/sqsdatasource'
-require 'bricolage/sqsmock'
-require 'bricolage/streamingload/dispatcher'
-require 'bricolage/streamingload/chunkrouter'
-require 'bricolage/streamingload/chunkbuffer'
-
-module Bricolage
-  module StreamingLoad
-
-    class TestDispatcher < Test::Unit::TestCase
-
-      test "checkpoint event" do
-        ctx = Context.for_application('.', environment: 'test', logger: NullLogger.new)
-        ctl_ds = ctx.get_data_source('sql', 'dwhctl')
-
-        event_queue = SQSDataSource.new_mock(queue: [
-          # 1st ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0003.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0004.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0005.json.gz')
-          ],
-          # 2nd ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0006.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0007.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0008.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0009.json.gz'),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'checkpoint'}),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.desttable/datafile-0010.json.gz')
-          ]
-        ])
-
-        task_queue = SQSDataSource.new_mock
-
-        chunk_buffer = ChunkBuffer.new(
-          control_data_source: ctl_ds,
-          logger: ctx.logger
-        )
-
-        chunk_router = ChunkRouter.for_config([
-          {
-            "url" => %r<\As3://test-bucket/testschema\.desttable/datafile-\d{4}\.json\.gz>.source,
-            "schema" => 'testschema',
-            "table" => 'desttable'
-          }
-        ])
-
-        task_logger = DummyLoadTaskLogger.new
-
-        dispatcher = Dispatcher.new(
-          event_queue: event_queue,
-          task_queue: task_queue,
-          chunk_buffer: chunk_buffer,
-          chunk_router: chunk_router,
-          task_logger: task_logger,
-          dispatch_interval: 600,
-          logger: ctx.logger
-        )
-
-        # FIXME: database cleaner
-        ctl_ds.open {|conn|
-          conn.update("truncate strload_tables")
-          conn.update("truncate strload_objects")
-          conn.update("truncate strload_task_objects")
-          conn.update("truncate strload_tasks")
-          conn.update("insert into strload_tables values (1, 'testschema.desttable', 'testschema', 'desttable', 100, 1800, false)")
-        }
-        dispatcher.event_loop
-
-        # Event Queue Call Sequence
-        hst = event_queue.client.call_history
-        assert_equal 6, hst.size
-        assert_equal :send_message, hst[0].name      # start flush timer
-        assert_equal :receive_message, hst[1].name
-        assert_equal :delete_message_batch, hst[2].name
-        assert_equal :receive_message, hst[3].name
-        assert_equal :delete_message, hst[4].name    # delete checkpoint
-        assert_equal :delete_message_batch, hst[5].name
-
-        # Task Queue Call Sequence
-        hst = task_queue.client.call_history
-        assert_equal 1, hst.size
-        assert_equal :send_message, hst[0].name
-        assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
-        task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
-        assert_not_equal 0, task_id
-
-        # Object Buffer
-        assert_equal [], unassigned_objects(ctl_ds)
-        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
-        assert_equal 'testschema', task.schema_name
-        assert_equal 'desttable', task.table_name
-        assert_equal 10, task.object_urls.size
-
-        # Task Logger
-        assert_not_nil task_logger.last_task
-        assert_equal task_id, task_logger.last_task.id.to_i
-      end
-
-      def unassigned_objects(ctl_ds)
-        ctl_ds.open {|conn|
-          conn.query_values(<<-EndSQL)
-              select
-                  object_url
-              from
-                  strload_objects
-              where
-                  object_id not in (select object_id from strload_task_objects)
-              ;
-          EndSQL
-        }
-      end
-
-      test "flushtable event" do
-        ctx = Context.for_application('.', environment: 'test', logger: NullLogger.new)
-        ctl_ds = ctx.get_data_source('sql', 'dwhctl')
-
-        event_queue = SQSDataSource.new_mock(queue: [
-          # 1st ReceiveMessage
-          [
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0001.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0002.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0003.json.gz'),
-            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0003.json.gz'),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'flushtable', tableName: 'testschema.bbb'}),
-            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'shutdown'})
-          ]
-        ])
-
-        task_queue = SQSDataSource.new_mock
-
-        chunk_buffer = ChunkBuffer.new(
-          control_data_source: ctl_ds,
-          logger: ctx.logger
-        )
-
-        chunk_router = ChunkRouter.for_config([
-          {
-            "url" => %r<\As3://test-bucket/testschema\.(?<table>\w+)/datafile-\d{4}\.json\.gz>.source,
-            "schema" => 'testschema',
-            "table" => '%table'
-          }
-        ])
-
-        task_logger = DummyLoadTaskLogger.new
-
-        dispatcher = Dispatcher.new(
-          event_queue: event_queue,
-          task_queue: task_queue,
-          chunk_buffer: chunk_buffer,
-          chunk_router: chunk_router,
-          task_logger: task_logger,
-          dispatch_interval: 600,
-          logger: ctx.logger
-        )
-
-        # FIXME: database cleaner
-        ctl_ds.open {|conn|
-          conn.update("truncate strload_tables")
-          conn.update("truncate strload_objects")
-          conn.update("truncate strload_task_objects")
-          conn.update("truncate strload_tasks")
-          conn.update("insert into strload_tables values (1, 'testschema.aaa', 'testschema', 'aaa', 100, 1800, false)")
-          conn.update("insert into strload_tables values (2, 'testschema.bbb', 'testschema', 'bbb', 100, 1800, false)")
-          conn.update("insert into strload_tables values (3, 'testschema.ccc', 'testschema', 'ccc', 100, 1800, false)")
-        }
-        dispatcher.event_loop
-
-        # Event Queue Call Sequence
-        hst = event_queue.client.call_history
-        assert_equal 5, hst.size
-        assert_equal :send_message, hst[0].name      # start dispatch timer
-        assert_equal :receive_message, hst[1].name
-        assert_equal :delete_message, hst[2].name    # delete flushtable event
-        assert_equal :delete_message, hst[3].name    # delete shutdown event
-        assert_equal :delete_message_batch, hst[4].name
-
-        # Task Queue Call Sequence
-        hst = task_queue.client.call_history
-        assert_equal 1, hst.size
-        assert_equal :send_message, hst[0].name
-        assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
-        task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
-        assert_not_equal 0, task_id
-
-        # Object Buffer
-        assert_equal [], unassigned_table_objects(ctl_ds, 'testschema.bbb')
-        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
-        assert_equal 'testschema', task.schema_name
-        assert_equal 'bbb', task.table_name
-        assert_equal 2, task.object_urls.size
-
-        # Task Logger
-        assert_not_nil task_logger.last_task
-        assert_equal task_id, task_logger.last_task.id.to_i
-      end
-
-      def unassigned_table_objects(ctl_ds, table_name)
-        ctl_ds.open {|conn|
-          conn.query_values(<<-EndSQL)
-              select
-                  object_url
-              from
-                  strload_objects
-              where
-                  data_source_id = '#{table_name}'
-                  and object_id not in (select object_id from strload_task_objects)
-              ;
-          EndSQL
-        }
-      end
-
-    end
-
-
-    class DummyLoadTaskLogger
-
-      def initialize
-        @task = nil
-      end
-
-      def log(task)
-        @task = task
-        nil
-      end
-
-      def last_task
-        @task
-      end
-
-    end
-
-  end
-end