bricolage-streamingload 0.15.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c74f043409945f4dc970472a1a7735c67a7ba930
-  data.tar.gz: 9ce73867f02aff49a7391d168c791ce4be4d3727
+SHA256:
+  metadata.gz: 3e5d795da31bb8940f14d95be7ae2710ffdf026ab07bed08c619c20b25ce91e6
+  data.tar.gz: ce467fbb158a9a9a3a42eaf07a23c9741cf0f8721511a2cf66295d3f59f0c131
 SHA512:
-  metadata.gz: 4a6a1a3c6fbb28d1048481072596fc944e0732218bcddfd09dd9555a4d0a43b900f7806627c2f4ec4685b36cbf3a93ded8b173bd7a75917d652bef7d7a6a0739
-  data.tar.gz: ffa357b2233b11b9820ffeda35480c74a328904d531a1f67b716fc115d584979e4eff2332239c557179687121527d3e38c90afcc02d102b4e9a3f1e3fe12de70
+  metadata.gz: 22008a5432ecf609084750804e39b5ac20e71f477c86cdb71066b4c5adea0a370ee15316bfba9972196ad431b0751afa4ea1f18b9c0d1524acd9f1a896994ad6
+  data.tar.gz: 84b45b8a683d971965ce2162ce5a2e9b47ffff015242290266dd64409e9c9890c7c52c5ed692a91010cbed076de79d1383d9f1c0d31619924f158dbb789ed76c

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+
+/config/development/
+/config/test/

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,54 @@
+PATH
+  remote: .
+  specs:
+    bricolage-streamingload (0.15.1)
+      aws-sdk-s3 (~> 1.8)
+      aws-sdk-sqs (~> 1.3)
+      bricolage (>= 5.29.2)
+      pg (~> 0.18.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.0.1)
+    aws-partitions (1.100.0)
+    aws-sdk-core (3.24.1)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.7.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.17.0)
+      aws-sdk-core (~> 3, >= 3.21.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-sns (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-sqs (1.4.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
+    bricolage (5.29.2)
+      aws-sdk-s3 (~> 1)
+      aws-sdk-sns (~> 1)
+      pg (~> 0.18.0)
+    jmespath (1.4.0)
+    pg (0.18.4)
+    power_assert (1.1.3)
+    rake (12.3.1)
+    test-unit (3.2.8)
+      power_assert
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bricolage-streamingload!
+  rake
+  test-unit
+
+BUNDLED WITH
+   1.16.1

data/LICENSES

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2016 Minero Aoki, Shimpei Kodama
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/RELEASE.md

@@ -0,0 +1,116 @@
+# Bricolage Streaming Load Release Note
+
+## version 0.15.1
+
+- [new] Use the schema name in the parameters table by default (no more "no such variable: XXXX_schema" error).
+
+## version 0.15.0
+
+- [CHANGE] Upgrade AWS-SDK to v3.  This version requires at least Bricolage 5.25.
+
+## version 0.14.2
+
+- Reduces loading retry count
+
+## version 0.14.1
+
+- [fix] task logging did not work
+
+## version 0.14.0
+
+- [new] Logs (task - object) relashonships to S3 on dispatch, as a DB backup.
+
+## version 0.13.0
+
+- grand refactoring
+
+## version 0.12.0
+
+- [CHANGE] Adds task_id column to the log table (strload_load_logs).
+
+## version 0.11.0
+
+- Loosen dependent Bricolage version
+
+## version 0.10.2
+
+- [new] New parameter dispatch-interval.
+
+## version 0.10.1
+
+- [fix] Fixes simple variable ref bug.
+
+## version 0.10.0
+
+- [new] Automatically complement strload_jobs status with Redshift-side log table.
+
+## version 0.9.0
+
+- [new] Introduces Redshift-side load log table (strload_load_logs) for load duplication checking.
+
+## version 0.8.1
+
+- [fix] tmp: Do not retry on data connection failure.
+
+## version 0.8.0
+
+- [CHANGE] Loader retries failed load tasks automatically.  Streaming loader does NOT delete a task from the queue on job failures.
+- [enhancement] Rewrites loader to get better error handling (ensure to write log record in the wider range of errornous situations).
+
+## version 0.7.1
+
+- fix utilities
+
+## version 0.7.0
+
+- [CHANGE] SQS data source requires "region" attribute.
+
+## version 0.6.2
+
+- [new] AWS access key id and secret key are now optional for SQS data sources (to allow using EC2 instance or ECS task attached IAM roles).
+- [new] New utility commands send-data-event, send-shutdown, send-checkpoint, send-load-task.
+- Adds sample config files.
+
+## version 0.6.1
+
+- [fix] dispatcher: Default ctl data source was wrong.
+- [fix] dispatcher: Detects S3 events by "s3" attribute instead of "eventSource" attribute, to allow fake S3 events (from non-S3 system).
+- [fix] dispatcher: SNS alert is now optional.
+- [fix] dispatcher: Correctly deletes unknown format messages.
+- [enhancement] Adds more logging messages.
+
+## version 0.6.0
+
+- [CHANGE] Adds loaded column to strload_objects table to record if the object is really loaded or not.
+- [CHANGE] Now strload_objects' object_url is unique.  Duplicated objects are stored in another table, strload_dup_objects.
+- [CHANGE] Now strload_table has table_id column, which is the primary key.
+- [new] Loader daemon supports new command line option --working-dir, to support symbolic linked path, such as Capistrano deploy target (current/).
+- [new] Keeps Redshift manifest file for later inspection.
+- [enhancement] Reduces the number of Redshift writer transactions (1 transaction for 1 loading).
+- [enhancement] Delay dispatching tasks until current event batch is processed, to avoid unexpected visibility timeout.
+- [enhancement] Adds more logging messages.
+
+## version 0.5.1
+
+- [fix] Fixes slow query
+
+## version 0.5.0
+
+- [new] Introduces FLUSHTABLE dispatcher event
+
+## version 0.4.0
+
+- [new] Introduces CHECKPOINT dispatcher event
+
+## version 0.3.0
+
+- [new] Supoprts SNS notification
+
+## version 0.2.0
+
+- not released
+- [fix] Fixes async delete timing
+
+## version 0.1.0
+
+- 2016-07-13 works 1 month

data/Rakefile

@@ -0,0 +1,3 @@
+task :test do
+  load "#{__dir__}/test/all.rb"
+end

data/bricolage-streamingload.gemspec

@@ -0,0 +1,26 @@
+require_relative 'lib/bricolage/streamingload/version'
+
+Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.name = 'bricolage-streamingload'
+  s.version = Bricolage::StreamingLoad::VERSION
+  s.summary = 'Bricolage Streaming Load Daemon'
+  s.description = 'Bricolage Streaming Load Daemon loads S3 data files to Redshift continuously.'
+  s.license = 'MIT'
+
+  s.author = ['Minero Aoki', 'Shimpei Kodama']
+  s.email = ['aamine@loveruby.net']
+  s.homepage = 'https://github.com/aamine/bricolage-streamingload'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.executables = s.files.grep(%r{bin/}).map {|path| File.basename(path) }
+  s.require_path = 'lib'
+
+  s.required_ruby_version = '>= 2.3.0'
+  s.add_dependency 'bricolage', '>= 5.29.2'
+  s.add_dependency 'pg', '~> 0.18.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1.8'
+  s.add_dependency 'aws-sdk-sqs', '~> 1.3'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'test-unit'
+end

data/config/production/database.yml

@@ -0,0 +1,66 @@
+### Databases
+
+db_ctl:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage
+  username: bricolage
+  # Get password from password.yml
+  password: <%= password 'postgres_bricolage_password' %>
+  encoding: utf8
+
+db_data: &db_data
+  type: psql
+  host: redshift.host
+  port: 5439
+  database: production
+  username: bricolage
+  # Get password from password.yml
+  password: <%= password 'redshift_bricolage_password' %>
+  encoding: utf8
+
+sql:
+  <<: *db_data
+
+### SQS
+
+sqs_event:
+  type: sqs
+  region: ap-northeast-1
+  url: https://sqs.ap-northeast-1.amazonaws.com/111111111111/bricolage-events
+  max_number_of_messages: 10
+  visibility_timeout: 600
+  wait_time_seconds: 10
+  # Enable following lines if you use access key explicitly.
+  # Otherwise Bricolage uses EC2 instance or ECS task attached IAM role.
+  #access_key_id: "<%%= password 'aws_access_key_id' %>"
+  #secret_access_key: "<%%= password 'aws_secret_access_key' %>"
+
+sqs_task:
+  type: sqs
+  region: ap-northeast-1
+  url: https://sqs.ap-northeast-1.amazonaws.com/111111111111/bricolage-tasks
+  max_number_of_messages: 1
+  visibility_timeout: 1800
+  wait_time_seconds: 10
+  # Enable following lines if you use access key explicitly.
+  # Otherwise Bricolage uses EC2 instance or ECS task attached IAM role.
+  #access_key_id: "<%%= password 'aws_access_key_id' %>"
+  #secret_access_key: "<%%= password 'aws_secret_access_key' %>"
+
+### S3
+
+s3_ctl: &s3_ctl
+  type: s3
+  region: ap-northeast-1
+  endpoint: s3-ap-northeast-1.amazonaws.com
+  bucket: bricolagectl.ap-northeast-1
+  prefix: development/strload
+  # Enable following lines if you use access key explicitly.
+  # Otherwise Bricolage uses EC2 instance or ECS task attached IAM role.
+  #access_key_id: "<%%= password 'aws_access_key_id' %>"
+  #secret_access_key: "<%%= password 'aws_secret_access_key' %>"
+
+s3:
+  <<: *s3_ctl

data/config/production/password.yml

@@ -0,0 +1,5 @@
+# Never commit this file
+redshift_bricolage_password: xxxxxxxxxxx
+postgres_bricolage_password: xxxxxxxxxxx
+aws_access_key_id: "AKIAAAAAAAAAAAAAAAAA"
+aws_secret_access_key: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

data/config/production/streamingload.yml

@@ -0,0 +1,20 @@
+#event-queue-ds: sqs_event
+#task-queue-ds: sqs_task
+
+#ctl-postgres-ds: db_ctl
+#ctl-s3-ds: s3_ctl
+
+#redshift-ds: db_data
+#log-table: strload_load_logs
+
+#dispatch-interval: 60
+
+# To Enable SNS notification
+#sns-ds: sns
+#alert-level: warn
+
+url_patterns:
+  -
+    url: "s3://some-log-bucket/\\w{4}\\.\\w+?\\.\\w+?\\.(?<schema>\\w+)\\.(?<table>\\w+)/\\d{4}/\\d{2}/\\d{2}/.*\\.gz"
+    schema: "%schema"
+    table: "%table"

data/config/production/variable.yml

@@ -0,0 +1,5 @@
+defaults:
+    redshift-ds: db_data
+    ctl-ds: s3_ctl
+
+dwh_schema: dwh

data/lib/bricolage/streamingload/jobparams.rb

@@ -35,7 +35,7 @@ module Bricolage
       end
 
       def JobParams.resolve_schema(ctx, schema)
-        ctx.global_variables["#{schema}_schema"] || schema
+        ctx.global_variables.get_force("#{schema}_schema") || schema
       end
       private_class_method :resolve_schema
 

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.15.0'
+    VERSION = '0.15.1'
   end
 end

data/sample/sqs-message.txt

@@ -0,0 +1,38 @@
+{
+  "Records" => [
+    {
+      "eventVersion" => "2.0",
+      "eventSource" => "aws:s3",
+      "awsRegion" => "ap-northeast-1",
+      "eventTime" => "2016-02-13T11:40:08.001Z",
+      "eventName" => "ObjectCreated:Put",
+      "userIdentity" => {
+        "principalId" => "AWS:AIXXXXXXXXXXXXXXXXX6A"
+      },
+      "requestParameters" => {
+        "sourceIPAddress" => "111.222.111.90"
+      },
+      "responseElements" =>  {
+        "x-amz-request-id" => "1111AAAA9999AAAA",
+        "x-amz-id-2" => "6p9IZG+R+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxFyXiNMMB4="
+      },
+      "s3" => {
+        "s3SchemaVersion" => "1.0",
+        "configurationId" => "ObjectCreatedEvents",
+        "bucket" => {
+          "name" => "some-bucket",
+          "ownerIdentity" => {
+            "principalId" => "AAAAIIIIBBBB11"
+          },
+          "arn" => "arn:aws:s3:::some-bucket"
+        },
+        "object" => {
+          "key" => "development/logs/schema.table/20160125_0000_0_de37c5ad-d26a-42cc-a141-475676b65f69.gz",
+          "size" => 1302957,
+          "eTag" => "d704db7f9cb77b1ecb981c534526b542",
+          "sequencer" => "111122223333444499"
+        }
+      }
+    }
+  ]
+}

data/sample/sqs-result.txt

@@ -0,0 +1,18 @@
+#<struct Aws::SQS::Types::ReceiveMessageResult
+  messages = [
+    #<struct Aws::SQS::Types::Message
+      message_id = "11112222-b37f-4281-aee2-696408c482be",
+      receipt_handle = "AQEBErp4Vv159Hvx5oXSioWu7Ov1Jj1ht423LwTTUlgLohmKh02gAMAZ7kN1J4+aLzM54BRuMatnifWPH3cBh6rtdSWlllysBVmXDRKj83VCaseKPpq+Vdw/bLZrPEYRSGPuhKsHa2DFjo67KeND4AqasZO7lOCJX+YcsbqsqAtcODlDWlVbfedPkKIGM3Scra3uY9ysGjNx/zWuJ88fCtvRpu9tgk0nG7srHAm84Mxj5ArakJnKnJMbtkD/+lvm0Vqi2YCkXonS8+BP8gK4kSTbl1uv/Gp7ZfxkjKQJBB8xQiYo7e7qKLwhbImMZtO5tqDKzIKFVfMi1G2ODF+tEj4Ce+ryMRGA50GhON2ETCyofsl1T7Wdr61IEOV0NFlonGTWFf4q/1r3OPhFAchp+tYnyA==",
+      md5_of_body = "00006bf43abdff178ca0ffa96205aaaa",
+      body = "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-northeast-1\",\"eventTime\":\"2016-02-13T11:40:07.268Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AAAAJKS3A4VEF45XCAAAA\"},\"requestParameters\":{\"sourceIPAddress\":\"111.222.111.90\"},\"responseElements\":{\"x-amz-request-id\":\"10AAAA31A1EDCCCC\",\"x-amz-id-2\":\"YT+U/PxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxzfFGPjqP5AQg=\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"LogStreamDev\",\"bucket\":{\"name\":\"redshift-copy-buffer\",\"ownerIdentity\":{\"principalId\":\"AAAAFIDWR40000\"},\"arn\":\"arn:aws:s3:::redshift-copy-buffer\"},\"object\":{\"key\":\"development/logs/schema.table/20160125_0000_0_2d0b43ad-0db9-4655-bd98-b7464b123763.gz\",\"size\":1238953,\"eTag\":\"aaaa196c3935f4957c7bb645f9780000\",\"sequencer\":\"0000BF161697C1AAAA\"}}}]}",
+      attributes = {
+        "SenderId" => "FACE0VEO02BJMF37H2JKW",
+        "ApproximateFirstReceiveTimestamp" => "1455364193429",
+        "ApproximateReceiveCount" => "1",
+        "SentTimestamp" => "1455363607341"
+      },
+      md5_of_message_attributes = nil,
+      message_attributes = {}
+    >
+  ]
+>

data/strload_load_logs.ct

@@ -0,0 +1,13 @@
+--dest-table: bricolage.strload_load_logs
+
+/*
+    Redshift-side log table
+*/
+create table $dest_table
+( job_id bigint encode raw
+, task_id bigint encode zstd
+, finish_time timestamp encode delta
+)
+distkey (job_id)
+sortkey (job_id)
+;