bricolage 5.16.1 → 5.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b49d4cd6667b360a4ef43d6a721718729a191aad
-  data.tar.gz: eaee40ab39ff69582b9fc10d1c2a4a16c028e23a
+  metadata.gz: f390623f428534408c149e45da7c23f11ea2f90f
+  data.tar.gz: 8857a1d90455b1fcac9f388e5662da69e87886e4
 SHA512:
-  metadata.gz: 4743a3a160b1c0cd43a5311df78750a5b37cc200498137e74cceb493a99bab3f0054b932e3daad13e6aa594de8aadb5c839313c45990881ce9c1a4b239bebcda
-  data.tar.gz: 5b46b650bbc8bd0e11fbbefeefdedf0d619abdc9beb4293fc8511016603bde59cc02ddb0e2adbbb1320eb19309ef1a8391d9c9ecfb1d9699af661e53b644f583
+  metadata.gz: 86f56d4df2cfaad3fbe71010c847846ed6d9a93d6b7aa341d01f4e42e24c9796f8ba2cd5b9645e5086d558e71e0675dd636581fc43bc34ec8629671ea14865dc
+  data.tar.gz: d41b89b4c9aea470177546a946ee14b00e8d8c1f4bb2e33ceb7ae7c7bba00e84ba8272609ff65ab7ca8a0b1801c5f7f1f9a0e57cf5ca42fa61464bdde3e6ed7a

data/jobclass/streaming_load.rb

@@ -222,6 +222,7 @@ class StreamingLoadJobClass < RubyJobClass
         from '#{manifest_url}'
         credentials '#{@src.credential_string}'
         manifest
+        compupdate false
         statupdate false
         #{@load_options}
       ;).gsub(/\s+/, ' ').strip
@@ -277,6 +278,7 @@ class StreamingLoadJobClass < RubyJobClass
         credentials '#{credential_string}'
         delimiter ','
         removequotes
+        compupdate false
       ;).gsub(/\s+/, ' ').strip
     end
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.16.1'
+  VERSION = '5.16.2'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.16.1)
+    bricolage (5.16.2)
       aws-sdk (~> 2)
       mysql2
       pg
@@ -22,8 +22,10 @@ GEM
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
     hirb (0.7.3)
     httpclient (2.7.1)
-    jmespath (1.1.3)
+    jmespath (1.2.3)
+      json_pure (>= 1.8.1)
     json (1.8.3)
+    json_pure (1.8.3)
     method_source (0.8.2)
     msgpack (0.5.12)
     mysql2 (0.4.3)

data/test/home/aamine/pv_log.job

@@ -0,0 +1,9 @@
+class: streaming_load_v3
+work-table: $activity_schema.pv_log_wk
+load-options:
+    json: "auto"
+    gzip: true
+    maxerror: 10
+    timeformat: "auto"
+    truncatecolumns: true
+    acceptinvchars: " "

data/test/home/aamine/variable.yml

@@ -0,0 +1,3 @@
+defaults:
+    redshift-ds: sql
+    ctl-ds: s3_strload

data/test/home/config/development/database.yml

@@ -49,8 +49,8 @@ s3:
   bucket: tabemiru-data.ap-northeast-1
   prefix: "/dev"
   s3cfg: <%= user_home_relative_path '.s3cfg' %>
-  access_key_id: "AKIAI5R5BL2W73PR3FXQ"
-  secret_access_key: "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"
+  access_key_id: "<%= password 'aws_access_key_id' %>"
+  secret_access_key: "<%= password 'aws_secret_access_key' %>"
   #master_symmetric_key: "12345678901234567890123456789012"
 
 mysql:
@@ -62,3 +62,28 @@ mysql:
   username: work_readonly
   password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
+
+s3_strload:
+  type: s3
+  region: "ap-northeast-1"
+  endpoint: "s3-ap-northeast-1.amazonaws.com"
+  bucket: tabemiru-data.ap-northeast-1
+  prefix: tmp/strload
+  access_key_id: "<%= password 'aws_access_key_id' %>"
+  secret_access_key: "<%= password 'aws_secret_access_key' %>"
+
+sqs_strload_event:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
+  max_number_of_messages: 10
+  visibility_timeout: 60
+  access_key_id: "<%= password 'aws_access_key_id' %>"
+  secret_access_key: "<%= password 'aws_secret_access_key' %>"
+
+sqs_strload_task:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
+  max_number_of_messages: 1
+  visibility_timeout: 180
+  access_key_id: "<%= password 'aws_access_key_id' %>"
+  secret_access_key: "<%= password 'aws_secret_access_key' %>"

data/test/home/config/development/password.yml

@@ -2,3 +2,5 @@ td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
 mysql_shared_work_readonly: "s0uL/M@9r1"
 mysql_shared_ckpd_readonly: "no3lnar;"
 mysql_local_work_readonly: "ckpdtest"
+aws_access_key_id: "AKIAI5R5BL2W73PR3FXQ"
+aws_secret_access_key: "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"

data/test/home/config/development/variable.yml

@@ -1,3 +1,4 @@
 data_dir: /tmp/bricolagedev
 test_schema: aamine
 test_group: aamine
+activity_schema: aamine

data/test/home/config/streamingload.yml

@@ -0,0 +1,8 @@
+redshift-ds: sql
+event-queue-ds: sqs_strload_event
+task-queue-ds: sqs_strload_task
+url_patterns:
+  -
+    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
+    schema: "aamine"
+    table: "pv_log"

data/test/home/sqs/delete_all.rb

@@ -0,0 +1,35 @@
+ENV['AWS_REGION'] = 'ap-northeast-1'
+
+require 'aws-sdk'
+require 'json'
+require 'pp'
+require 'pry'
+
+queue_id = ARGV[0] || "log-stream-dev"
+queue_url = "https://sqs.ap-northeast-1.amazonaws.com/789035092620/#{queue_id}"
+$stderr.puts queue_url
+
+sqs = Aws::SQS::Client.new
+
+deleted = true
+while deleted
+  deleted = false
+
+  result = sqs.receive_message(
+    queue_url: queue_url,
+    max_number_of_messages: 10,
+    visibility_timeout: 300,
+    wait_time_seconds: 10
+  )
+  unless result.successful?
+    $stderr.puts "receive_message failed: #{result.error.inspect}"
+    exit 1
+  end
+  result.messages.each do |msg|
+    body = JSON.parse(msg.body)
+    $stderr.puts body['Records'].inspect
+    res = sqs.delete_message(queue_url: queue_url, receipt_handle: msg.receipt_handle)
+    $stderr.puts "deleted: #{res.successful?}"
+    deleted = true if res.successful?
+  end
+end

data/test/home/sqs/graceful.rb

@@ -0,0 +1,15 @@
+ENV['AWS_REGION'] = 'ap-northeast-1'
+
+require 'aws-sdk'
+require 'json'
+require 'pp'
+require 'pry'
+
+sqs = Aws::SQS::Client.new
+message_body = {'Records' => [{'eventName' => 'shutdown'}]}.to_json
+result = sqs.send_message(
+  queue_url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev",
+  message_body: message_body,
+  delay_seconds: 0
+)
+pp result

data/test/home/sqs/messages.txt

@@ -0,0 +1,50 @@
+{"Records"=>
+  [{"eventVersion"=>"2.0",
+    "eventSource"=>"aws:s3",
+    "awsRegion"=>"ap-northeast-1",
+    "eventTime"=>"2016-02-13T11:40:08.001Z",
+    "eventName"=>"ObjectCreated:Put",
+    "userIdentity"=>{"principalId"=>"AWS:AIDAJKS3A4VEF45XC2N6A"},
+    "requestParameters"=>{"sourceIPAddress"=>"122.220.193.90"},
+    "responseElements"=>
+     {"x-amz-request-id"=>"9F82D41D69E958B5",
+      "x-amz-id-2"=>
+       "6p9IZG+R+dsHCAySGkg42ajt/ZdheE5RLChABqiUl1wL47DCqdu2xrweGQtkG7KR1zFyXiNMMB4="},
+    "s3"=>
+     {"s3SchemaVersion"=>"1.0",
+      "configurationId"=>"LogStreamDev",
+      "bucket"=>
+       {"name"=>"redshift-copy-buffer",
+        "ownerIdentity"=>{"principalId"=>"A391FIDWR49810"},
+        "arn"=>"arn:aws:s3:::redshift-copy-buffer"},
+      "object"=>
+       {"key"=>
+         "development/log_stream/20160125_0000_0_de37c5ad-d26a-42cc-a141-475676b65f69.gz",
+        "size"=>1302957,
+        "eTag"=>"d704db7f9cb77b3ecb981c534526b542",
+        "sequencer"=>"0056BF161746DA8807"}}}]}
+{"Records"=>
+  [{"eventVersion"=>"2.0",
+    "eventSource"=>"aws:s3",
+    "awsRegion"=>"ap-northeast-1",
+    "eventTime"=>"2016-02-13T11:40:08.319Z",
+    "eventName"=>"ObjectCreated:Put",
+    "userIdentity"=>{"principalId"=>"AWS:AIDAJKS3A4VEF45XC2N6A"},
+    "requestParameters"=>{"sourceIPAddress"=>"122.220.193.90"},
+    "responseElements"=>
+     {"x-amz-request-id"=>"9D3CAB675D7C2FFF",
+      "x-amz-id-2"=>
+       "wn1pP79+bv8811aTz3f0L682VhAo6YHInNzNSDrBo9kvlSsK0X31JcINHMiVH5kUEt4y4Gh5Y8M="},
+    "s3"=>
+     {"s3SchemaVersion"=>"1.0",
+      "configurationId"=>"LogStreamDev",
+      "bucket"=>
+       {"name"=>"redshift-copy-buffer",
+        "ownerIdentity"=>{"principalId"=>"A391FIDWR49810"},
+        "arn"=>"arn:aws:s3:::redshift-copy-buffer"},
+      "object"=>
+       {"key"=>
+         "development/log_stream/20160125_0001_0_bd519b56-301e-4f51-8585-c53b1e40ce52.gz",
+        "size"=>1228210,
+        "eTag"=>"bfa868a9f00376f6804f41d6a9e0da20",
+        "sequencer"=>"0056BF1617DEB23219"}}}]}

data/test/home/sqs/put-slow.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+srcdir=$1; shift
+desturl=$1; shift
+
+for rel in $(ls $srcdir/*.gz)
+do
+    aws s3 cp $rel $desturl
+    sleep 3
+done

data/test/home/sqs/recv.rb

@@ -0,0 +1,27 @@
+ENV['AWS_REGION'] = 'ap-northeast-1'
+
+require 'aws-sdk'
+require 'json'
+require 'pp'
+require 'pry'
+
+queue_id = ARGV[0] || "log-stream-dev"
+queue_url = "https://sqs.ap-northeast-1.amazonaws.com/789035092620/#{queue_id}"
+
+sqs = Aws::SQS::Client.new
+result = sqs.receive_message(
+  queue_url: queue_url,
+  attribute_names: ["All"],
+  message_attribute_names: ["All"],
+  max_number_of_messages: 10,
+  visibility_timeout: 21,
+  wait_time_seconds: 20
+)
+if result.messages.empty?
+  $stderr.puts 'empty'
+else
+  binding.pry
+end
+#result.messages.each do |msg|
+#  pp JSON.parse(msg.body)
+#end

data/test/home/sqs/result.txt

@@ -0,0 +1,16 @@
+#<struct Aws::SQS::Types::ReceiveMessageResult
+ messages=
+  [#<struct Aws::SQS::Types::Message
+    message_id="2339fc04-b37f-4283-aee2-696408c482be",
+    receipt_handle=
+     "AQEBErp4Vv059Hvx5oXSioWu7Ov1Jj1ht493LwTTUlgLohmKh02gAMAZ7kN1J4+aLzM54BRuMatnifWPH3cBh6rtdSWlllysBVmXDRKj83VCaseKPpq+Vdw/bLZrPEYRSGPuhKsHa2DFjo67KeND4AqasZO7lOCJX+YcsbqsqAtcODlDWlVbfedPkKIGM3Scra3uY9ysGjNx/zWuJ88fCtvRpu9tgk0nG7srHAm84Mxj5ArakJnKnJMbtkD/+lvm0Vqi2YCkXonS8+BP8gK4kSTbl1uv/Gp7ZfxkjKQJBB8xQiYo7e7qKLwhbImMZtO5tqDKzIKFVfMi1G2ODF+tEj4Ce+ryMRGA50GhON2ETCyofsl1T7Wdr61IEOV0NFlonGTWFf4q/1r3OPhFAchp+tYnyA==",
+    md5_of_body="3da46bf43abdff078ca0ffa9620534cb",
+    body=
+     "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-northeast-1\",\"eventTime\":\"2016-02-13T11:40:07.268Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AIDAJKS3A4VEF45XC2N6A\"},\"requestParameters\":{\"sourceIPAddress\":\"122.220.193.90\"},\"responseElements\":{\"x-amz-request-id\":\"60A61031A1ED5C31\",\"x-amz-id-2\":\"YT+U/PKVDOH6Abnd1o4KzOZaoXCdVRJs/ORAxn5EPrgjQqF2AwUQf9SAoEw5HKNzfFGPjqP5AQg=\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"LogStreamDev\",\"bucket\":{\"name\":\"redshift-copy-buffer\",\"ownerIdentity\":{\"principalId\":\"A391FIDWR49810\"},\"arn\":\"arn:aws:s3:::redshift-copy-buffer\"},\"object\":{\"key\":\"development/log_stream/20160125_0000_0_2d0b43ad-0db9-4655-bd98-b7464b123763.gz\",\"size\":1238953,\"eTag\":\"37bb696c3935f4957c7bb645f97801d0\",\"sequencer\":\"0056BF161697C1AA6C\"}}}]}",
+    attributes=
+     {"SenderId"=>"AIDAJVEO32BJMF27H2JKW",
+      "ApproximateFirstReceiveTimestamp"=>"1455364193429",
+      "ApproximateReceiveCount"=>"1",
+      "SentTimestamp"=>"1455363607341"},
+    md5_of_message_attributes=nil,
+    message_attributes={}>]>

data/test/home/sqs/send.rb

@@ -0,0 +1,16 @@
+ENV['AWS_REGION'] = 'ap-northeast-1'
+
+require 'aws-sdk'
+require 'json'
+require 'pp'
+require 'pry'
+
+queue_name = ARGV[0] || 'log-stream-dev'
+queue_url = "https://sqs.ap-northeast-1.amazonaws.com/789035092620/#{queue_name}"
+
+message_body = { 'Records' => [ { 'eventName' => 'shutdown' } ] }.to_json
+args = { queue_url: queue_url, message_body: message_body, delay_seconds: 0 }
+pp(**args)
+
+sqs = Aws::SQS::Client.new
+pp sqs.send_message(**args)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.16.1
+  version: 5.16.2
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-30 00:00:00.000000000 Z
+date: 2016-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -181,10 +181,12 @@ files:
 - test/all.rb
 - test/home/Gemfile
 - test/home/Gemfile.lock
-- test/home/config.yml
+- test/home/aamine/pv_log.job
+- test/home/aamine/variable.yml
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
 - test/home/config/development/variable.yml
+- test/home/config/streamingload.yml
 - test/home/data/20141002-1355_00.txt
 - test/home/data/20141002-1355_01.txt
 - test/home/data/20141002-1355_02.txt
@@ -192,6 +194,75 @@ files:
 - test/home/jobnet-test.rb
 - test/home/put.sh
 - test/home/revert.sh
+- test/home/sqs/data/20160125_0000_0_20f3aa7a-6b5b-4a19-9c8f-1972e820fbaf.gz
+- test/home/sqs/data/20160125_0000_0_25fe3a52-9477-4030-8038-c84636343f56.gz
+- test/home/sqs/data/20160125_0000_0_27361985-3142-4f9c-a2fe-6c864ff7e68d.gz
+- test/home/sqs/data/20160125_0000_0_2d0b43ad-0db9-4655-bd98-b7464b123763.gz
+- test/home/sqs/data/20160125_0000_0_2fe26d66-45b0-4668-b212-668bd44ebd71.gz
+- test/home/sqs/data/20160125_0000_0_62546a2e-c9be-46b1-b601-c25de138a377.gz
+- test/home/sqs/data/20160125_0000_0_6adcc23f-b073-48b4-a56f-2023866cd8e5.gz
+- test/home/sqs/data/20160125_0000_0_7403ecce-d9cc-43e0-b1ea-a35221a5b780.gz
+- test/home/sqs/data/20160125_0000_0_8d429746-2b60-48d0-8cac-59e6bc4d557e.gz
+- test/home/sqs/data/20160125_0000_0_a1ff5331-dc10-4072-ab95-b9fd8667ff88.gz
+- test/home/sqs/data/20160125_0000_0_b02e9cf1-cf0d-4c3c-a9cc-85773bfbcb44.gz
+- test/home/sqs/data/20160125_0000_0_de37c5ad-d26a-42cc-a141-475676b65f69.gz
+- test/home/sqs/data/20160125_0001_0_254f8131-84dd-43ec-a66f-d9a445194e49.gz
+- test/home/sqs/data/20160125_0001_0_2abcee36-4b4e-4758-b331-1ee4775e531e.gz
+- test/home/sqs/data/20160125_0001_0_68f856be-03ac-400c-b819-a680a965801b.gz
+- test/home/sqs/data/20160125_0001_0_90e7c6fb-f649-4fe8-9875-14734b34df04.gz
+- test/home/sqs/data/20160125_0001_0_959223b2-7376-40a8-aa27-b3e95f828099.gz
+- test/home/sqs/data/20160125_0001_0_a8f57916-73f5-4d19-8f83-cc1468e06368.gz
+- test/home/sqs/data/20160125_0001_0_b6091557-eadd-4239-b653-f9728b0c8b78.gz
+- test/home/sqs/data/20160125_0001_0_bd519b56-301e-4f51-8585-c53b1e40ce52.gz
+- test/home/sqs/data/20160125_0001_0_e96f1782-0c23-487a-87ec-6d41cd922999.gz
+- test/home/sqs/data/20160125_0001_0_ed414ea9-1b0b-4d23-ab37-3ddcd0c5368f.gz
+- test/home/sqs/data/20160125_0001_0_ef86d3d5-cf80-4b25-ad77-286b170fe8d8.gz
+- test/home/sqs/data/20160125_0001_0_efafcb99-368f-46dc-9c68-0d1d621e835f.gz
+- test/home/sqs/data/20160125_0001_0_f7732d73-156d-40a2-bbf6-94a285eda18d.gz
+- test/home/sqs/data/20160125_0002_0_012b9865-c7f9-4d37-bc0c-ffbea16be92f.gz
+- test/home/sqs/data/20160125_0002_0_0a986c8d-dee7-45fd-8f11-c9327d6d332a.gz
+- test/home/sqs/data/20160125_0002_0_147e37cb-b8a1-4e65-b6be-1dd57b33c575.gz
+- test/home/sqs/data/20160125_0002_0_533c7aec-d80d-435b-b292-8c5ca25eef44.gz
+- test/home/sqs/data/20160125_0002_0_5f44fde9-9760-4a61-8305-01b22aca1b07.gz
+- test/home/sqs/data/20160125_0002_0_6f4133b4-acca-4833-aa61-79b03fb22950.gz
+- test/home/sqs/data/20160125_0002_0_798c039e-75bc-4cd5-b862-9a062718766e.gz
+- test/home/sqs/data/20160125_0002_0_7bb967a0-040a-488f-bd45-ecb43ba17278.gz
+- test/home/sqs/data/20160125_0002_0_d7b8df8c-735a-47e0-bcfc-3b24f0de6991.gz
+- test/home/sqs/data/20160125_0002_0_e332458d-4cbf-496f-ac4f-7098dafee8da.gz
+- test/home/sqs/data/20160125_0002_0_e430e1d8-941b-45ba-b580-1d498ded9e2d.gz
+- test/home/sqs/data/20160125_0002_0_fc6a64f1-ba03-48a9-adb8-26c11307f8bb.gz
+- test/home/sqs/data/20160125_0003_0_06fdc0ed-73e8-4404-a442-9a2d444b1e82.gz
+- test/home/sqs/data/20160125_0003_0_13fbab94-7acb-4f6e-8bde-fd3270e4170b.gz
+- test/home/sqs/data/20160125_0003_0_24e13b65-fc76-4450-bc27-f43256e15cfa.gz
+- test/home/sqs/data/20160125_0003_0_4daafad3-e8fe-45c4-89c8-45531461650e.gz
+- test/home/sqs/data/20160125_0003_0_63db2f4b-5db7-42d0-96e2-202cad9e3ee7.gz
+- test/home/sqs/data/20160125_0003_0_7b709b4d-5ef7-4526-b619-44785fa54a0f.gz
+- test/home/sqs/data/20160125_0003_0_87d371f5-7e19-4c52-ba07-a0913536e52b.gz
+- test/home/sqs/data/20160125_0003_0_9f7c8d47-98a0-4756-8e41-ea4e50b995a5.gz
+- test/home/sqs/data/20160125_0003_0_b7e6e1ce-8157-4b84-8e40-463c35fca4d9.gz
+- test/home/sqs/data/20160125_0003_0_bb70daee-7616-4ec9-a223-adbbdef5ed02.gz
+- test/home/sqs/data/20160125_0003_0_cb761a01-9c07-404e-9780-d130d5a72831.gz
+- test/home/sqs/data/20160125_0003_0_db038e65-0d72-439f-9d2a-10de9ec1e253.gz
+- test/home/sqs/data/20160125_0004_0_01e9d44e-14f7-4a1b-805b-fb47cdaca50a.gz
+- test/home/sqs/data/20160125_0004_0_0e504783-4156-4826-8bf0-88a6b876b5dd.gz
+- test/home/sqs/data/20160125_0004_0_2555b915-f1bd-4598-aa9a-664e45848ac2.gz
+- test/home/sqs/data/20160125_0004_0_2fcd8e7a-0a2b-4a09-b441-9f8f34fef2ce.gz
+- test/home/sqs/data/20160125_0004_0_a36f0369-bfef-4860-8298-873130761575.gz
+- test/home/sqs/data/20160125_0004_0_af2a00e1-cdaf-4526-b2a0-40b566887743.gz
+- test/home/sqs/data/20160125_0004_0_af7d18f1-94ad-404a-99a1-598cefc6d2c4.gz
+- test/home/sqs/data/20160125_0004_0_db118a23-4af7-4589-b384-4c79b993df20.gz
+- test/home/sqs/data/20160125_0004_0_dd49a099-a716-46b6-9149-3d0791bce097.gz
+- test/home/sqs/data/20160125_0004_0_e0aeb75b-5b16-4875-9b93-abb84fc9ea79.gz
+- test/home/sqs/data/20160125_0004_0_e16cd2ca-7fbe-4724-8e41-69c4488f01c6.gz
+- test/home/sqs/data/20160125_0004_0_e8fe8417-9f92-4cd9-b83a-333f7990eb7c.gz
+- test/home/sqs/data/20160125_0005_0_1ebf35e8-47f6-43f9-a42e-e01fa6e24d9a.gz
+- test/home/sqs/delete_all.rb
+- test/home/sqs/graceful.rb
+- test/home/sqs/messages.txt
+- test/home/sqs/put-slow.sh
+- test/home/sqs/recv.rb
+- test/home/sqs/result.txt
+- test/home/sqs/send.rb
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
 - test/home/subsys/job1.job

data/test/home/config.yml

@@ -1,13 +0,0 @@
-event_queue:
-  sqs_url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
-  visibility_timeout: 60
-
-load_queue:
-  sqs_url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/XXXXXXXXXXXXXX"
-  visibility_timeout: 60
-
-url_patterns:
-  -
-    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
-    schema: "aamine"
-    table: "pv_log"