bricolage-streamingload 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2043736baf77c16a2df73b7233e7f1839ea8c44
-  data.tar.gz: 4418e6433eaee7eca223f073a8d684b6325ac2c8
+  metadata.gz: 5e6fd594b56fc33bc988e905de9cba98cd9c6c6e
+  data.tar.gz: 5379c4bee784641b7231b6c04a7ad81262943650
 SHA512:
-  metadata.gz: 078a0cb67b3d26a3f7dceedeedfb6e7631d39199d2d7dd762067be06e29dd1b26eb696d7d1f0fef305415b156b4b4507855170c065a7316e1d5325ee0bfb3608
-  data.tar.gz: 239686023066c23a915d68e313a7ed49fe015d2671c33f3e7a51f48ac1ed14d1354c6a117177dc66986da6d1a5ff20e65e34660f54b6bdc50edc203ed47295db
+  metadata.gz: 478636bb27db1be56dccce73424bdbe9d49ee66b50218f1ad7e416e6bffc7a1d59a8f715f2e12e7603e35c71a0d1f8f5dae7f20bfc78d0b7db24eb1d2649108a
+  data.tar.gz: 5600a8e5883aab13f05cd2386bb416d92bb36dfd716843f26d869db8dcad3d363fdf540d58daac74d65819005cf1f8f21b6234eaf78b957bcd67cecd2eef71d9

data/bin/send-checkpoint-event

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'bricolage/commandlineapplication'
+
+app = Bricolage::CommandLineApplication.define {|opts|
+  opts.data_source_option('--queue-ds', 'Target queue data soure.', short: '-D', kind: 'sqs')
+}
+queue = app.data_source('--queue-ds')
+queue.send_event('checkpoint')

data/bin/send-data-event

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'bricolage/commandlineapplication'
+
+no_dispatch = false
+app = Bricolage::CommandLineApplication.define {|opts|
+  opts.data_source_option('--queue-ds', 'Target queue data soure.', short: '-D', kind: 'sqs')
+  opts.on('--no-dispatch', 'Turn off dispatch flag.') {
+    no_dispatch = true
+  }
+}
+queue = app.data_source('--queue-ds')
+
+ARGF.each do |line|
+  url = line.strip
+  bucket, key = url.sub(%r<s3://>, '').split('/', 2)
+  $stderr.puts "PUT bucket=#{bucket} key=#{key}"
+  queue.send_event(
+    "ObjectCreated:Put",
+    eventVersion: "2.0",
+    awsRegion: queue.region,
+    s3: {
+      s3SchemaVersion: "1.0",
+      configurationId: "LogStreamDev",
+      bucket: {
+        name: bucket,
+        arn: "arn:aws:s3:::#{bucket}"
+      },
+      object: {
+        key: key,
+        size: 1
+      }
+    },
+    noDispatch: no_dispatch
+  )
+end

data/bin/send-load-task

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'bricolage/commandlineapplication'
+
+app = Bricolage::CommandLineApplication.define {|opts|
+  opts.data_source_option('--queue-ds', 'Target queue data soure.', short: '-D', kind: 'sqs')
+}
+queue = app.data_source('--queue-ds')
+
+ARGF.each do |line|
+  task_id = line.strip
+  queue.send_event('streaming_load_v3', taskId: task_id)
+end

data/bin/send-shutdown-event

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'bricolage/commandlineapplication'
+
+app = Bricolage::CommandLineApplication.define {|opts|
+  opts.data_source_option('--queue-ds', 'Target queue data soure.', short: '-D', kind: 'sqs')
+}
+queue = app.data_source('--queue-ds')
+queue.send_event('shutdown')

data/lib/bricolage/sqsdatasource.rb

@@ -10,7 +10,7 @@ module Bricolage
 
     declare_type 'sqs'
 
-    def initialize(region: 'ap-northeast-1', url:, access_key_id:, secret_access_key:,
+    def initialize(region: 'ap-northeast-1', url:, access_key_id: nil, secret_access_key: nil,
         visibility_timeout:, max_number_of_messages: 10, wait_time_seconds: 20, noop: false)
       @region = region
       @url = url
@@ -170,6 +170,17 @@ module Bricolage
       )
     end
 
+    def send_object(obj)
+      send_message(ObjectMessage.new(obj))
+    end
+
+    def send_event(name, source: SQSMessage::SQS_EVENT_SOURCE, time: Time.now, **attrs)
+      attrs['eventName'] = name
+      attrs['eventSource'] = source
+      attrs['eventTime'] = time.iso8601
+      send_object(attrs)
+    end
+
     def delete_message(msg)
       client.delete_message(
         queue_url: @url,
@@ -439,4 +450,18 @@ module Bricolage
 
   end   # class UnknownSQSMessage
 
+
+  # library private
+  class ObjectMessage
+
+    def initialize(body, delay_seconds: 0)
+      @body = body
+      @delay_seconds = delay_seconds
+    end
+
+    attr_reader :body
+    attr_reader :delay_seconds
+
+  end
+
 end   # module Bricolage

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.6.1'
+    VERSION = '0.6.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-05 00:00:00.000000000 Z
+date: 2016-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.16.9
+        version: 5.17.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.16.9
+        version: 5.17.1
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.4
+        version: 0.18.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.4
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.6
+        version: 2.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.6
+        version: 2.6.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -101,12 +101,20 @@ email:
 executables:
 - bricolage-streaming-dispatcher
 - bricolage-streaming-loader
+- send-checkpoint-event
+- send-data-event
+- send-load-task
+- send-shutdown-event
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
 - bin/bricolage-streaming-dispatcher
 - bin/bricolage-streaming-loader
+- bin/send-checkpoint-event
+- bin/send-data-event
+- bin/send-load-task
+- bin/send-shutdown-event
 - lib/bricolage/nulllogger.rb
 - lib/bricolage/snsdatasource.rb
 - lib/bricolage/sqsdatasource.rb