bricolage 5.12.5 → 5.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 741eaa97792bc5c5c755c6cd4e8c4f1545a8be6d
-  data.tar.gz: 0d69df37b2769c8c08e9ba5da41aeca3a9a9e5e3
+  metadata.gz: 8e56d9299cba57f0e086ad58140a5694aaa4d114
+  data.tar.gz: 63d3933d15359f340f9e7a7ffa0c50bfc39404b0
 SHA512:
-  metadata.gz: 8c956ddbfa6594810ced49c67c9cd5de6e0fe9ce514c50194e904802d2a8b2f5419dce600f160ccf99fcb52d2c0aeeb4b1515f761dc46c388957b177a6590abc
-  data.tar.gz: 2c0a615f183a1774abdebe80ec2c9d81595a02844f8b94584939f3268854fe421bc7da7c8968ce0acb03f0163e8ab63ca7fe0091d1b715497dfbbef58cceedd2
+  metadata.gz: 2a7cf7446da8ac747cba7cb41f56bb593a453bec20a57b6d785daca08314f27161a5a1db7ed9231b432bdedd417a8a9c06f9b6e49de5249371b2c840684d9d8f
+  data.tar.gz: 5d86674545cb5b8953e53c465bd5a5c12ab077a0e676c240967d0bcfe668418125c2f2c465622d2498bb6e7ce87109a406094b8a3d093e32fb1e5dfb3ed692cf

data/jobclass/streaming_load.rb

@@ -3,6 +3,7 @@ require 'bricolage/psqldatasource'
 require 'bricolage/exception'
 require 'json'
 require 'socket'
+require 'forwardable'
 
 class StreamingLoadJobClass < RubyJobClass
   job_class_id 'streaming_load'
@@ -53,13 +54,15 @@ class StreamingLoadJobClass < RubyJobClass
 
   def make_loader(params)
     ds = params['redshift-ds']
+    load_opts = params['load-options']
+    load_opts.provide_defaults(params['s3-ds'])
     RedshiftStreamingLoader.new(
       data_source: ds,
       queue: make_s3_queue(params),
       table: string(params['dest-table']),
       work_table: string(params['work-table']),
       log_table: string(params['log-table']),
-      load_options: params['load-options'],
+      load_options: load_opts,
       sql: params['sql-file'],
       logger: ds.logger,
       noop: params['noop'],
@@ -306,6 +309,8 @@ class StreamingLoadJobClass < RubyJobClass
   end
 
   class S3Queue
+    extend Forwardable
+
     def initialize(data_source:, queue_path:, persistent_path:, file_name:, logger:)
       @ds = data_source
       @queue_path = queue_path
@@ -318,13 +323,15 @@ class StreamingLoadJobClass < RubyJobClass
       @ds.credential_string
     end
 
+    def_delegator '@ds', :encryption
+
     attr_reader :queue_path
 
     def queue_url
       @ds.url(@queue_path)
     end
 
-    def object_url(key)
+    def object_url_direct(key)
       @ds.url(key, no_prefix: true)
     end
 
@@ -334,7 +341,7 @@ class StreamingLoadJobClass < RubyJobClass
 
     def put_control_file(name, data, noop: false)
       @logger.info "s3 put: #{control_file_url(name)}"
-      @ds.object(control_file_path(name)).write(data) unless noop
+      @ds.object(control_file_path(name)).put(body: data) unless noop
       control_file_url(name)
     end
 
@@ -344,39 +351,27 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def control_file_path(name)
-      "#{queue_path}/#{name}"
-    end
-
-    def consume_each(noop: false, &block)
-      each do |obj|
-        yield obj and obj.save(noop: noop)
-      end
+      "#{queue_path}/ctl/#{name}"
     end
 
     def each(&block)
       queued_objects.each(&block)
     end
 
-    def queue_directory
-      @ds.objects_with_prefix(queue_path)
-    end
-
-    def queued_file_nodes
-      queue_directory.as_tree.children.select {|node|
-        node.leaf? and
-          node.key[-1, 1] != '/' and
-          target_file_name?(File.basename(node.key))
-      }
-    end
-
     def queued_objects
-      queued_file_nodes.map {|node| LoadableObject.new(self, node, @logger) }
+      @ds.traverse(queue_path)
+          .select {|obj| target_file_name?(File.basename(obj.key)) }
+          .map {|obj| LoadableObject.new(self, obj, @logger) }
     end
 
     def target_file_name?(name)
       file_name_pattern =~ name
     end
 
+    def persistent_object(name)
+      @ds.object(persistent_path(name), no_prefix: true)
+    end
+
     def persistent_path(name)
       @ds.path("#{format_path(@persistent_path, name)}/#{name}")
     end
@@ -425,9 +420,9 @@ class StreamingLoadJobClass < RubyJobClass
   end
 
   class LoadableObject
-    def initialize(s3queue, node, logger)
+    def initialize(s3queue, object, logger)
       @s3queue = s3queue
-      @node = node
+      @object = object
       @logger = logger
     end
 
@@ -436,7 +431,7 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def path
-      @node.key
+      @object.key
     end
 
     def basename
@@ -444,20 +439,29 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def url
-      @s3queue.object_url(path)
+      @s3queue.object_url_direct(path)
     end
 
-    def save(noop = false)
-      @logger.info "s3 move: #{path} -> #{save_path}"
+    def dequeue(noop = false)
+      @logger.info "s3 move: #{path} -> #{persistent_path}"
       return if noop
-      @node.object.move_to save_path
+      @object.move_to persistent_object, dequeue_options
       @logger.info "file saved"
     end
 
-    alias dequeue save
+    def persistent_object
+      @s3queue.persistent_object(basename)
+    end
 
-    def save_path
+    def persistent_path
       @s3queue.persistent_path(basename)
     end
+
+    def dequeue_options
+      opts = {
+        server_side_encryption: @s3queue.encryption
+      }
+      opts.reject {|k,v| v.nil? }
+    end
   end
 end

data/lib/bricolage/psqldatasource.rb

@@ -322,7 +322,7 @@ module Bricolage
       unless src_ds.redshift_loader_source?
         raise ParameterError, "input data source does not support redshift as bulk loading source: #{src_ds.name}"
       end
-      provide_default_load_options opts, src_ds
+      opts.provide_defaults(src_ds)
       buf = StringIO.new
       buf.puts "copy #{dest_table}"
       buf.puts "from '#{src_ds.url(src_path)}'"
@@ -335,12 +335,6 @@ module Bricolage
       buf.string
     end
 
-    def provide_default_load_options(opts, src_ds)
-      if src_ds.encrypted? and not opts.key?('encrypted')
-        opts['encrypted'] = true
-      end
-    end
-
     def format_option(fmt, src_ds, jsonpath)
       case fmt
       when 'tsv'
@@ -477,6 +471,12 @@ module Bricolage
       buf.string
     end
 
+    def provide_defaults(src_ds)
+      if src_ds.encrypted? and not key?('encrypted')
+        self['encrypted'] = true
+      end
+    end
+
     class Option
       def initialize(name, value)
         @name = name

data/lib/bricolage/s3datasource.rb

@@ -8,22 +8,28 @@ module Bricolage
   class S3DataSource < DataSource
     declare_type 's3'
 
-    def initialize(endpoint: 's3-ap-northeast-1.amazonaws.com',
+    def initialize(
+        endpoint: 's3-ap-northeast-1.amazonaws.com',
+        region: 'ap-northeast-1',
         bucket: nil, prefix: nil,
         access_key_id: nil, secret_access_key: nil, master_symmetric_key: nil,
+        encryption: nil,
         s3cfg: nil)
-      @endpoint = endpoint
-      @bucket = bucket
+      @endpoint = (/\Ahttps?:/ =~ endpoint) ? endpoint : "https://#{endpoint}"
+      @region = region
+      @bucket_name = bucket
       @prefix = (prefix && prefix.empty?) ? nil : prefix
       @access_key_id = access_key_id
       @secret_access_key = secret_access_key
       @master_symmetric_key = master_symmetric_key
+      @encryption = encryption
       @s3cfg = s3cfg
       @configurations = @s3cfg ? load_configurations(@s3cfg) : nil
     end
 
     attr_reader :endpoint
-    attr_reader :bucket
+    attr_reader :region
+    attr_reader :bucket_name
     attr_reader :prefix
 
     def new_task
@@ -64,8 +70,10 @@ module Bricolage
       h
     end
 
+    attr_reader :encryption
+
     def encrypted?
-      !!@master_symmetric_key
+      !!(@master_symmetric_key or @encryption)
     end
 
     #
@@ -73,29 +81,30 @@ module Bricolage
     #
 
     def client
-      @client ||= AWS::S3.new(s3_endpoint: endpoint, access_key_id: access_key, secret_access_key: secret_key)
-    end
-
-    def objects
-      client.buckets[bucket].objects
+      @client ||= Aws::S3::Client.new(region: @region, endpoint: @endpoint, access_key_id: access_key, secret_access_key: secret_key)
     end
 
-    def objects_with_prefix(rel, no_prefix: false)
-      objects.with_prefix(path(rel, no_prefix: no_prefix))
+    def bucket
+      @resource ||= Aws::S3::Resource.new(client: client)
+      @bucket ||= @resource.bucket(@bucket_name)
     end
 
     def object(rel, no_prefix: false)
-      objects[path(rel, no_prefix: no_prefix)]
+      bucket.object(path(rel, no_prefix: no_prefix))
     end
 
     def url(rel, no_prefix: false)
-      "s3://#{@bucket}/#{path(rel, no_prefix: no_prefix)}"
+      "s3://#{@bucket_name}/#{path(rel, no_prefix: no_prefix)}"
     end
 
     def path(rel, no_prefix: false)
-      path = (no_prefix || !prefix) ? rel.to_s : "#{@prefix}/#{rel}"
+      path = (no_prefix || !@prefix) ? rel.to_s : "#{@prefix}/#{rel}"
       path.sub(%r<\A/>, '').gsub(%r<//>, '/')
     end
+
+    def traverse(rel, no_prefix: false)
+      bucket.objects(prefix: path(rel, no_prefix: no_prefix))
+    end
   end
 
   class S3Task < DataSourceTask
@@ -143,7 +152,7 @@ module Bricolage
         raise JobFailure, "no such file: #{@src}" if source_files.empty?
         each_src_dest do |src, dest|
           ds.logger.info command_line(src, dest)
-          ds.object(dest).write(file: src)
+          ds.object(dest).upload_file(src)
         end
         nil
       end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.12.5'
+  VERSION = '5.13.0'
 end

data/test/home/Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org/'
 gem 'bricolage', path: '../..'
+gem 'pry'

data/test/home/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.12.5)
-      aws-sdk (< 2)
+    bricolage (5.13.0)
+      aws-sdk (~> 2)
       mysql2
       pg
       td
@@ -10,35 +10,43 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (1.66.0)
-      aws-sdk-v1 (= 1.66.0)
-    aws-sdk-v1 (1.66.0)
-      json (~> 1.4)
-      nokogiri (>= 1.4.4)
+    aws-sdk (2.2.6)
+      aws-sdk-resources (= 2.2.6)
+    aws-sdk-core (2.2.6)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.6)
+      aws-sdk-core (= 2.2.6)
+    coderay (1.1.0)
     fluent-logger (0.4.10)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       yajl-ruby (~> 1.0)
     hirb (0.7.3)
     httpclient (2.5.3.3)
+    jmespath (1.1.3)
     json (1.8.3)
-    mini_portile (0.6.2)
+    method_source (0.8.2)
     msgpack (0.5.11)
-    mysql2 (0.4.1)
-    nokogiri (1.6.6.2)
-      mini_portile (~> 0.6.0)
+    mysql2 (0.4.2)
     parallel (0.6.5)
-    pg (0.18.3)
+    pg (0.18.4)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    ruby-progressbar (1.7.5)
     rubyzip (1.1.7)
-    td (0.12.0)
+    slop (3.6.0)
+    td (0.13.0)
       hirb (>= 0.4.5)
       msgpack (>= 0.4.4, < 0.5.12, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       parallel (~> 0.6.1)
+      ruby-progressbar (~> 1.7.5)
       rubyzip (~> 1.1.7)
-      td-client (~> 0.8.75)
+      td-client (~> 0.8.76)
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.75)
+    td-client (0.8.76)
       httpclient (>= 2.5.2, < 2.6.0)
       json (>= 1.7.6)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -55,6 +63,7 @@ PLATFORMS
 
 DEPENDENCIES
   bricolage!
+  pry
 
 BUNDLED WITH
    1.10.6

data/test/home/config/development/database.yml

@@ -45,6 +45,7 @@ td_search_log:
 s3:
   type: s3
   endpoint: "s3-ap-northeast-1.amazonaws.com"
+  region: "ap-northeast-1"
   bucket: tabemiru-data.ap-northeast-1
   prefix: "/dev"
   s3cfg: <%= user_home_relative_path '.s3cfg' %>

data/test/home/data/test.txt

@@ -0,0 +1 @@
+test

data/test/home/put.sh

@@ -8,5 +8,5 @@ in \
 do
     aws s3 cp \
         $(dirname $0)/data/$name \
-        $S3_TABEMIRU/dev/queue/$name
+        $S3_DEV/dev/queue/$name
 done

data/test/home/revert.sh

@@ -7,6 +7,6 @@ in \
     20141002-1355_02.txt
 do
     aws s3 mv \
-        s3://tabemiru-data.ap-northeast-1/tmp/save/year=2014/month=10/day=02/hour=13/$name \
-        s3://tabemiru-data.ap-northeast-1/tmp/queue/$name
+        $S3_DEV/dev/save/year=2014/month=10/day=02/hour=13/$name \
+        $S3_DEV/dev/queue/$name
 done

data/test/home/subsys/put.job

@@ -0,0 +1,3 @@
+class: s3-put
+dest-file: test-dest.txt
+src-file: $bricolage_home/data/test.txt

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.12.5
+  version: 5.13.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-27 00:00:00.000000000 Z
+date: 2015-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -28,14 +28,14 @@ dependencies:
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
@@ -169,6 +169,7 @@ files:
 - test/home/data/20141002-1355_00.txt
 - test/home/data/20141002-1355_01.txt
 - test/home/data/20141002-1355_02.txt
+- test/home/data/test.txt
 - test/home/jobnet-test.rb
 - test/home/put.sh
 - test/home/revert.sh
@@ -187,6 +188,7 @@ files:
 - test/home/subsys/migrate.job
 - test/home/subsys/net1.jobnet
 - test/home/subsys/net2.jobnet
+- test/home/subsys/put.job
 - test/home/subsys/raw-vacuum.jobnet
 - test/home/subsys/raw-vacuum.sql.job
 - test/home/subsys/rebuild.sql.job