bricolage 5.12.3 → 5.12.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 247e0b7b78e35324997d2ae53ddb4b346817c6f5
-  data.tar.gz: 3d3e8a97e0802a507fa71c6117d5626854cc77ff
+  metadata.gz: f1936717e92dd5c0ceab713deb5f4ece78b91f49
+  data.tar.gz: 16721287ca7a2eb341796c06e8b21dcf57f50809
 SHA512:
-  metadata.gz: 2aecb3e9be927e1914ad12ea86b438c6ba2e5d876368e4ea0befdaf69f640be5e92e298b117554b1681250d9dc1fae38c66fa6282cfd9756dbb4760368bc0b6b
-  data.tar.gz: 510d226a01d49707bed15859bf6d1593efe6a3b4058089e4ba693824b6bef3410852f0bcabd7a162657c463b1293da92a9185c128c2f802ef8fee0832b6f6fa9
+  metadata.gz: cac5b19704f17a36b208cdb8e557659bf1d76a9b24c5a8ad9d06ca5e50adafa4b89a994aa04d751776d9ca680d2e6c04c754641440309055b7b58e991c85782b
+  data.tar.gz: 2458de15c20a2439ea130919ab1245a907557c971d6c2e150cd9ed83c4527359f98ff98a2933b0e04575be4f5a311c12f602ec8b88a7ad7a6d1a412d4c3ce21e

data/jobclass/create.rb

@@ -4,6 +4,7 @@ JobClass.define('create') {
     params.add DestTableParam.new
     params.add OptionalBoolParam.new('drop', 'DROP table before CREATE.')
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
     params.add DataSourceParam.new('sql')
   }
 
@@ -16,6 +17,7 @@ JobClass.define('create') {
       task.drop_force_if params['drop']
       task.exec params['table-def']
       task.analyze_if params['analyze']
+      task.grant_if params['grant'], params['dest-table']
     }
   }
 }

data/jobclass/createview.rb

@@ -0,0 +1,22 @@
+JobClass.define('createview') {
+  parameters {|params|
+    params.add SQLFileParam.new('sql-file', 'PATH', 'CREATE VIEW file.')
+    params.add DestTableParam.new
+    params.add SrcTableParam.new
+    params.add OptionalBoolParam.new('drop', 'DROP table before CREATE.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
+    params.add DataSourceParam.new('sql')
+  }
+
+  declarations {|params|
+    params['sql-file'].declarations
+  }
+
+  script {|params, script|
+    script.task(params['data-source']) {|task|
+      task.drop_view_force_if params['drop']
+      task.exec params['sql-file']
+      task.grant_if params['grant'], params['dest-table']
+    }
+  }
+}

data/jobclass/my-migrate.rb

@@ -27,7 +27,7 @@ JobClass.define('my-migrate') {
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
     params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
     params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
 
     # All
     params.add OptionalBoolParam.new('export', 'Runs EXPORT task.')

data/jobclass/rebuild-drop.rb

@@ -7,7 +7,7 @@ JobClass.define('rebuild-drop') {
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
     params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
     params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: on, to)')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
     params.add DataSourceParam.new('sql')
   }
 

data/jobclass/rebuild-rename.rb

@@ -7,7 +7,7 @@ JobClass.define('rebuild-rename') {
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
     params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
     params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
-    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
     params.add DataSourceParam.new('sql')
   }
 

data/jobclass/sql.rb

@@ -7,6 +7,7 @@ JobClass.define('sql') {
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.')
     params.add OptionalBoolParam.new('vacuum', 'VACUUM table after SQL is executed.')
     params.add OptionalBoolParam.new('vacuum-sort', 'VACUUM SORT table after SQL is executed.')
+    params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
     params.add DataSourceParam.new('sql')
   }
 
@@ -24,6 +25,7 @@ JobClass.define('sql') {
       task.exec params['sql-file']
       task.vacuum_if params['vacuum'], params['vacuum-sort']
       task.analyze_if params['analyze']
+      task.grant_if params['grant'], params['dest-table']
     }
   }
 }

data/lib/bricolage/mysqldatasource.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'csv'
 require 'stringio'
 require 'open3'
+require 'etc'
 
 module Bricolage
 
@@ -66,7 +67,7 @@ module Bricolage
   end
 
   class MySQLTask < DataSourceTask
-    def export(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: sqldump)
+    def export(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: false)
       add Export.new(stmt, path: path, format: format, override: override, gzip: gzip, sqldump: sqldump)
     end
 
@@ -118,7 +119,7 @@ module Bricolage
       end
 
       def sqldump_available?
-        sqldump_path.executable?
+        sqldump_real_path.executable?
       end
 
       def sqldump_path
@@ -130,7 +131,7 @@ module Bricolage
       end
 
       def platform_name
-        @platform_name ||= `uname -s`
+        @platform_name ||= Etc.uname[:sysname]
       end
 
       def sqldump_usable?

data/lib/bricolage/psqldatasource.rb

@@ -232,18 +232,30 @@ module Bricolage
       drop '${dest_table}' if enabled
     end
 
-    def drop_force(target_table)
+    def drop_obj_force(type, name)
       exec SQLStatement.for_string(
         "\\set ON_ERROR_STOP false\n" +
-        "drop table #{target_table} cascade;\n" +
+        "drop #{type} #{name} cascade;\n" +
         "\\set ON_ERROR_STOP true\n"
       )
     end
 
+    def drop_force(target_table)
+      drop_obj_force('table', target_table)
+    end
+
+    def drop_view_force(target_view)
+      drop_obj_force('view', target_view)
+    end
+
     def drop_force_if(enabled)
       drop_force('${dest_table}') if enabled
     end
 
+    def drop_view_force_if(enabled)
+      drop_view_force('${dest_table}') if enabled
+    end
+
     def truncate_if(enabled, target = '${dest_table}')
       exec SQLStatement.for_string("truncate #{target};") if enabled
     end
@@ -310,6 +322,7 @@ module Bricolage
       unless src_ds.redshift_loader_source?
         raise ParameterError, "input data source does not support redshift as bulk loading source: #{src_ds.name}"
       end
+      provide_default_load_options opts, src_ds
       buf = StringIO.new
       buf.puts "copy #{dest_table}"
       buf.puts "from '#{src_ds.url(src_path)}'"
@@ -322,6 +335,12 @@ module Bricolage
       buf.string
     end
 
+    def provide_default_load_options(opts, src_ds)
+      if src_ds.encrypted? and not opts.key?('encrypted')
+        opts['encrypted'] = true
+      end
+    end
+
     def format_option(fmt, src_ds, jsonpath)
       case fmt
       when 'tsv'
@@ -420,10 +439,25 @@ module Bricolage
       @opts = opts
     end
 
+    def key?(name)
+      n = name.to_s
+      @opts.any? {|opt| opt.name == n }
+    end
+
     def each(&block)
       @opts.each(&block)
     end
 
+    def []=(name, value)
+      n = name.to_s
+      delete n
+      @opts.push Option.new(n, value)
+    end
+
+    def delete(name)
+      @opts.reject! {|opt| opt.name == name }
+    end
+
     def merge(pairs)
       h = {}
       @opts.each do |opt|
@@ -465,6 +499,8 @@ module Bricolage
           "#{@name} '#{@value}'"
         when Integer   # maxerror 10
           "#{@name} #{@value}"
+        when nil       # (explicitly disable boolean options)
+          ''
         else
           raise ParameterError, "unsupported type of option value for #{@name}: #{@value.inspect}"
         end

data/lib/bricolage/s3datasource.rb

@@ -10,13 +10,14 @@ module Bricolage
 
     def initialize(endpoint: 's3-ap-northeast-1.amazonaws.com',
         bucket: nil, prefix: nil,
-        access_key_id: nil, secret_access_key: nil,
+        access_key_id: nil, secret_access_key: nil, master_symmetric_key: nil,
         s3cfg: nil)
       @endpoint = endpoint
       @bucket = bucket
       @prefix = (prefix && prefix.empty?) ? nil : prefix
       @access_key_id = access_key_id
       @secret_access_key = secret_access_key
+      @master_symmetric_key = master_symmetric_key
       @s3cfg = s3cfg
       @configurations = @s3cfg ? load_configurations(@s3cfg) : nil
     end
@@ -31,7 +32,11 @@ module Bricolage
 
     # For Redshift
     def credential_string
-      "aws_access_key_id=#{access_key};aws_secret_access_key=#{secret_key}"
+      [
+        "aws_access_key_id=#{access_key}",
+        "aws_secret_access_key=#{secret_key}",
+        (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}")
+      ].compact.join(';')
     end
 
     def access_key
@@ -59,6 +64,10 @@ module Bricolage
       h
     end
 
+    def encrypted?
+      !!@master_symmetric_key
+    end
+
     #
     # Ruby Interface
     #

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.12.3'
+  VERSION = '5.12.4'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.12.3)
+    bricolage (5.12.4)
       aws-sdk (< 2)
       mysql2
       pg

data/test/home/config/development/database.yml

@@ -50,6 +50,7 @@ s3:
   s3cfg: <%= user_home_relative_path '.s3cfg' %>
   access_key_id: "AKIAI5R5BL2W73PR3FXQ"
   secret_access_key: "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"
+  #master_symmetric_key: "12345678901234567890123456789012"
 
 mysql:
   type: mysql

data/test/home/subsys/load.job

@@ -0,0 +1,9 @@
+class: load
+drop: true
+table-def: search_backends.ct
+dest-table: $test_schema.search_backends
+src-file: /search_backends/search_backends.json.gz
+format: json
+options:
+    gzip: true
+    #encrypted: null

data/test/home/subsys/some_view-cv.sql.job

@@ -0,0 +1,11 @@
+/*
+class: createview
+drop: true
+dest-table: some_view
+src-tables:
+    d: d
+*/
+
+create view $dest_table as
+select * from $d
+;

data/test/home/subsys/variable.yml

@@ -1,4 +1,5 @@
 defaults:
+    analyze: true
     grant:
         privilege: "select"
         to: "$test_group"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.12.3
+  version: 5.12.4
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2015-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -106,6 +106,7 @@ files:
 - bin/bricolage
 - bin/bricolage-jobnet
 - jobclass/create.rb
+- jobclass/createview.rb
 - jobclass/exec.rb
 - jobclass/insert-delta.rb
 - jobclass/insert.rb
@@ -180,6 +181,7 @@ files:
 - test/home/subsys/job5.job
 - test/home/subsys/job6.job
 - test/home/subsys/job7.job
+- test/home/subsys/load.job
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
 - test/home/subsys/migrate.job
@@ -191,6 +193,7 @@ files:
 - test/home/subsys/search_backends.ct
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
+- test/home/subsys/some_view-cv.sql.job
 - test/home/subsys/unified.jobnet
 - test/home/subsys/unified.sql.job
 - test/home/subsys/variable.yml