bricolage 5.27.2 → 5.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b910f08fd3e77241f8ac067c9f129f2b33a7b98bcd1d017a72149fbf228fa8a
-  data.tar.gz: 91f21bf4953bfbf1299df94700db1c7adf49000cb724b338da5a9b09e92da17e
+  metadata.gz: a3d6d789d6f3c437f1e3e9df962f92db5c7aac168952d633b72be38023d875c1
+  data.tar.gz: adbca861a3b9f811a22338422167f8945ba3d3894bfde294875ccb22c38e1a80
 SHA512:
-  metadata.gz: a80f093c8db79a91b1d10419b29a00043ec1f54893654365e4b8c7c7bae421ced67a1516ddb0bc69cd30e9426611be3e0c6480d1f819add3973689789ffd5154
-  data.tar.gz: 48861ccd45aff5c927e9b2f51587ac13e97b4c62a15de51d8ff181d38f1a606ed3e15c0dbc27259bc0010f554844aa0831864cc08ff20e39cdc1c99e144603e4
+  metadata.gz: 88a618757a3747efc8e846bb217c382044348b56f64e7f02c67f0e9e64c08fe56534ed4b282542a6639b07417dad40ac4c0a0afe78b14a273944c12d15ec1abf
+  data.tar.gz: a2ee05a0200bd069ff057b98d2bac2640d2dc9f2a984f90f50f844ebab07097961877038c3a7627f8d2adf641ca5b75c24ede6c5f024e959980782e8b7ebc63e

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+/dev
+/test/home/config
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSES

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2015-2018 Minero Aoki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/RELEASE.md

@@ -0,0 +1,271 @@
+# Bricolage Release Note
+
+## version 5.28.0
+
+- [new] bricolage command now accepts .rb.job, .py.job, .sh.job as a script-embedded job.
+- [new] new data sources file "datasource.yml", in addition to database.yml.  Users can continue to use database.yml, this does NOT introduce any incompatibility.
+- [new] exec: New option "env" to set additional environment variables.
+- [new] exec: Accepts single string value for args.
+
+## version 5.27.2
+
+- [fix] Supports `encoding` load option.
+
+## version 5.27.1
+
+- [fix] SNS data source did not work.
+
+## version 5.27.0
+
+- [new] New option --disable-queue, to disable job queue explicitly.  This option overrides enable-queue option in bricolage.yml.
+- [new] New option --clear-queue, to clear job queue.
+
+## version 5.26.0
+
+- [CHANGE] MySQL or Redis related job classes are separated into other gems, bricolage-mysql and bricolage-redis.
+  Use those gems if you need them.
+
+## version 5.25.1
+
+- [fix] Adds dependency to aws-sdk-sns.
+
+## version 5.25.0
+
+- Upgrade aws-sdk to v3 (aws-sdk-s3 v1).
+
+## version 5.24.6
+
+- [fix] __FILE__ or __dir__ was not correct in config/prelude.rb.
+
+## version 5.24.5
+
+- (skipped)
+
+## version 5.24.4
+
+- Improve error message with my-import job failure
+
+## version 5.24.3
+
+- Support ECS task role with my-import job class
+
+## version 5.24.2
+
+- [fix] --log-dir, --log-path and --s3-log options are wrongly not ommittable
+
+## version 5.24.1
+
+- [fix] --enable-queue did not work
+
+## version 5.24.0
+
+- [new] New config file config/bricolage.yml to save command line options in the file.
+- [new] New option --s3-log to upload log files to S3.
+- [fix] Strips ".sql" from job ID, when the job is executed via *.sql.job file.
+- [fix] Strips all file extensions from jobnet ID, including ".job" or ".sql.job".
+
+## version 5.23.3
+
+- [fix] mys3dump creates empty object even though if source table has no records.
+
+## version 5.23.2
+
+- [new] new job class: adhoc.
+  This job class have only one parameter, sql-file, so instance jobs are never affected by
+  defaults value such as analyze or grant.
+
+## version 5.23.1
+
+- [new] streaming_load: new option --skip-work
+
+## version 5.23.0
+
+- [CHANGE] Drops TD data source support from core.  Use separated bricolage-td gem.
+
+## version 5.22.3
+
+- [new] load, insert: Reduces the number of transactions.
+
+## version 5.22.2
+
+- [new] new option -Q, -L
+
+## version 5.22.1
+
+- [new] bricolage-jobnet: new options --enable-queue and --local-state-dir, for auto-named job queue.
+
+## version 5.22.0
+
+- [new] bricolage: new option --log-path.
+- [new] bricolage, bricolage-jobnet: new option --log-dir.
+- [new] bricolage, bricolage-jobnet: new env BRICOLAGE_LOG_PATH.
+- [new] bricolage, bricolage-jobnet: new env BRICOLAGE_LOG_DIR.
+
+## version 5.21.0
+
+- [new] bricolage-jobnet command accepts .job file as a single job jobnet.
+
+## version 5.20.5
+
+- [fix] my-migrate, my-import: Do not exposure passwords in command line arguments or log files.
+
+## version 5.20.4
+
+- [fix] my-migrate, my-import: should not drop old tables in the RENAME transaction, to avoid "table dropped by concurrent transaction" error.
+
+## version 5.20.3
+
+- [new] mysql data source: new option "collation".
+
+## version 5.20.2
+
+- [fix] AWS S3 API ListObjectsV2 may return corrupted XML, retry it
+
+## version 5.20.1
+
+- [new] new job class my-import-delta.
+
+## version 5.20.0
+
+- [new] streaming_load: Reduces the number of transaction.
+
+## version 5.19.1
+
+- [new] streaming_load: new option --ctl-prefix and --keep-ctl (both is optional).
+
+## version 5.19.0
+
+- [new] bricolage, bricolage-jobnet, Bricolage::CommandLineApplication now do not block on executing queries in PostgreSQL-like DBs (including Redshift).
+- [CHANGE] Removes (maybe) unused method PostgresConnection#streaming_execute_query.  Use #query_batch instead.
+
+## version 5.18.1
+
+- new class SNSDataSource.
+- new class NullLogger.
+- new exception S3Exception.
+- new exception SNSException.
+
+## version 5.18.0
+
+- [new] New parameter "no-backup" for my-import and my-migrate job classes.
+- [new] New parameter "sql_log_level" for the psql data source.
+- [new] Shows SQL source location before the query.
+- Raises ConnectionError for all connection-level problems, while it raises SQLError for SQL-level errors.
+
+## version 5.17.2
+
+- [fix] Using CommandLineApplication with --environment option causes unexpected option error
+
+## version 5.17.1
+
+- [fix] --dry-run option did not work for my-import job class.
+- [new] AWS access key id & secret key are now optional for S3 data sources (to allow using EC2 instance or ECS task attached IAM role)
+
+## version 5.17.0
+
+- [new] Supports Redshift attached IAM role for COPY and UNLOAD.
+
+## version 5.16.9
+
+- code-level change only: [new] new method Transaction#truncate_and_commit
+
+## version 5.16.8
+
+- code-level change only
+
+## version 5.16.7
+
+- [fix] require 'bricolage/context' wrongly caused NameError.
+- [new] PostgresConnection#drop_table_force utilizes DROP TABLE IF EXISTS.
+
+## version 5.16.6
+
+- rebuild-rename, rebuild-drop, my-import, my-migrate, create, createview: Reduces the number of transactions for speed.
+
+## version 5.16.5
+
+- [fix] my-import: mys3dump: Fixes buffer size problem.
+- [fix] my-import: mys3dump: Escapes more meta characters (e.g. \n, \r, \v, ...).
+
+## version 5.16.4
+
+- [fix] Adds dependency to rake
+- [fix] my-import: Reduces warning log messages.
+
+## version 5.16.3
+
+- [fix] streaming_load: Disables statupdate for log staging table, it is useless.
+
+## version 5.16.2
+
+- [fix] streaming_load: Disables compupdate on COPY.   This *might* cause Assert error on some clusters.
+
+## version 5.16.1
+
+- [fix] Fixes syntax error on ruby 2.1
+
+## version 5.16.0
+
+- [CHANGE][EXPERIMENTAL] streaming_load: Always reuse same temporary log table xxxx_l_wk instead of temporary xxxx_l_tmpNNNN.  This *might* cause Redshift DDL slow down, I try to reduce the number of drop-create.
+
+## version 5.15.2
+
+- [fix] redis-export: remove un-required error check.
+
+## version 5.15.1
+
+- [new] redis-export: make faster using cursor and Redis pipeline.
+
+## version 5.15.0
+
+- [new][EXPERIMENTAL] new job class redis-export.
+
+## version 5.14.0
+
+- [new] streaming_load: Fast log check by temporary load log table.
+- [new] streaming_load: Ignores all S3 key-does-not-exist errors; they are caused by S3 eventual consistency.
+
+## version 5.13.1
+
+- [fix] load, streaming_load: "encrypted" load option should not be used for SSE-KMS
+
+## version 5.13.0
+
+- [new] streaming_load: Supports S3 server-side encryption with AWS KMS (Key Management Service).
+- Now Bricolage requires Ruby AWS-SDK v2 for AWS signature v4.
+
+## version 5.12.5
+
+- [fix] Ruby 2.1 does not have Etc.uname, use uname command instead.
+
+## version 5.12.4
+
+- [new] Supports loading from encrypted S3 data source.
+- [new] New job class "createview".
+- [new] Now "create" and "sql" job class support "grant" parameter.
+
+## version 5.12.3
+
+- [new] my-migrate job class supports sql-file parameter for export.
+
+## version 5.12.2
+
+- [new] td-export job class supports .sql.job file
+
+## version 5.12.1
+
+- [fix] ensure unlocking VACUUM lock, also when VACUUM statement was failed.
+
+## version 5.12.0
+
+- [new] Introduces subsystem-wise variable file (SUBSYS/variable.yml)
+- [new] Allows providing default options by "defaults" global variable (e.g. enabling "grant" option by default)
+
+## version 5.11.0
+
+- [fix] Supports jobnet which has both a job and a jobnet
+
+## version 5.10.0
+
+- streaming_load: new option --sql-file
+

data/Rakefile

@@ -0,0 +1,3 @@
+task :test do
+  load "#{__dir__}/test/all.rb"
+end

data/bricolage.gemspec

@@ -0,0 +1,26 @@
+require_relative 'lib/bricolage/version'
+
+Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.name = 'bricolage'
+  s.version = Bricolage::VERSION
+  s.summary = 'SQL Batch Framework'
+  s.description = 'Redshift-oriented Data Warehouse Batch Framework'
+  s.license = 'MIT'
+
+  s.author = ['Minero Aoki']
+  s.email = 'aamine@loveruby.net'
+  s.homepage = 'https://github.com/bricolages/bricolage'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.executables = s.files.grep(%r{bin/}).map {|path| File.basename(path) }
+  s.require_path = 'lib'
+
+  s.required_ruby_version = '>= 2.0.0'
+  s.add_dependency 'pg', '~> 0.18.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1'
+  s.add_dependency 'aws-sdk-sns', '~> 1'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'mocha'
+end

data/jobclass/exec.rb

@@ -1,13 +1,16 @@
 JobClass.define('exec') {
   parameters {|params|
-    params.add StringListParam.new('args', 'ARG', 'Command line arguments.')
+    params.add StringListParam.new('args', 'ARG', 'Command line arguments.', allow_string: true)
+    params.add KeyValuePairsParam.new('env', 'ENVS', 'Environment variables.')
   }
 
   script {|params, script|
     script.task(params.generic_ds) {|task|
       task.action(params['args'].join(' ')) {|ds|
         ds.logger.info '[CMD] ' + params['args'].join(' ')
-        system(*params['args'])
+        environ = params['env'] || {}
+        argv = params['args']
+        system(environ, *argv)
         st = $?
         ds.logger.info "status=#{st.exitstatus}"
         JobResult.for_bool(st.exitstatus == 0)

data/lib/bricolage/configloader.rb

@@ -16,11 +16,19 @@ module Bricolage
     end
 
     def load_yaml(path)
-      parse_yaml(load_text(path), path)
+      parse_yaml(load_eruby(path), path)
     end
 
-    def load_text(path)
-      eruby(read_file(path), path)
+    def load_eruby(path)
+      eruby(load_file(path), path)
+    end
+
+    alias load_text load_eruby   # FIXME: obsolete
+
+    def load_file(path)
+      File.read(path)
+    rescue SystemCallError => err
+      raise ParameterError, "could not read file: #{err.message}"
     end
 
     def eruby(text, path)
@@ -39,12 +47,6 @@ module Bricolage
       raise ParameterError, "#{path}: config file syntax error: #{err.message}"
     end
 
-    def read_file(path)
-      File.read(path)
-    rescue SystemCallError => err
-      raise ParameterError, "could not read file: #{err.message}"
-    end
-
     #
     # For embedded code
     #
@@ -71,7 +73,7 @@ module Bricolage
     # $base_dir + "vars.yml" -> "$base_dir/vars.yml"
     # $base_dir + "/abs/path/vars.yml" -> "/abs/path/vars.yml"
     def read_config_file(path)
-      load_text(relative_path(Pathname(path)))
+      load_eruby(relative_path(Pathname(path)))
     end
   end
 

data/lib/bricolage/datasource.rb

@@ -5,15 +5,20 @@ require 'bricolage/exception'
 module Bricolage
 
   class DataSourceFactory
+    DATA_SOURCE_FILE_NAME_1 = 'datasource.yml'
+    DATA_SOURCE_FILE_NAME_2 = 'database.yml'
+    PASSWORD_FILE_NAME = 'password.yml'
+
     def DataSourceFactory.load(context, logger)
       loader = Loader.new(context, logger)
-      loader.load_passwords
-      loader.load
+      loader.load_passwords(PASSWORD_FILE_NAME)
+      begin
+        return loader.load(DATA_SOURCE_FILE_NAME_1)
+      rescue ParameterError
+        return loader.load(DATA_SOURCE_FILE_NAME_2)
+      end
     end
 
-    DEFAULT_CONFIG_FILE_NAME = 'database.yml'
-    DEFAULT_PASSWORD_FILE_NAME = 'password.yml'
-
     class Loader < ConfigLoader
       def initialize(context, logger)
         super context.home_path
@@ -22,7 +27,7 @@ module Bricolage
         @passwords = nil
       end
 
-      def load_passwords(basename = DEFAULT_PASSWORD_FILE_NAME)
+      def load_passwords(basename)
         @context.config_pathes(basename).each do |path|
           if path.exist?
             @passwords = load_yaml(path)
@@ -31,11 +36,11 @@ module Bricolage
         end
       end
 
-      def load(basename = DEFAULT_CONFIG_FILE_NAME)
-        database_yml = @context.config_pathes(basename).detect {|path| path.exist? }
-        raise ParameterError, "database.yml does not exist" unless database_yml
-        @config_dir = database_yml.parent
-        DataSourceFactory.new(load_yaml(database_yml), @context, @logger)
+      def load(basename)
+        yml_path = @context.config_pathes(basename).detect(&:exist?)
+        raise ParameterError, "database.yml does not exist" unless yml_path
+        @config_dir = yml_path.parent
+        DataSourceFactory.new(load_yaml(yml_path), @context, @logger)
       end
 
       def password(name)

data/lib/bricolage/job.rb

@@ -23,6 +23,9 @@ module Bricolage
       f = JobFile.load(ctx, path)
       instantiate(f.job_id, f.class_id, ctx).tap {|job|
         job.bind_parameters f.values
+        f.global_variables.each do |name, value|
+          job.global_variables[name.to_s] = value
+        end
       }
     end
 

data/lib/bricolage/jobfile.rb

@@ -1,23 +1,29 @@
 require 'bricolage/exception'
 require 'pathname'
+require 'stringio'
 
 module Bricolage
 
   class JobFile
     def JobFile.load(ctx, path)
-      values = if /\.sql\.job\z/ =~ path.to_s
-        load_embedded_definition(ctx, path)
-      else
-        ctx.parameter_file_loader.load_yaml(path)
-      end
+      values =
+        case path.to_s
+        when /\.sql\.job\z/
+          load_embedded_definition(ctx, path)
+        when /\.(?:rb|py|sh)\.job\z/
+          load_program_embedded_definition(ctx, path)
+        else
+          ctx.parameter_file_loader.load_yaml(path)
+        end
       parse(values, path)
     end
 
     class << JobFile
       private
 
+      # For .sql.job
       def load_embedded_definition(ctx, path)
-        sql = ctx.parameter_file_loader.load_text(path)
+        sql = ctx.parameter_file_loader.load_eruby(path)
         block = sql.slice(%r{\A/\*.*?^\*/}m) or
             raise ParameterError, "missing embedded job definition block: #{path}"
         yaml = block.sub(%r{\A/\*}, '').sub(%r{^\*/\s*\z}, '')
@@ -70,6 +76,30 @@ module Bricolage
         end
         decls
       end
+
+      # For .rb.job, .py.job, .sh.job ("#" comment style files)
+      # This method does NOT expand variables of parameter files.
+      def load_program_embedded_definition(ctx, path)
+        script = StringIO.new(ctx.parameter_file_loader.load_file(path))
+        first_line = script.gets
+        if first_line.start_with?('#!')
+          yaml_lines = []
+        else
+          yaml_lines = [first_line]
+        end
+        script.each_line do |line|
+          break unless line.start_with?('#')
+          yaml_lines.push line[1..-1]
+        end
+        yaml = yaml_lines.join
+
+        begin
+          values = YAML.load(yaml)
+        rescue => err
+          raise ParameterError, "#{path}: embedded job definition syntax error: #{err.message}"
+        end
+        values
+      end
     end
 
     def JobFile.parse(values, path)
@@ -97,6 +127,12 @@ module Bricolage
     def subsystem
       @path.parent.basename.to_s
     end
+
+    def global_variables
+      {
+        'script' => @path.to_s
+      }
+    end
   end
 
 end

data/lib/bricolage/parameters.rb

@@ -604,8 +604,9 @@ module Bricolage
   end
 
   class StringListParam < Param
-    def initialize(name, arg_spec, description, optional: false, publish: false)
+    def initialize(name, arg_spec, description, optional: false, publish: false, allow_string: false)
       super name, arg_spec, description, optional: optional, publish: publish
+      @allow_string = allow_string
     end
 
     def parse_option_value(value, list)
@@ -614,7 +615,10 @@ module Bricolage
     end
 
     def parse_value(vals)
-      raise ParameterError, "bad type for parameter #{name}: #{vals.class}" unless vals.kind_of?(Array)
+      unless vals.kind_of?(Array)
+        raise ParameterError, "bad type for parameter #{name}: #{vals.class}" unless @allow_string
+        vals = [vals]
+      end
       vals.empty? ? nil : vals
     end
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.27.2'
+  VERSION = '5.28.0'
 end

data/streaming_load_log.ct

@@ -0,0 +1,9 @@
+create table $dest_table
+( job_process_id varchar(256) encode lzo
+, start_time timestamp encode delta
+, end_time timestamp encode delta
+, target_table varchar(256) encode lzo
+, data_file varchar(2048) encode lzo
+)
+sortkey (job_process_id)
+;