bricolage 5.16.7 → 5.16.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a8efbc939f402ccf882643c4a90ca12ea640c30
-  data.tar.gz: e062facaaae1162a6f7a191d83b11a2df6c00f93
+  metadata.gz: b81ae1279889983f0a8c1f0637024aa4dd0e47b9
+  data.tar.gz: 62451f57e167a845b5b838cf2e2708137f284e80
 SHA512:
-  metadata.gz: 7566e80daca754ce975ad7ea047a468661670e6364f7784631fac49795c2ad2f2239bd115a55d1d0724db057ecae0bdd7bc31200baded0f58d7b92129d0860ea
-  data.tar.gz: 049efc4e14ca96ed81dca065ab177ee590b498c50de57b9be737e6ea264fd5c9e153bb33b2b5bf7762f89127b229e981abe74a0dc0101c48d24084a4dccd8e34
+  metadata.gz: 7966c1ebdc742ea017cffef2a0f5cbec9bd8994af325e8c60159ecbc3d2ae6b1d6f9b47f4f5b38cf29556b0f07710edbdc9ba5e09f7ac96e05ad4bda0d0f1f63
+  data.tar.gz: dd6d3178526464a47edc4eaa02db6bda76d2cd3e8c053ca12570b45393d5be9aa10fed71938f2e89a27fcb5f78218ebe0d1c5950c3c86bc57189da7eb8e5d315

data/lib/bricolage/job.rb

@@ -108,11 +108,13 @@ module Bricolage
       @script.bind @context, @variables
     end
 
+    def provide_default(name, value)
+      @param_vals[name] ||= value if @param_vals
+    end
+
     # Called from jobclasses (parameters_filter)
     def provide_sql_file_by_job_id
-      if @param_vals and @id
-        @param_vals['sql-file'] ||= @id
-      end
+      provide_default 'sql-file', @id if @id
     end
 
     def declarations

data/lib/bricolage/logger.rb

@@ -4,8 +4,14 @@ require 'logger'
 module Bricolage
   class Logger < ::Logger
     def Logger.default
-      logger = new($stderr)
-      logger.level = $stderr.tty? ? Logger::DEBUG : Logger::INFO
+      @default ||= new
+    end
+
+    DEFAULT_ROTATION_SIZE = 1024 ** 2 * 100   # 100MB
+
+    def Logger.new(device: $stderr, rotation_period: nil, rotation_size: DEFAULT_ROTATION_SIZE)
+      logger = super(device, (rotation_period || 0), rotation_size)
+      logger.level = (device == $stderr && $stderr.tty?) ? Logger::DEBUG : Logger::INFO
       logger.formatter = -> (sev, time, prog, msg) {
         "#{time}: #{sev}: #{msg}\n"
       }

data/lib/bricolage/postgresconnection.rb

@@ -17,13 +17,13 @@ module Bricolage
     end
 
     def execute_update(query)
-      @logger.info "[#{@ds.name}] #{query}"
-      log_elapsed_time {
-        rs = @connection.exec(query)
-        result = rs.to_a
-        rs.clear
-        result
+      log_query query
+      rs = log_elapsed_time {
+        @connection.exec(query)
       }
+      result = rs.to_a
+      rs.clear
+      result
     rescue PG::Error => ex
       raise PostgreSQLException.wrap(ex)
     end
@@ -35,8 +35,21 @@ module Bricolage
       execute_query("select * from #{table}", &block)
     end
 
+    def query_value(query)
+      row = query_row(query)
+      row ? row.values.first : nil
+    end
+
+    def query_values(query)
+      execute_query(query) {|rs| rs.to_a }.flat_map {|rec| rec.values }
+    end
+
+    def query_row(query)
+      execute_query(query) {|rs| rs.to_a.first }
+    end
+
     def execute_query(query, &block)
-      @logger.info "[#{@ds.name}] #{query}"
+      log_query query
       rs = log_elapsed_time {
         @connection.exec(query)
       }
@@ -56,7 +69,7 @@ module Bricolage
     end
 
     def streaming_execute_query(query, &block)
-      @logger.info "[#{@ds.name}] #{query}"
+      log_query query
       log_elapsed_time {
         @connection.send_query(query)
       }
@@ -79,8 +92,40 @@ module Bricolage
 
     def transaction
       execute 'begin transaction'
-      yield
-      execute 'commit'
+      txn = Transaction.new(self)
+      begin
+        yield txn
+      rescue
+        begin
+          txn.abort unless txn.committed?
+        rescue => ex
+          @logger.error "SQL error on transaction abort: #{ex.message} (ignored)"
+        end
+        raise
+      ensure
+        txn.commit unless txn.committed?
+      end
+    end
+
+    class Transaction
+      def initialize(conn)
+        @conn = conn
+        @committed = false
+      end
+
+      def committed?
+        @committed
+      end
+
+      def commit
+        @conn.execute 'commit'
+        @committed = true
+      end
+
+      def abort
+        @conn.execute 'abort'
+        @committed = true
+      end
     end
 
     def open_cursor(query, name = nil, &block)
@@ -140,7 +185,18 @@ module Bricolage
       execute "analyze #{table};"
     end
 
-    private
+    def lock(table)
+      execute("lock #{table}")
+    end
+
+    def log_query(query)
+      @logger.info "[#{@ds.name}] #{mask_secrets query}"
+    end
+
+    def mask_secrets(msg)
+      msg.gsub(/\bcredentials\s+'.*?'/mi, "credentials '****'")
+    end
+    private :mask_secrets
 
     def log_elapsed_time
       b = Time.now
@@ -150,6 +206,7 @@ module Bricolage
       t = e - b
       @logger.info "#{'%.1f' % t} secs"
     end
+
   end
 
 end

data/lib/bricolage/sqlutils.rb

@@ -0,0 +1,29 @@
+module Bricolage
+
+  module SQLUtils
+
+    private
+
+    def sql_string_literal(s)
+      %Q('#{escape_sql_string s}')
+    end
+
+    alias s sql_string_literal
+
+    def escape_sql_string(s)
+      s.gsub(/'/, "''")
+    end
+
+    def sql_timestamp_literal(time)
+      %Q(timestamp '#{sql_timestamp_format(time)}')
+    end
+
+    alias t sql_timestamp_literal
+
+    def sql_timestamp_format(time)
+      time.strftime('%Y-%m-%d %H:%M:%S')
+    end
+
+  end
+
+end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.16.7'
+  VERSION = '5.16.8'
 end

data/test/all.rb

@@ -1,3 +1,3 @@
-Dir.glob("#{__dir__}/test_*.rb").each do |path|
+Dir.glob("#{__dir__}/**/test_*.rb").each do |path|
   load path
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.16.7)
+    bricolage (5.16.8)
       aws-sdk (~> 2)
       mysql2
       pg
@@ -11,17 +11,17 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.3.2)
-      aws-sdk-resources (= 2.3.2)
-    aws-sdk-core (2.3.2)
+    aws-sdk (2.3.15)
+      aws-sdk-resources (= 2.3.15)
+    aws-sdk-core (2.3.15)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.3.2)
-      aws-sdk-core (= 2.3.2)
+    aws-sdk-resources (2.3.15)
+      aws-sdk-core (= 2.3.15)
     coderay (1.1.0)
     fluent-logger (0.5.1)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
     hirb (0.7.3)
-    httpclient (2.7.1)
+    httpclient (2.8.0)
     jmespath (1.2.4)
       json_pure (>= 1.8.1)
     json (1.8.3)
@@ -29,7 +29,7 @@ GEM
     method_source (0.8.2)
     msgpack (0.5.12)
     mysql2 (0.4.4)
-    parallel (0.6.5)
+    parallel (1.8.0)
     pg (0.18.4)
     pry (0.10.3)
       coderay (~> 1.1.0)
@@ -39,17 +39,17 @@ GEM
     ruby-progressbar (1.7.5)
     rubyzip (1.1.7)
     slop (3.6.0)
-    td (0.13.2)
+    td (0.14.0)
       hirb (>= 0.4.5)
       msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
-      parallel (~> 0.6.1)
+      parallel (~> 1.8.0)
       ruby-progressbar (~> 1.7.5)
       rubyzip (~> 1.1.7)
-      td-client (~> 0.8.78)
+      td-client (~> 0.8.79)
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.78)
+    td-client (0.8.81)
       httpclient (~> 2.7)
       json (>= 1.7.6)
       msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -69,4 +69,4 @@ DEPENDENCIES
   pry
 
 BUNDLED WITH
-   1.11.2
+   1.12.5

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.16.7
+  version: 5.16.8
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-14 00:00:00.000000000 Z
+date: 2016-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -122,7 +122,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: SQL batch framework
+description: Redshift-oriented Data Warehouse Batch Framework
 email: aamine@loveruby.net
 executables:
 - bricolage
@@ -182,6 +182,7 @@ files:
 - lib/bricolage/s3datasource.rb
 - lib/bricolage/script.rb
 - lib/bricolage/sqlstatement.rb
+- lib/bricolage/sqlutils.rb
 - lib/bricolage/taskqueue.rb
 - lib/bricolage/tddatasource.rb
 - lib/bricolage/vacuumlock.rb
@@ -335,5 +336,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: SQL batch framework
+summary: SQL Batch Framework
 test_files: []