bricolage 5.17.2 → 5.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a9370caedc8884fd3923af7d5c4c970fbe3e2b5
-  data.tar.gz: 38084d6817ed52e925ff100bc651caf0ca990004
+  metadata.gz: 12e309ad5b258e1f273a35fb9b71e8f556501555
+  data.tar.gz: 861d23ff01a1b07c4c97c8f4f8cce26ac50a8a0f
 SHA512:
-  metadata.gz: 3fe83ffc1f604fa2b7a1bd704bd0652893412b52999abdd0af84716a76e691e333122b4de0d9e5ad2861331e7cdd3f73f62b1e9547ea0fc4b1768aacf9f2a247
-  data.tar.gz: 99f10efff0298ad6a4f3e48c2b54de3b93faa4fb79e6e6f393379ae5724e0b799cfb28996d6b41cb67f2ca6c96224ff814e27fa2930f411793afb5dfe864f3e3
+  metadata.gz: f70ce6284e2e5c0e5b8d5b101dd6b1ea79545aeee2cb97970aa499107a75dd86326526f4ad3c6adcf9ceea29926e1ba8b3d84733f6055a3fe6430c19725ef36c
+  data.tar.gz: e4d2b5141b3afabe8d5e93b559876c637382f2d4b53b614bb2476f5e7c62329db18c3e90e21e60d2b60ae158aeb7b9a3cc06e244bf86b01bbd0aa1c13aa7a45c

data/jobclass/my-import.rb

@@ -18,6 +18,7 @@ JobClass.define('my-import') {
         optional: true, default: PSQLLoadOptions.new,
         value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
     params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+    params.add OptionalBoolParam.new('no-backup', 'Do not backup current table with suffix "_old".', default: false)
 
     # Misc
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
@@ -73,8 +74,12 @@ JobClass.define('my-import') {
 
         # RENAME
         task.transaction {
-          task.create_dummy_table '${dest_table}'
-          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          if params['no-backup']
+            task.drop_force params['dest-table'].to_s
+          else
+            task.create_dummy_table '${dest_table}'
+            task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          end
           task.rename_table work_table, params['dest-table'].name
         }
       }

data/jobclass/my-migrate.rb

@@ -22,6 +22,7 @@ JobClass.define('my-migrate') {
         optional: true, default: PSQLLoadOptions.new,
         value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
     params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+    params.add OptionalBoolParam.new('no-backup', 'Do not backup current table with suffix "_old".', default: false)
 
     # Misc
     params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
@@ -95,8 +96,12 @@ JobClass.define('my-migrate') {
 
         # RENAME
         task.transaction {
-          task.create_dummy_table '${dest_table}'
-          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          if params['no-backup']
+            task.drop_force params['dest-table'].to_s
+          else
+            task.create_dummy_table '${dest_table}'
+            task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          end
           task.rename_table work_table, params['dest-table'].name
         }
       }

data/lib/bricolage/exception.rb

@@ -1,19 +1,15 @@
 module Bricolage
 
-  ##
-  # Common super-class of handlable bricolage exceptions
+  # Common super class of handleable Bricolage exceptions
   class ApplicationError < StandardError; end
 
-  ##
   # Job failure.
   # This exception may occur in production environment and is temporary.
   # e.g. Source data error, SQL error
   class JobFailure < ApplicationError; end
 
-  ##
-  # various SQL exception
-  class SQLException < JobFailure
-    def SQLException.wrap(ex)
+  class JobFailureByException < JobFailure
+    def JobFailureByException.wrap(ex)
       new(ex.message, ex)
     end
 
@@ -25,26 +21,27 @@ module Bricolage
     attr_reader :original
   end
 
-  ##
+  # Various SQL exception, except connection problem.
+  class SQLException < JobFailureByException; end
+
+  # Database connection problems (not established, closed unexpectedly, invalid state)
+  class ConnectionError < JobFailureByException; end
+
   # Aquiring lock takes too long (e.g. VACUUM lock)
   class LockTimeout < JobFailure; end
 
-  ##
   # Job error.
   # This exception should NOT be thrown in production environment.
-  # Developer must fix source code or configuration, not to be get this exception.
+  # You must fix source code or configuration not to be get this exception.
   class JobError < ApplicationError; end
 
-  ##
   # Command-line option errors (should NOT be thrown in production environment)
   class OptionError < JobError; end
 
-  ##
   # User parameter errors (should NOT be thrown in production environment)
   class ParameterError < JobError; end
 
-  ##
-  # Bad code in bricolage core or job classes.
+  # Bad code in Bricolage core or job classes.
   # This exception should NOT be thrown in ANY user environment.
   class FatalError < Exception; end
 

data/lib/bricolage/logger.rb

@@ -3,6 +3,15 @@ require 'logger'
 
 module Bricolage
   class Logger < ::Logger
+    def Logger.intern_severity(sev)
+      if sev.kind_of?(Integer)
+        sev
+      else
+        SEV_LABEL.index(sev.to_s.upcase) or
+            raise ParameterError, "no such log level: #{sev}"
+      end
+    end
+
     def Logger.default
       @default ||= new
     end

data/lib/bricolage/postgresconnection.rb

@@ -6,16 +6,54 @@ module Bricolage
   class PostgreSQLException < SQLException; end
 
   class PostgresConnection
+
+    def PostgresConnection.open_data_source(ds)
+      conn = _open_ds(ds)
+      if block_given?
+        begin
+          yield conn
+        ensure
+          conn.close_force
+        end
+      else
+        return conn
+      end
+    end
+
+    def PostgresConnection._open_ds(ds)
+      conn = PG::Connection.open(host: ds.host, port: ds.port, dbname: ds.database, user: ds.user, password: ds.password)
+      new(conn, ds, ds.logger)
+    rescue PG::ConnectionBad, PG::UnableToSend => ex
+      raise ConnectionError.wrap(ex)
+    end
+    private_class_method :_open_ds
+
     def initialize(connection, ds, logger)
       @connection = connection
       @ds = ds
       @logger = logger
+      @closed = false
+      @connection_failed = false
     end
 
     def source
       @connection
     end
 
+    def close
+      @connection.close
+      @closed = true
+    end
+
+    def close_force
+      close
+    rescue
+    end
+
+    def closed?
+      @closed
+    end
+
     def execute_update(query)
       log_query query
       rs = log_elapsed_time {
@@ -24,6 +62,9 @@ module Bricolage
       result = rs.to_a
       rs.clear
       result
+    rescue PG::ConnectionBad, PG::UnableToSend => ex
+      @connection_failed = true
+      raise ConnectionError.wrap(ex)
     rescue PG::Error => ex
       raise PostgreSQLException.wrap(ex)
     end
@@ -56,6 +97,9 @@ module Bricolage
       result = yield rs
       rs.clear
       result
+    rescue PG::ConnectionBad, PG::UnableToSend => ex
+      @connection_failed = true
+      raise ConnectionError.wrap(ex)
     rescue PG::Error => ex
       raise PostgreSQLException.wrap(ex)
     end
@@ -82,6 +126,9 @@ module Bricolage
           rs.clear
         end
       end
+    rescue PG::ConnectionBad, PG::UnableToSend => ex
+      @connection_failed = true
+      raise ConnectionError.wrap(ex)
     rescue PG::Error => ex
       raise PostgreSQLException.wrap(ex)
     end
@@ -97,7 +144,9 @@ module Bricolage
         yield txn
       rescue
         begin
-          txn.abort unless txn.committed?
+          if not txn.committed? and not @connection_failed
+            txn.abort
+          end
         rescue => ex
           @logger.error "SQL error on transaction abort: #{ex.message} (ignored)"
         end
@@ -195,7 +244,7 @@ module Bricolage
     end
 
     def log_query(query)
-      @logger.info "[#{@ds.name}] #{mask_secrets query}"
+      @logger.log(@ds.sql_log_level) { "[#{@ds.name}] #{mask_secrets query}" }
     end
 
     def mask_secrets(msg)
@@ -209,7 +258,7 @@ module Bricolage
     ensure
       e = Time.now
       t = e - b
-      @logger.info "#{'%.1f' % t} secs"
+      @logger.log(@ds.sql_log_level) { "#{'%.1f' % t} secs" }
     end
 
   end

data/lib/bricolage/psqldatasource.rb

@@ -5,6 +5,7 @@ require 'bricolage/commandutils'
 require 'bricolage/postgresconnection'
 require 'bricolage/vacuumlock'
 require 'bricolage/exception'
+require 'bricolage/logger'
 require 'pathname'
 
 module Bricolage
@@ -23,6 +24,7 @@ module Bricolage
         pgpass: nil,
         encoding: nil,
         psql: 'psql',
+        sql_log_level: Logger::INFO,
         tmpdir: Dir.tmpdir)
       @host = host
       @port = port
@@ -32,6 +34,7 @@ module Bricolage
       @pgpass = pgpass
       @encoding = encoding
       @psql = psql
+      @sql_log_level = Logger.intern_severity(sql_log_level)
       @tmpdir = tmpdir
       raise ParameterError, "missing psql host" unless @host
       raise ParameterError, "missing psql port" unless @port
@@ -47,6 +50,8 @@ module Bricolage
     attr_reader :database
     attr_reader :user
 
+    attr_reader :sql_log_level
+
     def new_task
       PSQLTask.new(self)
     end
@@ -96,10 +101,7 @@ module Bricolage
     end
 
     def open(&block)
-      conn = PG::Connection.open(host: @host, port: @port, dbname: @database, user: @user, password: password)
-      yield PostgresConnection.new(conn, self, logger)
-    ensure
-      conn.close if conn
+      PostgresConnection.open_data_source(self, &block)
     end
 
     def query_batch(query, batch_size = 5000, &block)
@@ -178,7 +180,7 @@ module Bricolage
       buf.puts '\timing on'
       each_statement do |stmt|
         buf.puts
-        buf.puts "-- #{stmt.location}" if stmt.location
+        buf.puts "/* #{stmt.location} */" if stmt.location
         buf.puts stmt.stripped_source
       end
       buf.string

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.17.2'
+  VERSION = '5.18.0'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.17.2)
+    bricolage (5.18.0)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)
@@ -11,12 +11,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.6.2)
-      aws-sdk-resources (= 2.6.2)
-    aws-sdk-core (2.6.2)
+    aws-sdk (2.6.4)
+      aws-sdk-resources (= 2.6.4)
+    aws-sdk-core (2.6.4)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.2)
-      aws-sdk-core (= 2.6.2)
+    aws-sdk-resources (2.6.4)
+      aws-sdk-core (= 2.6.4)
     coderay (1.1.0)
     fluent-logger (0.5.1)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -47,7 +47,7 @@ GEM
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
       zip-zip (~> 0.3)
-    td-client (0.8.82)
+    td-client (0.8.83)
       httpclient (~> 2.7)
       json (>= 1.7.6)
       msgpack (>= 0.4.4, < 0.8.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
@@ -67,4 +67,4 @@ DEPENDENCIES
   pry
 
 BUNDLED WITH
-   1.12.5
+   1.13.2

data/test/home/config/development/database.yml

@@ -32,6 +32,16 @@ sql_app2:
   pgpass: <%= user_home_relative_path '.pgpass' %>
   encoding: utf8
 
+postgres:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage
+  username: bricolage
+  password: bricolage
+  encoding: utf8
+  sql_log_level: DEBUG
+
 td:
   type: td
   database: logs

data/test/home/sqltest.rb

@@ -0,0 +1,16 @@
+require 'bricolage/commandlineapplication'
+require 'pp'
+
+app = Bricolage::CommandLineApplication.define {|opts|
+  opts.data_source_option('--ds', 'Target data source', short: '-D', kind: 'sql')
+}
+ds = app.data_source('--ds')
+ds.open {|conn|
+  #task_ids = conn.query_values('select task_id from strload_tasks')
+  #p [task_ids.class, task_ids.size]
+
+  conn.update('create table t (x int)')
+  pp conn.query_value('select count(*) from t')
+  conn.update('drop table t')
+  pp conn.query_value('select count(*) from t')
+}

data/test/home/subsys/migrate.job

@@ -12,6 +12,7 @@ remove-tmp: true
 dest-ds: sql
 dest-table: $test_schema.search_backends
 table-def: search_backends.ct
+no-backup: true
 options:
     statupdate: false
     compupdate: false

data/test/home/subsys/my-import.job

@@ -14,6 +14,7 @@ dump-options:
 dest-ds: sql
 dest-table: $test_schema.users
 table-def: users.ct
+#no-backup: true
 options:
     statupdate: false
     compupdate: false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.17.2
+  version: 5.18.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-29 00:00:00.000000000 Z
+date: 2016-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -206,6 +206,7 @@ files:
 - test/home/data/20141002-1355_02.txt
 - test/home/data/test.txt
 - test/home/jobnet-test.rb
+- test/home/sqltest.rb
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
 - test/home/subsys/job1.job