bricolage 5.9.0 → 5.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d8fe450bc754d0074c361ebdec9b3f0119cf440
-  data.tar.gz: a12ba4bf06eb22f598e4d8e7df8b1028c84e0c52
+  metadata.gz: bf741a3e8446d11bfb5e9b0f6dcc891f1cae8495
+  data.tar.gz: c17962a361321f6b19d976f62aace9e7b63544da
 SHA512:
-  metadata.gz: 801a0ba6a167068141d446e2d8e598ef6bcafcfec591c9011dcedf8aa126afdfb2dd6b830dfc9786f530e2608a48d5c69faee6b510c1b0e833feca6ae661df6a
-  data.tar.gz: cbb2b228f85c74f5b1a4f83a9d55eddcccfbcf9d9ea91ab5b2e4564614a11ecb512222448bf2e71eebbb17237eb792368e780f5ca3104f63e3278543cf8bce28
+  metadata.gz: c81b950eef56f09a3a1dc7e682a45eabb6c0d25a610d1925ec7409de54f6f11643ddd1153d9d78a25c338160c23c83084293ac7960e1a0634b97bd835cb164d5
+  data.tar.gz: 3068f48c6134aab0c219f826d74b57bc43e077bd486b39140c23925631fd97c9debff45422be35034a82aaa80822c26d14edc4a69249707c38e87f40656e8e1a

data/lib/bricolage/commandutils.rb

@@ -10,7 +10,7 @@ module Bricolage
       sargs.unshift env if env
       system(*sargs)
       st = $?
-      logger.info "status: #{st.exitstatus} (#{st})"
+      logger.info "status: #{st.exitstatus || 'nil'} (#{st})"
       st
     end
 

data/lib/bricolage/jobresult.rb

@@ -47,7 +47,8 @@ module Bricolage
 
     def status
       if @process_status
-        @process_status.exitstatus
+        # Process::Status#exitstatus may be nil when the command did not exited normally
+        @process_status.exitstatus || EXIT_FAILURE
       else
         case @type
         when :success then EXIT_SUCCESS

data/lib/bricolage/psqldatasource.rb

@@ -4,6 +4,7 @@ require 'bricolage/sqlstatement'
 require 'bricolage/commandutils'
 require 'bricolage/postgresconnection'
 require 'bricolage/exception'
+require 'pathname'
 
 module Bricolage
 
@@ -243,12 +244,33 @@ module Bricolage
 
     def vacuum_if(enable_vacuum, enable_vacuum_sort, target = '${dest_table}')
       if enable_vacuum
-        exec SQLStatement.for_string("vacuum #{target};")
+        serialize_vacuum {
+          exec SQLStatement.for_string("vacuum #{target};")
+        }
       elsif enable_vacuum_sort
-        exec SQLStatement.for_string("vacuum sort only #{target};")
+        serialize_vacuum {
+          exec SQLStatement.for_string("vacuum sort only #{target};")
+        }
       end
     end
 
+    DEFAULT_VACUUM_LOCK_FILE = '/tmp/bricolage.vacuum.lock'
+    DEFAULT_VACUUM_LOCK_TIMEOUT = 3600   # 60min
+
+    def serialize_vacuum
+      if ENV['BRICOLAGE_VACUUM_LOCK']
+        path, tm = ENV['BRICOLAGE_VACUUM_LOCK'].split(':', 2)
+        timeout = tm.to_i
+      else
+        path = DEFAULT_VACUUM_LOCK_FILE
+        timeout = DEFAULT_VACUUM_LOCK_TIMEOUT
+      end
+      lock_file_cmd = Pathname(__FILE__).parent.parent.parent + 'libexec/create-lockfile'
+      exec SQLStatement.for_string "\\! #{lock_file_cmd} #{path} #{timeout}"
+      yield
+      exec SQLStatement.for_string "\\! rm #{path}"
+    end
+
     def analyze_if(enabled, target = '${dest_table}')
       exec SQLStatement.for_string("analyze #{target};") if enabled
     end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.9.0'
+  VERSION = '5.9.1'
 end

data/libexec/create-lockfile

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+DEFAULT_TIMEOUT = 900    # 15 minutes
+MINIMUM_TIMEOUT = 1
+
+def main
+  unless ARGV.size == 2
+    $stderr.puts "Usage: #{$0} PATH TIMEOUT_SEC"
+    exit 1
+  end
+  path, tm = ARGV
+  timeout = [tm.to_i, MINIMUM_TIMEOUT].max
+  create_lock_file path, timeout
+end
+
+def create_lock_file(path, timeout)
+  start_time = Time.now
+  print_message "trying to create lock file: #{path} (psql #{parent_is_psql? ? 'detected' : 'NOT detected'})"
+  begin
+    File.open(path, File::WRONLY | File::CREAT | File::EXCL) {|f|
+      f.puts "#{Time.now}: created by process \##{Process.ppid}"
+    }
+  rescue Errno::EEXIST
+    if Time.now - start_time > timeout
+      emit_error_to_psql
+      error_exit "could not create lock file: #{path} (timeout #{timeout} seconds)"
+    end
+    sleep 3
+    retry
+  rescue
+    emit_error_to_psql
+    raise
+  end
+  print_message "lock file created: #{path}"
+end
+
+def emit_error_to_psql
+  # psql does not stop on error of external command,
+  # we must explicitly stop that.
+  if parent_is_psql?
+    Process.kill 'TERM', Process.ppid
+  end
+end
+
+def parent_is_psql?
+  # psql set this environment variable
+  !!ENV['PGSYSCONFDIR']
+end
+
+def error_exit(msg)
+  $stderr.puts "#{$0}: error: #{msg}"
+  exit 1
+end
+
+def print_message(msg)
+  $stderr.puts "#{Time.now}: #{msg}"
+end
+
+main

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.8.7)
+    bricolage (5.9.0)
       aws-sdk (< 2)
       mysql2
       pg
@@ -52,3 +52,6 @@ PLATFORMS
 
 DEPENDENCIES
   bricolage!
+
+BUNDLED WITH
+   1.10.2

data/test/home/subsys/d.ct

@@ -0,0 +1,7 @@
+--dest-table: d
+
+create table $dest_table
+( x int
+, y int
+, z int
+);

data/test/home/subsys/rebuild.sql.job

@@ -0,0 +1,13 @@
+/*
+    class: rebuild-drop
+    table-def: d.ct
+    dest-table: d
+    src-tables:
+        s: s
+    analyze: true
+    vacuum-sort: true
+*/
+
+insert into $dest_table
+select * from $s
+;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.9.0
+  version: 5.9.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
+date: 2015-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -153,6 +153,7 @@ files:
 - lib/bricolage/tddatasource.rb
 - lib/bricolage/variables.rb
 - lib/bricolage/version.rb
+- libexec/create-lockfile
 - libexec/sqldump
 - libexec/sqldump.Darwin
 - libexec/sqldump.Linux
@@ -166,8 +167,10 @@ files:
 - test/home/data/20141002-1355_02.txt
 - test/home/put.sh
 - test/home/revert.sh
+- test/home/subsys/d.ct
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
+- test/home/subsys/rebuild.sql.job
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
 - test/home/subsys/unified.jobnet