bricolage 5.9.1 → 5.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf741a3e8446d11bfb5e9b0f6dcc891f1cae8495
-  data.tar.gz: c17962a361321f6b19d976f62aace9e7b63544da
+  metadata.gz: d90b31340ca5e8f083b1398ea38ec9ad8e5f48b7
+  data.tar.gz: e24c75f6c80dba6d318ee7c23c81167c69790ba8
 SHA512:
-  metadata.gz: c81b950eef56f09a3a1dc7e682a45eabb6c0d25a610d1925ec7409de54f6f11643ddd1153d9d78a25c338160c23c83084293ac7960e1a0634b97bd835cb164d5
-  data.tar.gz: 3068f48c6134aab0c219f826d74b57bc43e077bd486b39140c23925631fd97c9debff45422be35034a82aaa80822c26d14edc4a69249707c38e87f40656e8e1a
+  metadata.gz: 25c123224e59677fb1515b4be8000744d09850571190e5d32a31193f122e838cfe5f60ab4efb847cbd2d87de457f8c99889dea924bc74d94f4b018ed29c8f251
+  data.tar.gz: bfb3624f71f581cdbf37a1cc05c71be6007ee5c585406b7dc87e5c022d1cd14a7c062b26aa9687083bee568e653abcff961e105033ea23f0c61232a2ea5324d6

data/lib/bricolage/configloader.rb

@@ -1,5 +1,6 @@
 require 'bricolage/sqlstatement'
 require 'bricolage/resource'
+require 'bricolage/embeddedcodeapi'
 require 'bricolage/exception'
 require 'pathname'
 require 'yaml'
@@ -58,6 +59,8 @@ module Bricolage
 
     private
 
+    include EmbeddedCodeAPI
+
     def app_home
       @app_home or raise ParameterError, "app_home is not given in this file"
     end
@@ -76,51 +79,4 @@ module Bricolage
     end
   end
 
-  module EmbeddedCodeAPI
-    private
-
-    def user_home
-      Pathname(ENV['HOME'])
-    end
-
-    def user_home_relative_path(rel)
-      user_home + rel
-    end
-
-    def app_home_relative_path(rel)
-      app_home + rel
-    end
-
-    def relative_path(rel)
-      base_dir + rel
-    end
-
-    def read_file_if_exist(path)
-      return nil unless File.exist?(path)
-      File.read(path)
-    end
-
-    def date(str)
-      Date.parse(str)
-    end
-
-    def ymd(date)
-      date.strftime('%Y-%m-%d')
-    end
-
-    def attribute_tables(attr)
-      all_tables.select {|table| table.attributes.include?(attr) }
-    end
-
-    def all_tables
-      Dir.glob("#{app_home}/*/*.ct").map {|path|
-        SQLStatement.new(FileResource.new(path))
-      }
-    end
-  end
-
-  class ConfigLoader
-    include EmbeddedCodeAPI
-  end
-
 end

data/lib/bricolage/embeddedcodeapi.rb

@@ -0,0 +1,50 @@
+require 'bricolage/vacuumlock'
+
+module Bricolage
+
+  module EmbeddedCodeAPI
+    private
+
+    def user_home
+      Pathname(ENV['HOME'])
+    end
+
+    def user_home_relative_path(rel)
+      user_home + rel
+    end
+
+    def app_home_relative_path(rel)
+      app_home + rel
+    end
+
+    def relative_path(rel)
+      base_dir + rel
+    end
+
+    def read_file_if_exist(path)
+      return nil unless File.exist?(path)
+      File.read(path)
+    end
+
+    def date(str)
+      Date.parse(str)
+    end
+
+    def ymd(date)
+      date.strftime('%Y-%m-%d')
+    end
+
+    def attribute_tables(attr)
+      all_tables.select {|table| table.attributes.include?(attr) }
+    end
+
+    def all_tables
+      Dir.glob("#{app_home}/*/*.ct").map {|path|
+        SQLStatement.new(FileResource.new(path))
+      }
+    end
+
+    include VacuumLock
+  end
+
+end

data/lib/bricolage/exception.rb

@@ -25,6 +25,10 @@ module Bricolage
     attr_reader :original
   end
 
+  ##
+  # Aquiring lock takes too long (e.g. VACUUM lock)
+  class LockTimeout < JobFailure; end
+
   ##
   # Job error.
   # This exception should NOT be thrown in production environment.

data/lib/bricolage/psqldatasource.rb

@@ -3,6 +3,7 @@ require 'bricolage/s3datasource'
 require 'bricolage/sqlstatement'
 require 'bricolage/commandutils'
 require 'bricolage/postgresconnection'
+require 'bricolage/vacuumlock'
 require 'bricolage/exception'
 require 'pathname'
 
@@ -116,12 +117,18 @@ module Bricolage
       open {|conn| conn.select(table, &block) }
     end
 
+    include VacuumLock
+
     def vacuum(table)
-      open {|conn| conn.vacuum(table) }
+      serialize_vacuum {
+        open {|conn| conn.vacuum(table) }
+      }
     end
 
     def vacuum_sort_only(table)
-      open {|conn| conn.vacuum_sort_only(table) }
+      serialize_vacuum {
+        open {|conn| conn.vacuum_sort_only(table) }
+      }
     end
 
     def analyze(table)
@@ -254,21 +261,12 @@ module Bricolage
       end
     end
 
-    DEFAULT_VACUUM_LOCK_FILE = '/tmp/bricolage.vacuum.lock'
-    DEFAULT_VACUUM_LOCK_TIMEOUT = 3600   # 60min
+    include VacuumLock
 
-    def serialize_vacuum
-      if ENV['BRICOLAGE_VACUUM_LOCK']
-        path, tm = ENV['BRICOLAGE_VACUUM_LOCK'].split(':', 2)
-        timeout = tm.to_i
-      else
-        path = DEFAULT_VACUUM_LOCK_FILE
-        timeout = DEFAULT_VACUUM_LOCK_TIMEOUT
-      end
-      lock_file_cmd = Pathname(__FILE__).parent.parent.parent + 'libexec/create-lockfile'
-      exec SQLStatement.for_string "\\! #{lock_file_cmd} #{path} #{timeout}"
+    def serialize_vacuum   # override
+      exec SQLStatement.for_string psql_serialize_vacuum_begin
       yield
-      exec SQLStatement.for_string "\\! rm #{path}"
+      exec SQLStatement.for_string psql_serialize_vacuum_end
     end
 
     def analyze_if(enabled, target = '${dest_table}')
@@ -467,5 +465,4 @@ module Bricolage
     end
   end
 
-
 end

data/lib/bricolage/vacuumlock.rb

@@ -0,0 +1,77 @@
+require 'bricolage/exception'
+
+module Bricolage
+
+  module VacuumLock
+    def enable_vacuum_lock?
+      !!ENV['BRICOLAGE_VACUUM_LOCK']
+    end
+
+    DEFAULT_VACUUM_LOCK_FILE = '/tmp/bricolage.vacuum.lock'
+    DEFAULT_VACUUM_LOCK_TIMEOUT = 3600   # 60min
+
+    def vacuum_lock_parameters
+      return nil unless enable_vacuum_lock?
+      path, tm = ENV['BRICOLAGE_VACUUM_LOCK'].split(':', 2)
+      timeout = tm ? [tm.to_i, 1].max : DEFAULT_VACUUM_LOCK_TIMEOUT
+      return path, timeout
+    end
+    module_function :vacuum_lock_parameters
+
+    def psql_serialize_vacuum_begin
+      if enable_vacuum_lock?
+        path, timeout = vacuum_lock_parameters
+        "\\! #{create_lockfile_cmd} #{path} #{timeout}"
+      else
+        ';'
+      end
+    end
+    module_function :psql_serialize_vacuum_begin
+
+    def psql_serialize_vacuum_end
+      if enable_vacuum_lock?
+        path, timeout = vacuum_lock_parameters
+        "\\! rm #{path}"
+      else
+        ';'
+      end
+    end
+    module_function :psql_serialize_vacuum_end
+
+    def create_lockfile_cmd
+      Pathname(__FILE__).parent.parent.parent + 'libexec/create-lockfile'
+    end
+    module_function :create_lockfile_cmd
+
+    def serialize_vacuum
+      return yield unless enable_vacuum_lock?
+      path, timeout = vacuum_lock_parameters
+      create_vacuum_lock_file path, timeout
+      begin
+        yield
+      ensure
+        FileUtils.rm_f path
+      end
+    end
+    module_function :serialize_vacuum
+
+    def create_vacuum_lock_file(path, timeout)
+      start_time = Time.now
+      begin
+        File.open(path, File::WRONLY | File::CREAT | File::EXCL) {|f|
+          f.puts "#{Time.now}: created by bricolage [#{Process.pid}]"
+        }
+      rescue Errno::EEXIST
+        if Time.now - start_time > timeout
+          raise LockTimeout, "could not create lock file: #{path} (timeout #{timeout} seconds)"
+        end
+        sleep 1
+        retry
+      rescue
+        raise
+      end
+    end
+    module_function :create_vacuum_lock_file
+  end
+
+end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.9.1'
+  VERSION = '5.9.2'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.9.0)
+    bricolage (5.9.2)
       aws-sdk (< 2)
       mysql2
       pg

data/test/home/subsys/raw-vacuum.sql.job

@@ -0,0 +1,7 @@
+/*
+    class: sql
+*/
+
+<%= psql_serialize_vacuum_begin %>
+vacuum s;
+<%= psql_serialize_vacuum_end %>

data/test/vacuum-test.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift '../../lib'
+require 'bricolage/psqldatasource'
+
+ds = Bricolage::PSQLDataSource.new(port: 5444, database: 'production', username: 'aamine', pgpass: "#{ENV['HOME']}/.pgpass")
+ds.__send__ :initialize_base, 'tmp', nil, Logger.new($stderr)
+p ds
+ds.vacuum 's'
+puts 'OK'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.9.1
+  version: 5.9.2
 platform: ruby
 authors:
 - Minero Aoki
@@ -128,6 +128,7 @@ files:
 - lib/bricolage/configloader.rb
 - lib/bricolage/context.rb
 - lib/bricolage/datasource.rb
+- lib/bricolage/embeddedcodeapi.rb
 - lib/bricolage/eventhandlers.rb
 - lib/bricolage/exception.rb
 - lib/bricolage/filedatasource.rb
@@ -151,6 +152,7 @@ files:
 - lib/bricolage/sqlstatement.rb
 - lib/bricolage/taskqueue.rb
 - lib/bricolage/tddatasource.rb
+- lib/bricolage/vacuumlock.rb
 - lib/bricolage/variables.rb
 - lib/bricolage/version.rb
 - libexec/create-lockfile
@@ -170,6 +172,7 @@ files:
 - test/home/subsys/d.ct
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
+- test/home/subsys/raw-vacuum.sql.job
 - test/home/subsys/rebuild.sql.job
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
@@ -178,6 +181,7 @@ files:
 - test/test_filesystem.rb
 - test/test_parameters.rb
 - test/test_variables.rb
+- test/vacuum-test.rb
 homepage: https://github.com/aamine/bricolage
 licenses:
 - MIT