bricolage 5.29.0 → 6.0.0beta2

data/lib/bricolage/jobnet.rb

@@ -51,6 +51,10 @@ module Bricolage
 
     attr_reader :start_jobnet
 
+    def ref
+      @start_jobnet.ref
+    end
+
     def each_jobnet(&block)
       @jobnets.each_value(&block)
     end
@@ -351,7 +355,7 @@ module Bricolage
         unless node_subsys
           raise ParameterError, "missing subsystem: #{ref}"
         end
-        ref_class.new(node_subsys, name, location)
+        ref_class.new(node_subsys.to_s, name.to_s, location)
       end
 
       def initialize(subsys, name, location)
@@ -389,7 +393,7 @@ module Bricolage
 
     class JobRef < Ref
       def JobRef.for_path(path)
-        new(path.parent.basename, JobRef.strip_exts(path), Location.dummy)
+        new(path.parent.basename.to_s, JobRef.strip_exts(path), Location.dummy)
       end
 
       def JobRef.strip_exts(path)
@@ -398,7 +402,7 @@ module Bricolage
         until (ext = basename.extname).empty?
           basename = basename.basename(ext)
         end
-        basename
+        basename.to_s
       end
 
       def net?
@@ -408,11 +412,11 @@ module Bricolage
 
     class JobNetRef < Ref
       def JobNetRef.for_path(path)
-        new(path.parent.basename, path.basename('.jobnet'), Location.dummy)
+        new(path.parent.basename.to_s, path.basename('.jobnet').to_s, Location.dummy)
       end
 
       def JobNetRef.for_job_path(path)
-        new(path.parent.basename, JobRef.strip_exts(path), Location.dummy)
+        new(path.parent.basename.to_s, JobRef.strip_exts(path).to_s, Location.dummy)
       end
 
       def initialize(subsys, name, location)
@@ -444,7 +448,6 @@ module Bricolage
         @end ||= JobRef.new(subsystem, "@#{name}@end", location)
       end
 
-
       def start
         @jobnet.start
       end

data/lib/bricolage/jobnetrunner.rb

@@ -15,6 +15,9 @@ require 'bricolage/version'
 require 'fileutils'
 require 'pathname'
 require 'optparse'
+require 'socket'
+require 'net/http'
+require 'json'
 
 module Bricolage
 
@@ -39,7 +42,7 @@ module Bricolage
       @hooks.run_before_option_parsing_hooks(opts)
       opts.parse!(ARGV)
 
-      @ctx = Context.for_application(job_path: opts.jobnet_files.first, environment: opts.environment, global_variables: opts.global_variables)
+      @ctx = Context.for_application(job_path: opts.jobnet_files.first, environment: opts.environment, option_variables: opts.option_variables)
       opts.merge_saved_options(@ctx.load_system_options)
 
       jobnet = RootJobNet.load_auto(@ctx, opts.jobnet_files)
@@ -54,16 +57,18 @@ module Bricolage
         exit EXIT_SUCCESS
       end
 
+      queue = make_queue(opts)
+      if queue.locked?(jobnet)
+        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help(jobnet)}"
+      end
       if opts.clear_queue?
-        clear_queue(opts)
+        queue.cancel_jobnet(jobnet, 'cancelled by --clear-queue')
+        logger.info "queue is unlocked and cleared"
         exit EXIT_SUCCESS
       end
-      queue = get_queue(opts)
-      if queue.locked?
-        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
-      end
-      unless queue.queued?
-        enqueue_jobs jobnet, queue
+      queue.restore_jobnet(jobnet)
+      if queue.empty?
+        queue.enqueue_jobnet(jobnet)
       end
 
       if opts.list_jobs?
@@ -91,18 +96,30 @@ module Bricolage
       @ctx.logger
     end
 
-    def clear_queue(opts)
-      if path = get_queue_file_path(opts)
-        FileUtils.rm_f path
+    def make_queue(opts)
+      if opts.db_name
+        logger.info "Enables DB queue: datasource=#{opts.db_name}"
+        datasource = @ctx.get_data_source('psql', opts.db_name)
+        executor_id = get_executor_id(opts.executor_type)
+        DatabaseTaskQueue.new(datasource: datasource, executor_id: executor_id, enable_lock: false)
+      elsif path = get_queue_file_path(opts)
+        logger.info "Enables file queue: #{path}"
+        FileTaskQueue.new(path: path)
+      else
+        MemoryTaskQueue.new
       end
     end
 
-    def get_queue(opts)
-      if path = get_queue_file_path(opts)
-        logger.info "queue path: #{path}"
-        FileTaskQueue.restore_if_exist(path)
+    def get_executor_id(executor_type)
+      # executor_id is 'TaskID:PID' or 'Hostname:PID'
+      if executor_type == 'ecs'
+        uri = URI.parse("#{ENV['ECS_CONTAINER_METADATA_URI']}/task")
+        response = Net::HTTP.get_response(uri)
+        task_id = JSON.parse(response.body)['TaskARN'].split('/').last
+        "#{task_id}:#{$$}"
       else
-        TaskQueue.new
+        hostname = Socket.gethostname
+        "#{hostname}:#{$$}"
       end
     end
 
@@ -124,45 +141,38 @@ module Bricolage
       path.basename.to_s
     end
 
-    def enqueue_jobs(jobnet, queue)
-      seq = 1
-      jobnet.sequential_jobs.each do |ref|
-        queue.enq JobTask.new(ref)
-        seq += 1
-      end
-      queue.save
-    end
-
     def list_jobs(queue)
-      queue.each do |task|
-        puts task.job
+      queue.each do |job|
+        puts job
       end
     end
 
     def check_jobs(queue)
-      queue.each do |task|
-        Job.load_ref(task.job, @ctx).compile
+      queue.each do |job|
+        Job.load_ref(job, @ctx).compile
       end
     end
 
     def run_queue(queue)
+      result = nil
+      job = nil
       @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
-      queue.consume_each do |task|
-        result = execute_job(task.job, queue)
-        unless result.success?
-          logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
-          logger.error "[job #{task.job}] #{result.message}"
-          @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
-          exit result.status
-        end
+      queue.consume_each do |job|
+        result = execute_job(job, queue)
       end
-      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
+      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(result.success?, queue))
       logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
-      logger.info "status all green"
+
+      if result.success?
+        logger.info "status all green"
+      else
+        logger.error "[job #{job}] #{result.message}"
+        exit result.status
+      end
     end
 
     def execute_job(ref, queue)
-      logger.debug "job #{ref}"
+      logger.info "job #{ref}"
       job_start_time = Time.now
       job = Job.load_ref(ref, @ctx)
       job.compile
@@ -211,7 +221,7 @@ module Bricolage
       def initialize(app)
         @app = app
         @environment = nil
-        @global_variables = Variables.new
+        @option_variables = Variables.new
         @jobnet_files = nil
 
         @dump_options = false
@@ -226,7 +236,9 @@ module Bricolage
         super.merge({
           'local-state-dir' => OptionValue.new('default value', '/tmp/bricolage'),
           'enable-queue' => OptionValue.new('default value', false),
-          'queue-path' => OptionValue.new('default value', nil)
+          'queue-path' => OptionValue.new('default value', nil),
+          'db-name' => OptionValue.new('default value', nil),
+          'ecs-executor' => OptionValue.new('default value', false)
         })
       end
       private :opts_default
@@ -277,15 +289,22 @@ Options:
         parser.on('--queue-path=PATH', 'Enables job queue with this path.') {|path|
           @opts_cmdline['queue-path'] = OptionValue.new('--queue-path option', path)
         }
+        parser.on('--db-name=DB_NAME', 'Enables job queue with this database.') {|db_name|
+          @opts_cmdline['db-name'] = OptionValue.new('--db-name option', db_name)
+        }
+        parser.on('--ecs-executor', 'Set executor type as ECS ') {
+          @opts_cmdline['ecs-executor'] = OptionValue.new('--ecs-executor option', true)
+        }
+
         parser.on('-c', '--check-only', 'Checks job parameters and quit without executing.') {
           @check_only = true
         }
         parser.on('-l', '--list-jobs', 'Lists target jobs without executing.') {
           @list_jobs = true
         }
-        parser.on('-v', '--variable=NAME=VALUE', 'Defines global variable.') {|name_value|
+        parser.on('-v', '--variable=NAME=VALUE', 'Defines option variable.') {|name_value|
           name, value = name_value.split('=', 2)
-          @global_variables[name] = value
+          @option_variables[name] = value
         }
         parser.on('--dump-options', 'Shows option parsing result and quit.') {
           @dump_options = true
@@ -317,7 +336,7 @@ Options:
 
       attr_reader :jobnet_files
 
-      attr_reader :global_variables
+      attr_reader :option_variables
 
       def dump_options?
         @dump_options
@@ -352,6 +371,24 @@ Options:
         end
       end
 
+      def db_name
+        opt = @opts['db-name']
+        if opt.value
+          opt.value
+        else
+          nil
+        end
+      end
+
+      def executor_type
+        opt = @opts['ecs-executor']
+        if opt.value
+          'ecs'
+        else
+          'ec2'
+        end
+      end
+
       def clear_queue?
         @clear_queue
       end

data/lib/bricolage/logger.rb

@@ -20,9 +20,9 @@ module Bricolage
 
     DEFAULT_ROTATION_SIZE = 1024 ** 2 * 100   # 100MB
 
-    def Logger.new(device: $stderr, rotation_period: nil, rotation_size: DEFAULT_ROTATION_SIZE)
+    def Logger.new(device: $stderr, level: nil, rotation_period: nil, rotation_size: DEFAULT_ROTATION_SIZE)
       logger = super(device, (rotation_period || 0), rotation_size)
-      logger.level = (device == $stderr && $stderr.tty?) ? Logger::DEBUG : Logger::INFO
+      logger.level = level || Logger::INFO
       logger.formatter = -> (sev, time, prog, msg) {
         "#{time}: #{sev}: #{msg}\n"
       }

data/lib/bricolage/loglocator.rb

@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Bricolage
   class LogLocator
     def LogLocator.empty
@@ -63,9 +65,25 @@ module Bricolage
       puts "bricolage: S3 log: #{s3_url}"
       begin
         @s3_writer.upload(path)
+        # tmp: Prints & removes local file if S3 upload is succeeded.
+        # It seems leaving local files causes unexpected Docker failure, I try to remove this.
+        puts File.read(path)
+        FileUtils.rm_f(path)
+        cleanup_local_dirs(File.dirname(path))
       rescue => ex
-        puts "warning: S3 upload failed: #{s3_url}"
+        $stderr.puts "warning: S3 upload failed: #{ex.class} #{ex.message}: #{s3_url}"
+      end
+    end
+
+    # Removes empty directories recursively
+    def cleanup_local_dirs(path)
+      dir_path = path
+      until dir_path == '/' or dir_path == '.'
+        Dir.rmdir(dir_path)
+        dir_path = File.dirname(dir_path)
       end
+    rescue SystemCallError
+      return   # ignore
     end
   end
 end

data/lib/bricolage/postgresconnection.rb

@@ -87,7 +87,7 @@ module Bricolage
     private :querying
 
     def execute_update(query)
-      log_query query
+      log_query query, @ds.update_sql_log_level
       rs = log_elapsed_time {
         querying {
           @connection.async_exec(query)
@@ -97,6 +97,8 @@ module Bricolage
     rescue PG::ConnectionBad, PG::UnableToSend => ex
       @connection_failed = true
       raise ConnectionError.wrap(ex)
+    rescue PG::UniqueViolation => ex
+      raise UniqueViolationException.wrap(ex)
     rescue PG::Error => ex
       raise PostgreSQLException.wrap(ex)
     ensure
@@ -128,7 +130,7 @@ module Bricolage
     end
 
     def execute_query(query, &block)
-      log_query query
+      log_query query, @ds.query_sql_log_level
       rs = log_elapsed_time {
         querying {
           @connection.async_exec(query)
@@ -262,8 +264,8 @@ module Bricolage
       execute("lock #{table}")
     end
 
-    def log_query(query)
-      @logger.log(@ds.sql_log_level) { "[#{@ds.name}] #{mask_secrets query}" }
+    def log_query(query, log_level = @ds.sql_log_level)
+      @logger.log(log_level) { "[#{@ds.name}] #{mask_secrets query}" }
     end
 
     def mask_secrets(msg)

data/lib/bricolage/psqldatasource.rb

@@ -15,6 +15,8 @@ module Bricolage
 
     include CommandUtils
 
+    DEFAULT_RETRY_LIMIT = 3
+
     def initialize(
         host: 'localhost',
         port: 5439,
@@ -24,7 +26,9 @@ module Bricolage
         pgpass: nil,
         encoding: nil,
         psql: 'psql',
-        sql_log_level: Logger::INFO,
+        sql_log_level: nil,
+        query_sql_log_level: nil,
+        update_sql_log_level: nil,
         tmpdir: Dir.tmpdir)
       @host = host
       @port = port
@@ -34,8 +38,11 @@ module Bricolage
       @pgpass = pgpass
       @encoding = encoding
       @psql = psql
-      @sql_log_level = Logger.intern_severity(sql_log_level)
+      @sql_log_level = Logger.intern_severity(sql_log_level || Logger::DEBUG)
+      @query_sql_log_level = Logger.intern_severity(query_sql_log_level || sql_log_level || Logger::DEBUG)
+      @update_sql_log_level = Logger.intern_severity(update_sql_log_level || sql_log_level || Logger::INFO)
       @tmpdir = tmpdir
+      @connection_pool = []
       raise ParameterError, "missing psql host" unless @host
       raise ParameterError, "missing psql port" unless @port
       raise ParameterError, "missing psql database" unless @database
@@ -51,6 +58,8 @@ module Bricolage
     attr_reader :user
 
     attr_reader :sql_log_level
+    attr_reader :query_sql_log_level
+    attr_reader :update_sql_log_level
 
     def new_task
       PSQLTask.new(self)
@@ -71,7 +80,15 @@ module Bricolage
             '--no-password',
             *options,
             env: get_psql_env
-        msg = LogLocator.slice_last_stderr(/^psql:.*?:\d+: ERROR: (.*)/, 1) unless st.success?
+        unless st.success?
+          begin
+            msg = LogLocator.slice_last_stderr(/^psql:.*?:\d+: ERROR: (.*)/, 1)
+          rescue IOError => ex
+            # slice_last_stderr may fail if stderr is not a file
+            logger.error ex.message
+            msg = nil
+          end
+        end
         JobResult.for_process_status(st, msg)
       }
     end
@@ -101,7 +118,48 @@ module Bricolage
     end
 
     def open(&block)
-      PostgresConnection.open_data_source(self, &block)
+      retries = (ENV['BRICOLAGE_OPEN_RETRY_LIMIT'] || DEFAULT_RETRY_LIMIT).to_i
+      begin
+        conn = PostgresConnection.open_data_source(self)
+        conn.execute_query('select 1'){}
+      rescue PG::ConnectionBad, PG::UnableToSend => ex
+        retries -= 1
+        if retries >= 0
+          logger.warn "Retry PG connection for execute query: #{ex.message}"
+          sleep 1
+          retry
+        end
+      end
+      if block_given?
+        yield conn
+      else
+        return conn
+      end
+    end
+
+    def open_shared_connection
+      raise ParameterError, 'open_shared_connection require block' unless block_given?
+      conn = nil
+      if @connection_pool.empty?
+        conn = open
+      else
+        begin
+          conn = @connection_pool.shift
+          conn.execute_query('select 1'){}
+        rescue
+          conn.close
+          conn = open
+        end
+      end
+
+      yield conn
+    ensure
+      @connection_pool.push(conn)
+    end
+
+    def clear_connection_pool
+      @connection_pool.map(&:close)
+      @connection_pool = []
     end
 
     def query_batch(query, batch_size = 5000, &block)
@@ -320,7 +378,6 @@ module Bricolage
       unless src_ds.redshift_loader_source?
         raise ParameterError, "input data source does not support redshift as bulk loading source: #{src_ds.name}"
       end
-      opts.provide_defaults(src_ds)
       buf = StringIO.new
       buf.puts "copy #{dest_table}"
       buf.puts "from '#{src_ds.url(src_path)}'"
@@ -340,12 +397,23 @@ module Bricolage
       when 'csv'
         %q(delimiter ',')
       when 'json'
-        jsonpath ? "json \'#{src_ds.url(jsonpath)}\'" : %q(json 'auto')
+        "json '#{json_param(jsonpath)}'"
       else
         raise ParameterError, "unsupported format: #{fmt}"
       end
     end
 
+    def json_param(jsonpath)
+      case jsonpath
+      when nil
+        'auto'
+      when %r{\As3://}
+        jsonpath
+      else
+        src_ds.url(jsonpath)
+      end
+    end
+
     def unload(stmt, dest_ds, dest_path, format, opts)
       exec unload_statement(stmt, dest_ds, dest_path, format, opts)
     end
@@ -376,7 +444,7 @@ module Bricolage
     end
 
     def format_query(query)
-      query.gsub(/^--.*/, '').strip.gsub(/[ \t]*\n[ \t]*/, ' ').gsub("'", "\\\\'")
+      query.gsub(/^--.*/, '').strip.gsub(/[ \t]*\n[ \t]*/, ' ').gsub(/\\/,"\\\\\\\\").gsub("'", "\\\\'")
     end
   end