bricolage 5.8.7

data/lib/bricolage/jobnetrunner.rb

@@ -0,0 +1,264 @@
+require 'bricolage/application'
+require 'bricolage/context'
+require 'bricolage/jobflow'
+require 'bricolage/taskqueue'
+require 'bricolage/job'
+require 'bricolage/jobresult'
+require 'bricolage/datasource'
+require 'bricolage/variables'
+require 'bricolage/eventhandlers'
+require 'bricolage/logger'
+require 'bricolage/exception'
+require 'bricolage/version'
+require 'pathname'
+require 'optparse'
+
+module Bricolage
+
+  class JobNetRunner
+    def JobNetRunner.main
+      new.main
+    end
+
+    def initialize
+      Signal.trap('PIPE', 'IGNORE')
+      @hooks = ::Bricolage
+      @flow_id = nil
+      @flow_start_time = Time.now
+      @log_path = nil
+    end
+
+    EXIT_SUCCESS = JobResult::EXIT_SUCCESS
+    EXIT_FAILURE = JobResult::EXIT_FAILURE
+    EXIT_ERROR = JobResult::EXIT_ERROR
+
+    def main
+      opts = Options.new(self)
+      @hooks.run_before_option_parsing_hooks(opts)
+      opts.parse ARGV
+      @ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
+      @flow_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
+      @log_path = opts.log_path
+      flow = RootJobFlow.load(@ctx, opts.jobnet_file)
+      queue = get_queue(opts)
+      if queue.locked?
+        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
+      end
+      unless queue.queued?
+        enqueue_jobs flow, queue
+        logger.info "jobs are queued." if opts.queue_exist?
+      end
+      if opts.list_jobs?
+        list_jobs queue
+        exit EXIT_SUCCESS
+      end
+      check_jobs queue
+      if opts.check_only?
+        puts "OK"
+        exit EXIT_SUCCESS
+      end
+      run_queue queue
+      exit EXIT_SUCCESS
+    rescue OptionError => ex
+      raise if $DEBUG
+      usage_exit ex.message, opts.help
+    rescue ApplicationError => ex
+      raise if $DEBUG
+      error_exit ex.message
+    end
+
+    def logger
+      @ctx.logger
+    end
+
+    def get_queue(opts)
+      if opts.queue_path
+        FileTaskQueue.restore_if_exist(opts.queue_path)
+      else
+        TaskQueue.new
+      end
+    end
+
+    def enqueue_jobs(flow, queue)
+      flow.each_subnet_sequence do |subnet|
+        seq = 1
+        subnet.sequential_jobs.each do |ref|
+          queue.enq JobTask.new(subnet.ref, seq, ref)
+          seq += 1
+        end
+      end
+      queue.save
+    end
+
+    def list_jobs(queue)
+      prev = nil
+      queue.each do |task|
+        if not prev or prev.jobnet != task.jobnet
+          puts "---- jobnet #{task.jobnet} ---"
+        end
+        puts task.job
+        prev = task
+      end
+    end
+
+    def check_jobs(queue)
+      queue.each do |task|
+        Job.load_ref(task.job, @ctx).compile
+      end
+    end
+
+    def run_queue(queue)
+      @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@flow_id, queue))
+      queue.consume_each do |task|
+        result = execute_job(task.job, queue)
+        unless result.success?
+          logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
+          logger.error "[job #{task.job}] #{result.message}"
+          @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
+          exit result.status
+        end
+      end
+      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
+      logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
+      logger.info "status all green"
+    end
+
+    def execute_job(ref, queue)
+      logger.debug "job #{ref}"
+      job = Job.load_ref(ref, @ctx)
+      job.compile
+      @hooks.run_before_job_hooks(BeforeJobEvent.new(ref))
+      result = job.execute_in_process(make_log_path(ref))
+      @hooks.run_after_job_hooks(AfterJobEvent.new(result))
+      result
+    rescue Exception => ex
+      logger.exception ex
+      logger.error "unexpected error: #{ref} (#{ex.class}: #{ex.message})"
+      JobResult.error(ex)
+    end
+
+    def make_log_path(job_ref)
+      return nil unless @log_path
+      start_time = Time.now
+      @log_path.gsub(/%\{\w+\}/) {|var|
+        case var
+        when '%{flow_start_date}' then @flow_start_time.strftime('%Y%m%d')
+        when '%{flow_start_time}' then @flow_start_time.strftime('%Y%m%d_%H%M%S%L')
+        when '%{job_start_date}' then start_time.strftime('%Y%m%d')
+        when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
+        when '%{flow}', '%{flow_id}' then @flow_id.gsub('/', '::')
+        when '%{subsystem}' then job_ref.subsystem
+        when '%{job}', '%{job_id}' then job_ref.name
+        else
+          raise ParameterError, "bad log path variable: #{var}"
+        end
+      }
+    end
+
+    def usage_exit(msg, usage)
+      print_error msg
+      $stderr.puts usage
+      exit 1
+    end
+
+    def error_exit(msg)
+      print_error msg
+      exit 1
+    end
+
+    def print_error(msg)
+      $stderr.puts "#{program_name}: error: #{msg}"
+    end
+
+    def program_name
+      File.basename($PROGRAM_NAME, '.*')
+    end
+
+    class Options
+      def initialize(app)
+        @app = app
+        @environment = nil
+        @jobnet_files = nil
+        @log_path = nil
+        @queue_path = nil
+        @check_only = false
+        @list_jobs = false
+        @global_variables = Variables.new
+        @parser = OptionParser.new
+        define_options @parser
+      end
+
+      attr_reader :environment
+      attr_reader :jobnet_file
+      attr_reader :log_path
+      attr_reader :queue_path
+
+      def queue_exist?
+        !!@queue_path
+      end
+
+      def check_only?
+        @check_only
+      end
+
+      def list_jobs?
+        @list_jobs
+      end
+
+      attr_reader :global_variables
+
+      def help
+        @parser.help
+      end
+
+      def define_options(parser)
+        parser.banner = <<-EndBanner
+Synopsis:
+  #{@app.program_name} [options] JOB_NET_FILE
+Options:
+        EndBanner
+        parser.on('-e', '--environment=NAME', "Sets execution environment. [default: #{Context::DEFAULT_ENV}]") {|env|
+          @environment = env
+        }
+        parser.on('--log-path=PATH', 'Log file path template.') {|path|
+          @log_path = path
+        }
+        parser.on('--queue=PATH', 'Use job queue.') {|path|
+          @queue_path = Pathname(path)
+        }
+        parser.on('-c', '--check-only', 'Checks job parameters and quit without executing.') {
+          @check_only = true
+        }
+        parser.on('-l', '--list-jobs', 'Lists target jobs without executing.') {
+          @list_jobs = true
+        }
+        parser.on('-v', '--variable=NAME=VALUE', 'Defines global variable.') {|name_value|
+          name, value = name_value.split('=', 2)
+          @global_variables[name] = value
+        }
+        parser.on('--help', 'Shows this message and quit.') {
+          @app.puts parser.help
+          @app.exit 0
+        }
+        parser.on('--version', 'Shows program version and quit.') {
+          @app.puts "#{APPLICATION_NAME} version #{VERSION}"
+          @app.exit 0
+        }
+      end
+
+      def on(*args, &block)
+        @parser.on(*args, &block)
+      end
+
+      def parse(argv)
+        @parser.parse! argv
+        raise OptionError, "missing jobnet file" if argv.empty?
+        raise OptionError, "too many jobnet file" if argv.size > 1
+        @jobnet_file = argv.map {|path| Pathname(path) }.first
+      rescue OptionParser::ParseError => ex
+        raise OptionError, ex.message
+      end
+    end
+  end
+
+end

data/lib/bricolage/jobresult.rb

@@ -0,0 +1,74 @@
+require 'bricolage/exception'
+
+module Bricolage
+
+  class JobResult
+    def JobResult.success
+      new(:success)
+    end
+
+    def JobResult.failure(ex)
+      new(:failiure, exception: ex)
+    end
+
+    def JobResult.error(ex)
+      new(:error, exception: ex)
+    end
+
+    def JobResult.for_bool(is_success, msg = nil)
+      new((is_success ? :success : :failure), message: msg)
+    end
+
+    def JobResult.for_process_status(st, msg = nil)
+      new((st.success? ? :success : :failure), process_status: st, message: msg)
+    end
+
+    EXIT_SUCCESS = 0
+    EXIT_FAILURE = 1    # production time errors; expected / unavoidable job error
+    EXIT_ERROR = 2      # development time errors (e.g. bad option, bad parameter, bad configuration)
+
+    def initialize(type, exception: nil, process_status: nil, message: nil)
+      @type = type
+      @exception = exception
+      @process_status = process_status
+      @message = message
+    end
+
+    def success?
+      @type == :success
+    end
+
+    attr_reader :exception
+    attr_reader :process_status
+
+    def status_string
+      @type.to_s.upcase
+    end
+
+    def status
+      if @process_status
+        @process_status.exitstatus
+      else
+        case @type
+        when :success then EXIT_SUCCESS
+        when :failure then EXIT_FAILURE
+        when :error then EXIT_ERROR
+        else EXIT_ERROR
+        end
+      end
+    end
+
+    alias to_i status
+
+    def message
+      if @message
+        @message
+      elsif @exception
+        @exception.message
+      else
+        success? ? 'suceeded' : 'failed'
+      end
+    end
+  end
+
+end

data/lib/bricolage/logger.rb

@@ -0,0 +1,52 @@
+require 'stringio'
+require 'logger'
+
+module Bricolage
+  class Logger < ::Logger
+    def Logger.default
+      logger = new($stderr)
+      logger.level = $stderr.tty? ? Logger::DEBUG : Logger::INFO
+      logger.formatter = -> (sev, time, prog, msg) {
+        "#{time}: #{sev}: #{msg}\n"
+      }
+      logger
+    end
+
+    def exception(ex)
+      buf = StringIO.new
+      buf.puts "#{ex.class}: #{ex.message}"
+      ex.backtrace.each do |trace|
+        buf.puts "\t" + trace
+      end
+      error buf.string
+    end
+
+    def with_elapsed_time(label = '')
+      start_time = Time.now
+      begin
+        return yield
+      ensure
+        elapsed_time(label, Time.now - start_time)
+      end
+    end
+
+    def elapsed_time(label, t)
+      info "#{label}#{pretty_interval(t)}"
+    end
+
+    private
+
+    def pretty_interval(seconds)
+      case
+      when seconds > 60 * 60
+        h, secs = seconds.divmod(60 * 60)
+        m, s = secs.divmod(60)
+        "%d hours %d minutes" % [h, m]
+      when seconds > 60
+        "%d minutes %d seconds" % seconds.divmod(60)
+      else
+        "%.2f secs" % seconds
+      end
+    end
+  end
+end

data/lib/bricolage/mysqldatasource.rb

@@ -0,0 +1,223 @@
+require 'bricolage/datasource'
+require 'mysql2'
+require 'json'
+require 'csv'
+require 'stringio'
+require 'open3'
+
+module Bricolage
+
+  class MySQLDataSource < DataSource
+    declare_type 'mysql'
+
+    def initialize(**mysql_options)
+      @mysql_options = mysql_options
+      @client = nil
+    end
+
+    def host
+      @mysql_options[:host]
+    end
+
+    def port
+      @mysql_options[:port]
+    end
+
+    def username
+      @mysql_options[:username]
+    end
+
+    def password
+      @mysql_options[:password]
+    end
+
+    def database
+      @mysql_options[:database]
+    end
+
+    def new_task
+      MySQLTask.new(self)
+    end
+
+    def open
+      @client = Mysql2::Client.new(**@mysql_options)
+      begin
+        yield self
+      ensure
+        c = @client
+        @client = nil
+        c.close
+      end
+    end
+
+    def query(sql, **opts)
+      logger.info "[SQL] #{sql}"
+      connection_check
+      @client.query(sql, **opts)
+    end
+
+    private
+
+    def connection_check
+      unless @client
+        raise FatalError, "#{self.class} used outside of \#open block"
+      end
+    end
+  end
+
+  class MySQLTask < DataSourceTask
+    def export(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: sqldump)
+      add Export.new(stmt, path: path, format: format, override: override, gzip: gzip, sqldump: sqldump)
+    end
+
+    class Export < Action
+      def initialize(stmt, path: nil, format: nil, override: false, gzip: false, sqldump: false)
+        @statement = stmt
+        @path = path
+        @format = format
+        @override = override
+        @gzip = gzip
+        @sqldump = sqldump
+      end
+
+      def bind(*args)
+        @statement.bind(*args)
+      end
+
+      def source
+        @statement.stripped_source
+      end
+
+      def run
+        if @sqldump and sqldump_available? and sqldump_usable?
+          export_by_sqldump
+        else
+          export_by_ruby
+        end
+        JobResult.success
+      end
+
+      def export_by_sqldump
+        cmds = [[sqldump_path.to_s, "--#{@format}", ds.host, ds.port.to_s, ds.username, ds.password, ds.database, @statement.stripped_source]]
+        cmds.push [GZIP_COMMAND] if @gzip
+        cmds.last.push({out: @path.to_s})
+        ds.logger.info '[CMD] ' + format_pipeline(cmds)
+        statuses = Open3.pipeline(*cmds)
+        statuses.each_with_index do |st, idx|
+          unless st.success?
+            cmd = cmds[idx].first
+            raise JobFailure, "#{cmd} failed (status #{st.to_i})"
+          end
+        end
+      end
+
+      def format_pipeline(cmds)
+        cmds = cmds.map {|args| args.dup }
+        cmds.first[5] = '****'
+        cmds.map {|args| %Q("#{args.join('" "')}") }.join(' | ')
+      end
+
+      def sqldump_available?
+        sqldump_path.executable?
+      end
+
+      def sqldump_path
+        Pathname(__dir__).parent.parent + "libexec/sqldump"
+      end
+
+      def sqldump_real_path
+        Pathname("#{sqldump_path}.#{platform_name}")
+      end
+
+      def platform_name
+        @platform_name ||= `uname -s`
+      end
+
+      def sqldump_usable?
+        %w[json tsv].include?(@format)
+      end
+
+      def export_by_ruby
+        ds.logger.info "exporting table into #{@path} ..."
+        count = 0
+        open_target_file(@path) {|f|
+          writer_class = WRITER_CLASSES[@format] or raise ArgumentError, "unknown export format: #{@format.inspect}"
+          writer = writer_class.new(f)
+          rs = ds.query(@statement.stripped_source, as: writer_class.record_format, stream: true, cache_rows: false)
+          ds.logger.info "got result set, writing..."
+          rs.each do |values|
+            writer.write_record values
+            count += 1
+            ds.logger.info "#{count} records exported..." if count % 10_0000 == 0
+          end
+        }
+        ds.logger.info "#{count} records exported; export finished"
+      end
+
+      private
+
+      # FIXME: parameterize
+      GZIP_COMMAND = 'gzip'
+
+      def open_target_file(path, &block)
+        unless @override
+          raise JobFailure, "destination file already exists: #{path}" if File.exist?(path)
+        end
+        if @gzip
+          ds.logger.info "enable compression: gzip"
+          IO.popen(%Q(#{GZIP_COMMAND} > "#{path}"), 'w', &block)
+        else
+          File.open(path, 'w', &block)
+        end
+      end
+    end
+
+    WRITER_CLASSES = {}
+
+    class JSONWriter
+      def JSONWriter.record_format
+        :hash
+      end
+
+      def initialize(f)
+        @f = f
+      end
+
+      def write_record(values)
+        @f.puts JSON.dump(values)
+      end
+    end
+    WRITER_CLASSES['json'] = JSONWriter
+
+    class TSVWriter
+      def TSVWriter.record_format
+        :array
+      end
+
+      def initialize(f)
+        @f = f
+      end
+
+      def write_record(values)
+        @f.puts values.join("\t")
+      end
+    end
+    WRITER_CLASSES['tsv'] = TSVWriter
+
+    class CSVWriter
+      def CSVWriter.record_format
+        :array
+      end
+
+      def initialize(f)
+        @csv = CSV.new(f)
+      end
+
+      def write_record(values)
+        @csv.add_row values
+      end
+    end
+    WRITER_CLASSES['csv'] = CSVWriter
+  end
+
+end