bricolage 5.8.7

data/lib/bricolage/eventhandlers.rb

@@ -0,0 +1,47 @@
+module Bricolage
+  class EventHandlers
+    def initialize
+      @handlers = []
+    end
+
+    def add(handler)
+      @handlers.push handler
+    end
+
+    def run(event)
+      @handlers.each do |h|
+        begin
+          h.call(event)
+        rescue => err
+          $stderr.puts "hook error: #{err.class}: #{err.message}"
+          err.backtrace.each do |line|
+            $stderr.puts "\t#{line}"
+          end
+        end
+      end
+    end
+  end
+
+  %w[
+    before_option_parsing
+    before_all_jobs
+    before_job
+    after_job
+    after_all_jobs
+  ].each do |type|
+    handlers = EventHandlers.new
+    instance_variable_set "@#{type}".intern, handlers
+    cc = (class << self; self; end)
+    cc.__send__(:define_method, type.intern) {|&action|
+      handlers.add(action)
+    }
+    cc.__send__(:define_method, "run_#{type}_hooks".intern) {|event|
+      handlers.run(event)
+    }
+  end
+
+  BeforeAllJobsEvent = Struct.new(:flow_id, :queue)
+  BeforeJobEvent = Struct.new(:job)
+  AfterJobEvent = Struct.new(:result)
+  AfterAllJobsEvent = Struct.new(:succeeded, :queue)
+end

data/lib/bricolage/exception.rb

@@ -0,0 +1,47 @@
+module Bricolage
+
+  ##
+  # Common super-class of handlable bricolage exceptions
+  class ApplicationError < StandardError; end
+
+  ##
+  # Job failure.
+  # This exception may occur in production environment and is temporary.
+  # e.g. Source data error, SQL error
+  class JobFailure < ApplicationError; end
+
+  ##
+  # various SQL exception
+  class SQLException < JobFailure
+    def SQLException.wrap(ex)
+      new(ex.message, ex)
+    end
+
+    def initialize(msg, orig = nil)
+      super msg
+      @original = orig
+    end
+
+    attr_reader :original
+  end
+
+  ##
+  # Job error.
+  # This exception should NOT be thrown in production environment.
+  # Developer must fix source code or configuration, not to be get this exception.
+  class JobError < ApplicationError; end
+
+  ##
+  # Command-line option errors (should NOT be thrown in production environment)
+  class OptionError < JobError; end
+
+  ##
+  # User parameter errors (should NOT be thrown in production environment)
+  class ParameterError < JobError; end
+
+  ##
+  # Bad code in bricolage core or job classes.
+  # This exception should NOT be thrown in ANY user environment.
+  class FatalError < Exception; end
+
+end

data/lib/bricolage/filedatasource.rb

@@ -0,0 +1,42 @@
+require 'bricolage/datasource'
+require 'stringio'
+
+module Bricolage
+
+  class FileDataSource < DataSource
+    declare_type 'file'
+
+    def initialize(opts)
+    end
+
+    def new_task
+      FileTask.new(self)
+    end
+  end
+
+  class FileTask < DataSourceTask
+    def remove(src)
+      add Remove.new(src)
+    end
+
+    class Remove < Action
+      def initialize(src)
+        @src = src
+      end
+
+      def source_files
+        Dir.glob(@src)
+      end
+
+      def source
+        "rm -f #{@src}"
+      end
+
+      def run
+        FileUtils.rm_f source_files, verbose: true
+        nil
+      end
+    end
+  end
+
+end

data/lib/bricolage/filesystem.rb

@@ -0,0 +1,165 @@
+require 'bricolage/resource'
+require 'bricolage/exception'
+require 'pathname'
+
+module Bricolage
+
+  class FileSystem
+    def FileSystem.for_option_pathes(home_path, job_path, env)
+      if job_path
+        home, subsys_id = extract_home_dirs(job_path)
+        if home_path and home_path.realpath.to_s != home.realpath.to_s
+          raise OptionError, "--home option and job file is exclusive"
+        end
+        new(home, env).subsystem(subsys_id)
+      elsif home_path
+        new(home_path, env)
+      elsif home = ENV['BRICOLAGE_HOME']
+        new(home, env)
+      else
+        new(Pathname.getwd, env)
+      end
+    end
+
+    # job_path -> [home_path, subsys_id]
+    def FileSystem.extract_home_dirs(job_path)
+      subsys_path = Pathname(job_path).realpath.parent
+      return subsys_path.parent, subsys_path.basename
+    rescue SystemCallError => err
+      raise ParameterError, "failed to access job file: #{err.message}"
+    end
+
+    def initialize(path, env)
+      @path = Pathname(path)
+      @environment = env
+      @subsystems = {}
+    end
+
+    def scoped?
+      false
+    end
+
+    attr_reader :path
+    attr_reader :environment
+
+    def inspect
+      "\#<#{self.class} #{@path}>"
+    end
+
+    def root
+      self
+    end
+
+    def subsystem(id)
+      @subsystems[id.to_s] ||= begin
+        unless root.relative(id).directory?
+          raise ParameterError, "no such subsystem: #{id}"
+        end
+        ScopedFileSystem.new(root, id)
+      end
+    end
+
+    def subsystems
+      root.subdirectories
+          .map {|path| path.basename.to_s }
+          .select {|name| /\A\w+\z/ =~ name }
+          .reject {|name| name == 'config' }
+          .map {|name| subsystem(name) }
+    end
+
+    def subdirectories
+      @path.children(true).select {|path| path.directory? }
+    end
+
+    def home_path
+      root.path
+    end
+
+    def job_dir
+      scoped? ? @path : nil
+    end
+
+    def exist?(name)
+      relative(name).exist?
+    end
+
+    def file(name)
+      FileResource.new(relative(name))
+    end
+
+    def root_relative_path(rel)
+      root.relative_path(rel)
+    end
+
+    alias root_relative root_relative_path
+
+    def relative_path(name)
+      path = Pathname(name)
+      if path.absolute?
+        path
+      else
+        @path + path
+      end
+    end
+
+    alias relative relative_path
+
+    # typed_name("make_master", "sql") -> Pathname("$prefix/make_master.sql")
+    def typed_name(name, type)
+      relative(name.count('.') > 0 ? name : "#{name}.#{type}")
+    end
+
+    # typed_file("make_master", "sql") -> FileResource("$prefix/make_master.sql")
+    def typed_file(name, type)
+      FileResource.new(typed_name(name, type))
+    end
+
+    def parameter_file(name, type)
+      name.count('/') == 0 ? typed_file(name, type) : root.file(name)
+    end
+
+    def parameter_file_loader
+      ConfigLoader.new(home_path)
+    end
+
+    def config_pathes(name)
+      [ "config/#{name}", "config/#{@environment}/#{name}" ].map {|rel| root.relative(rel) }
+    end
+
+    def job_file(id)
+      path = typed_name(id, 'job')
+      return path if path.exist?
+      glob("#{id}.*.job").first or path
+    end
+
+    def glob(pattern)
+      Dir.glob("#{@path}/#{pattern}").map {|path| Pathname(path) }
+    end
+
+    ## */*.TYPE
+    def all_typed_pathes(type)
+      subsystems.map {|subsys| subsys.typed_pathes(type) }.flatten
+    end
+
+    def typed_pathes(type)
+      @path.children.select {|path| path.file? and path.extname.to_s == ".#{type}" }
+    end
+  end
+
+  class ScopedFileSystem < FileSystem
+    def initialize(parent, id)
+      super parent.relative(id), parent.environment
+      @parent = parent
+      @id = id
+    end
+
+    def scoped?
+      true
+    end
+
+    def root
+      @parent.root
+    end
+  end
+
+end

data/lib/bricolage/genericdatasource.rb

@@ -0,0 +1,37 @@
+require 'bricolage/datasource'
+
+module Bricolage
+  class GenericDataSource < DataSource
+    declare_type 'generic'
+
+    # FIXME: keyword argument placeholder is required
+    def initialize(**)
+    end
+
+    def new_task
+      GenericTask.new(self)
+    end
+  end
+
+  class GenericTask < DataSourceTask
+    def action(label = nil, &block)
+      raise FatalError, "no block" unless block
+      add AnyAction.new(label, block)
+    end
+
+    class AnyAction < Action
+      def initialize(label, block)
+        @label = label
+        @block = block
+      end
+
+      def source
+        @label || @block.to_s
+      end
+
+      def run
+        @block.call(ds)
+      end
+    end
+  end
+end

data/lib/bricolage/job.rb

@@ -0,0 +1,212 @@
+require 'bricolage/jobfile'
+require 'bricolage/jobclass'
+require 'bricolage/jobresult'
+require 'bricolage/parameters'
+require 'bricolage/variables'
+require 'bricolage/configloader'
+require 'bricolage/exception'
+require 'fileutils'
+
+module Bricolage
+  
+  class Job
+    # For JobNetRunner
+    def Job.load_ref(ref, jobnet_context)
+      ctx = jobnet_context.subsystem(ref.subsystem)
+      path = ctx.job_file(ref.name)
+      load_file(path, ctx)
+    end
+
+    # For standalone job (.job file mode)
+    def Job.load_file(path, ctx)
+      f = JobFile.load(ctx, path)
+      instantiate(f.job_id, f.class_id, ctx).tap {|job|
+        job.bind_parameters f.values
+      }
+    end
+
+    # For standalone job (command line mode)
+    def Job.instantiate(id, class_id, ctx)
+      new(id, JobClass.get(class_id), ctx).tap {|job|
+        job.init_global_variables
+      }
+    end
+
+    def initialize(id, job_class, context)
+      @id = id
+      @job_class = job_class
+      @context = context
+      @global_variables = nil
+      @param_decls = @job_class.get_parameters
+      @param_vals = nil      # Parameters::IntermediateValues by *.job
+      @param_vals_opt = nil  # Parameters::IntermediateValues by options
+      @params = nil
+      @variables = nil
+    end
+
+    attr_reader :id
+
+    def class_id
+      @job_class.id
+    end
+
+    def init_global_variables
+      # Context#global_variables loads file on each call, fix global variables here.
+      @global_variables = @context.global_variables
+      @global_variables['bricolage_cwd'] = Dir.pwd
+      @global_variables['bricolage_job_dir'] = @context.job_dir.to_s
+    end
+
+    attr_reader :params
+    attr_reader :global_variables   # valid after #init_global_variables
+    attr_reader :variables          # valid after #compile
+    attr_reader :script             # valid after #compile
+
+    # For job file
+    def bind_parameters(values)
+      @param_vals = @param_decls.parse_direct_values(values)
+    end
+
+    # For command line options
+    def parsing_options(&block)
+      @param_vals_opt = @param_decls.parsing_options(&block)
+    end
+
+    def compile
+      @job_class.invoke_parameters_filter(self)
+
+      job_file_rest_vars = @param_vals ? @param_vals.variables : Variables.new
+      job_v_opt_vars = @param_vals_opt ? @param_vals_opt.variables : Variables.new
+
+      # We use different variable set for paramter expansion and
+      # SQL variable expansion.  Parameter expansion uses global
+      # variables and "-v" option variables (both of global and job).
+      base_vars = Variables.union(
+        #          ^ Low precedence
+        @global_variables,
+        job_v_opt_vars
+        #          v High precedence
+      )
+      pvals = @param_decls.union_intermediate_values(*[@param_vals, @param_vals_opt].compact)
+      @params = pvals.resolve(@context, base_vars.resolve)
+
+      # Then, expand SQL variables and check with declarations.
+      vars = Variables.union(
+        #          ^ Low precedence
+        declarations.default_variables,
+        @global_variables,
+        @params.variables,   # Like $dest_table
+        job_file_rest_vars,
+        job_v_opt_vars
+        #          v High precedence
+      )
+      @variables = vars.resolve
+      @variables.bind_declarations declarations
+
+      @script = @job_class.get_script(@params)
+      @script.bind @context, @variables
+    end
+
+    # Called from jobclasses (parameters_filter)
+    def provide_sql_file_by_job_id
+      if @param_vals and @id
+        @param_vals['sql-file'] ||= @id
+      end
+    end
+
+    def declarations
+      @declarations ||= @job_class.get_declarations(@params)
+    end
+
+    def script_source
+      raise 'Job#script_source called before #compile' unless @script
+      @script.source
+    end
+
+    def explain
+      raise 'Job#explain called before #compile' unless @script
+      @script.run_explain
+    end
+
+    def execute
+      logger = @context.logger
+      logger.info "#{@context.environment} environment"
+      result = logger.with_elapsed_time {
+        script.run
+      }
+      logger.info result.status_string
+      result
+    rescue JobFailure => ex
+      logger.error ex.message
+      logger.error "failure: #{ex.message}"
+      return JobResult.failure(ex)
+    rescue Exception => ex
+      logger.exception ex
+      logger.error "error: #{ex.class}: #{ex.message}"
+      return JobResult.error(ex)
+    end
+
+    def execute_in_process(log_path)
+      isolate(log_path) {
+        execute
+      }
+    end
+
+    private
+
+    def isolate(log_path)
+      cpid = Process.fork {
+        Process.setproctitle "bricolage [#{@id}]"
+        redirect_stdouts_to log_path if log_path
+        result = yield
+        save_result result, log_path
+        exit result.status
+      }
+      _, st = Process.waitpid2(cpid)
+      restore_result(st, log_path)
+    end
+
+    def redirect_stdouts_to(path)
+      FileUtils.mkdir_p File.dirname(path)
+      # make readable for retrieve_last_match_from_stderr
+      File.open(path, 'w+') {|f|
+        $stdout.reopen f
+        $stderr.reopen f
+      }
+    end
+
+    def save_result(result, log_path)
+      return if result.success?
+      return unless log_path
+      begin
+        File.open(error_log_path(log_path), 'w') {|f|
+          f.puts result.message
+        }
+      rescue
+      end
+    end
+
+    def restore_result(st, log_path)
+      JobResult.for_process_status(st, restore_message(log_path))
+    end
+
+    def restore_message(log_path)
+      return nil unless log_path
+      msg = read_if_exist(error_log_path(log_path))
+      msg ? msg.strip : nil
+    ensure
+      FileUtils.rm_f error_log_path(log_path) if log_path
+    end
+
+    def error_log_path(log_path)
+      "#{log_path}.error"
+    end
+
+    def read_if_exist(path)
+      File.read(path)
+    rescue
+      nil
+    end
+  end
+
+end

data/lib/bricolage/jobclass.rb

@@ -0,0 +1,98 @@
+require 'bricolage/parameters'
+require 'bricolage/script'
+require 'bricolage/sqlstatement'
+require 'bricolage/exception'
+require 'pathname'
+
+module Bricolage
+
+  class JobClass
+    CLASSES = {}
+
+    def JobClass.define(id, &block)
+      id = id.to_s
+      raise FatalError, "duplicated job class: #{@id.inspect}" if CLASSES[id]
+      c = new(id)
+      c.instance_exec(c, &block)
+      CLASSES[id] = c
+    end
+
+    srcdir = Pathname(__FILE__).realpath.parent.parent.parent.cleanpath
+    LOAD_PATH = [srcdir + 'jobclass']
+
+    def JobClass.get(id)
+      unless CLASSES[id.to_s]
+        begin
+          path = LOAD_PATH.map {|prefix| prefix + "#{id}.rb" }.detect(&:exist?)
+          raise ParameterError, "no such job class: #{id}" unless path
+          ::Bricolage.module_eval File.read(path), path.to_path, 1
+        rescue SystemCallError => err
+          raise FatalError, "could not load job class: #{id}: #{err.message}"
+        end
+        raise FatalError, "job class file loaded but required job class is not defined: #{id}" unless CLASSES[id.to_s]
+      end
+      CLASSES[id.to_s]
+    end
+
+    def JobClass.list
+      LOAD_PATH.map {|dir|
+        Dir.glob("#{dir}/*.rb").map {|path| File.basename(path, '.rb') }
+      }.flatten.uniq.sort
+    end
+
+    def JobClass.each(&block)
+      CLASSES.each_value(&block)
+    end
+
+    def initialize(id)
+      @id = id
+      @parameters = nil
+      @parameters_filter = nil   # optional
+      @declarations = nil
+      @script = nil
+    end
+
+    attr_reader :id
+
+    def inspect
+      "\#<#{self.class} #{@id}>"
+    end
+
+    def parameters(&block)
+      @parameters = block
+    end
+
+    def get_parameters
+      Parameters::Declarations.new.tap {|params|
+        @parameters.call(params)
+      }
+    end
+
+    def parameters_filter(&block)
+      @parameters_filter = block
+    end
+
+    def invoke_parameters_filter(job)
+      @parameters_filter.call(job) if @parameters_filter
+    end
+
+    def declarations(&block)
+      @declarations = block
+    end
+
+    def get_declarations(params)
+      @declarations ? @declarations.call(params) : Declarations.new
+    end
+
+    def script(&block)
+      @script = block
+    end
+
+    def get_script(params)
+      Script.new.tap {|script|
+        @script.call(params, script)
+      }
+    end
+  end
+
+end