bricolage 5.8.7

data/lib/bricolage/application.rb

@@ -0,0 +1,260 @@
+require 'bricolage/context'
+require 'bricolage/job'
+require 'bricolage/jobclass'
+require 'bricolage/jobresult'
+require 'bricolage/variables'
+require 'bricolage/datasource'
+require 'bricolage/eventhandlers'
+require 'bricolage/logger'
+require 'bricolage/exception'
+require 'bricolage/version'
+require 'pathname'
+require 'optparse'
+
+module Bricolage
+
+  class Application
+    def Application.main
+      new.main
+    end
+
+    def initialize
+      Signal.trap('PIPE', 'IGNORE')
+      @hooks = Bricolage
+    end
+
+    def main
+      opts = GlobalOptions.new(self)
+      @hooks.run_before_option_parsing_hooks(opts)
+      opts.parse ARGV
+      @ctx = Context.for_application(opts.home, opts.job_file, environment: opts.environment, global_variables: opts.global_variables)
+      if opts.list_global_variables?
+        list_variables @ctx.global_variables.resolve
+        exit 0
+      end
+      job = load_job(@ctx, opts)
+      process_job_options job, opts
+      job.compile
+      if opts.list_declarations?
+        list_declarations job.declarations
+        exit 0
+      end
+      if opts.list_variables?
+        list_variables job.variables
+        exit 0
+      end
+      if opts.dry_run?
+        puts job.script_source
+        exit 0
+      end
+      if opts.explain?
+        job.explain
+        exit 0
+      end
+
+      @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(job.id, [job]))
+      @hooks.run_before_job_hooks(BeforeJobEvent.new(job))
+      result = job.execute
+      @hooks.run_after_job_hooks(AfterJobEvent.new(result))
+      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(result.success?, [job]))
+      exit result.status
+    rescue OptionError => ex
+      raise if $DEBUG
+      usage_exit ex.message, opts.help
+    rescue ApplicationError => ex
+      raise if $DEBUG
+      error_exit ex.message
+    end
+
+    def load_job(ctx, opts)
+      if opts.file_mode?
+        Job.load_file(opts.job_file, ctx)
+      else
+        usage_exit "no job class given", opts.help if ARGV.empty?
+        job_class_id = ARGV.shift
+        Job.instantiate(nil, job_class_id, ctx)
+      end
+    rescue ParameterError => ex
+      raise if $DEBUG
+      usage_exit ex.message, opts.help
+    end
+
+    def process_job_options(job, opts)
+      parser = OptionParser.new
+      parser.banner = "Usage: #{program_name} #{job.class_id} [job_class_options]"
+      job.parsing_options {|job_opt_defs|
+        job_opt_defs.define_options parser
+        parser.on_tail('--help', 'Shows this message and quit.') {
+          puts parser.help
+          exit 0
+        }
+        parser.on_tail('--version', 'Shows program version and quit.') {
+          puts "#{APPLICATION_NAME} version #{VERSION}"
+          exit 0
+        }
+        parser.parse!
+      }
+      unless ARGV.empty?
+        msg = opts.file_mode? ? "--job-file and job class argument is exclusive" : "bad argument: #{ARGV.first}"
+        usage_exit msg, parser.help
+      end
+    rescue OptionError => ex
+      raise if $DEBUG
+      usage_exit ex.message, parser.help
+    end
+
+    def list_variables(vars)
+      vars.each_variable do |var|
+        puts "#{var.name}=#{var.value.inspect}"
+      end
+    end
+
+    def list_declarations(decls)
+      decls.each do |decl|
+        if decl.have_default_value?
+          puts "#{decl.name}\t= #{decl.default_value.inspect}"
+        else
+          puts decl.name
+        end
+      end
+    end
+    
+    def usage_exit(msg, usage)
+      print_error msg
+      $stderr.puts usage
+      exit 1
+    end
+
+    def error_exit(msg)
+      print_error msg
+      exit 1
+    end
+
+    def print_error(msg)
+      $stderr.puts "#{program_name}: error: #{msg}"
+    end
+
+    def program_name
+      File.basename($PROGRAM_NAME, '.*')
+    end
+  end
+
+  class GlobalOptions
+    def initialize(app)
+      @app = app
+      @job_file = nil
+      @environment = nil
+      @home = nil
+      @global_variables = Variables.new
+      @dry_run = false
+      @explain = false
+      @list_global_variables = false
+      @list_variables = false
+      @list_declarations = false
+      @parser = OptionParser.new
+      define_options @parser
+    end
+
+    attr_reader :parser
+
+    def help
+      @parser.help
+    end
+
+    def define_options(parser)
+      parser.banner = <<-EndBanner
+Synopsis:
+  #{@app.program_name} [global_options] JOB_CLASS [job_options]
+  #{@app.program_name} [global_options] --job=JOB_FILE -- [job_options]
+Global Options:
+      EndBanner
+      parser.on('-f', '--job=JOB_FILE', 'Give job parameters via job file (YAML).') {|path|
+        @job_file = path
+      }
+      parser.on('-e', '--environment=NAME', "Sets execution environment [default: #{Context::DEFAULT_ENV}]") {|env|
+        @environment = env
+      }
+      parser.on('-C', '--home=PATH', 'Sets application home directory.') {|path|
+        @home = Pathname(path)
+      }
+      parser.on('-n', '--dry-run', 'Shows job script without executing it.') {
+        @dry_run = true
+      }
+      parser.on('-E', '--explain', 'Applies EXPLAIN to the SQL.') {
+        @explain = true
+      }
+      parser.on('--list-job-class', 'Lists job class name and (internal) class path.') {
+        JobClass.list.each do |name|
+          puts name
+        end
+        exit 0
+      }
+      parser.on('--list-global-variables', 'Lists global variables.') {
+        @list_global_variables = true
+      }
+      parser.on('--list-variables', 'Lists all variables.') {
+        @list_variables = true
+      }
+      parser.on('--list-declarations', 'Lists script variable declarations.') {
+        @list_declarations = true
+      }
+      parser.on('-r', '--require=FEATURE', 'Requires ruby library.') {|feature|
+        require feature
+      }
+      parser.on('-v', '--variable=NAME=VALUE', 'Set global variable (is different from job-level -v !!).') {|name_value|
+        name, value = name_value.split('=', 2)
+        @global_variables[name] = value
+      }
+      parser.on('--help', 'Shows this message and quit.') {
+        puts parser.help
+        exit 0
+      }
+      parser.on('--version', 'Shows program version and quit.') {
+        puts "#{APPLICATION_NAME} version #{VERSION}"
+        exit 0
+      }
+    end
+
+    def on(*args, &block)
+      @parser.on(*args, &block)
+    end
+
+    def parse(argv)
+      @parser.order! argv
+      @rest_args = argv.dup
+    rescue OptionParser::ParseError => ex
+      raise OptionError, ex.message
+    end
+
+    attr_reader :environment
+    attr_reader :home
+    attr_reader :global_variables
+
+    attr_reader :job_file
+
+    def file_mode?
+      !!@job_file
+    end
+
+    def dry_run?
+      @dry_run
+    end
+
+    def explain?
+      @explain
+    end
+
+    def list_global_variables?
+      @list_global_variables
+    end
+
+    def list_variables?
+      @list_variables
+    end
+
+    def list_declarations?
+      @list_declarations
+    end
+  end
+
+end

data/lib/bricolage/commandutils.rb

@@ -0,0 +1,52 @@
+require 'fileutils'
+require 'tmpdir'
+
+module Bricolage
+
+  module CommandUtils
+    def command(*args, env: nil)
+      logger.info "command: #{args.join(' ')}"
+      sargs = args.map {|a| a.to_s }
+      sargs.unshift env if env
+      system(*sargs)
+      st = $?
+      logger.info "status: #{st.exitstatus} (#{st})"
+      st
+    end
+
+    def make_tmpfile(content, tmpdir: Dir.tmpdir)
+      path = new_tmpfile_path(tmpdir)
+      File.open(path, 'w') {|f|
+        f.write content
+      }
+      yield path
+    ensure
+      FileUtils.rm_f path
+    end
+
+    def new_tmpfile_path(tmpdir = Dir.tmpdir)
+      "#{tmpdir}/#{Time.now.to_i}_#{$$}_#{'%x' % Thread.current.object_id}_#{rand(2**16)}"
+    end
+
+    # CLUDGE: FIXME: bricolage-jobnet command writes stderr to the file, we can find error messages from there.
+    # Using a temporary file or Ruby SQL driver is **MUCH** better.
+    def retrieve_last_match_from_stderr(re, nth = 0)
+      return unless $stderr.stat.file?
+      $stderr.flush
+      f = $stderr.dup
+      matched = nil
+      begin
+        f.seek(0)
+        f.each do |line|
+          m = line.slice(re, nth)
+          matched = m if m
+        end
+      ensure
+        f.close
+      end
+      matched = matched.to_s.strip
+      matched.empty? ? nil : matched
+    end
+  end
+
+end

data/lib/bricolage/configloader.rb

@@ -0,0 +1,126 @@
+require 'bricolage/sqlstatement'
+require 'bricolage/resource'
+require 'bricolage/exception'
+require 'pathname'
+require 'yaml'
+require 'erb'
+require 'date'
+
+module Bricolage
+
+  class ConfigLoader
+    def ConfigLoader.load_eruby_yaml(path)
+      new(nil).load_eruby_yaml(path)
+    end
+
+    def initialize(app_home)
+      @app_home = app_home
+      @base_dir = Pathname('.')
+    end
+
+    def load_eruby(path)
+      eruby(read_file(path), path)
+    end
+
+    def load_eruby_yaml(path)
+      parse_yaml(load_eruby(path), path)
+    end
+
+    def load_yaml(path)
+      parse_yaml(read_file(path), path)
+    end
+
+    def parse_yaml(text, path)
+      YAML.load(text)
+    rescue => err
+      raise ParameterError, "#{path}: config file syntax error: #{err.message}"
+    end
+
+    def eruby(text, path)
+      erb = ERB.new(text, nil, '%-')
+      erb.filename = path.to_s
+      push_base_dir(path) {
+        erb.result(binding())
+      }
+    end
+
+    def eval_file(path)
+      push_base_dir(path) {
+        instance_eval(File.read(path), path.to_s, 1)
+      }
+    end
+
+    def read_file(path)
+      File.read(path)
+    rescue SystemCallError => err
+      raise ParameterError, "could not read file: #{err.message}"
+    end
+
+    private
+
+    def app_home
+      @app_home or raise ParameterError, "app_home is not given in this file"
+    end
+
+    def base_dir
+      @base_dir
+    end
+
+    def push_base_dir(path)
+      saved, @base_dir = @base_dir, Pathname(path).parent
+      begin
+        yield
+      ensure
+        @base_dir = saved
+      end
+    end
+  end
+
+  module EmbeddedCodeAPI
+    private
+
+    def user_home
+      Pathname(ENV['HOME'])
+    end
+
+    def user_home_relative_path(rel)
+      user_home + rel
+    end
+
+    def app_home_relative_path(rel)
+      app_home + rel
+    end
+
+    def relative_path(rel)
+      base_dir + rel
+    end
+
+    def read_file_if_exist(path)
+      return nil unless File.exist?(path)
+      File.read(path)
+    end
+
+    def date(str)
+      Date.parse(str)
+    end
+
+    def ymd(date)
+      date.strftime('%Y-%m-%d')
+    end
+
+    def attribute_tables(attr)
+      all_tables.select {|table| table.attributes.include?(attr) }
+    end
+
+    def all_tables
+      Dir.glob("#{app_home}/*/*.ct").map {|path|
+        SQLStatement.new(FileResource.new(path))
+      }
+    end
+  end
+
+  class ConfigLoader
+    include EmbeddedCodeAPI
+  end
+
+end

data/lib/bricolage/context.rb

@@ -0,0 +1,108 @@
+require 'bricolage/filesystem'
+require 'bricolage/datasource'
+require 'bricolage/configloader'
+require 'bricolage/logger'
+require 'forwardable'
+
+module Bricolage
+
+  class Context
+    DEFAULT_ENV = 'development'
+
+    def Context.environment(opt_env)
+      opt_env || ENV['BRICOLAGE_ENV'] || DEFAULT_ENV
+    end
+
+    def Context.for_application(home_path, job_path = nil, environment: nil, global_variables: nil, logger: nil)
+      env = environment(environment)
+      fs = FileSystem.for_option_pathes(home_path, job_path, env)
+      load(fs, env, global_variables: global_variables, logger: logger)
+    end
+
+    def Context.load(fs, env, global_variables: nil, data_sources: nil, logger: nil)
+      new(fs, env, global_variables: global_variables, logger: logger).tap {|ctx|
+        ctx.load_configurations
+      }
+    end
+    private_class_method :load
+
+    def initialize(fs, env, global_variables: nil, data_sources: nil, logger: nil)
+      @logger = logger || Logger.default
+      @filesystem = fs
+      @environment = env
+      @opt_global_variables = global_variables || Variables.new
+      @data_sources = data_sources
+    end
+
+    def load_configurations
+      @filesystem.config_pathes('prelude.rb').each do |path|
+        EmbeddedCodeAPI.module_eval(File.read(path)) if path.exist?
+      end
+      @data_sources = DataSourceFactory.load(self, @logger)
+    end
+
+    attr_reader :environment
+    attr_reader :logger
+
+    def get_data_source(type, name)
+      @data_sources.get(type, name)
+    end
+
+    def subsystem(id)
+      self.class.new(@filesystem.subsystem(id), @environment,
+        global_variables: @opt_global_variables,
+        data_sources: @data_sources,
+        logger: @logger)
+    end
+
+    extend Forwardable
+    def_delegators '@filesystem',
+      :scoped?,
+      :home_path,
+      :root_relative_path,
+      :config_path,
+      :config_pathes,
+      :job_dir,
+      :job_file,
+      :parameter_file,
+      :parameter_file_loader
+
+    #
+    # Variables
+    #
+
+    def global_variables
+      Variables.union(
+        builtin_variables,
+        load_global_variables,
+        @opt_global_variables
+      )
+    end
+
+    def builtin_variables
+      Variables.define {|vars|
+        vars['bricolage_env'] = @environment
+        vars['bricolage_home'] = home_path.to_s
+      }
+    end
+
+    def load_global_variables
+      vars_list = config_pathes(GLOBAL_VARIABLE_FILE).map {|path|
+        path.exist? ? load_variables(path) : nil
+      }
+      Variables.union(*vars_list.compact)
+    end
+
+    GLOBAL_VARIABLE_FILE = 'variable.yml'
+
+    def load_variables(path)
+      Variables.define {|vars|
+        parameter_file_loader.load_eruby_yaml(path).each do |name, value|
+          vars[name] = value
+        end
+      }
+    end
+    private :load_variables
+  end
+
+end

data/lib/bricolage/datasource.rb

@@ -0,0 +1,144 @@
+require 'bricolage/script'
+require 'bricolage/configloader'
+require 'bricolage/exception'
+
+module Bricolage
+
+  class DataSourceFactory
+    def DataSourceFactory.load(context, logger)
+      loader = Loader.new(context, logger)
+      loader.load_passwords
+      loader.load
+    end
+
+    DEFAULT_CONFIG_FILE_NAME = 'database.yml'
+    DEFAULT_PASSWORD_FILE_NAME = 'password.yml'
+
+    class Loader < ConfigLoader
+      def initialize(context, logger)
+        super context.home_path
+        @context = context
+        @logger = logger
+        @passwords = nil
+      end
+
+      def load_passwords(basename = DEFAULT_PASSWORD_FILE_NAME)
+        @context.config_pathes(basename).each do |path|
+          if path.exist?
+            @passwords = load_yaml(path)
+            break
+          end
+        end
+      end
+
+      def load(basename = DEFAULT_CONFIG_FILE_NAME)
+        database_yml = @context.config_pathes(basename).detect {|path| path.exist? }
+        raise ParameterError, "database.yml does not exist" unless database_yml
+        @config_dir = database_yml.parent
+        DataSourceFactory.new(load_eruby_yaml(database_yml), @context, @logger)
+      end
+
+      def password(name)
+        (@passwords || {})[name] or raise ParameterError, "no such password entry: #{name}"
+      end
+    end
+
+    def initialize(configs, context, logger)
+      @configs = configs
+      @context = context
+      @logger = logger
+    end
+
+    BUILTIN_TYPES = %w(generic file)
+
+    # For job classes
+    def get(kind, name)
+      if BUILTIN_TYPES.include?(kind)
+        return DataSource.new_for_type(kind, kind, {}, @context, @logger)
+      end
+      entry_name = name || kind
+      conf = config(entry_name)
+      type = conf.delete(:type)
+      DataSource.new_for_type(type, entry_name, conf, @context, @logger)
+    end
+
+    # Ruby API
+    def [](name)
+      if BUILTIN_TYPES.include?(name)
+        return DataSource.new_for_type(name, name, {}, @context, @logger)
+      end
+      conf = config(name)
+      type = conf.delete(:type)
+      DataSource.new_for_type(type, name, conf, @context, @logger)
+    end
+
+    private
+
+    def config(key)
+      ent = @configs[key] or raise ParameterError, "no such data source entry: #{key}"
+      canonicalize(ent)
+    end
+
+    def canonicalize(config)
+      h = {}
+      config.each do |k, v|
+        h[k.intern] = v
+      end
+      h
+    end
+  end
+
+  class DataSource
+    def DataSource.new_for_type(type, name, config, context, logger)
+      ds = get_class(type).new(**config)
+      ds.__send__ :initialize_base, name, context, logger
+      ds
+    rescue ArgumentError => err
+      # FIXME: do not rely on error message
+      ent = err.message.slice(/unknown keyword: (\S+)/, 1) or raise
+      raise ParameterError, "unknown config entry in database.yml: #{name}.#{ent}"
+    end
+
+    CLASSES = {}
+
+    class << self
+      private
+
+      def declare_type(type)
+        CLASSES[type.to_s] = self
+      end
+    end
+
+    def DataSource.get_class(type)
+      unless CLASSES[type.to_s]
+        begin
+          require "bricolage/#{type}datasource"
+        rescue LoadError
+          raise ParameterError, "no such SQL client type: #{type}"
+        end
+        raise FatalError, "DataSource class does not exist: #{type}" unless CLASSES[type.to_s]
+      end
+      CLASSES[type.to_s]
+    end
+
+    def initialize_base(name, context, logger)
+      @name = name
+      @context = context
+      @logger = logger
+    end
+    private :initialize_base
+
+    attr_reader :name
+    attr_reader :context
+    attr_reader :logger
+
+    def open
+      yield nil
+    end
+
+    def open_for_batch(&block)
+      open(&block)
+    end
+  end
+
+end