bricolage 5.8.7

data/lib/bricolage/script.rb

@@ -0,0 +1,120 @@
+require 'bricolage/resource'
+require 'bricolage/exception'
+require 'forwardable'
+require 'stringio'
+
+module Bricolage
+
+  class Script
+    def initialize
+      @tasks = []
+      @in_task_block = false
+    end
+
+    def task(ds)
+      raise FatalError, "nested task is not supported" if @in_task_block
+      raise ParameterError, "no data source" unless ds
+      @in_task_block = true
+      begin
+        task = ds.new_task
+        yield task
+        @tasks.push task
+      ensure
+        @in_task_block = false
+      end
+    end
+
+    def bind(ctx, vars)
+      raise "[BUG] unresolved variables given" unless vars.resolved?
+      @tasks.each do |task|
+        task.bind(ctx, vars)
+      end
+    end
+
+    def source
+      buf = StringIO.new
+      first = true
+      @tasks.each do |task|
+        buf.puts unless first; first = false
+        buf.puts task.source
+      end
+      buf.string
+    end
+
+    def run
+      result = nil
+      @tasks.each do |task|
+        result = task.run
+      end
+      result || JobResult.success
+    end
+
+    def run_explain
+      @tasks.each do |task|
+        if task.respond_to?(:run_explain)
+          task.run_explain
+        else
+          puts "-- task #{task.class} does not support explain; show source"
+          puts task.source
+        end
+      end
+    end
+  end
+
+  class DataSourceTask
+    def initialize(ds)
+      @ds = ds
+      @actions = []
+    end
+
+    attr_reader :ds
+
+    def bind(*args)
+      @actions.each do |action|
+        action.bind(*args)
+      end
+    end
+
+    def run
+      result = nil
+      @ds.open_for_batch {
+        @actions.each do |action|
+          result = action.run
+        end
+      }
+      result
+    end
+
+    def source
+      buf = StringIO.new
+      @actions.each do |action|
+        buf.puts action.source
+      end
+      buf.string
+    end
+
+    private
+
+    def add(action)
+      action.ds = @ds
+      @actions.push action
+    end
+
+    def each_action(&block)
+      @actions.each(&block)
+    end
+
+    class Action
+      extend Forwardable
+
+      attr_accessor :ds
+
+      def bind(context, variables)
+      end
+
+      # abstract def run
+      # abstract def source
+    end
+  end
+
+end

data/lib/bricolage/sqlstatement.rb

@@ -0,0 +1,351 @@
+require 'bricolage/filesystem'
+require 'bricolage/exception'
+
+module Bricolage
+
+  class SQLStatement
+    def SQLStatement.delete_where(cond, location = nil)
+      for_string("delete from $dest_table where #{cond};", location)
+    end
+
+    def SQLStatement.for_string(sql, location = nil)
+      new(StringResource.new(sql, location))
+    end
+
+    def initialize(resource, declarations = nil)
+      @resource = resource
+      @declarations = declarations
+      @code = nil
+      @replace = nil
+    end
+
+    attr_reader :resource
+
+    def ==(other)
+      return false unless other.kind_of?(SQLStatement)
+      @resource == other.resource
+    end
+
+    alias eql? ==
+
+    def hash
+      @resource.hash
+    end
+
+    def declarations
+      @declarations ||= Declarations.parse(@resource)
+    end
+
+    attr_writer :declarations
+
+    def replace(re, subst)
+      @replace = [re, subst]
+      self
+    end
+
+    def bind(ctx, vars)
+      raise FatalError, "already bound SQL statement: #{@resource.name}" if @code
+      src = ctx.parameter_file_loader.eruby(@resource.content, @resource.name || '-')
+      @code = Variable.expand_string(apply_replace(src)) {|name|
+        val = vars[name]
+        val.respond_to?(:to_sql) ? val.to_sql : val.to_s
+      }
+    end
+
+    def location
+      @resource.name
+    end
+
+    def source
+      @code or raise FatalError, "unbound SQL statement: #{@resource.name}"
+    end
+
+    def stripped_source
+      strip_sql(source)
+    end
+
+    def kind
+      # This implementation is not complete but useful
+      first_word = @resource.content.gsub(%r<'(?:[^']+|'')*'|"(?:[^"]+|"")*"|--.*|/\*(?m:.*?)\*/>, '').strip.slice(/\A\w+/)
+      op = first_word ? first_word.downcase : nil
+      case op
+      when 'with' then 'select'
+      else op
+      end
+    end
+
+    def inspect
+      "\#<#{self.class} #{@resource.inspect}>"
+    end
+
+    def raw_content
+      @resource.content
+    end
+
+    def stripped_raw_content
+      strip_sql(raw_content)
+    end
+
+    def meta_data
+      SQLMetaDataParser.new.parse(raw_content)
+    end
+
+    def attributes
+      Array(meta_data['attributes'])
+    end
+
+    private
+
+    def apply_replace(content)
+      return content unless @replace
+      re, subst = @replace
+      content.gsub(re, subst)
+    end
+
+    def strip_sql(src)
+      src.sub(%r{\A/\*.*?^\*/}m, '').gsub(/^--.*/, '').strip
+    end
+  end
+
+  # Variable declarations in parameter files
+  class Declarations
+    def Declarations.parse(source)
+      SQLMetaDataParser.new.parse_declarations(source)
+    end
+
+    def initialize(vars = {})
+      @vars = {}
+      vars.each do |name, default_value|
+        declare name, default_value
+      end
+    end
+
+    def inspect
+      "\#<#{self.class} #{@vars.inspect}>"
+    end
+
+    def declared?(name)
+      @vars.key?(name.to_s)
+    end
+
+    def [](name)
+      @vars[name.to_s] or raise ParameterError, "undeclared variable: #{name}"
+    end
+
+    def each(&block)
+      @vars.each_value(&block)
+    end
+
+    def default_variables
+      Variables.define {|vars|
+        each do |decl|
+          if decl.have_default_value?
+            vars[decl.name.to_s] = decl.default_value
+          end
+        end
+      }
+    end
+
+    def freeze
+      @vars.freeze
+      super
+    end
+
+    def declare(name, value)
+      raise ParameterError, "duplicated variable declaration: #{name}" if declared?(name)
+      varname = name.to_s.tr('-', '_')
+      @vars[varname] = Declaration.new(varname, value)
+    end
+
+    def declare_table(name, value)
+      case value
+      when String
+        declare name, value
+      when nil
+        declare name, name
+      else
+        raise ParameterError, "bad default value for table variable: #{value.class} (#{value.inspect})"
+      end
+    end
+
+    def declare_tables(label, value)
+      case value
+      when String
+        declare value, value
+      when Array
+        value.each do |name|
+          declare name, name
+        end
+      when Hash
+        value.each do |name, val|
+          declare name, val
+        end
+      else
+        raise ParameterError, "'#{label}' entry value must be a string, a list or a map but a #{value.class}"
+      end
+    end
+
+    def declare_map(label, value)
+      case value
+      when String
+        declare value, nil
+      when Array
+        value.each do |name|
+          declare name, nil
+        end
+      when Hash
+        value.each do |name, val|
+          declare name, val
+        end
+      else
+        raise ParameterError, "'#{label}' entry value must be a string, a list or a map but a #{value.class}"
+      end
+    end
+
+    def declare_none(label, value)
+      # ignore
+    end
+  end
+
+  # Accepts all variable as declared dynamically
+  class DynamicDeclarations
+    def declared?(name)
+      true
+    end
+
+    def each
+    end
+
+    def [](name)
+      Declaration.new(name.to_s, nil)
+    end
+  end
+
+  class Declaration
+    def initialize(name, default_value)
+      @name = name
+      @default_value = default_value
+    end
+
+    attr_reader :name
+    attr_reader :default_value
+
+    def have_default_value?
+      not @default_value.nil?
+    end
+
+    def inspect
+      "\#<#{self.class} #{@name}#{have_default_value? ? '=' + @default_value.inspect : ''}>"
+    end
+  end
+
+  class SQLMetaDataParser
+    def parse_declarations(source)
+      build_decls(parse(source))
+    end
+
+    def parse(source)
+      parse_meta(read_mata(source))
+    end
+
+    private
+
+    DECLARATIONS = {
+      'dest-table' => 'table',
+      'src-tables' => 'tables',
+      'params' => 'map',
+      'attributes' => 'none'
+    }
+
+    def build_decls(map)
+      decls = Declarations.new
+      DECLARATIONS.each do |key, type|
+        value = map.delete(key)
+        decls.send "declare_#{type}", key, value if value
+      end
+      unless map.empty?
+        raise ParameterError, "unknown SQL parameter declaration: #{map.keys.join(', ')}"
+      end
+      decls.freeze
+      decls
+    end
+
+    def parse_meta(source)
+      return Hash.new if source.empty?
+      YAML.load(source) || {}
+    rescue YAML::SyntaxError => ex
+      raise ParameterError, "SQL meta data syntax error: #{ex.message}"
+    end
+
+    KEYS_RE = /\A--(?:#{DECLARATIONS.keys.join('|')}):/
+
+    def read_mata(source)
+      lines = ''
+      source.each_line do |line|
+        case line
+        when KEYS_RE
+          lines.concat line.sub(/\A--/, '')
+        when /\A--([\w\-]+):/
+          key = $1
+          raise ParameterError, "unknown SQL meta data: #{source.name}: #{key}"
+        when /\A--/
+          # skip non-metadata comments
+          lines.concat "\n"
+        else
+          break
+        end
+      end
+      lines
+    end
+  end
+
+  class TableSpec
+    # "[SCHEMA.]TABLE" -> TableSpec(SCHEMA, TABLE)
+    def TableSpec.parse(spec)
+      new(*split_name(spec))
+    end
+
+    # "TABLE" -> [nil, "TABLE"]
+    # "SCHEMA.TABLE" -> ["SCHEMA", "TABLE"]
+    def TableSpec.split_name(name_pair)
+      raise ParameterError, "table spec is empty" if name_pair.strip.empty?
+      components = name_pair.split('.', 2)
+      if components.size == 1
+        return nil, components.first
+      else
+        s, t = components
+        raise ParameterError, "schema name is blank" if not s or s.empty?
+        raise ParameterError, "table name is blank" if not t or t.empty?
+        return s, t
+      end
+    end
+
+    def initialize(schema, name)
+      @schema = schema
+      @name = name
+    end
+
+    attr_reader :schema
+    attr_reader :name
+
+    def to_s
+      @schema ? "#{@schema}.#{@name}" : @name
+    end
+
+    def inspect
+      "\#<#{self.class} #{to_s}>"
+    end
+
+    def ==(other)
+      return false unless other.kind_of?(TableSpec)
+      @schema == other.schema and @name == other.name
+    end
+
+    alias eql? ==
+
+    def hash
+      @schema.hash ^ @name.hash
+    end
+  end
+
+end

data/lib/bricolage/taskqueue.rb

@@ -0,0 +1,156 @@
+require 'bricolage/jobflow'
+require 'bricolage/exception'
+require 'pathname'
+
+module Bricolage
+
+  class TaskQueue
+    def initialize
+      @queue = []
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def size
+      @queue.size
+    end
+
+    def queued?
+      not empty?
+    end
+
+    def each(&block)
+      @queue.each(&block)
+    end
+
+    def consume_each
+      lock
+      save
+      while task = self.next
+        yield task
+        deq
+      end
+    ensure
+      unlock
+    end
+
+    def enq(task)
+      @queue.push task
+    end
+
+    def next
+      @queue.first
+    end
+
+    def deq
+      task = @queue.shift
+      save
+      task
+    end
+
+    def save
+    end
+
+    def restore
+    end
+
+    def locked?
+      false
+    end
+
+    def lock
+    end
+
+    def unlock
+    end
+
+    def unlock_help
+      "[MUST NOT HAPPEN] this message must not be shown"
+    end
+  end
+
+  class FileTaskQueue < TaskQueue
+    def FileTaskQueue.restore_if_exist(path)
+      q = new(path)
+      q.restore if q.queued?
+      q
+    end
+
+    def initialize(path)
+      super()
+      @path = path
+    end
+
+    def queued?
+      @path.exist?
+    end
+
+    def save
+      if empty?
+        @path.unlink if @path.exist?
+        return
+      end
+      tmpname = "#{@path}.tmp.#{Process.pid}"
+      begin
+        File.open(tmpname, 'w') {|f|
+          each do |task|
+            f.puts task.serialize
+          end
+        }
+        File.rename tmpname, @path
+      ensure
+        FileUtils.rm_f tmpname
+      end
+    end
+
+    def restore
+      File.foreach(@path) do |line|
+        enq JobTask.deserialize(line)
+      end
+    end
+
+    def locked?
+      lock_file_path.exist?
+    end
+
+    def lock
+      FileUtils.touch lock_file_path
+    end
+
+    def unlock
+      FileUtils.rm_f lock_file_path
+    end
+
+    def lock_file_path
+      Pathname.new("#{@path}.LOCK")
+    end
+
+    def unlock_help
+      "remove the file: #{lock_file_path}"
+    end
+  end
+
+  class JobTask
+    def initialize(jobnet, seq, job)
+      @jobnet = jobnet
+      @seq = seq
+      @job = job
+    end
+
+    attr_reader :jobnet
+    attr_reader :seq
+    attr_reader :job
+
+    def serialize
+      [@jobnet, @seq, @job].join("\t")
+    end
+
+    def JobTask.deserialize(str)
+      jobnet, seq, job = str.strip.split("\t", 3)
+      new(JobFlow::Ref.parse(jobnet), seq.to_i, JobFlow::Ref.parse(job))
+    end
+  end
+
+end