masamune 0.11.0

data/lib/masamune/data_plan.rb

@@ -0,0 +1,31 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  module DataPlan
+    require 'masamune/data_plan/builder'
+    require 'masamune/data_plan/elem'
+    require 'masamune/data_plan/engine'
+    require 'masamune/data_plan/rule'
+    require 'masamune/data_plan/set'
+  end
+end

data/lib/masamune/data_plan/builder.rb

@@ -0,0 +1,66 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'singleton'
+
+class Masamune::DataPlan::Builder
+  include Singleton
+
+  def build(namespaces, commands, sources, targets)
+    Masamune::DataPlan::Engine.new.tap do |engine|
+      sources_for, sources_anon = partition_by_for(sources)
+      targets_for, targets_anon = partition_by_for(targets)
+
+      commands.each do |name|
+        command_name = "#{namespaces.shift}:#{name}"
+
+        source_options = sources_for[name] || sources_anon.shift or next
+        target_options = targets_for[name] || targets_anon.shift or next
+        next if source_options[:skip] || target_options[:skip]
+
+        engine.add_source_rule(command_name, source_options)
+        engine.add_target_rule(command_name, target_options)
+
+        engine.add_command_rule(command_name, thor_command_wrapper)
+      end
+    end
+  end
+
+  private
+
+  def partition_by_for(annotations)
+    with_for, anon = annotations.partition { |opts| opts.has_key?(:for) }
+    decl = {}
+    with_for.each do |opts|
+      decl[opts[:for]] = opts.reject { |k,_| k == :for }
+    end
+    [decl, anon]
+  end
+
+  def thor_command_wrapper
+    Proc.new do |engine, rule, _|
+      engine.environment.with_exclusive_lock(rule) do
+        engine.environment.parent.invoke(rule)
+      end
+    end
+  end
+end

data/lib/masamune/data_plan/elem.rb

@@ -0,0 +1,190 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+class Masamune::DataPlan::Elem
+  MISSING_MODIFIED_AT = Time.new(0)
+
+  include Masamune::Accumulate
+  include Comparable
+
+  attr_reader :rule, :options
+
+  def initialize(rule, start_time, options = {})
+    @rule = rule
+    self.start_time = start_time
+    @options = options
+  end
+
+  def input
+    @input ||= start_time.strftime(strftime_format)
+  end
+  alias :path :input
+  alias :table :input
+
+  def partition
+    input.split('_').last
+  end
+  alias :suffix :partition
+
+  def exists?
+    if rule.for_path?
+      rule.engine.filesystem.exists?(path)
+    elsif rule.for_table_with_partition?
+      rule.engine.postgres_helper.table_exists?(table)
+    elsif rule.for_table?
+      table
+    end
+  end
+
+  def complete?
+    if rule.for_targets?
+      sources.existing.map(&:start_date).uniq.length == sources.map(&:start_date).uniq.length
+    end
+  end
+
+  def last_modified_at
+    if rule.for_path?
+      rule.engine.filesystem.stat(path).try(:mtime)
+    elsif rule.for_table?
+      rule.engine.postgres_helper.table_last_modified_at(table, @options)
+    end || MISSING_MODIFIED_AT
+  end
+
+  def explode(&block)
+    if rule.for_path?
+      file_glob = path
+      file_glob += '/' unless path.include?('*') || path.include?('.')
+      file_glob += '*' unless path.include?('*')
+      rule.engine.filesystem.glob(file_glob) do |new_path|
+        yield rule.bind_input(new_path)
+      end
+    elsif rule.for_table_with_partition?
+      yield self if exists?
+    end
+  end
+  method_accumulate :explode
+
+  def targets(&block)
+    return Masamune::DataPlan::Set::EMPTY if @rule.for_targets?
+    rule.engine.targets_for_source(rule.name, self) do |target|
+      yield target
+    end
+  end
+  method_accumulate :targets, lambda { |elem| Masamune::DataPlan::Set.new(elem.rule.engine.get_target_rule(elem.rule.name)) }
+
+  def target
+    targets.first
+  end
+
+  def sources(&block)
+    return Masamune::DataPlan::Set::EMPTY if @rule.for_sources?
+    rule.engine.sources_for_target(rule.name, self) do |source|
+      yield source
+    end
+  end
+  method_accumulate :sources, lambda { |elem| Masamune::DataPlan::Set.new(elem.rule.engine.get_source_rule(elem.rule.name)) }
+
+  def source
+    sources.first
+  end
+
+  def start_time
+    @start_time.to_time.utc
+  end
+
+  def start_time=(start_time)
+    @start_time =
+    case start_time
+    when Time
+      rule.time_round(start_time.utc)
+    when Date, DateTime
+      rule.time_round(start_time.to_time.utc)
+    end
+  end
+
+  def start_date
+    @start_time.to_date
+  end
+
+  def stop_time
+    start_time.advance(@rule.time_step => 1)
+  end
+
+  def stop_date
+    stop_time.to_date
+  end
+
+  def glob
+    @options[:glob]
+  end
+
+  def next(i = 1)
+    self.class.new(@rule, start_time.advance(@rule.time_step => +1*i), @options)
+  end
+
+  def prev(i = 1)
+    self.class.new(@rule, start_time.advance(@rule.time_step => -1*i), @options)
+  end
+
+  def round(grain)
+    self.class.new(@rule.round(grain), start_time, @options)
+  end
+
+  def ==(other)
+    uniq_constraint == other.uniq_constraint
+  end
+
+  def eql?(other)
+    self == other
+  end
+
+  def hash
+    uniq_constraint.hash
+  end
+
+  # FIXME should consider stop_time for correctness
+  def <=>(other)
+    if start_time < other.start_time
+      1
+    elsif start_time > other.start_time
+      -1
+    elsif start_time == other.start_time
+      0
+    end
+  end
+
+  def inspect
+    {rule: rule, input: input, start_date: start_time.to_s, stop_date: stop_time.to_s, :options => options}.to_s
+  end
+
+  protected
+
+  def uniq_constraint
+    [rule, options, rule.for_table? ? start_time : input]
+  end
+
+  private
+
+  def strftime_format
+    @strftime_format ||= glob ? @rule.strftime_format.sub('*', glob) : @rule.strftime_format
+  end
+end

data/lib/masamune/data_plan/engine.rb

@@ -0,0 +1,162 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support'
+require 'active_support/core_ext/numeric/time'
+
+class Masamune::DataPlan::Engine
+  MAX_DEPTH = 10
+
+  include Masamune::HasEnvironment
+  include Masamune::Accumulate
+
+  def initialize
+    @target_rules = Hash.new
+    @source_rules = Hash.new
+    @command_rules = Hash.new
+    @targets = Hash.new { |set,rule| set[rule] = Masamune::DataPlan::Set.new(@target_rules[rule]) }
+    @sources = Hash.new { |set,rule| set[rule] = Masamune::DataPlan::Set.new(@source_rules[rule]) }
+    @set_cache = Hash.new { |cache,level| cache[level] = Hash.new }
+    @current_depth = 0
+  end
+
+  def filesystem
+    @filesystem ||= Masamune::CachedFilesystem.new(environment.filesystem)
+  end
+
+  def add_target_rule(rule, target_options = {})
+    @target_rules[rule] = Masamune::DataPlan::Rule.new(self, rule, :target, target_options)
+  end
+
+  def get_target_rule(rule)
+    @target_rules[rule]
+  end
+
+  def add_source_rule(rule, source_options = {})
+    @source_rules[rule] = Masamune::DataPlan::Rule.new(self, rule, :source, source_options)
+  end
+
+  def get_source_rule(rule)
+    @source_rules[rule]
+  end
+
+  def add_command_rule(rule, command)
+    @command_rules[rule] = command
+  end
+
+  # TODO use constructed reference instead
+  def rule_for_target(target)
+    target_matches = @target_rules.select { |rule, matcher| matcher.primary? && matcher.matches?(target) }
+    source_matches = @source_rules.select { |rule, matcher| matcher.matches?(target) }
+
+    if target_matches.empty?
+      if source_matches.empty?
+        raise "No rule matches target #{target}"
+      else
+        Masamune::DataPlan::Rule::TERMINAL
+      end
+    else
+      logger.error("Multiple rules match target #{target}") if target_matches.length > 1
+      target_matches.map(&:first).first
+    end
+  end
+
+  def targets_for_date_range(rule, start, stop, &block)
+    target_template = @target_rules[rule]
+    return unless target_template
+    target_template.generate(start.to_time.utc, stop.to_time.utc) do |target|
+      yield target
+    end
+  end
+  method_accumulate :targets_for_date_range, lambda { |engine, rule, _, _| Masamune::DataPlan::Set.new(engine.get_target_rule(rule)) }
+
+  def targets_for_source(rule, source, &block)
+    source_template = @source_rules[rule]
+    target_template = @target_rules[rule]
+    source_instance = source_template.bind_input(source)
+    source_template.generate_via_unify(source_instance, target_template) do |target|
+      yield target
+    end
+  end
+  method_accumulate :targets_for_source, lambda { |engine, rule, source| Masamune::DataPlan::Set.new(engine.get_target_rule(rule)) }
+
+  def sources_for_target(rule, target, &block)
+    source_template = @source_rules[rule]
+    target_template = @target_rules[rule]
+    target_instance = target_template.bind_input(target)
+    target_template.generate_via_unify(target_instance, source_template) do |source|
+      yield source
+    end
+  end
+  method_accumulate :sources_for_target, lambda { |engine, rule, target| Masamune::DataPlan::Set.new(engine.get_source_rule(rule)) }
+
+  def targets(rule)
+    @set_cache[:targets_for_rule][rule] ||= @targets[rule].union(@sources[rule].targets)
+  end
+
+  def sources(rule)
+    @set_cache[:sources_for_rule][rule] ||= @sources[rule].union(@targets[rule].sources).adjacent
+  end
+
+  def prepare(rule, options = {})
+    @targets[rule].merge options.fetch(:targets, [])
+    @sources[rule].merge options.fetch(:sources, [])
+
+    constrain_max_depth(rule) do
+      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|
+        prepare(derived_rule, targets: sources.map(&:input)) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+      end
+    end if options.fetch(:resolve, true)
+    clear!
+  end
+
+  def execute(rule, options = {})
+    return if targets(rule).actionable.empty?
+
+    constrain_max_depth(rule) do
+      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|
+        execute(derived_rule, options) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+      end
+    end if options.fetch(:resolve, true)
+
+    @command_rules[rule].call(self, rule, options)
+    clear!
+  end
+
+  def executing?
+    @current_depth > 0
+  end
+
+  def constrain_max_depth(rule, &block)
+    @current_depth += 1
+    raise "Max depth of #{MAX_DEPTH} exceeded for rule '#{rule}'" if @current_depth > MAX_DEPTH
+    yield
+  ensure
+    @current_depth -= 1
+  end
+
+  def clear!
+    @set_cache.clear
+    filesystem.clear!
+    environment.postgres_helper.clear!
+  end
+end