tap 0.18.0 → 0.19.0

data/lib/tap/schema.rb

@@ -1,374 +0,0 @@
-require 'tap/app'
-require 'tap/schema/utils'
-require 'tap/schema/parser'
-
-module Tap
-  class App
-    def build(schema, options={})
-      options = {
-        :clean => true,
-        :validate => true
-      }.merge(options)
-      
-      unless schema.kind_of?(Schema)
-        schema = Schema.new(schema)
-      end
-      
-      if resources = options[:resources]
-        schema.resolve! do |type, id|
-          resources[type][id]
-        end
-      end
-      
-      if options[:clean]
-        reset
-      end
-      
-      schema.build!(self, options[:validate])
-    end
-    
-    def to_schema
-      schema = Schema.new
-      queue.to_a.each do |task, inputs|
-        schema.add(task, inputs)
-      end
-      
-      middleware.reverse_each do |m|
-        schema.use(m)
-      end
-      
-      index = 0
-      schema.tasks.keys.each do |task|
-        schema.rename(task, index)
-        index += 1
-      end
-      
-      if block_given?
-        schema.resources.each_pair do |type, resource|
-          yield(type, resource)
-        end
-      end
-      
-      schema
-    end
-  end
-  
-  class Schema
-    class << self
-      def load(str)
-        schema = YAML.load(str)
-        new(schema ? Utils.symbolize(schema) : {})
-      end
-      
-      def load_file(path)
-        load(File.read(path))
-      end
-    end
-    
-    include Utils
-    
-    # A hash of task schema describing individual tasks in a workflow.  Tasks
-    # only require a class, but may contain configurations and even arguments
-    # for enque.  Individual tasks may be a hash or an array.  The tasks are
-    # resolved if they take one of these forms:
-    #
-    #   tasks:
-    #     key: {:class: TaskClass, ...}
-    #     key: [TaskClass, ...]
-    #   
-    attr_reader :tasks
-    
-    # An array of join schema that describe how to join tasks together.  Joins
-    # have arrays of inputs and outputs that reference task keys.  Individual
-    # joins may be a hash or an array.  The joins are resolved if they take
-    # one of these forms:
-    #
-    #   joins:
-    #   - [[inputs], [outputs], {:class: JoinClass, ...}]
-    #   - [[inputs], [outputs], [JoinClass, ...]]
-    #
-    attr_reader :joins
-    
-    # An array of [key, [args]] data that indicates the tasks and arguments
-    # to be added to an application during build.  A key may be specified
-    # alone if tasks[key] is an array; in that case, the arguments remaining
-    # in tasks[key] after instantiation will be used.
-    #
-    #   queue:
-    #   - key                  # uses tasks[key]
-    #   - [key, [1, 2, 3]]     # enques tasks[key] with [1, 2, 3]
-    #
-    attr_reader :queue
-    
-    # An array of middleware to build onto the app.
-    attr_reader :middleware
-    
-    # The app used to build self
-    attr_reader :app
-    
-    def initialize(schema={})
-      @tasks = schema[:tasks] || {}
-      @joins = schema[:joins] || []
-      @queue = schema[:queue] || []
-      @middleware = schema[:middleware] || []
-      
-      @app = nil
-    end
-    
-    def add(node, inputs=nil)
-      collect_tasks(node).collect do |task|
-        tasks[task] = task.to_hash
-        task.joins
-      end.flatten.uniq.each do |join|
-        joins << [join.inputs, join.outputs, join.to_hash]
-      end
-      
-      if inputs
-        queue << [node, inputs]
-      end
-      
-      self
-    end
-    
-    def use(middleware)
-      self.middleware << middleware.to_hash
-      self
-    end
-    
-    def resources
-      {
-        :task => tasks.values,
-        :join => joins.collect {|join| join[2] },
-        :middleware => middleware
-      }
-    end
-    
-    # Renames the current_key task to new_key.  References in joins and
-    # queue are updated by rename.  Raises an error if built? or if the
-    # specified task does not exist.
-    def rename(current_key, new_key)
-      if built?
-        raise "cannot rename if built"
-      end
-      
-      # rename task
-      unless task = tasks.delete(current_key)
-        raise "unknown task: #{current_key.inspect}"
-      end
-      tasks[new_key] = task
-      
-      # update join references
-      joins.each do |inputs, outputs, join|
-        inputs.each_index do |index|
-          inputs[index] = new_key if inputs[index] == current_key
-        end
-        
-        outputs.each_index do |index|
-          outputs[index] = new_key if outputs[index] == current_key
-        end
-      end
-      
-      # update queue references, note both array and 
-      # reference-style entries must be handled
-      queue.each_index do |index|
-        if queue[index].kind_of?(Array)
-          if queue[index][0] == current_key
-            queue[index][0] = new_key
-          end
-        else
-          if queue[index] == current_key
-            queue[index] = new_key
-          end
-        end
-      end
-      
-      self
-    end
-    
-    def resolve!
-      tasks.each_pair do |key, task|
-        task ||= {}
-        tasks[key] = resolve(task) do |id|
-          yield(:task, id || key)
-        end
-      end
-      
-      joins.collect! do |inputs, outputs, join|
-        join ||= {}
-        join = resolve(join) do |id|
-          yield(:join, id || 'join')
-        end
-        [inputs, outputs, join]
-      end
-      
-      middleware.collect! do |m|
-        resolve(m) do |id|
-          yield(:middleware, id)
-        end
-      end
-      
-      queue.collect! do |(key, inputs)|
-        [key, inputs || tasks[key]]
-      end
-      
-      self
-    end
-    
-    def validate!
-      errors = []
-      tasks.each_value do |task|
-        unless resolved?(task)
-          errors << "unresolvable task: #{task.inspect}"
-        end
-      end
-      
-      joins.each do |inputs, outputs, join|
-        unless resolved?(join)
-          errors << "unresolvable join: #{join.inspect}"
-        end
-        
-        inputs.each do |key| 
-          unless tasks.has_key?(key)
-            errors << "missing join input: #{key.inspect}"
-          end
-        end
-        
-        outputs.each do |key| 
-          unless tasks.has_key?(key)
-            errors << "missing join output: #{key.inspect}"
-          end
-        end
-      end
-      
-      queue.each do |(key, args)| 
-        if tasks.has_key?(key)
-          unless args.kind_of?(Array)
-            errors << "non-array args: #{args.inspect}"
-          end
-        else
-          errors << "missing task: #{key}"
-        end
-      end
-      
-      middleware.each do |m|
-        unless resolved?(m)
-          errors << "unresolvable middleware: #{m.inspect}"
-        end
-      end
-      
-      unless errors.empty?
-        prefix = if errors.length > 1
-          "#{errors.length} schema errors\n"
-        else
-           ""
-        end
-
-        raise "#{prefix}#{errors.join("\n")}\n"
-      end
-      
-      self
-    end
-    
-    def cleanup!
-      joins.delete_if do |inputs, outputs, join|
-        
-        # remove missing inputs
-        inputs.delete_if  {|key| !tasks.has_key?(key) }
-        
-        # remove missing outputs
-        outputs.delete_if {|key| !tasks.has_key?(key) }
-        
-        # remove orphan joins
-        inputs.empty? || outputs.empty?
-      end
-      
-      # remove inputs without a task
-      queue.delete_if do |(key, inputs)|
-        !tasks.has_key?(key)
-      end
-      
-      self
-    end
-    
-    def build!(app, validate=true)
-      validate! if validate
-      
-      # instantiate tasks
-      tasks.each_pair do |key, task|
-        tasks[key] = instantiate(task, app)
-      end
-      tasks.freeze
-      
-      # build the workflow
-      joins.collect! do |inputs, outputs, join|
-        inputs = inputs.collect {|key| tasks[key] }
-        outputs = outputs.collect {|key| tasks[key] }
-        instantiate(join, app).join(inputs, outputs)
-      end
-      joins.freeze
-      
-      # utilize middleware
-      middleware.collect! do |middleware|
-        instantiate(middleware, app)
-      end
-      middleware.freeze
-      
-      # enque tasks
-      queue.each do |(key, inputs)|
-        app.enq(tasks[key], *inputs)
-      end
-      queue.clear.freeze
-      
-      @app = app
-      self
-    end
-    
-    def built?
-      @app != nil
-    end
-    
-    def enque(key, *inputs)
-      unless built?
-        raise "cannot enque unless built"
-      end
-      
-      unless task = tasks[key]
-        raise "unknown task: #{key.inspect}"
-      end
-      
-      app.queue.enq(task, inputs)
-      task
-    end
-    
-    # Creates an hash dump of self.
-    def to_hash
-      { :tasks => tasks, 
-        :joins => joins, 
-        :queue => queue, 
-        :middleware => middleware
-      }
-    end
-    
-    # Converts self to a hash and serializes it to YAML.
-    def dump(io=nil)
-      YAML.dump(to_hash, io)
-    end
-    
-    protected
-    
-    # helper to collect all tasks and tasks joined to task
-    def collect_tasks(task, collection=[]) # :nodoc:
-      unless collection.include?(task)
-        collection << task
-        
-        task.joins.each do |join|
-          (join.inputs + join.outputs).each do |input|
-            collect_tasks(input, collection)
-          end
-        end
-      end
-      
-      collection
-    end
-  end
-end

data/lib/tap/schema/parser.rb

@@ -1,425 +0,0 @@
-require 'shellwords'
-require 'tap/schema'
-
-module Tap
-  class Schema
-    class << self
-      def parse(argv=ARGV)
-        Parser.new(argv).schema
-      end
-    end
-    
-    # A parser for workflow schema defined on the command line.
-    #
-    # == Syntax
-    #
-    # The command line syntax can be thought of as a series of ARGV arrays
-    # connected by breaks.  The arrays define tasks (ie nodes) in a workflow
-    # while the breaks define joins.  These are the available breaks:
-    #
-    #   break          meaning
-    #   --             default delimiter, no join
-    #   --:            sequence join
-    #   --[][]         multi-join (sequence, fork, merge)
-    #
-    # As an example, this defines three tasks (a, b, c) and sequences the
-    # b and c tasks:
-    #
-    #   schema = Parser.new("a -- b --: c").schema
-    #   schema.tasks                  # => [["a"], ["b"], ["c"]]
-    #   schema.joins                  # => [['join', [1],[2]]]
-    #
-    # In the example, the indicies of the tasks participating in the sequence
-    # are inferred as the last and next tasks in the schema.  Alternatively
-    # the tasks participating in the sequence may be written out directly;
-    # these also sequence b to c.
-    #
-    #   schema = Parser.new("a -- b -- c --1:2").schema
-    #   schema.tasks                  
-    #   # => {
-    #   # 0 => ["a"], 
-    #   # 1 => ["b"], 
-    #   # 2 => ["c"]
-    #   # }
-    #   schema.joins                  
-    #   # => [
-    #   # [[1],[2]]
-    #   # ]
-    #
-    #   schema = Parser.new("a --1:2 b -- c").schema
-    #   schema.tasks
-    #   # => {
-    #   # 0 => ["a"], 
-    #   # 1 => ["b"], 
-    #   # 2 => ["c"]
-    #   # }
-    #   schema.joins                  
-    #   # => [
-    #   # [[1],[2]]
-    #   # ]
-    #
-    # ==== Multi-Join Syntax
-    #
-    # The multi-join syntax allows the specification of arbitrary joins.
-    # Starting with a few examples:
-    #
-    #   example        meaning
-    #   --[][]         last.sequence(next)
-    #   --[1][2]       1.sequence(2)
-    #   --[1][2,3]     1.fork(2,3)
-    #   --[1,2][3]     3.merge(1,2)
-    #
-    # The meaning of the bracket breaks seems to be changing but note that
-    # the sequences, forks, and (unsynchronized) merges are all variations
-    # of a multi-way join.  Internally the breaks are interpreted like this:
-    #
-    #   join = Join.new
-    #   join.join(inputs, outputs)
-    #
-    # To specify another class of join, or to specify join configurations,
-    # add a string in the format "configs.class" where the configs are the
-    # single-letter configuration flags and class is a lookup for the join
-    # class.
-    #
-    #   example        interpretation
-    #   --:s           Join.new(:splat => true)
-    #   --1:2is        Join.new(:iterate => true, :splat => true)
-    #   --[][]q.sync   Sync.new(:enq => true)
-    #   --[][].sync    Sync.new
-    #
-    # If you can stand the syntax, you can also specify a full argv after
-    # the bracket, just be sure to enclose the whole break in quotes.
-    #
-    #   example                interpretation
-    #   "--1:2 join -i -s"     Join.new(:iterate => true, :splat => true)
-    #   "--[][] sync --enq"    Sync.new(:enq => true)
-    #
-    # ==== Escapes and End Flags
-    #
-    # Breaks can be escaped by enclosing them in '-.' and '.-' delimiters;
-    # any number of arguments may be enclosed within the escape. After the 
-    # end delimiter, breaks are active once again.
-    #
-    #   schema = Parser.new("a -- b -- c").schema
-    #   schema.tasks
-    #   # => {
-    #   # 0 => ["a"], 
-    #   # 1 => ["b"], 
-    #   # 2 => ["c"]
-    #   # }
-    # 
-    #   schema = Parser.new("a -. -- b .- -- c").schema
-    #   schema.tasks
-    #   # => {
-    #   # 0 => ["a", "--", "b"], 
-    #   # 1 => ["c"]
-    #   # }
-    #
-    # Parsing continues until the end of argv, or a an end flag '---' is 
-    # reached.  The end flag may also be escaped.
-    #
-    #   schema = Parser.new("a -- b --- c").schema
-    #   schema.tasks
-    #   # => {
-    #   # 0 => ["a"], 
-    #   # 1 => ["b"]
-    #   # }
-    #
-    class Parser
-      
-      # A set of parsing routines used internally by Tap::Schema::Parser,
-      # modularized for ease of testing, and potential re-use. These methods 
-      # require that <tt>current_index</tt> and <tt>previous_index</tt> be 
-      # implemented in the including class.
-      module Utils
-        module_function
-
-        # The escape begin argument
-        ESCAPE_BEGIN = "-."
-
-        # The escape end argument
-        ESCAPE_END = ".-"
-
-        # The parser end flag
-        END_FLAG = "---"
-
-        # Matches any breaking arg. Examples:
-        #
-        #   --
-        #   --1:2
-        #   --[1][2]
-        #   --[1,2,3][4,5,6]is.join
-        #   --.middleware
-        #
-        # After the match:
-        #
-        #   $1:: The string after the break
-        #        (ex: '--' => '', '--:' => ':', '--[1,2][3,4]is.join' => '[1,2][3,4]is.join')
-        #
-        BREAK =  /\A--(\z|[\d\:\[\.].*\z)/
-
-        # Matches a sequence break. Examples:
-        #
-        #   :
-        #   1:
-        #   :2
-        #   1:2:3
-        #
-        # After the match:
-        #
-        #   $1:: The sequence string after the break. 
-        #        (ex: ':' => ':', '1:2' => '1:2', '1:' => '1:', ':2' => ':2')
-        #   $2:: The modifier string.
-        #        (ex: ':i' => 'i', '1:2is' => 'is')
-        #
-        SEQUENCE = /\A(\d*(?::\d*)+)(.*)\z/
-        
-        # Matches a generic join break. Examples:
-        #
-        #   "[1,2,3][4,5,6] join -i -s"
-        #   [1,2,3][4,5,6]is.join
-        #   [1,2][3,4]
-        #   [1][2]
-        #
-        # After the match:
-        #
-        #   $1:: The inputs string.
-        #        (ex: '[1,2,3][4,5,6]' => '1,2,3')
-        #   $2:: The outputs string.
-        #        (ex: '[1,2,3][4,5,6]' => '4,5,6')
-        #   $3:: The modifier string.
-        #        (ex: '[][]is' => 'is')
-        #
-        JOIN = /\A\[([\d,]*)\]\[([\d,]*)\](.*)\z/
-        
-        # Matches a join modifier. After the match:
-        #
-        #   $1:: The modifier flag string.
-        #        (ex: 'is.sync' => 'is')
-        #   $2:: The class string.
-        #        (ex: 'is.sync' => 'sync')
-        #
-        JOIN_MODIFIER = /\A([A-z]*)(?:\.(.*))?\z/
-        
-        # Matches a generic middleware break. Examples:
-        #
-        #   ". middleware --flag"
-        #   .middleware
-        #
-        # After the match:
-        #
-        #   $1:: The modifier string.
-        #        (ex: '.middleware' => 'middleware')
-        #
-        MIDDLEWARE = /\A\.(.*)\z/
-        
-        # Parses an indicies str along commas, and collects the indicies
-        # as integers. Ex:
-        #
-        #   parse_indicies('')                  # => []
-        #   parse_indicies('1')                 # => [1]
-        #   parse_indicies('1,2,3')             # => [1,2,3]
-        #
-        def parse_indicies(str, regexp=/,+/)
-          indicies = []
-          str.split(regexp).each do |n|
-            indicies << n.to_i unless n.empty?
-          end
-          indicies
-        end
-
-        # Parses the match of a SEQUENCE regexp an array of [input_indicies,
-        # output_indicies, metadata] arrays. The inputs corresponds to $1 and
-        # $2 for the match. The previous and current index are assumed if $1
-        # starts and/or ends with a semi-colon.
-        #
-        #   parse_sequence("1:2:3", '')
-        #   # => [
-        #   # [[1], [2]],
-        #   # [[2], [3]],
-        #   # ]
-        #
-        #   parse_sequence(":1:2:", 'is')
-        #   # => [
-        #   # [[:previous_index], [1], ['join', '-i', '-s']],
-        #   # [[1], [2], ['join', '-i', '-s']]],
-        #   # [[2], [:current_index], ['join', '-i', '-s']],
-        #   # ]
-        #
-        def parse_sequence(one, two)
-          indicies = parse_indicies(one, /:+/)
-          indicies.unshift previous_index if one[0] == ?:
-          indicies << current_index if one[-1] == ?:
-          
-          sequences = []
-          while indicies.length > 1
-            sequences << [[indicies.shift], [indicies[0]]]
-          end
-          
-          if argv = parse_join_modifier(two)
-            sequences.each do |sequence|
-              sequence << argv
-            end
-          end
-          
-          sequences
-        end
-
-        # Parses the match of a JOIN regexp into a [input_indicies,
-        # output_indicies, metadata] array. The inputs corresponds to $1, $2,
-        # and $3 for the match.  A join type of  'join' is assumed unless
-        # otherwise specified.
-        #
-        #   parse_join("1", "2,3", "")         # => [[1], [2,3]]
-        #   parse_join("", "", "is.type")      # => [[], [], ['type', '-i', '-s']]
-        #   parse_join("", "", "type -i -s")   # => [[], [], ['type', '-i', '-s']]
-        #
-        def parse_join(one, two, three)
-          join = [parse_indicies(one), parse_indicies(two)]
-          
-          if argv = parse_join_modifier(three)
-            join << argv
-          end
-          
-          join
-        end
-        
-        # Parses a join modifier string into an argv.
-        def parse_join_modifier(modifier)
-          case modifier
-          when ""
-            nil
-          when JOIN_MODIFIER
-            argv = [$2 == nil || $2.empty? ? 'join' : $2]
-            $1.split("").each {|char| argv << "-#{char}"}
-            argv
-          else
-            Shellwords.shellwords(modifier)
-          end
-        end
-        
-        # Parses the match of a MIDDLEWARE regexp into metadata array.
-        # The input corresponds to $1 for the match. Currently this
-        # method is an alias for Shellwords.shellwords.
-        def parse_middleware(one)
-          Shellwords.shellwords(one)
-        end
-      end
-      
-      include Utils
-      
-      # The schema into which tasks are being parsed
-      attr_reader :schema
-      
-      def initialize(argv=[])
-        parse(argv)
-      end
-
-      # Iterates through the argv splitting out task and join definitions.
-      # Parse is non-destructive to argv.  If a string argv is provided, parse
-      # splits it into an array using Shellwords; if a hash argv is provided,
-      # parse converts it to an array using Parser::Utils#parse_argh.
-      def parse(argv)
-        parse!(argv.kind_of?(String) ? argv : argv.dup)
-      end
-
-      # Same as parse, but removes parsed args from argv.
-      def parse!(argv)
-        @schema = Schema.new
-        
-        # prevent the addition of an empty task to schema
-        return schema if argv.empty?
-        
-        argv = Shellwords.shellwords(argv) if argv.kind_of?(String)
-        argv.unshift('--') unless argv[0] =~ BREAK
-        
-        @current_index = -1
-        @current = nil
-        escape = false
-        while !argv.empty?
-          arg = argv.shift
-
-          # if escaping, add escaped arguments 
-          # until an escape-end argument
-          if escape
-            if arg == ESCAPE_END
-              escape = false
-            else
-              current << arg
-            end
-
-            next
-          end
-          
-          case arg
-          when ESCAPE_BEGIN
-            # begin escaping if indicated
-            escape = true
-            
-          when END_FLAG
-            # break on an end-flag
-            break
-          
-          when BREAK
-            # a breaking argument was reached
-            @current_index += 1
-            @current = nil
-            
-            # parse the break string for any
-            # schema modifications
-            parse_break($1)
-            
-          else
-            # add all other non-breaking args to
-            # the current argv; this includes
-            # both inputs and configurations
-            current << arg
-            
-          end
-        end
-        
-        # determine the queue as all tasks not
-        # used as a join output
-        queue = schema.tasks.keys
-        schema.joins.each {|join| queue -= join[1] }
-        schema.queue.concat(queue)
-        
-        schema
-      end
-      
-      protected
-      
-      # The index of the task currently being parsed.
-      attr_reader :current_index # :nodoc:
-      
-      def current
-        @current ||= task(current_index)
-      end
-      
-      # helper to initialize a task at the specified index
-      def task(index) # :nodoc:
-        schema.tasks[index] ||= []
-      end
-      
-      # returns current_index-1, or raises an error if current_index < 1.
-      def previous_index # :nodoc:
-        current_index - 1
-      end
-      
-      # determines the type of break and modifies self appropriately
-      def parse_break(arg) # :nodoc:
-        case arg
-        when ""
-        when SEQUENCE
-          schema.joins.concat parse_sequence($1, $2)
-        when JOIN
-          schema.joins << parse_join($1, $2, $3)
-        when MIDDLEWARE
-          schema.middleware << parse_middleware($1)
-        else
-          raise ArgumentError, "invalid break argument: #{arg}"
-        end
-      end
-    end
-  end
-end