tap 0.11.1 → 0.12.0

data/lib/tap/support/parser.rb

@@ -1,6 +1,3 @@
-require 'tap/support/schema'
-autoload(:Shellwords, 'shellwords')
-
 module Tap
   module Support
     
@@ -20,7 +17,8 @@ module Tap
     # final argument reassigns e to round 3.
     #
     #   schema = Parser.new("a -- b --+ c -- d -- e --+3[4]").schema
-    #   schema.rounds(true)             # => [[0,1,3],[2], nil, [4]]
+    #   a, b, c, d, e = schema.nodes
+    #   schema.rounds                   # => [[a,b,d],[c], nil, [e]]
     #
     # ==== Workflow Assignment
     # All simple workflow patterns except switch can be specified within
@@ -49,16 +47,19 @@ module Tap
     # workflow until the final argument which sequenced b and c.
     #
     #   schema = Parser.new("a --: b -- c --1:2i").schema
-    #   schema.argvs                    # => [["a"], ["b"], ["c"], []]
-    #   schema.joins(true)              # => [[:sequence,0,[1],{}], [:sequence,1,[2],{:iterate => true}]]
+    #   a, b, c = schema.nodes
+    #   schema.joins.collect do |join, inputs, outputs|
+    #     [join.options, inputs, outputs]
+    #   end                             # => [[{},[a],[b]], [{:iterate => true},[b],[c]]]
     #
     # ==== Globals
     # Global instances of task (used, for example, by dependencies) may
     # be assigned in the parse syntax as well.  The break for a global
     # is '--*'.
     #
-    #   schema = Parser.new("a -- b --* global_name --config for --global").schema
-    #   schema.globals(true)            # => [2]
+    #   schema = Parser.new("a -- b --* c").schema
+    #   a, b, c = schema.nodes
+    #   schema.globals                  # => [c]
     #
     # ==== Escapes and End Flags
     # Breaks can be escaped by enclosing them in '-.' and '.-' delimiters;
@@ -167,12 +168,12 @@ module Tap
           indicies
         end
 
-        # Parses the match of a ROUND regexp into a round index
-        # and an array of task indicies that should be added to the
-        # round. The inputs correspond to $2 and $5 for the match.
+        # Parses the match of a ROUND regexp into a round index and an array
+        # of task indicies that should be added to the round. The inputs
+        # correspond to $2 and $5 for the match.
         #
-        # If $2 is nil, a round index of zero is assumed; if $5 is
-        # nil or empty, then indicies of [:current_index] are assumed.
+        # If $2 is nil, a round index of zero is assumed; if $5 is nil or
+        # empty, then indicies of [:current_index] are assumed.
         #
         #   parse_round("+", "")                # => [1, [:current_index]]
         #   parse_round("+2", "1,2,3")          # => [2, [1,2,3]]
@@ -213,20 +214,26 @@ module Tap
           one.empty? ? current_index : one.to_i
         end
 
-        # Parses the match of an bracket_regexp into a [source_index,
-        # target_indicies, options] array. The inputs corresponds to $1,
-        # $2, and $3 for the match. The previous and current index are
-        # assumed if $1 and/or $2 is empty.
+        # Parses the match of an bracket_regexp into a [input_indicies,
+        # output_indicies, options] array. The inputs corresponds to $1,
+        # $2, and $3 for a match to a bracket regexp. The previous and
+        # current index are assumed if $1 and/or $2 is empty.
         #
-        #   parse_bracket("1", "2,3", "")       # => [1, [2,3], {}]
-        #   parse_bracket("", "", "")           # => [:previous_index, [:current_index], {}]
-        #   parse_bracket("1", "", "")          # => [1, [:current_index], {}]
-        #   parse_bracket("", "2,3", "")        # => [:previous_index, [2,3], {}]
+        #   parse_bracket("1", "2,3", "")       # => [[1], [2,3], {}]
+        #   parse_bracket("", "", "")           # => [[:previous_index], [:current_index], {}]
+        #   parse_bracket("1", "", "")          # => [[1], [:current_index], {}]
+        #   parse_bracket("", "2,3", "")        # => [[:previous_index], [2,3], {}]
         #
         def parse_bracket(one, two, three)
           targets = parse_indicies(two)
           targets << current_index if targets.empty?
-          [one.empty? ? previous_index : one.to_i, targets, parse_options(three)]
+          [[one.empty? ? previous_index : one.to_i], targets, parse_options(three)]
+        end
+        
+        # Same as parse_bracket but reverses the input and output indicies.
+        def parse_reverse_bracket(one, two, three)
+          inputs, outputs, options = parse_bracket(one, two, three)
+          [outputs, inputs, options]
         end
         
         # Parses an options string into a hash.  The input corresponds
@@ -234,17 +241,20 @@ module Tap
         # if the options string contains unknown options.
         #
         #   parse_options("")                   # => {}
-        #   parse_options("is")                 # => {:iterate => true, :stack => true}
+        #   parse_options("ik")                 # => {:iterate => true, :stack => true}
         #
         def parse_options(three)
           options = {}
           0.upto(three.length - 1) do |char_index|
             char = three[char_index, 1]
-            unless index = Join::SHORT_FLAGS.index(char)
-              raise "unknown option in: #{three} (#{char})"
+            
+            entry = Join.configurations.find do |key, config| 
+              config.attributes[:short] == char
             end
+            key, config = entry
             
-            options[Join::FLAGS[index]] = true
+            raise "unknown option in: #{three} (#{char})" unless key 
+            options[key] = true
           end
           options
         end
@@ -326,6 +336,9 @@ module Tap
 
       # Same as parse, but removes parsed args from argv.
       def parse!(argv)
+        # prevent the addition of an empty node to schema
+        return if argv.empty?
+        
         argv = case argv
         when Array then argv
         when String then Shellwords.shellwords(argv) 
@@ -335,8 +348,7 @@ module Tap
         argv.unshift('--')
         
         escape = false
-        current_argv = schema[current_index].argv        
-        
+        current_argv = nil
         while !argv.empty?
           arg = argv.shift
 
@@ -346,7 +358,7 @@ module Tap
             if arg == ESCAPE_END
               escape = false
             else
-              current_argv << arg
+              (current_argv ||= schema[current_index].argv) << arg
             end
 
             next
@@ -365,9 +377,9 @@ module Tap
             # a breaking argument was reached:
             # unless the current argv is empty,
             # append and start a new definition
-            unless current_argv.empty?
+            if current_argv && !current_argv.empty?
               self.current_index += 1
-              current_argv = schema[current_index].argv
+              current_argv = nil
             end
             
             # parse the break string for any
@@ -378,26 +390,28 @@ module Tap
             # add all other non-breaking args to
             # the current argv; this includes
             # both inputs and configurations
-            current_argv << arg
+            (current_argv ||= schema[current_index].argv) << arg
             
           end
         end
-
+        
         schema
       end
       
       def load(argv)
-         argv.each do |arg|
-          case arg
+        argv.each do |args|
+          case args
           when Array
-            schema.nodes << arg
+            schema.nodes << Node.new(args, 0)
             self.current_index += 1
           else
-            parse_break(arg)
+            parse_break(args)
           end
         end
         
-        schema
+        # cleanup is currently required if terminal joins like
+        # --0[] are allowed (since it's interpreted as --0[next])
+        schema.cleanup
       end
       
       protected
@@ -421,13 +435,13 @@ module Tap
         when SEQUENCE
           indicies, options = parse_sequence($1, $3)
           while indicies.length > 1
-            schema.set(Joins::Sequence, indicies.shift, indicies[0], options)
+            schema.set(Join, [indicies.shift], [indicies[0]], options)
           end
 
         when INSTANCE    then schema[parse_instance($1)].globalize
-        when FORK        then schema.set(Joins::Fork,       *parse_bracket($1, $2, $3))
-        when MERGE       then schema.set(Joins::Merge,      *parse_bracket($1, $2, $3))
-        when SYNC_MERGE  then schema.set(Joins::SyncMerge,  *parse_bracket($1, $2, $3))    
+        when FORK        then schema.set(Join, *parse_bracket($1, $2, $3))
+        when MERGE       then schema.set(Join, *parse_reverse_bracket($1, $2, $3))
+        when SYNC_MERGE  then schema.set(Joins::SyncMerge, *parse_reverse_bracket($1, $2, $3))
         else raise ArgumentError, "invalid break argument: #{arg}"
         end
       end

data/lib/tap/support/schema.rb

@@ -42,8 +42,8 @@ module Tap
         #
         #   format_sequence(1, [2,3], {})       # => "1:2:3"
         #
-        def format_sequence(source_index, target_indicies, options)
-          ([source_index] + target_indicies).join(":") + format_options(options)
+        def format_sequence(source_index, outputs, options)
+          ([source_index] + outputs).join(":") + format_options(options)
         end
 
         # Formats a global instance string.
@@ -58,8 +58,8 @@ module Tap
         #
         #   format_fork(1, [2,3],{})            # => "1[2,3]"
         #
-        def format_fork(source_index, target_indicies, options)
-          "#{source_index}[#{target_indicies.join(',')}]#{format_options(options)}"
+        def format_fork(source_index, outputs, options)
+          "#{source_index[0]}[#{outputs.join(',')}]#{format_options(options)}"
         end
 
         # Formats a merge string (note the target index is
@@ -67,8 +67,8 @@ module Tap
         #
         #   format_merge(1, [2,3],{})           # => "1{2,3}"
         #
-        def format_merge(target_index, source_indicies, options)
-          "#{target_index}{#{source_indicies.join(',')}}#{format_options(options)}"
+        def format_merge(target_index, inputs, options)
+          "#{target_index[0]}{#{inputs.join(',')}}#{format_options(options)}"
         end
 
         # Formats a sync_merge string (note the target index 
@@ -76,8 +76,8 @@ module Tap
         #
         #   format_sync_merge(1, [2,3],{})      # => "1(2,3)"
         #
-        def format_sync_merge(target_index, source_indicies, options)
-          "#{target_index}(#{source_indicies.join(',')})#{format_options(options)}"
+        def format_sync_merge(target_index, inputs, options)
+          "#{target_index[0]}(#{inputs.join(',')})#{format_options(options)}"
         end
 
         # Formats an options hash into a string.  Raises an error
@@ -88,12 +88,12 @@ module Tap
         def format_options(options)
           options_str = []
           options.each_pair do |key, value|
-            unless index = Join::FLAGS.index(key)
+            unless config = Join.configurations[key]
               raise "unknown key in: #{options} (#{key})"
             end
             
             if value
-              options_str << Join::SHORT_FLAGS[index]
+              options_str << config.attributes[:short]
             end
           end
           options_str.sort.join
@@ -112,59 +112,39 @@ module Tap
           parser.load(argv)
           parser.schema
         end
-
+        
+        # Loads a schema from the specified path.  Raises an error if no such
+        # file existts.
         def load_file(path)
           argv = YAML.load_file(path)
-          load(argv)
+          argv ? load(argv) : new
         end
       end
       
       # An array of the nodes registered in self.
       attr_reader :nodes
-
+      
       def initialize(nodes=[])
         @nodes = nodes
-        @current_index = 1
       end
       
-      # Retrieves the node at index, or instantiates
-      # a new Node if one does not already exists.
+      # Retrieves the node at index, or instantiates a new Node if one does
+      # not already exists.
       def [](index)
         nodes[index] ||= Node.new
       end
       
-      # Sets a join between the source and targets.  
-      # Returns the new join.
-      def set(join_class, source_indicies, target_indicies, options={})
-        join = join_class.new(options)
-
-        [*source_indicies].each {|source_index| self[source_index].output = join }
-        [*target_indicies].each {|target_index| self[target_index].input = join  }
-
-        join
+      # Returns the index of the node in nodes.
+      def index(node)
+        nodes.index(node)
       end
       
-      # Removes all nil nodes, and nodes with empty argvs.
-      # Additionally reassigns rounds by shifting later
-      # rounds up to fill any nils in the rounds array.
-      #
-      # Returns self.
-      def compact
-        # remove nil and empty nodes
-        nodes.delete_if do |node|
-          node == nil || node.argv.empty?
-        end
-        
-        # reassign rounds
-        index = 0
-        rounds.compact.each do |round|
-          round.each {|node| node.round = index }
-          index += 1
-        end
-        
-        self
+      # Shortcut to collect the indicies of each node in nodes.  Returns nil if
+      # nodes is nil.
+      def indicies(nodes)
+        nodes ? nodes.collect {|node| index(node) } : nodes
       end
-      
+       
       # Returns an array of the argvs for each nodes.
       def argvs
         nodes.collect do |node|
@@ -172,72 +152,126 @@ module Tap
         end
       end
       
-      # Returns a collection of nodes sorted  
-      # into arrays by node.round.
-      def rounds(as_indicies=false)
+      # Returns a collection of nodes sorted into arrays by round.
+      def rounds
         rounds = []
         nodes.each do |node|
-          (rounds[node.round] ||= []) << node if node && node.round
+          next unless node
+          round = node.round
+          (rounds[round] ||= []) << node if round
         end
-        
-        rounds.each do |round|
-          next unless round
-          round.collect! {|node| nodes.index(node) }
-        end if as_indicies
-
         rounds
       end
       
-      # Returns a collection of global nodes
-      # (nodes with no input or output set).
-      def globals(as_indicies=false)
+      # Returns a collection of nodes sorted into arrays by natural round.
+      def natural_rounds
+        rounds = []
+        nodes.each do |node|
+          next unless node
+          round = node.natural_round
+          (rounds[round] ||= []) << node if round
+        end
+        rounds
+      end
+      
+      # Returns a collection of global nodes.
+      def globals
         globals = []
         nodes.each do |node|
-          globals << node if node && node.global?
+          if node && node.global?
+            globals << node
+          end
         end
-        
-        globals.collect! do |node| 
-          nodes.index(node)
-        end if as_indicies
-        
         globals
       end
       
-      # Returns a hash of [join, [source_node, target_nodes]] pairs
-      # across all nodes.
-      def joins(as_indicies=false)
-        joins = {}
+      # Returns an array of joins among nodes in self.
+      def joins
+        joins = []
+        
         nodes.each do |node|
           next unless node
-          
-          case node.input
-          when Join, ReverseJoin
-            (joins[node.input] ||= [nil,[]])[1] << node
-          end
-          
-          case node.output
-          when Join, ReverseJoin
-            (joins[node.output] ||= [nil,[]])[0] = node
+          if join = node.output_join
+            joins << join
           end
         end
         
-        if as_indicies
-          summary = []
-          joins.each_pair do |join, (source_node, target_nodes)|
-            target_indicies = target_nodes.collect {|node| nodes.index(node) }
-            summary << [join.name, nodes.index(source_node), target_indicies, join.options]
+        nodes.each do |node|
+          next unless node
+          if join = node.input_join
+            joins << join
           end
+        end
+        
+        joins.uniq
+      end
+      
+      # Sets a join between the nodes at the input and output indicies.
+      # Returns the new join.
+      def set(join_class, inputs, outputs, options={})
+        unless inputs && !inputs.empty?
+          raise ArgumentError, "no input nodes specified"
+        end
+        
+        join = [join_class.new(options), [],[]]
+        
+        inputs.each {|index| self[index].output = join }
+        outputs.each {|index| self[index].input = join }
+        
+        join
+      end
+      
+      # Removes all nil nodes, nodes with empty argvs, and orphaned joins.
+      # Additionally reassigns rounds by shifting later rounds up to fill
+      # any nils in the rounds array.
+      #
+      # Returns self.
+      #
+      #-- 
+      # Note: the algorithm for cleaning up joins can likely be optimized.
+      def cleanup
+        # remove nil and empty nodes
+        nodes.delete_if do |node|
+          node == nil || node.argv.empty?
+        end
+        
+        # cleanup joins
+        joins.each do |join, input_nodes, output_nodes|
           
-          summary.sort_by {|entry| entry[1] || -1 }
-        else
-          joins
+          # remove missing output nodes
+          output_nodes.delete_if {|node| !nodes.include?(node) }
+
+          # remove missing input nodes; the removed nodes need
+          # to be preserved in case an orphan join results and
+          # the natural round before cleanup needs to be
+          # determined.
+          remaining_nodes, removed_nodes = input_nodes.partition {|node| nodes.include?(node) }
+          
+          case
+          when remaining_nodes.empty?
+            # orphan join: reassign output nodes to natural round
+            orphan_round = Node.natural_round(removed_nodes)
+            output_nodes.dup.each {|node| node.round = orphan_round }
+          else
+            input_nodes.replace(remaining_nodes)
+          end
+        end
+        
+        # reassign rounds
+        index = 0
+        rounds.compact.each do |round|
+          round.each {|node| node.round = index }
+          index += 1
         end
+        
+        self
       end
       
       def build(app)
-        tasks = {}
+        cleanup
         
         # instantiate the nodes
+        tasks = {}
         nodes.each do |node|
           tasks[node] = yield(node.argv) if node
         end
@@ -258,102 +292,94 @@ module Tap
         end
 
         # build the workflow
-        joins.each_pair do |join, (source_node, target_nodes)|
-          raise "unassigned join: #{join}" if source_node == nil || target_nodes.empty?
-
-          targets = target_nodes.collect do |target_node|
-            tasks[target_node][0]
-          end
-          source = tasks[source_node][0]
+        joins.each do |join, input_nodes, output_nodes|
+          sources = input_nodes.collect {|node| tasks[node][0] }
+          targets = output_nodes.collect {|node| tasks[node][0] }
           
-          join.join(source, targets)
+          join.join(sources, targets)
         end
 
-        # build queues
+        # build rounds
         queues = rounds.compact.collect do |round|
-          round.each do |node|
-            task, args = tasks.delete(node)
-            task.enq(*args)
-          end
-
-          app.queue.clear
+          round.collect {|node| tasks.delete(node) }
         end
         
         # notify any args that will be overlooked
         tasks.each_pair do |node, (task, args)|
           next if args.empty?
-          warn "warning: ignoring args for node (#{nodes.index(node)}) #{task} [#{args.join(' ')}]"
+          warn "warning: ignoring args for node (#{index(node)}) #{task} [#{args.join(' ')}]"
         end
-
-        queues
+        
+        # enque
+        queues.each {|queue| app.queue.concat(queue) }
+        app
       end
       
       # Creates an array dump of the contents of self.
       def dump
-        segments = argvs
-        each_schema_str {|str| segments << str }
-        segments
-      end
-      
-      # Constructs a command-line string for the schema, ex:
-      # '-- a -- b --0:1'.
-      def to_s
-        segments = []
-        nodes.each do |node|
-          segments << "--"
-          
-          node.argv.each do |arg| 
-            segments << shell_quote(arg)
-          end unless node == nil
-        end
+        cleanup
         
-        each_schema_str {|str| segments << "--#{str}" }
-        segments.join(' ')
-      end
-      
-      protected
-      
-      # Yields each formatted schema string (global, round, and join).
-      def each_schema_str # :nodoc:
-        each_globals_str {|str| yield str }
-        each_round_str   {|str| yield str }
-        each_join_str    {|str| yield str }
-      end
-      
-      # Yields globals formatted as a string.
-      def each_globals_str # :nodoc:
+        # add argvs
+        array = argvs
+        
+        # add global declarations
         globals.each do |node|
-          yield format_instance(nodes.index(node))
+          array << format_instance(index(node))
         end
-      end
-      
-      # Yields each round formatted as a string.
-      def each_round_str # :nodoc
+        
+        # add round declarations
         index = 0
-        rounds.each do |indicies|
-          unless indicies == nil || index == 0
-            indicies = indicies.collect {|node| nodes.index(node) }
-            yield format_round(index, indicies)
+        rounds.each do |nodes|
+          
+          # skip round 0 as it is implicit
+          if index > 0
+            indicies = nodes.collect {|node| index(node) }
+            array << format_round(index, indicies)
           end
+          
           index += 1
         end
-      end
+        
+        # add join declarations
+        joins.each do |join, input_nodes, output_nodes|
+          inputs = input_nodes.collect {|node| nodes.index(node) }
+          outputs = output_nodes.collect {|node| nodes.index(node) }
 
-      # Yields each join formatted as a string.
-      def each_join_str # :nodoc
-        joins.each_pair do |join, (source_node, target_nodes)|
-          source_index = nodes.index(source_node)
-          target_indicies = target_nodes.collect {|node| nodes.index(node) }
-          
-          yield case join
-          when Joins::Sequence   then format_sequence(source_index, target_indicies, join.options)
-          when Joins::Fork       then format_fork(source_index, target_indicies, join.options)
-          when Joins::Merge      then format_merge(source_index, target_indicies, join.options)
-          when Joins::SyncMerge  then format_sync_merge(source_index, target_indicies, join.options)
-          else raise "unknown join type: #{join.class} (#{source_index}, [#{target_indicies.join(',')}])"
+          array << case join
+          when Joins::SyncMerge  then format_sync_merge(outputs, inputs, join.options)
+          when Join
+            if inputs.length == 1
+              if outputs.length == 1
+                format_sequence(inputs, outputs, join.options)
+              else
+                format_fork(inputs, outputs, join.options)
+              end
+            else
+              format_merge(outputs, inputs, join.options)
+            end
+          else raise "unknown join type: #{join.class} ([#{inputs.join(',')}], [#{outputs.join(',')}])"
+          end
+        end
+        
+        array
+      end
+      
+      # Constructs a command-line string for the schema, ex:
+      # '-- a -- b --0:1'.
+      def to_s
+        args = []
+        dump.each do |obj|
+          if obj.kind_of?(Array)
+            args << "--"
+            args.concat obj.collect {|arg| shell_quote(arg) }
+          else
+            args << "--#{obj}"
           end
         end
+        
+        args.join(' ')
       end
+      
     end
   end
 end