nodepile 0.1.1 → 0.1.2

data/lib/nodepile/pile_organizer.rb

@@ -0,0 +1,258 @@
+require 'nodepile/colspecs.rb'
+require 'nodepile/rec_source.rb'
+require 'nodepile/pragmas.rb'
+require 'nodepile/rule_eval.rb'
+
+module Nodepile
+    
+# Container class for managing a Nodepile.  A nodepile consists of a set of
+# entities including nodes, edges, and rules.  It includes methods for 
+# enumerating the various items in the collection, filtering, and
+# deducing the existence of implied edges using rules.
+class PileOrganizer
+
+    # see nodepile/base_structs.rb for definition of an Nodepile::EntityPacket
+    class ERecStack; end #ERecStack  defined further down
+    class RuleCache; end # defined further down
+    SourceData = Struct.new(:source_name,:highest_sequence_num) 
+
+
+    def initialize()
+        @nodes = Hash.new{|h,k| h[k] = ERecStack.new}
+        @edges = Hash.new{|h,k| h[k] = ERecStack.new}
+        @rules = Array.new  # not subject to overlaying with themselves
+        @pragmas = Nodepile::Pragmas.new
+        @sources = Hash.new{|h,k| h[k] = SourceData.new(k,0)} # insert a dummy source for unspecifie
+        @last_source_name = nil
+        @dirty = true
+    end
+    
+    # If a source name is not specified, then the source is assumed to be
+    # the last source that was used to append.  If no sequence number is provided
+    # then the sequence_number is assumed to be one more than the highest 
+    # sequence number that was specified.  If callers are manually specifying
+    # sequence numbers for a source, they should do so consistently to avoid
+    # repeats.
+    # @param kaa [KeyedArrayAccessor] Includes metadata about @type, @key, @is_implied
+    # @return [self]
+    def append(kaa)
+        @last_source_name = kaa.source || @last_source_name
+        source_data = @sources[@last_source_name]
+        # note that the way things work below deliberately "overlays" items
+        # when a matching key is encountered.  Rule recalculation is deferred
+        case kaa['@type']
+            when :node
+                @nodes[kaa['@key']] << kaa
+            when :edge
+                @edges[kaa['@key']] << kaa
+            when :rule
+                @rules << RuleCache.new(kaa)
+            when :pragma
+                @pragmas.parse(kaa['_id'])
+            else
+                raise "Unhandled entity entity type #{kaa['@type'].inspect}"
+        end #case
+        return self
+    end
+    
+    def node_count() = @nodes.length
+    def rule_count() = @rules.length
+    def edge_count() = @edges.length  
+    def pragmas() = @pragmas
+        
+    def entity_record(key)
+        _update_rule_impacts
+        case key
+            when String
+                return @nodes[key]
+            when Array
+                return @edges[key]
+            else
+                raise "Unrecognized key structure/type"
+        end
+    end
+        
+    # Provide summarized records in order
+    def edge_records
+        return enum_for(:edge_records) unless block_given?
+        _update_rule_impacts
+        @edges.each_value{|erstack| yield(erstack.summary) }
+    end
+    
+    # Provide the summarized records
+    def node_records
+        return enum_for(:node_records) unless block_given?
+        _update_rule_impacts
+        @nodes.each_value{|erstack| yield(erstack.summary) }
+    end
+        
+    
+    # Alias for #append()
+    # @param entity_record [Nodepile::KeyedArrayAccessor]
+    def <<(entity_record) = append(entity_record)
+    
+    # Loads the given file (on top of anything already stored in this object)
+    def load_from_file(tabular_filepath)
+        source = Nodepile::TabularRecordSource.new(tabular_filepath,format: :guess)
+        specs = nil
+        loaded_entity_count = 0
+        rec_src_meta = {'path' => tabular_filepath,'rec_num' => nil}
+        metadata = Hash.new
+        source.each{|(rec,rec_num)|
+            rec_src_meta['rec_num'] = rec_num
+            if specs.nil? #first row is header
+                specs = Nodepile::InputColumnSpecs.new(rec,metadata_key_prefix: '@')
+            else
+                begin
+                    specs.parse(rec,source: tabular_filepath,
+                                ref_num: rec_num,
+                                metadata: metadata,
+                               ){|keyed_array_accessor|
+                        append(keyed_array_accessor)
+                        loaded_entity_count += 1
+                       }
+                rescue Nodepile::InputColumnSpecs::InvalidRecordError => err 
+                    # re-raise but add info about the record number that triggered the error
+                    err.rec_num = rec_num
+                    err.file_path = tabular_filepath
+                    raise   # re-raise
+                end
+            end #if
+           }
+        return loaded_entity_count
+    end
+    
+    
+    private
+        # This sledgehammer approach deletes all calculated impacts that may
+        # have previously been applied to the @edges and @nodes and recalculates
+        # all of them.
+        #
+        # Method is a no-op if the structrues are up-to-date
+        def _update_rule_impacts(force = false)
+            return nil unless force || @dirty
+            @nodes.each_value(&:purge_rule_overlays)
+            @edges.each_value(&:purge_rule_overlays)
+            @rules.each{|rulecache|
+                recs = (rulecache.relevant_entity_type == :node ?  @nodes : @edges).each_value
+                recs.each{|erstack|
+                                if rulecache.match?(erstack.summary)
+                                    # calculate the rule as applied to the given node/edge
+                                    calculated_rule_erec = rulecache.eval_using(erstack.summary)
+                                    erstack << calculated_rule_erec 
+                                end
+                           }
+               } #loop over rules
+            @dirty = false 
+        end
+ 
+    # An ERecStack is a data structure used for holding and summarizing
+    # overlay-able records related to a given Node or Edge which can include
+    # "rules" that apply to that node/stack
+    class ERecStack
+        def initialize()
+            @a = Array.new
+            @summary = nil
+            @mc = CrudeCalculationCache.new
+        end
+        
+        def inspect 
+            "#<#{self.class}:0x#{object_id} type= #{type} key=#{self.key.inspect} depth=#{@a.length}> "
+        end
+        
+        
+        def type = @a.first['@type']
+        def key() = @a.first['@key']
+
+        def is_node? = self.type == :node
+        def is_edge? = self.type == :edge
+        def summary() = @summary
+        def to_a = @a
+              
+          # A stack of type :node or :edge is implied if it contains 
+          # no ERec records where the is_implied attribute  is false.
+          # The return value of this method is undefined for types
+        def is_implied
+            @a.each{|kaa| return false if !kaa['@is_implied'] && 
+                                              [:node,:edge].include?(kaa['@type']) }
+            return true
+        end
+        
+        # Delete overlayed rule records
+        # @return [void]
+        def purge_rule_overlays()
+            @a.delete_if{|rec| rec['@type'] == :rule}
+            @a.each{|rec| @summary = self.class._update_summary(@summary,rec)}  #recalc
+        end
+        
+            
+        # Note that this method does not verify whether it is appropriate to stack the new
+        # record and assumes callers have alreay done this due-diligence.
+        # @param erec [KeyedArrayAccessor]
+        def <<(rec)
+            raise "ERecStack may only hold objects of type Nodepile::KeyedArrayAccessor" unless rec.is_a?(Nodepile::KeyedArrayAccessor)
+            # Keep the summary up-to-date if we've got one.
+            @a << rec
+            if @a.length == 1
+                @summary = rec.dup
+            else
+                @summary = self.class._update_summary(@summary,rec) 
+            end
+            return self
+        end
+        
+        def each_keyed_array()
+            return enum_for(:each_keyed_array) unless block_given?
+            @a.each{|erec| yield erec }
+        end
+
+        private
+        def self._update_summary(cur_summary,new_overlay)
+             return new_overlay if cur_summary.nil?
+             cur_summary.underlay!(new_overlay) 
+             cur_summary.source = nil if cur_summary.source != new_overlay.source
+             cur_summary.ref_num = nil #summary no longer represents a single ref_num
+             cur_summary.update_metadata('@is_implied',false) unless new_overlay['@is_implied']
+             return cur_summary
+        end
+        
+    end #class Nodepile::PileOrganizer::ERecStack
+
+    # Represents cached information about a single specific rule
+    class RuleCache
+
+        # @param rule_erec[KeyedArrayAccessor] created from this rule record
+        def initialize(rule_erec)
+            @er = rule_erec.freeze  #just for to avoid casual alteration
+            raise "Only ERecs of type :rule may be stored in this structure" unless self.type == :rule
+            @verifiers = [*self.key].map{|s| Nodepile::InputColumnSpecs.make_pattern_match_verifier(s)}
+            @rule_eval = RuleRecordEvaluator.new(@er)
+        end
+        
+        def inspect 
+            "#<#{self.class}:0x#{object_id} key=#{self.key.inspect} > "
+        end
+        
+        def type = @er['@type']
+        def is_implied = @er['@is_implied']
+        def key = @er['@key']
+            
+        # Note that a rule that uses dynamic matching cannot precalculate which
+        # records it matches and must (to be safe) reclculate the match in response
+        # to any changes
+        def uses_dynamic_match? = @rule_eval.uses_dynamic_match?
+            
+        # @param kaa [KeyedArrayAccessor] a set of field values that will
+        #               tested against this rule for matching.
+        def match?(kaa) = @rule_eval.match_record?(kaa)
+        def eval_using(kaa) = @rule_eval.calculate_rule(kaa)
+
+            
+        def relevant_entity_type = @er['@key'].is_a?(String) ? :node : :edge
+        
+    end #class Nodepile::PileOrganizer::RuleCache
+    
+end #class PileOrganizer
+    
+    
+end #module Nodepile

data/lib/nodepile/pragmas.rb

@@ -0,0 +1,97 @@
+
+
+module Nodepile
+    
+# Pragmas is a parser and organizer class used to interpret the meaning
+# of pragmas that may appear in source files.  By default, pragmas are
+# coded instructions for rendering, parsing, or layout that may be embedded
+# in input files.  They are often instructions stored in the "_id" field
+# of an input file that begin with a specific indicating string '#pragma '
+# Pragmas may be used to control things like the specific layout engine that
+# is used to visualize a graph (e.g. dot versus neato)
+# Example pragma lines are.  
+#
+#     #pragma neato
+#     #pragma unflatten
+#
+# Create an instance of a Nodepile::Pragmas object in order to track and 
+# interpret the collective pragmas of a given graph.  Note that the most
+# common rule in #pragma interpretation is that if two pragmas contradict each
+# other, the furthest down in the file/parsing stream dominates.
+class Pragmas
+    DEFAULT_PRAGMA_MARKER = "#pragma "
+    
+    @@indicator_patterns = Array.new # array of [pragma_sym,regexp]
+    
+    def initialize(pragma_marker: DEFAULT_PRAGMA_MARKER)
+        @marker = pragma_marker.freeze
+        @indicators = Hash.new  #name mapped to value
+        # if you make this method more complex, remember to update #dup
+    end
+    
+    def dup
+        c = self.class.new(pragma_marker: @marker)
+        c._indicators.merge!(@indicators)  
+        return c
+    end
+    
+    def [](pragma_sym)  
+        @indicators[pragma_sym]
+    end
+    
+    # @yield [pragma_sym,pragma_val] provides pairs of name values as they have
+    #     been set.  Only set values will appear in the yielded set.
+    def each_setting_pair
+        return enum_for(__method__) unless block_given
+        @indicators.each_pair{|k,v| yield(k,v) }
+        return @indicators.length
+    end
+    
+    # Parse the given pragma and store the meaning for access via square bracket
+    # method or the #each_setting_pair method.
+    # @return [void]
+    def parse(pragma_string)
+        raise "Expecting pragma_string to start with [#{@marker}]" unless pragma_string.start_with?(@marker)
+        
+        #TODO: I there are more complicated parsing rules, they should go before
+        #      the simple fallthrough case of whitespace separated indicators
+        
+        # Simple indicators are single "word" values where each value is delimited
+        # by whitespace.  If two indicators apply to the same pragma_sym,
+        pragma_string[@marker.length..-1].split(/\s+/).each{|s| 
+                                prag_sym,_ = @@indicator_patterns.find{|(prag_sym,rx)| rx.match(s) }
+                                if prag_sym
+                                    @indicators[prag_sym] = $1
+                                else
+                                    raise "Unrecognized pragma encountered [#{s}]"
+                                end
+                               }
+        return nil
+    end
+    
+    private 
+    
+    # Declare simple indicators are pragmas whose presence or absence is the indicator.
+    # such indicators can be stacked (multiple per pragma expression)
+    # For example:
+    #              #pragma neato unflatten
+    #
+    # The above line would mean that two sepearate effects were being invoked,
+    # the use of the "neato" rendering engine and the use of the unflatten
+    # setting to improve the aspect ratio of some directed graphs
+    def self._decl_simple_indicator(prag_name,alt_regexp)
+        @@indicator_patterns<< [prag_name,alt_regexp].freeze
+    end
+    
+    _decl_simple_indicator(:layout_engine,/^(dot|neato|fdp|sfdp|circo|twopi|nop2|osage|patchwork)$/)
+    _decl_simple_indicator(:directionality,/^(graph|digraph)$/)
+    #_decl_simple_indicators(:unflatten,/^(unflatten)$/)  # not sure this is supported
+    
+    # Crude accessor (protected) for duplication and merge
+    def _indicators = @indicator
+    
+    
+end #class Nodepile::Pragmas
+    
+    
+end #module Nodepile

data/lib/nodepile/rec_source.rb

@@ -0,0 +1,329 @@
+require 'stringio'
+
+module Nodepile
+    
+# Generates "Factories" for harvesting tabular data from a source stream/file.
+# Includes facilities for parsing common file formats (CSV/TSV).
+# Includes facilities for handling common problems encountered when parsing
+# manually-created tabular data files such as: relevant tabular data is not
+# aligned "top-left", tabular data includes blank or repeated columns,
+# tabular data ends before end of file
+# summary rows appear in the tabular data that need to be ignored.
+class TabularRecordSource
+    include Enumerable
+    
+    DEFAULT_LOADING_GUIDELINES = {
+            mandatory_headers: [], # this can be extremely important to correctly finding tables
+            format: :csv||:tsv||:guess, #assume CSV unless told otherwise 
+            allow_leading_skip_rows: 10, # arbitrary content that may appear before table
+            allow_gap_rows: 2||nil,  # entirely blank rows appearing mid-tabl, nil indicates allow infinite
+            allow_gap_columns: 1, # columns which have a blank header within the table
+            allow_left_offset:  5, # blank columnns allowed left of table 
+            duplicate_header_rule: :first||:last||:ignore||:rename||:fail, #keep the first
+            ignored_header_char: '#', # header names starting with this are just plain ignored
+            emit_blank_records: false, # unless true, entirely blank records are not returned
+            trim_headers: true, #strip leading and trailing spaces
+           }.freeze
+    
+    # Create a new RecordSource intended to read from the specified input
+    # and using the parsing strategy specified by the loading guidelines.
+    def initialize(source,**loading_guidelines)
+        (loading_guidelines.keys - DEFAULT_LOADING_GUIDELINES.keys).tap{|x| raise <<~ERRMSG unless x.empty?}
+               Unrecognized named parameters used for RecordSource creation #{x.inspect}
+              ERRMSG
+        @loading_guidelines = DEFAULT_LOADING_GUIDELINES.merge(loading_guidelines).freeze
+        raise "The source must be non-nil" if source.nil?
+        @source = source # will lazy load
+        @is_mid_read = false  # only relevant for non-parallel sources
+        @replayable_flag = if @source.is_a?(String) 
+                              :parallel # simultaneous each() is okay
+                           elsif @source.respond_to?(:rewind) 
+                               :single # can't guarantee simultaneous each() safe
+                           else
+                               nil
+                           end
+    end #new
+    
+    
+    # Yields the "records" of the first "table" encountered in the bound data
+    # source according to the parameters it was given.  First row yielded is
+    # always the header.  Raises an error if a header is not found.
+    # Beware... depending on the type of data source used at creation, it
+    # may not be possible to rewind or retrieve data in parallel.
+    # With that said, a filename or String both allow parallel retrieval.
+    #
+    # Also note that blank strings will be passed through until the specified
+    # allow_gap_rows are exceeded.  This can mean trailing blanks in long files.
+    #
+    # @yieldparam [Array] Array includes at least two elements.  The first is
+    #               an Array of "fields".  The second element is the record
+    #               number within the source (zero index).  It's important
+    #               to note if any field contains embedded newlines, the record
+    #               number is not the same as the line number
+    # @return [Integer,Enumerator] Returns enumerator if no block is given.
+    #               Otherwise returns the count of records yielded excluding
+    #               the header line.
+    def each(&block)
+        return enum_for(:each) unless block_given?
+        raise "This data source type may only be read once." if @source.nil?
+        raise <<~ERRMSG if @is_mid_read && @replayable_flag != :parallel
+            For this type of data source, you may not read simultaneously.
+           ERRMSG
+        @is_mid_read = true 
+        scanner = self.class._make_record_stream(@source,format: @loading_guidelines[:format])
+        scanner = self.class._reposition_to_header_rec(scanner,@loading_guidelines)
+        raw_header,header_pos = scanner.next
+        header_range = self.class._calc_header_range(raw_header,@loading_guidelines[:allow_gap_columns])
+        # process the header line to create a "mask"
+        yield [raw_header[header_range],header_pos]  # return the trimmed header
+        rec_count = self.class._emit_rows(scanner,header_range,
+                                          @loading_guidelines[:emit_blank_records],
+                                          trim_headers: @loading_guidelines[:trim_headers],
+                                          tolerate_blanks: @loading_guidelines[:allow_gap_rows],
+                                          &block
+                                         )
+        @is_mid_read = false
+        @source = nil if @replayable_flag.nil? # release resources
+        return rec_count 
+    end #each
+    
+
+    ###########################################################################
+    private
+    
+    SEPARATOR_CHAR_LIST = {tsv: "\t", csv: ','}.freeze
+    
+    # Note, due to the need to terminate if too many blanks, this may not read 
+    # to the end of the file
+    def self._emit_rows(raw_rec_enum,range_mask,emit_blank_records,trim_headers:,
+                        tolerate_blanks: nil,
+                        &block
+                       )
+        contig_blank_count = 0
+        emitted_record_count = 0
+        need_to_trim_row = trim_headers  # trim the first one
+        loop do
+            begin
+                rec,pos = raw_rec_enum.next
+                masked_rec = rec&.[](range_mask)
+                next if masked_rec.nil?
+                is_blank_record = masked_rec.all?{|s| s.nil? || /^\s*$/.match?(s)}
+                contig_blank_count = is_blank_record ? (contig_blank_count+1) : 0
+                if tolerate_blanks && contig_blank_count > tolerate_blanks 
+                    return emitted_record_count # end of records
+                else
+                    if emit_blank_records || !is_blank_record
+                        if need_to_trim_row # only done once (for header) if at all
+                            masked_rec.map!{|s| s&.strip}
+                            need_to_trim_row = false # only first emitted row 
+                        end
+                        yield [masked_rec,pos]
+                        emitted_record_count += 1
+                    end
+                end
+            rescue StopIteration
+                return emitted_record_count  # running out of records is an okay end
+            end # rescuing
+        end #loop over records
+        raise "Unexpected issue encountered."  # should never get here
+    end # self._emit_rows()
+ 
+    MIN_NONBLANK_HEADER_BEFORE_GAP_ALLOWED = 3 
+ 
+    # Given a presumed header, identify the position of the largest contiguous
+    # block of non-blank fields and return the range information to be used
+    # for scraping successive rows.
+    def self._calc_header_range(raw_header_record,max_blank_cols)
+        max_blank_cols ||= 0  # default is no blank columns tolerated in header
+        blank_tol = max_blank_cols
+        ix0 = nil
+        runs = Array.new
+        (0..(raw_header_record.length-1)).each{|ix|
+            if raw_header_record[ix].nil? || /^\s*$/.match?(raw_header_record[ix])
+                if ix0.nil?
+                    # deliberate no-op, in middle of blank run
+                elsif  blank_tol >= 0 && ix-ix0 >= MIN_NONBLANK_HEADER_BEFORE_GAP_ALLOWED 
+                    # at least two content-filled columns must be found before tolerating blanks
+                    blank_tol -= 1
+                else 
+                    runs << [ix0,ix-1]
+                    ix0 = nil
+                    blank_tol = max_blank_cols
+                end
+            else  # non blank
+                ix0 ||= ix  # record start of run
+                blank_tol = max_blank_cols  # reset tolerance for blanks
+            end
+           }
+        runs << [ix0,raw_header_record.length-1-(max_blank_cols-blank_tol)] if ix0
+        widest = runs.max{|a,b| a[1]-a[0] <=> b[1]-b[0] }
+        return widest && (widest[0]..widest[1])  # range defines
+    end                     
+    
+ 
+    # Opens up a record stream based on the source provided and the format rule
+    # specified.  
+    # @param source [] Many different values are possible 
+    #  1) a filepath to a readable text file
+    #  2) a string variable (must contain at least one newline)
+    #  3) An enumerable of strings where each string is a record "line" to be parsed individually
+    #  4) An enumerable of arrays of strings (where the inner array is the column values).
+    #     In this case the format parameter is ignored
+    # @param format [:csv,:tsv,:guess,Regexp] Indicates how to interpret column delimiters
+    #                 and row delimiters.  The format parameter is ignored if the source
+    #                 is an enumerable of 
+    # @return [Enumerator<Array>] Whose next() method returns two-element arrays
+    #                 Where the first element is the fields of the record
+    #                 (in the form of an array of strings) and the second element
+    #                 is the zero based index indicating the record number within the source
+    #                 Note that the Enumerator returned may not be rewindable/replayable.
+    def self._make_record_stream(source,format: :csv||:tsv||:guess||nil)
+        col_sep = case format
+                    when nil
+                        nil
+                    when :csv,:tsv 
+                        SEPARATOR_CHAR_LIST[format]
+                    when :guess
+                        # in the future, we might be able to guess based on reading
+                        # the first line and looking for tabs or commas
+                        if source.is_a?(String) && /\.(csv|tsv)$/i.match(source)
+                            SEPARATOR_CHAR_LIST[$1.downcase.to_sym]
+                        else
+                            raise "Format specified as :guess but unable to deduce format"
+                        end
+                    else
+                        raise "Currently unhandled format specifier [#{format}]"
+                  end
+        case source
+          in Enumerable if source.first.is_a?(String)
+            # This is the most manual case because of the need to try and detect
+            # lines that are split by a quoted multiline string.
+            return _chunk_lines(source,col_sep)
+          in Enumerable if source.first.is_a?(Array) and source.first.first.is_a?(String)
+            # no need for further processing, assume it already is a record source
+            return source.each_with_index 
+          in String if source.include?("\n") 
+            # if passed a string, it must be multiline to be treated as the data source
+            return CSV.parse(source,col_sep: col_sep).each_with_index
+          in String if !File.exist?(source)
+            raise "Unable to find the indicated file:  #{source}"
+          in String if File.exist?(source) # presumed to be valid filepath
+            return CSV.foreach(source,col_sep: col_sep).each_with_index
+        end #case source
+        raise "Unable to convert the provided source into a record stream"
+    end # self.make_record_stream()
+    
+    
+    
+    # Tests a string to see if the last field looks like it might have a
+    # mutiline field.  This is detected by checking for whether the rightmost
+    # field has unbalanced quote characters. 
+    # IMPORTANT NOTE: It relies on being told whether the line begins within a 
+    # quote
+    # (meaning that a complete line contains at least one unquoted separator)
+    def self._is_dangling?(line,sep_char,started_in_quote,quot_char: '"')
+        qc = Array.new #quote counts
+        qc << (start_in_quot ? 1 : 0)
+        # count quotes in each field to identify unbalanced quotes
+        line.each_char{|c|
+            if c == quot_char
+                qc[-1] += 1
+            elsif c == sep_char && qc.last.even?
+                qc << 0
+            end
+           }
+        return qc.last.odd?
+    end
+    
+    
+    # bunches up groups of lines when they look like the last column may
+    # contain a multiline value (with embedded carriage return)
+    def self._chunk_lines(line_enum,sep_char,&is_continued)
+        return enum_for(:_chunk_lines,line_enum,sep_char,&is_continued) unless block_given?
+        buf = ""
+        ix = 0  # will be one-based counter
+        is_in_quote = false 
+        line_enum.each{|line|
+            if _is_dangling?(line,sep_char,is_in_quote)
+                is_in_quote = true
+                buf.concat(line,line.last == "\n" ? '' : "\n")
+            else
+                is_in_quote = false
+                rec = CSV.parse((buf.empty? ? line : buf.concat(line)),col_sep: sep_char)
+                buf.clear
+                yield [rec,(ix+=1)-1]
+            end
+           }
+        yield [CSV.parse(buf,col_sep: sep_char),(ix+=1)-1] unless buf.empty?
+        return nil  #meaningless return value
+    end
+    
+    
+    # Assuming we are starting from the absolute top of the source, scan
+    # forward looking for the header row according to the provided guidelines.
+    # Note, it may have to read past the header row to ensure it's made the best
+    # possible choice for that header row.
+    #
+    # @param raw_rec_enum [Enumerator] record enumerator for "raw" records such
+    #                       as is generated by _make_record_stream
+    # @param guidelines [Hash] guidelines package as generated during 
+    # instantiation of a class.  Note this method is not intended to be called
+    # publicly.
+    # @return [Enumerator] Enumerator that should replace the enumerator passed
+    #                 in and whose first record is the header row.
+    # Important Note:  The position of the raw_rec_enum is almost certain to be
+    #        changed by calling next() on it.  It should not be used after this
+    #        call because of this and other buffer considerations
+    def self._reposition_to_header_rec(raw_rec_enum,guidelines)
+        buffer = Array.new  
+        begin 
+            loop do  
+                buffer << raw_rec_enum.next
+                break if buffer.length > guidelines[:allow_leading_skip_rows]
+            end
+        rescue StopIteration #deliberately a no-op
+            return nil if buffer.empty?
+        end   
+        scores = Hash.new{|h,ix| h[ix] = 0} # scoring for possible header row
+        mand_cols = guidelines[:mandatory_headers]
+        buffer.each_with_index{|(rec,_),buf_pos|
+                hdr_range = _calc_header_range(rec,guidelines[:allow_gap_columns]) # best possible header range
+                next if hdr_range.nil?
+                if mand_cols.empty? || (mand_cols - rec[hdr_range]).empty?
+                    scores[buf_pos] = 10*(hdr_range.size) +                     # prefer wide columns
+                                      (mand_cols.empty? ? 0 : 99000) +  # huge bonus for having mandatory columns
+                                      (1- buf_pos.to_f/buffer.length)   # slight preference for early records
+                end
+                # possible other factors for future consideration:
+                #   preceding blank line
+                #   containing a non-blank row immediately beneath it
+               } # end examination of rows in the buffer
+        best_guess = scores.max{|a,b| a[1] <=> b[1] }
+        if best_guess.nil?
+            raise "Unable to find header record within the first #{[buffer.length,guidelines[:allow_leading_skip_rows]].min} records examined!"
+        end
+        #buffer[best_guess[0]..-1].to_enum + raw_rec_enum  # chain enumerators to include buffer
+        return self._joined_enum(buffer,best_guess[0]..buffer.length, raw_rec_enum)
+    end
+    
+    # This hack-method was added because for some reason Enumerator::Chain
+    # does not seem to support the next() method as I had to hand-roll
+    # the Chain
+    # return [nil,Enumerator]
+    def self._joined_enum(buffer1,buffer1_range,record_enum)
+        return enum_for(:_joined_enum,buffer1,buffer1_range,record_enum) unless block_given?
+        buffer1_range.each{|ix| yield buffer1[ix] }
+        buffer1 = nil  # release (in case it matters)
+        begin 
+            loop do  
+                yield record_enum.next
+            end
+        rescue StopIteration #deliberately a no-op
+        end  
+        return nil # meaningless return value
+    end
+    
+end #class TabularRecordSource
+    
+    
+end #module Nodepile