nodepile 0.1.1 → 0.1.2

data/lib/nodepile/colspecs.rb

@@ -0,0 +1,562 @@
+#
+# colspecs.rb
+# 
+# Specification of the column names that have predefined meaning
+# when encountered in a input file.
+#
+
+require 'set'
+require 'yaml'
+require 'nodepile/keyed_array.rb'
+require 'nodepile/base_structs.rb'
+
+module Nodepile
+
+# This class provides information about the valid columns for potential use
+# in documentation and also provides facilities for doing per-line verification
+# of column values within a single line.
+# that can appear on a non-header line of an input file.  
+# Note that the best way to think of this class is as a scanner which is in some sense
+# stateless
+#
+# Records generated by the #parse method and related methods will by default
+# set metadata fields, particularly including:
+#   '@type' = :node, :edge, :rule, :pragma
+#   '@key' = String or [String,String] for node or edge respectively
+class InputColumnSpecs
+  
+  class InvalidRecordError < StandardError
+      attr_accessor :rec_num,:file_path  # use to add error detail
+      def initialize(msg) = @msg = msg
+      def message
+          prefix = "Nodepile parsing error at record [#{self.rec_num||'?'}] from source [#{self.file_path||'?'}]: "
+          return (!self.rec_num.nil? || !self.file_path.nil?) ? (prefix + @msg) : @msg
+      end
+  end  # parsing errors throw this
+  
+  DEFAULT_ID_DELIMITER = ',' # may be used in _link_from and _link_to for multiple edges
+  DEFAULT_PRAGMA_MARKER = "#pragma "
+
+
+  public
+  # Provide a simple hash of field names and their meaning/use.
+  def self.coldefs
+    @@class_mcache.cache(__method__){||
+         h = YAML.load(defined?(DATA) ? DATA.read : 
+                                                 /__END__\s+(.*)/m.match(File.read(__FILE__))[1]
+                                )['data']['fields']
+         h  # this value is cached
+       }
+  end
+
+  def self.val_is_pattern?(s)
+    s[0] == '/' ? :pattern : nil
+  end
+
+  # List the most crucial columns that indicate the existence of
+  # nodes, edges, and styling instructions.
+  def self.id_cols; %w(_id _links_from _links_to); end #do not reorder
+  def self.all_cols; coldefs().keys; end
+
+  # Defines the characters that will be interpreted as delimiting entity "id"
+  # values.
+  attr_accessor :id_delimiter
+
+  # Creates a customized InputColumnSpecs object based on the column names and order
+  # that are included in one specific file.  That object can then be used
+  # ONLY to validate that specific file.  See the #coldefs
+  # @param col_names[Array<String>] Order of column data expected from calls to
+  #                             #validate
+  # @param id_delimiter[String] Indicates a character that will be considered
+  #                     a delimiter between ids so that multiple may occupy the
+  #                     field
+  # @param pragmas[String,nil] If nil, "pragmas" are not identified.
+  #                  If true, then when the _id field is started with the
+  #                  "#pragma", it is identified as a pragma and made available
+  #                  through the #each_pragma method.  If a string, then
+  #                  any record whose _id column starts with that string is
+  #                  considered a pragma.  Note that ONLY the _id column of
+  #                  a pragma record is captured.
+  # @param metadata_key_prefix [String,nil] During #parse and related methods
+  #                  records are yielded in the form of KeyedArrayAccessor objects
+  #                  that have both the loaded data and also metadata about the 
+  #                  records such as the type of the entity and whether its
+  #                  existence was triggered explicitly or implicitly.  This
+  #                  value is is passed to the KeyedArrayAccessor.
+  # @raise InvalidRecordError
+  def initialize(col_names,id_delimiter: DEFAULT_ID_DELIMITER,pragmas: DEFAULT_PRAGMA_MARKER,
+                  metadata_key_prefix: '@')
+    @col_names = col_names.dup.freeze
+    @id_cols_indices = self.class.id_cols.map{|cnm| @col_names.find_index(cnm)}.freeze
+    @id_delimiter = id_delimiter
+    @pragma_marker = pragmas
+    @empty_kv_array = KeyedArrayAccessor.new(@col_names,Array.new(@col_names.length).freeze)
+    raise InvalidRecordError.new(<<~ERRMSG) if @id_cols_indices[0].nil?
+        A valid record set must contain an '_id' column
+      ERRMSG
+    @metadata_key_prefix = metadata_key_prefix
+    @md_pfxs = [(@metadata_key_prefix||'')+'type',
+               (@metadata_key_prefix||'')+'key',
+               (@metadata_key_prefix||'')+'is_implied',
+              ]      
+      
+    @mc = CrudeCalculationCache.new
+  end
+
+
+
+  # Given a string representing the contents of the "_id", "_links_to", or "_links_from" field,
+  # this method will split it into zero or more tokens representing either ids or
+  # or else patterns.  Patterns start with the question mark character. 
+  # Leading and trailing spaces are stripped before return.
+  # @param id_containing_field [String] Any of the possible id containing fields
+  # @return [Array<String>] zero or more
+  def split_ids(id_containing_field, &block)
+    # very simple implementation (make smarter later???)
+    return [] if id_containing_field.nil?
+    return enum_for(:split_ids,id_containing_field) unless block_given?
+    raise "A field containing a rule calculation may not contain other ids" if /,\s*\?/ =~ id_containing_field
+    id_containing_field.split(@id_delimiter).tap{|a2| 
+                                                 a2.each{|s| 
+                                                            s.strip!
+                                                            yield s unless s == ''
+                                                        }
+                                             }
+  end
+  
+  # Given a single "record" (which may define zero or more entities or contain errors)
+  # this method will yield once for each "entity" or "rule" that may be inferred
+  # by that record.  The "entities" defined by a given record are determined by
+  # three fields: _id, _links_from, and _links_to.
+  #
+  # The entries in these fields can indicate several things:
+  # 1) The explicit existence and attribute values for a node 
+  # 2) Override values for a node or pattern of nodes
+  # 3) The implicit existence of a node (because an explicit node links explicitly to/from it)
+  # 4) The explicit existence of an edge (because an edge is explicitly in the to/from fields) 
+  #    and attribute values for the edge
+  # 5) The implicit existence of an edge (because an edge is implied by a rule in the to/from)
+  #    and attribute values for the edge
+  #    
+  # Note that when metadata is attached to the KeyedArrayAccessors, it the metada will
+  # be updated to include the following key-values.
+  #  * 'type' = :node, :edge, :rule, :pragma
+  #  * 'key' = either a single String of nodes/node-rules or an array of two strings for edges
+  #            and edge-rules
+  #  * 'is_implied' = true,nil to indicate whether the entity is implied
+  #
+  # @param col_value_array [Array] Column values in exact order of column names
+  #                                provided when this object was constructed.
+  # @return [Integer] Number of entities encountered.  Note that zero is valid.
+  # @param metadata [Hash,nil] If provided, the given metadata will be attached to each of the
+  #             KeyedArrayAccessors that are yielded along with metadata about
+  #             this particular entity.  Note that the hash passed in will be altered
+  #             in two ways.  Firstly, if a @metadata_key_prefix is specified, all keys
+  #             will be changed to include this prefix (if they aren't already).
+  #             Secondly, the three additional metadata key-values will be added
+  #             (type, key, is_implied).
+  # @param metadata_key_prefix [String,nil] See KeyedArrayAccessor#initialize for detail.
+  #             If provided, this string will be foreced to appear at the beginning
+  #             of every metadata key.
+  # @param source [String,nil,Object] see KeyedArrayAccessor#initialize for detail
+  # @param ref_num [Integer,nil] see KeyedArrayAccessor#iniialize for detail
+  # @raise [InvalidRecordError] If errors or omissions in data make it uninterpretable
+  # @yieldparam [Nodepile::KeyedArrayAccessor] A single node, edge, or rule taken extracted 
+  #                             from the record.  Note that the id, links_to, and links_from
+  #                             fields may be altered in the return value.  
+  def parse(col_value_array,source: nil, ref_num: nil,metadata: nil,&entity_receiver)
+        #see below in this file for the various preprocessing defined
+        _preprocs.each{|(ix,preproc_block)|
+               col_value_array[ix] = preproc_block.call(col_value_array[ix]) 
+              }
+        _validators.each{|(vl_col_nums,val_block)|
+               errmsg = val_block.call(*vl_col_nums.map{|i| i && col_value_array[i]}) # test the specified column values
+               raise InvalidRecordError.new(errmsg) if errmsg 
+              }
+        if metadata && (@metadata_key_prefix||'') != ''
+            # if necessary, facilitate quick attachment of metadata to KeyedArrayAccessor
+            metadata.transform_keys{|k| k.start_with?(@metadata_key_prefix) ? k : @metadata_key_prefix + k}
+        end
+        metadata ||= Hash.new
+        # following proc is used to package up the return value at multiplel places below
+        yieldval_bldr =  Proc.new{|kaa,*three_md_fields|
+                                  (0..(@md_pfxs.length-1)).each{|i| metadata[@md_pfxs[i]] = three_md_fields[i]}
+                                  kaa.reset_metadata(metadata,metadata_key_prefix: @metadata_key_prefix)
+                                  kaa  
+                                 }
+        ids, links_from, links_to = @id_cols_indices.map{|i| i && col_value_array[i]}
+        return 0 if ids&.start_with?('#')  # ignore these records
+        base_kva = KeyedArrayAccessor.new(@col_names, col_value_array, source: source, ref_num: ref_num)
+        if @pragma_marker && ids&.start_with?(@pragma_marker) 
+            # pragmas get shortcut treatment, not keyed, ignore all other columns
+            yield yieldval_bldr(base_kva,:pragma,nil,false) if block_given?
+            return 1  # pragmas do not have links, or multiple ids 
+        end
+        entity_count = 0
+        lf_list = split_ids(links_from).to_a
+        lt_list = split_ids(links_to).to_a
+        if !ids.nil?
+            edge_list = Array.new
+        else
+            # for pure edges, add them to list for later yielding
+            edge_list = lf_list.to_a.product(lt_list.to_a)
+                                             .map{|(lf,lt)| 
+                                                    kva = base_kva.dup
+                                                    kva['_links_from'] = lf
+                                                    kva['_links_to'] = lt
+                                                    [lf,lt,kva ]
+                                                 }
+        end  #detecting pure edges
+
+        split_ids(ids).each{|id| 
+                 kva = base_kva.dup.tap{|kva| 
+                                             kva['_id'] = id
+                                             kva['_links_from'] = nil 
+                                             kva['_links_to'] = nil
+                                       }
+                 entity_count += 1
+                 yield yieldval_bldr.call(kva,id[0] == '?' ? :rule : :node,id.freeze,false) if block_given?
+                 # emit any implicitly existing nodes
+                 (lf_list + lt_list).each{|link|
+                                 if !link.start_with?('?')
+                                    entity_count += 1
+                                    # implied nodes have cleared value except their key
+                                    kva = base_kva.dup.tap{|x| 
+                                                              x['_id'] = link
+                                                              x['_links_from'] = nil
+                                                              x['_links_to'] = nil
+                                                          }
+                                    yield yieldval_bldr.call(kva,:node,link.freeze,true) if block_given?
+                                 end
+                                }
+                 # Flag edges the go from/to _id.  Note, you can't define rules this way.
+                 (lf_list.product([id]) + [id].product(lt_list)).each{|a|
+                                                 next if a.any?{|v| v.start_with?('?')} # rules can't imply an edge
+                                                 kva = @empty_kv_array.dup
+                                                 kva['_links_from'] = a[0]
+                                                 kva['_links_to'] = a[1]
+                                                 kva.source = base_kva.source
+                                                 kva.ref_num = base_kva.ref_num
+                                                 edge_list << [a[0],a[1],kva]
+                                                }
+                }
+          edge_list.each{|(n1,n2,kva)|
+                  entity_count += 1
+                  et = (n1.start_with?('?') || n2.start_with?('?')) ? :rule : :edge
+                  yield yieldval_bldr.call(kva,et,[n1,n2].freeze,false) if block_given?
+                 }
+       return entity_count
+  end
+
+
+  # Bulk parse is a convenience method for parsing a source of records.  It is essentially
+  # the same as instantiating an object using the first record and then calling parse multiple times
+  #  
+  # For information on most of the parameters, see the #parse method
+  #
+  # @param rec_source [Enumerable<Array<String>>] first record is presumed to be
+  #                      the header and all other lines will be forced into the #parse
+  #                      method.
+  # @return [Integer, Enumerator]  If a block is passed in, returns the total of all
+  #                      entities that were yielded from the source.  Otherwise
+  #                      returns an enumerator.
+  #
+  def self.bulk_parse(rec_source,source: nil,metadata: nil, metadata_key_prefix: nil, &entity_receiver)
+      return enum_for(:bulk_parse,rec_source, source:, metadata:, metadata_key_prefix:) unless block_given?
+      hdr_vals = rec_source.next
+      specs = InputColumnSpecs.new(hdr_vals)
+      rec_count = 0
+      begin
+          loop do
+              next_rec = rec_source.next
+              rec_count += specs.parse(next_rec,source:, ref_num: rec_count+2,metadata:,&entity_receiver)
+          end
+      rescue StopIteration
+          #no-op
+      end
+      return rec_count
+  end
+  
+  # Utility class returned by the #make_pattern_match_verifier() method
+  #
+  # It holds tests that can be used to confirm whether a pattern matches
+  # aspects of a given node.
+  #
+  # Example Pattern Strings:
+  #    1) "?/^alpha/"   matches type == :node where key starts with "alpha"
+  #    2) "beta"        mates type == :node where key is exactly "beta"
+  #
+  class PatternMatchVerifier
+      ALWAYS_TRUE_PROC = Proc.new{true}
+      def initialize(pattern_string)
+          @non_id_test = ALWAYS_TRUE_PROC 
+          @id_test = nil 
+          @pattern_string = pattern_string
+          case pattern_string
+            when /^\s*\?\s*\/(.*)\/\s*$/   
+                rx = Regexp.new($1)
+                @id_test = Proc.new{|id| rx.match?(id)}
+            else
+                exact_id = pattern_string.strip # match with the exact (trimmed) string
+                @id_test = Proc.new{|id| id == exact_id }
+          end   
+      end #initialize
+    
+        
+      def inspect = "#<#{self.class} 0x#{object_id} pattern_string=#{@pattern_string.inspect}> "
+        
+    
+        # Exclusively test whether the given node id would be acceptable for this 
+        # verifier.
+        #
+        # @param test_id_string [String]
+        def id_match?(test_id_string) = @id_test.call(test_id_string)
+          
+        # Exclusively test whether any of the non-id aspects of the node would be
+        # acceptable for this verifier.
+        # @param node_entity_packet [Nodepile::EntityPacket]
+        def non_id_match?(node_entity_packet) = @non_id_test.call(node_entity_packet)
+        
+        # Perform both the id_match?() and return their logical AND
+        def match?(nep) = id_match?(nep.key) && non_id_match?(nep)
+  end #class PatternMatchVerifier
+  
+  # "Rule" type entities are characterized by having one or more "patterns"
+  # that are used to determine which of the nodes a given rule should apply to.
+  # Most often, the patterns specify sets of node IDs would satisfy them
+  # such as through regular expression matching.  However, future instances
+  # may use field values to determine matching.
+  #
+  # For explanation of pattern logic see the PatternMatchVerifier class
+  #
+
+  def self.make_pattern_match_verifier(pattern_string)
+      return PatternMatchVerifier.new(pattern_string)
+  end
+  
+  
+  
+
+  private
+  
+
+  def _preprocs
+      @mc.cache(__method__){||
+          # collect preproc relevant for the columns present
+          my_preprocs = Array.new
+          @col_names.each_with_index{|nm,ix|
+                self.class._all_preprocs[nm]&.tap{|(skip_nil,block)|
+                           my_preprocs << [ix,block].freeze
+                          }
+               }
+          my_preprocs.freeze  # will get cached
+         } # end cache calculator
+         
+  end # _preprocs()
+  
+  def _validators
+      @mc.cache(__method__){||
+            my_validators = Array.new
+            # collect validators relevant for the columns present
+            self.class._all_validators.each{|(always,vl_col_names,block)|
+                   vl_col_nums = vl_col_names.map{|nm| @col_names.find_index(nm) } 
+                   if always || vl_col_nums.none?{|v| v.nil?}
+                     my_validators << [vl_col_nums.freeze,block].freeze
+                   end
+                  }
+          my_validators.freeze  # this should get cached
+         } # end cache calculator
+  end
+
+  
+
+  # A validator is a block used to verify the values in a specific
+  # set of fields.  The blocks registered here are compiled into calls
+  # to InputColumnSpecs#new.  A validator block should evaluate to nil
+  # if everything is okay.  If it evaluates to a string, that string may be
+  # communicated to users as a validation failure.
+  #
+  # @param always [true,false,nil] Indicates that the validator should
+  #                              be run regardless of whether all fields
+  #                              are present.  Nils will be passed
+  #                              to the validator for missing fields.
+  # @param col_name_array [Array<String>] These fields must be passed
+  #                                       in this order to the block to
+  #                                       perform the validation.
+  def self._make_validator(col_name_array,always: nil, &validator_block)
+    [always,col_name_array.dup.freeze,validator_block].freeze
+  end
+
+  # Package up field preprocessing into a record for later use
+  # @param skip_nil [Boolean] if true, does no preprocessing if the field value
+  #                     is nil
+  def self._make_field_preproc(col_name, skip_nil: true, &preproc_block)
+    return [col_name,skip_nil ? Proc.new{|s| s && preproc_block.(s)} : preproc_block]
+  end
+
+  # Package up preprocs for a field using some standard rules.  Multiple
+  # rules may apply to the same field.
+  #
+  #    * :strip  will cause leading and trailing spaces to be removed and
+  #              blank fields will be set to nil
+  #    * :downcase will cause field contents to be downcased
+  # @param col_name [String] name of the column the preproc applies to
+  # @param std_syms [Array<Symbol>,nil] one or more symbols representing
+  #                    the preprocs that should be combined.  They area applied
+  #                    in the specified order although is :strip is present it
+  #                    must appear first.  Method is a no-op of the std_syms is
+  #                    nil
+  def self._make_standard_preproc(col_name,std_syms)
+    return nil if std_syms.nil?
+    nproc = Proc.new{|s|
+        std_syms.each{|instr|
+            case instr
+                when :downcase then s&.downcase!
+                when :strip then  s = nil if (s.strip!||s) == ''
+                else
+                  raise "Unrecognized preproc found [#{proc_sym.inspect}]"
+            end
+           } #each instruction
+        next s  # "return value"
+       } #nproc
+    _make_field_preproc(col_name,skip_nil: true,&nproc)
+  end
+  
+  @@class_mcache = CrudeCalculationCache.new
+    
+  def self._all_preprocs
+      @@class_mcache.cache(__method__){||
+          h = Hash.new  # append to this array
+          # generate preprocs using the flags in the YAML at bottom of this file
+          coldefs.each_pair{|fieldname,fielddata|
+                          h[fieldname] = _make_standard_preproc(fieldname,fielddata['preproc']&.map(&:to_sym)) 
+                       }
+          h.freeze # this Hash will get cached
+         } # end cache calculator
+  end # _all_preprocs()
+  
+  def self._all_validators
+      @@class_mcache.cache(__method__){||
+          a = Array.new
+          a << _make_validator(['_id','_links_from','_links_to'],always: true){|id,lf,lt|
+                  if id.nil? && (lf.nil? ^ lt.nil?)
+                      next 'If the _id field is blank, both _links_from and links_to fields must be blank or both populated'
+                  end
+                  if id&.start_with?('?') && (lf || lt)
+                      next 'If the _id field indicates a :rule, _links_from and _links_to must be blank'
+                  end
+                  if id && (lf&.start_with?('?') || lt&.start_with?('?'))
+                      next "If the _id field is populated, you may not put a rule formula in _links_from or _links_to"
+                  end
+                  next nil
+                }
+          a.freeze # this Array will get cached
+         } # end cache calculator
+    
+  end # _all_validators()
+  
+    
+
+
+end # class InputColumnSpecs
+
+end # module Nodepile
+
+  # Below are the column spec to be used for documentation and to some degree 
+__END__
+---
+data:
+  fields:
+      _id:
+        description: >
+          Required column in any input file. Can be one of three value types.
+          If it starts with a literal asterisk character or with a literal
+          forward slash character, it indicates the line is a
+          style instruction.  If it is blank or whitespace, it indicates the
+          line defines an edge or edge style instruction.  Any other value 
+          indicates that this line defines a node and the value in this column
+          is interpreted as a unique identifier (node_id) that can be used on 
+          other lines to reference this node.  Unless otherwise overridden, the 
+          _id is used to label the node.  Note that your life may be happier
+          if you forbid using commas as part of _id values although it is not
+          forbidden.
+        preproc:
+          - strip
+    
+      _links_from: 
+        description: >
+          Required if the _id field has been left blank.  Specifies one or
+          more node_id values separated by valid delimiter characters.  If the
+          first character is an asterisk or forward slash, it indicates that this
+          is a edge styling instruction.  Otherwise, this is used to indicate
+          the existence of one or more edges originating from the specified node.
+        preproc:
+          - strip
+    
+      _links_to: 
+        description: Follows same protocol as _link_from.
+        preproc:
+          - strip
+    
+      _label: 
+        description: >
+          Indicates a (typically short) label that should appear rather than 
+          value of the _id for nodes and edges.  For nodes, see also _labelNN
+          which allows specifying node labels in a line-by-line format.
+        #preproc:   # no preproc for this one... deliberate blanks may be meaningful
+    
+      _labelNN: 
+        description: >
+          If present and non-blank, this value supercedes any text in _label
+          column.
+          When a column with this pattern is specified it should replace
+          _labelNN with a integer such as _label3 or _label22 to indicate
+          that the provided text appears on line 3 or line 22 respectively.
+    
+      _color:
+        decription: >
+          For nodes, color is the border color of the shape.  For edges, this is
+          the actual edge color.  There are a very wide variety of ways that
+          color can be specified.  Any format supported by the DOT language is
+          permitted.  The rock bottom simplest is to use the supported set of
+          simple color words like red, blue, etc.
+        dot_ref: https://graphviz.org/docs/attrs/color/
+        preproc:
+          - strip
+          - downcase
+      
+      _fillcolor:
+        description: >
+          For nodes, fillcolor is the background color of the shape.
+        dot_ref: https://graphviz.org/docs/attrs/fillcolor/
+        preproc:
+          - strip
+          - downcase
+          
+      _fontcolor:
+        description: >
+          For many entities, defines the text color.
+        dot_ref: https://graphviz.org/docs/attrs/fontcolor/ 
+        preproc:
+          - strip
+          - downcase
+          
+          
+      _shape:
+        description: >
+          For nodes, determines the shape of the node.  Shape names tend to be
+          either simple things like (box, plain, plaintext, circle, ellipse, 
+          etc.) or else it is a record type that is meant to render data
+          in a structured layout.
+        dot_ref: https://graphviz.org/docs/attr-types/shape/
+        preproc:
+          - strip
+          - downcase
+          
+          
+  
+
+
+
+
+

data/lib/nodepile/gross_actions.rb

@@ -0,0 +1,38 @@
+require 'nodepile/gviz.rb'
+require 'nodepile/pile_organizer.rb'
+
+module Nodepile
+  
+# A set of large scale, batch-like operations   
+module GrossActions
+    
+# Given an output filepath and one or more tabular datafile inputs, render
+# the visualization of the input datafiles.
+# @param output_filepath [String] location where the output file should be
+#                              written.  Note, if the file already exists it
+#                              will be overwritten.  Note that the file
+#                              suffix indicates the output format that will
+#                              be used.  Valid extensions are: jpg, dot, gif,
+#                              svg, png, json, pdf
+#            
+# @param input_filepaths [Array<String>] One or more input files.  Note that the
+#                              program will take a best guess at the text file
+#                              format with file suffix (csv, tsv) being the
+#                              a strong signal.  The order of the filenames
+#                              is used as their load order.
+def self.render_to_file(output_filepath, *input_filepaths)
+    pile = Nodepile::PileOrganizer.new
+    gviz = Nodepile::GraphVisualizer.new
+    input_filepaths.each{|fpath|
+        raise "File not found: #{fpath}" unless File.exist?(fpath)
+        pile.load_from_file(fpath)
+        gviz.load(pile.node_records,pile.edge_records,configs: pile.pragmas)
+        gviz.emit_file(output_filepath,configs: pile.pragmas)
+       }
+    return nil  # for now, return is meaningless
+end #render_to_file()
+    
+    
+end #module GrossActions
+
+end #module Nodepile