suffix_tree 0.0.1

data/lib/node_factory.rb

@@ -0,0 +1,169 @@
+require_relative 'node'
+
+class NodeFactory
+  attr_reader :nextNodeId, :root
+  attr_reader :dataSource
+  attr_reader :configuration
+  attr_reader :db
+
+  def initialize(dataSource, persister=nil)
+    @dataSource = dataSource
+    @suffixOffset = 0
+    @configuration = {
+        :leafCount => false,
+        :valueDepth => false,
+        :previousValue => false,
+        :dataSourceBit => false
+    }
+    @db = persister
+    self.reset
+  end
+
+  def reset
+    @nextNodeId = 1
+  end
+
+  def nextDataSourceBit
+    @dataSourceBit = (@dataSourceBit << 1) if ((@configuration[:dataSourceBit]) && (@dataSource != nil))
+  end
+
+  def extendDataSource(dataSource, startOffset)
+    self.nextDataSourceBit
+    if (@dataSource == nil) then
+      @dataSource = dataSource
+    else
+      @dataSource.extendWith(dataSource, startOffset)
+    end
+  end
+
+  def nextDataSourceSetSize(modForSwitch)
+    @nextDataSourceSwitch = modForSwitch
+  end
+
+  def setConfiguration configurationHash
+    configurationHash.each do |key, value|
+      @configuration[key] = value
+    end
+    self
+  end
+
+  def newRoot
+    self.reset
+    result = newNode
+    result.children = {}
+    @root = result
+    @configuration.each do |key, value|
+      if (value) then
+        @root.createAccessor(key.to_s)
+      end
+    end
+
+    # configuration controlled accessors
+    @root.valueDepth = 0 if @configuration[:valueDepth]
+    @root.leafCount = 0 if @configuration[:leafCount]
+    @dataSourceBit = 1 if @configuration[:dataSourceBit]
+    @root.dataSourceBit = @dataSourceBit if @configuration[:dataSourceBit]
+
+    persist(result)
+  end
+
+
+  #
+  #  The algorithm adds leaf nodes in order
+  #
+  def addLeaf(node, value, offset)
+    result = newChild(node, value, @suffixOffset, offset, Node::CURRENT_ENDING_OFFSET)
+
+    # optional configuration based properties
+    result.leafCount = 1 if (@configuration[:leafCount])
+    result.previousValue = (@dataSource.valueAt(@suffixOffset - 1)) if ((@suffixOffset > 0) && @configuration[:previousValue])
+    result.dataSourceBit = @dataSourceBit if @configuration[:dataSourceBit]
+    @suffixOffset += 1
+    if ((@nextDataSourceSwitch != nil) && ((@suffixOffset % @nextDataSourceSwitch) == 0)) then
+      self.nextDataSourceBit
+    end
+
+    persist(result)
+  end
+
+  def splitEdgeAt(node, incomingEdgeOffset)
+    result = newChild(node.parent, @dataSource.valueAt(node.incomingEdgeStartOffset), node.suffixOffset, node.incomingEdgeStartOffset, incomingEdgeOffset - 1)
+    node.incomingEdgeStartOffset = incomingEdgeOffset
+    addChild(result, @dataSource.valueAt(incomingEdgeOffset), node)
+
+    # optional configuration based properties
+    result.valueDepth = (result.parent.valueDepth + result.incomingEdgeLength) if @configuration[:valueDepth]
+    result.dataSourceBit = (node.dataSourceBit | @dataSourceBit) if @configuration[:dataSourceBit]
+
+    persist(node)
+    persist(result)
+  end
+
+  #
+  # return a sequence of all values on the path to this node
+  #
+  def valuePath(node, delimiter=' ')
+    result = []
+    while (node.parent != nil) do
+      reverseAddValues(result, node.incomingEdgeStartOffset, node.incomingEdgeEndOffset)
+      node = node.parent
+    end
+    result.reverse!
+    return result.join(delimiter)
+  end
+
+  #
+  # internal private methods
+  #
+  private
+
+  # return edge value sequence in reverse (used when getting path to root from a node)
+  def reverseAddValues(result, startOffset, endOffset)
+    if (endOffset == Node::CURRENT_ENDING_OFFSET) then
+      result << @dataSource.valueAt(startOffset)
+    else
+      scanner = endOffset
+      while (scanner >= startOffset) do
+        result << @dataSource.valueAt(scanner)
+        scanner -= 1
+      end
+    end
+  end
+
+  def newChild(node, key, suffixOffset, incomingEdgeStartOffset, incomingEdgeEndOffset)
+    child = newNode
+    child.suffixOffset = suffixOffset
+    child.incomingEdgeStartOffset = incomingEdgeStartOffset
+    child.incomingEdgeEndOffset = incomingEdgeEndOffset
+    addChild(node, key, child)
+    child.valueDepth = 0 if @configuration[:valueDepth]
+    return child
+  end
+
+  def newNode
+    result = Node.new(@nextNodeId)
+
+    # newRoot defines leafCount accessor, so that case is handled in newRoot after the node is created
+    result.leafCount = 0 if (@configuration[:leafCount] && (@nextNodeId > 1))
+    result.dataSourceBit = @dataSourceBit if (@configuration[:dataSourceBit] && (@nextNodeId > 1))
+    @nextNodeId += 1
+    return result
+  end
+
+  def addChild(parentNode, value, childNode)
+    if (parentNode.children == nil) then
+      parentNode.children = {}
+    end
+    parentNode.children[value] = childNode
+    childNode.parent = parentNode
+    persist(parentNode)
+    persist(childNode)
+  end
+
+  def persist(node)
+    if (@db != nil) then
+      @db.persist(node)
+    end
+    node
+  end
+end

data/lib/persist/suffix_tree_db.rb

@@ -0,0 +1,148 @@
+require_relative '../node'
+
+class SuffixTreeDB
+  def initialize(textFile)
+    @textFile = File.open(textFile, "w")
+    @dataValues = []
+    @dataValueIdx = 0
+  end
+
+  def val(node)
+    if (node == nil) then
+      return 0
+    else
+      return node.nodeId
+    end
+  end
+
+  def persist(node)
+    @textFile.print "#{node.nodeId} #{val(node.parent)} #{node.incomingEdgeStartOffset} #{node.incomingEdgeEndOffset} #{node.suffixOffset} #{val(node.suffixLink)}"
+    if (node.children != nil) then
+      node.children.values.each do |childNode|
+        @textFile.print " #{childNode.nodeId}"
+      end
+    end
+    @textFile.print " 0\n"
+  end
+
+  def readInt()
+    if (@dataValueIdx >= @dataValues.length) then
+      if (@textFile.eof?) then
+        return 0
+      end
+      line = @textFile.readline()
+      if (line == nil) then
+        return 0
+      else
+        line.chomp!
+        @dataValueIdx = 0
+        @dataValues = line.split
+      end
+    end
+
+    result = @dataValues[@dataValueIdx].to_i
+    @dataValueIdx += 1
+    return result
+  end
+end
+
+class SuffixTreeBuilder
+  attr_reader :suffixCount
+
+  def initialize(stdb, dataSource)
+    @suffxTreeDB = stdb
+    @dataSource = dataSource
+    @root = nil
+    @unresolvedParents = {}
+    @unresolvedSuffixLinks = {}
+    @unresolvedChildren = {}
+    @allNodes = {}
+  end
+
+  def buildNode
+    nodeId = @suffxTreeDB.readInt()
+    if (nodeId > 0) then
+      node = resolveNodeId(nodeId)
+      resolve(nodeId, node)
+      @allNodes[nodeId] = node
+      @root = node if (@root == nil)
+      resolveParent(node, @suffxTreeDB.readInt())
+      node.incomingEdgeStartOffset = @suffxTreeDB.readInt()
+      node.incomingEdgeEndOffset = @suffxTreeDB.readInt()
+      @suffixCount = node.suffixOffset = @suffxTreeDB.readInt()
+      resolveSuffixLink(node, @suffxTreeDB.readInt())
+      childNodeId = @suffxTreeDB.readInt()
+      while (childNodeId != 0) do
+        resolveChild(node, childNodeId)
+        childNodeId = @suffxTreeDB.readInt()
+      end
+      return node
+    end
+    return false
+  end
+
+  private
+
+  def resolveParent(node, nodeId)
+    if (@allNodes.has_key?(nodeId)) then
+      node.parent = @allNodes[nodeId]
+    else
+      resolveEntry(node, nodeId, @unresolvedParents)
+    end
+  end
+
+  def resolveSuffixLink(node, nodeId)
+    if (@allNodes.has_key?(nodeId)) then
+      node.suffixLink = @allNodes[nodeId]
+    else
+      resolveEntry(node, nodeId, @unresolvedSuffixLinks)
+    end
+  end
+
+  def resolveChild(node, nodeId)
+    if (@allNodes.has_key?(nodeId)) then
+      childNode = @allNodes[nodeId]
+      if (node.children == nil) then
+        node.children = {}
+      end
+      node.children[@dataSource.valueAt(childNode.incomingEdgeStartOffset)] = childNode
+    else
+      resolveEntry(node, nodeId, @unresolvedChildren)
+    end
+  end
+
+  def resolveEntry(node, nodeId, theList)
+    if (nodeId > 0) then
+      theList[nodeId] = node
+    end
+  end
+
+  def resolveNodeId(nodeId)
+    if @allNodes.has_key?(nodeId) then
+      @allNodes[nodeId]
+    else
+      Node.new(nodeId)
+    end
+  end
+
+  def resolve(nodeId, node)
+    if (@unresolvedParents.has_key?(nodeId)) then
+      print "Unresolved parent value #{nodeId}\n"
+      @unresolvedParents[nodeId].parent = node
+      @unresolvedParents.delete(nodeId)
+    end
+    if (@unresolvedChildren.has_key?(nodeId) && (node.incomingEdgeStartOffset >= 0)) then
+      unfinishedNode = @unresolvedChildren[nodeId]
+      if (unfinishedNode.children == nil) then
+        unfinishedNode.children = {}
+      end
+      unfinishedNode.children[@dataSource.valueAt(node.incomingEdgeStartOffset)] = node
+      @unresolvedChildren.delete(nodeId)
+    end
+    if (@unresolvedSuffixLinks.has_key?(nodeId)) then
+      unfinishedNode = @unresolvedSuffixLinks[nodeId]
+      unfinishedNode.suffixLink = node
+      @unresolvedSuffixLinks.delete(nodeId)
+    end
+  end
+end

data/lib/search/searcher.rb

@@ -0,0 +1,68 @@
+require_relative '../visitor/bfs'
+require_relative '../visitor/suffix_offset_visitor'
+require_relative '../data/string_data_source'
+
+#
+#  Searcher finds matches in a tree
+#
+#  It needs the tree root, and the data source used to make the tree
+#  This assumes the tree was made with a single data source.
+#
+#  "find" really should be finding matches from a different data source (not a string)
+#
+class Searcher
+  def initialize(treeDataSource, treeRoot)
+    @dataSource = treeDataSource
+    @root = treeRoot
+  end
+
+  #
+  #  match dataSource values, return location in the suffix tree where the match stopped
+  #
+  def matchDataSource(dataSource)
+    location = Location.new(@root)
+    location.matchDataSource(@dataSource, dataSource)
+    location
+  end
+
+  def findNode(dataSource)
+    location = Location.new(@root)
+    if (location.matchDataSource(@dataSource, dataSource).depth == dataSource.numberValues) then
+      return location.node
+    else
+      return nil
+    end
+  end
+
+  #
+  #  returns the list of suffix offset values where the searchString has been found
+  #
+  def findString(searchString)
+    node = self.findNode(StringDataSource.new(searchString))
+    return self.findResults(node)
+  end
+
+
+  def findWord(searchString)
+    node = self.findNode(SingleWordDataSource.new(searchString))
+    return self.findResults(node)
+  end
+
+  def findResults(node)
+    if (node != nil) then
+      soCollector = SuffixOffsetVisitor.new
+      so = BFS.new(soCollector)
+      so.traverse(node)
+      return soCollector.result.sort
+    else
+      return []
+    end
+  end
+
+  # match a string starting at a specific location,
+  # returning the character depth of the resulting match
+  def findAtLocation(location, s)
+    location.matchString(@dataSource, s)
+    return location.depth
+  end
+end

data/lib/suffix_linker.rb

@@ -0,0 +1,16 @@
+class SuffixLinker
+
+  def update(location)
+    if ((@nodeNeedingSuffixLink != nil) && (location.node != @nodeNeedingSuffixLink) && location.onNode) then
+      @nodeNeedingSuffixLink.suffixLink = location.node
+      @nodeNeedingSuffixLink = nil
+    end
+  end
+
+  def nodeNeedingSuffixLink(node)
+    if (@nodeNeedingSuffixLink != nil) then
+      @nodeNeedingSuffixLink.suffixLink = node
+    end
+    @nodeNeedingSuffixLink = node
+  end
+end

data/lib/suffix_tree.rb

@@ -0,0 +1,122 @@
+require_relative 'location'
+require_relative 'node_factory'
+require_relative 'suffix_linker'
+
+#
+# Builds a suffix tree from one or more DataSource instances
+#
+class SuffixTree
+  NO_SUFFIX_OFFSET = -1
+
+  # first data source we use
+  attr_reader :rootDataSource
+
+  # when there are a sequence of data sources, treat them as one long one, this is where next source starts
+  attr_reader :startOffset
+
+  # where we are in the implicit tree building process
+  attr_reader :location
+
+  attr_reader :nodeFactory
+
+  # the root of the tree, and the terminal value (for making implicit trees explicit)
+  attr_reader :root, :terminalValue
+
+  # keep track of which nodes need suffix links
+  attr_reader :suffixLinker
+
+  def initialize(terminalValue = nil, configuration = nil, persister = nil)
+    @nextNodeId = 0
+    @nodeFactory = NodeFactory.new(nil, persister)
+    @nodeFactory.setConfiguration(configuration) if (configuration != nil)
+    @root = @nodeFactory.newRoot()
+    @rootDataSource = nil
+    @location = Location.new(@root)
+    @startOffset = 0
+    @suffixOffset = 0
+    @suffixLinker = SuffixLinker.new
+    @terminalValue = terminalValue
+  end
+
+  #
+  #  Set the data source, but do not add any values from the data source
+  #
+  def setDataSource(dataSource)
+    if (@rootDataSource == nil) then
+      @rootDataSource = dataSource
+    end
+    @nodeFactory.extendDataSource(dataSource, @startOffset)
+  end
+
+  #
+  #  Add all values in a given dataSource
+  #
+  def addDataSource(dataSource)
+    @suffixOffset = 0
+    self.setDataSource(dataSource)
+    dataSource.each_with_index(@startOffset) do |value, offset|
+      self.addValue(value, offset)
+    end
+    if (@terminalValue != nil) then
+      @lastOffsetAdded += 1
+      self.addValue(@terminalValue, @lastOffsetAdded)
+    end
+    @startOffset = @lastOffsetAdded + 1
+  end
+
+  #
+  #  Adding one value at a time, rootDataSource must be set for this to work
+  #
+  def addValue(value, offset)
+    while (extend(value, offset)) do
+      @suffixLinker.update(@location)
+    end
+    @lastOffsetAdded = offset
+  end
+
+  #
+  #  Finish building the tree by adding a value that is not part of the data source
+  #
+  def finish()
+    if (@rootDataSource.has_terminator?) then
+      self.addValue(@rootDataSource.terminator, @startOffset)
+    end
+  end
+
+  #
+  #  Extend a single suffix at the current location, returns true if there is another
+  #  suffix to extend.
+  #
+  #  Handles these cases:
+  #
+  #     On a node:
+  #        if there is a child starting with the extension value, traverse down that one value, return FALSE
+  #        if no child has the extension value, add a leaf,
+  #            if we are at root, return FALSE,
+  #            otherwise traverse to the next suffix and return TRUE
+  #
+  #     On an edge:
+  #        if next character has the value, traverse past it, return FALSE
+  #        if next character is not the value, split edge at that location, locate at the new node, and return TRUE
+  #
+  def extend(value,offset)
+    if (@location.onNode)
+      if (@location.node.children.has_key?(value)) then
+        @location.traverseDownChildValue(value)
+        return false  # rule 3
+      else
+        @nodeFactory.addLeaf(@location.node, value, offset)
+        return @location.traverseToNextSuffix(@rootDataSource)  # rule 1, traverse returns false when at root
+      end
+    elsif (@rootDataSource.valueAt(@location.incomingEdgeOffset) == value) then
+      @location.traverseDownEdgeValue()
+      return false   # found value on edge, rule 3
+    else
+      newNode = @nodeFactory.splitEdgeAt(@location.node, @location.incomingEdgeOffset)
+      @suffixLinker.nodeNeedingSuffixLink(newNode)
+      @location.jumpToNode(newNode)
+      return true   # rule 2
+    end
+  end
+
+end

data/lib/visitor/base_visitor.rb

@@ -0,0 +1,17 @@
+class BaseVisitor
+  attr_accessor :preCounter, :postCounter
+
+  def initialize
+    @preCounter = 0
+    @postCounter = 0
+  end
+
+  def preVisit(node)
+    @preCounter += 1
+    return true
+  end
+
+  def postVisit(node)
+    @postCounter += 1
+  end
+end

data/lib/visitor/bfs.rb

@@ -0,0 +1,22 @@
+class BFS
+  def initialize(visitor)
+    @visitor = visitor
+    @q = Array.new
+  end
+
+  def traverse(node)
+    @q.unshift(node)
+
+    while (@q.size > 0) do
+      node = @q.pop
+      if (@visitor.preVisit(node)) then
+        if (node.children != nil)  then
+          node.children.values.each do |child|
+            @q.unshift(child)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/visitor/data_source_visitor.rb

@@ -0,0 +1,15 @@
+require_relative 'base_visitor'
+
+class DataSourceVisitor < BaseVisitor
+  def initialize
+    super
+  end
+
+  def postVisit(node)
+    if (node.children != nil) then
+      node.children.values.each do |child|
+        node.dataSourceBit |= child.dataSourceBit
+      end
+    end
+  end
+end

data/lib/visitor/dfs.rb

@@ -0,0 +1,34 @@
+class DFS
+  def initialize(visitor)
+    @visitor = visitor
+  end
+
+  def traverseChildren(children)
+    if (children != nil)
+      children.each do |key,value|
+        self.traverse(value)
+      end
+    end
+  end
+
+  def traverse(node)
+    if (@visitor.preVisit(node)) then
+      self.traverseChildren(node.children)
+      @visitor.postVisit(node)
+    end
+  end
+end
+
+class OrderedDFS < DFS
+  def initialize(visitor)
+    super(visitor)
+  end
+
+  def traverseChildren(children)
+    if (children != nil)
+      children.keys.sort.each do |key|
+        self.traverse(children[key])
+      end
+    end
+  end
+end

data/lib/visitor/k_common_visitor.rb

@@ -0,0 +1,71 @@
+class ValueRange
+
+  attr_accessor :startOffset, :endOffset
+
+  def initialize(startOffset, endOffset)
+    @startOffset = startOffset
+    @endOffset = endOffset
+  end
+
+  def length
+    return @endOffset - @startOffset + 1
+  end
+
+end
+
+class KCommonVisitor < BaseVisitor
+
+  def initialize(dataSource)
+    @dataSource = dataSource
+
+    #
+    # key = common to at least this many (2, 3, ...)
+    # value = [ startOffset, endOffset ] of value sequence
+    #
+    @commonTo = {}
+
+    #
+    # set up initial values
+    #
+    (0..64).each do |value|
+      @commonTo[value] = ValueRange.new(0,-1)
+    end
+    super()
+  end
+
+  def postVisit(node)
+    nCommon = self.countCommon(node.dataSourceBit)
+    currentCommonLength = @commonTo[nCommon].endOffset - @commonTo[nCommon].startOffset + 1
+    if (node.valueDepth > currentCommonLength) then
+      @commonTo[nCommon].startOffset = node.incomingEdgeEndOffset - node.valueDepth + 1
+      @commonTo[nCommon].endOffset = node.incomingEdgeEndOffset
+      if (nCommon > 2) then
+        longestLength = node.valueDepth
+        (1..(nCommon-1)).each do |offset|
+          testLength = @commonTo[offset].endOffset - @commonTo[offset].startOffset + 1
+          if (testLength < longestLength) then
+            @commonTo[offset].startOffset = @commonTo[nCommon].startOffset
+            @commonTo[offset].endOffset = @commonTo[nCommon].endOffset
+          end
+        end
+      end
+    end
+  end
+
+  def longestStringCommonTo(numberInCommon)
+    return @commonTo[numberInCommon].length, @dataSource.valueSequence(@commonTo[numberInCommon].startOffset, @commonTo[numberInCommon].endOffset)
+  end
+
+  def countCommon(bits)
+    result = 0
+    scanner = 1
+    bits = bits.to_i
+    (1..32).each do
+      if ((scanner & bits) != 0) then
+        result += 1
+      end
+      scanner = scanner << 1
+    end
+    result
+  end
+end

data/lib/visitor/leaf_count_visitor.rb

@@ -0,0 +1,15 @@
+require_relative 'base_visitor'
+
+class LeafCountVisitor < BaseVisitor
+  def initialize
+    super
+  end
+
+  def postVisit(node)
+    if (node.children != nil) then
+      node.children.values.each do |child|
+        node.leafCount += child.leafCount
+      end
+    end
+  end
+end