dawg 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 243a3162eee48baf49a16fc03b7950b55b6583fb
-  data.tar.gz: 21e7e03637c9418a297dc218b37449989e0da18d
+  metadata.gz: 44e60a8bd509c089b4b3d8d251491d798a5fac8b
+  data.tar.gz: 69e7885dfcbaadda4167d55abc7c678227a9c395
 SHA512:
-  metadata.gz: 283153800b269b13e87b169b0972221612eaa9c4e76b9ebb783985ce7b62dad3a4bcde2c17a33364a653daec504376c828816774d37d13765b7a647df193d653
-  data.tar.gz: e5ef8892af3655be6eff55abbbac3701b2e51aa0dd67b8a3b3063ca5ea6f7bdc5b61f0bac865d173d8769c861022cad04ec4f00cbd6f17cd0e12768134752fe2
+  metadata.gz: ecd9e5bf52c4211e139933f64bf769444d3370f41e74163b3868cfa6959afec72798cf86e300da850a63b4eed4eec88e3507592ee744fc145c98836bea47c8fa
+  data.tar.gz: c7d5bec04d7d9567f1cb0db9002172b4fa3a8879a5116ea8bb5fc29492a144faef3f3a6d2d36232c3759e59ef03022a4af041b34ce82280b651e08a577362ab4

data/lib/dawg.rb

@@ -1,167 +1,30 @@
-class DawgNode
-  @@next_id = 0
-  attr_accessor :edges,:final,:id
-  def initialize
-    @id = @@next_id
-    @@next_id += 1
-    @final = false
-    @edges = {}
-  end
-
-  def to_s
-    arr = []
-    if @final
-        arr<<"1"
-    else
-        arr<<"0"
-    end
+require 'stringio'
+require_relative 'dawg/serialization'
+require_relative 'dawg/finder'
+require_relative 'dawg/word'
+require_relative 'dawg/node/node'
+require_relative 'dawg/node/memory_node'
+require_relative 'dawg/dawg/dawg'
+require_relative 'dawg/dawg/memory_dawg'
 
-    for (label, node) in @edges
-        arr << label
-        arr << node.id.to_s
-    end
+module Dawg
+  NODE_START = 8
+  NODE_SIZE = 21
+  EDGE_SIZE = 16
+  extend self
 
-    arr.join("_")
+  def new
+    Dawg.new
   end
 
-  def hash
-      to_s.hash
-  end
-
-  def eql?(other)
-      to_s == other.to_s
-  end
-  def inspect
-    "to_s"
-  end
-end
-
-class Dawg
-  def initialize
-    @previousWord = ""
-    @root = DawgNode.new
-
-    # Here is a list of nodes that have not been checked for duplication.
-    @uncheckedNodes = []
-
-    # Here is a list of unique nodes that have been checked for
-    # duplication.
-    @minimizedNodes = {}
-  end
-
-  def save(filename)
-    data = Marshal.dump(self)
-    File.open(filename, 'w') { |file| file.write(data) }
-  end
-
-  def self.load(filename)
-    dawg = Marshal.load( File.open(filename).read )
-  end
-
-  def insert( word )
-    if word < @previousWord
-        raise "Error: Words must be inserted in alphabetical order."
-    end
-
-    # find common prefix between word and previous word
-    commonPrefix = 0
-    for i in 0..[word.length-1, @previousWord.length-1].min
-      break if word[i] != @previousWord[i]
-      commonPrefix += 1
-    end
-
-    # Check the uncheckedNodes for redundant nodes, proceeding from last
-    # one down to the common prefix size. Then truncate the list at that
-    # point.
-    _minimize( commonPrefix )
-
-    # add the suffix, starting from the correct node mid-way through the
-    # graph
-    if @uncheckedNodes.length == 0
-      node = @root
-    else
-      node = @uncheckedNodes[-1][2]
-    end
-
-    for letter in word.split("")[commonPrefix..-1]
-      nextNode = DawgNode.new
-      node.edges[letter] = nextNode
-      @uncheckedNodes<< [node, letter, nextNode]
-      node = nextNode
+  def load(filename, type = :small)
+    return case type
+    when :small
+      MemoryDawg.load(filename)
+    when :fast
+      Dawg.load(filename)
     end
-
-    node.final = true
-    @previousWord = word
-  end
-  def finish
-    # minimize all uncheckedNodes
-    _minimize( 0 )
-  end
-
-  def _minimize(downTo)
-    # proceed from the leaf up to a certain point
-    for i in (@uncheckedNodes.length - 1).downto(downTo)
-        parent, letter, child = @uncheckedNodes[i]
-        if @minimizedNodes.has_key? child
-            # replace the child with the previously encountered one
-            parent.edges[letter] = @minimizedNodes[child]
-        else
-            # add the state to the minimized nodes.
-            @minimizedNodes[child] = child
-        end
-        @uncheckedNodes.pop
-    end
-  end
-
-  def lookup(word)
-    node = @root
-    for letter in word.split("")
-        return false if !node.edges.has_key? letter
-        node = node.edges[letter]
-    end
-    node.final
-  end
-
-  def find_similar(word)
-    node = @root
-    for letter in word.split("")
-        return [] if !node.edges.has_key? letter
-        node = node.edges[letter]
-    end
-    results = get_recuirsively_all(node)
-
-    return [word].product(results).map(&:join)
+    dawg
   end
- 
-  def get_recuirsively_all(node)
-    suffixes = []
 
-    node.edges.each do |key,value|
-      results = get_recuirsively_all(value)
-
-      # result.flatten! if result.length==2
-      results.each do |result|
-        suffixes << [[key] + [result]].flatten.join
-      end
-
-      suffixes << key if results.empty?
-
-
-    end
-    return suffixes
-  end
-  def nodeCount
-    @minimizedNodes.length
-  end
-
-  def edgeCount
-    count = 0
-    for key,node in @minimizedNodes
-      count += node.edges.length
-    end
-    count
-  end
-  def inspect
-    "Dawg"    
-  end
-end
+end

data/lib/dawg/dawg/dawg.rb

@@ -0,0 +1,143 @@
+module Dawg
+  class Dawg
+    extend Serialization
+    include Serialization
+    include Finder
+
+    attr_accessor :minimized_nodes, :root
+
+    def initialize
+      @previous_word = ''
+      @root = Node.new
+      @unchecked_nodes = []
+      @minimized_nodes = {}
+      set_the_node(@root)
+    end
+
+    def insert(word)
+      if word < @previous_word #TODO there's should be the way to make adding without this
+        raise 'Error: Words must be inserted in alphabetical order.'
+      end
+
+      # find common prefix between word and previous word
+      common_prefix = 0
+      (0..[word.length-1, @previous_word.length-1].min).each do |i|
+        break if word[i] != @previous_word[i]
+        common_prefix += 1
+      end
+
+      # Check the uncheckedNodes for redundant nodes, proceeding from last
+      # one down to the common prefix size. Then truncate the list at that
+      # point.
+      minimize(common_prefix)
+
+      # add the suffix, starting from the correct node mid-way through the
+      # graph
+      if @unchecked_nodes.length == 0
+        node = @root
+      else
+        node = @unchecked_nodes[-1][2]
+      end
+
+      word.split('')[common_prefix..-1].each do |letter|
+        next_node = Node.new
+        node.edges[letter] = next_node
+        @unchecked_nodes << [node, letter, next_node]
+        node = next_node
+      end
+
+      node.final = true
+      @previous_word = word
+    end
+
+    def finish
+      minimize 0
+      @minimized_nodes[@root.hash] = @root
+    end
+
+    def minimize(down_to)
+      (@unchecked_nodes.length - 1).downto(down_to) do |i|
+        parent, letter, child = @unchecked_nodes[i]
+        if @minimized_nodes.has_key? child.hash
+          parent.edges[letter] = @minimized_nodes[child.hash]
+        else
+          child.index = @minimized_nodes.size
+          @minimized_nodes[child.hash] = child
+        end
+        @unchecked_nodes.pop
+      end
+    end
+
+    def node_count
+      @minimized_nodes.length
+    end
+
+    def edge_count
+      count = 0
+      @minimized_nodes.each do |hash, node|
+        count += node.edges.length
+      end
+      count
+    end
+
+    def inspect
+      'Dawg'
+    end
+
+
+    def save(filename)
+      dawg = self
+      File.open(filename,'w') do |f|
+        write_int(dawg.node_count, f) # overall nodes count
+        write_int(dawg.edge_count, f) # overall edge count
+        edges_pos = 0
+        dawg.minimized_nodes.each do |hash, node|
+          write_int(edges_pos, f)
+          write_int(node.edges.keys.length, f)
+          write_int(node.id, f)
+          write_bool(node.final, f)
+          write_bigint(hash, f)
+          edges_pos += EDGE_SIZE * node.edges.keys.length # position of node's edges in a file
+        end
+        dawg.minimized_nodes.each do |hash, node|
+          node.edges.each do |letter, n|
+            write_bigint(n.hash, f)
+            write_char(letter, f)
+            write_int(n.index,f)
+          end
+        end
+      end
+    end
+
+    def self.load(filename)
+      dawg = Dawg.new
+      File.open(filename) do |f|
+        minimized_nodes_count = load_int(f)
+        overall_edges_count = load_int(f)
+        minimized_nodes_count.times do
+          edges_pos = load_int(f)
+          edge_count = load_int(f)
+          id = load_int(f)
+          final = load_bool(f)
+          hash = load_bigint(f)
+          node = Node.new(id: id, final: final, edge_count: edge_count)
+          dawg.minimized_nodes[hash] = node
+        end
+
+        dawg.minimized_nodes.each do |hash, node|
+          node.edge_count.times do
+            hash2 = load_bigint(f)
+            letter = load_char(f)
+            node_index = load_int(f)
+            node.edges[letter] = dawg.minimized_nodes[hash2]
+          end
+        end
+        root_key = dawg.minimized_nodes.keys.last
+        dawg.minimized_nodes[root_key].edges.each do |letter, node|
+          dawg.root.edges[letter] = node
+        end
+      end
+      dawg
+    end
+  end
+end

data/lib/dawg/dawg/memory_dawg.rb

@@ -0,0 +1,69 @@
+module Dawg
+  class MemoryDawg
+    include Serialization
+    include Finder
+    attr_accessor :slice, :node_count, :edge_count
+
+    def initialize(slice)
+      @slice = slice
+      @node_count = get_node_count
+      @edge_count = get_edge_count
+      set_the_node(root)
+    end
+
+
+
+    def root
+      @root = get_node_by_index(@node_count - 1)
+    end
+
+    def make_io(start, size)
+      @slice.pos = start
+      StringIO.new(@slice.read(size))
+    end
+
+    def get_node_count
+      load_int(make_io(0,4))
+    end
+
+    def get_edge_count
+      load_int(make_io(4,4))
+    end
+
+    def get_node_by_index(index)
+      pos = NODE_START + (NODE_SIZE * index)
+      io = make_io(pos, NODE_SIZE)
+      edges_pos = load_int(io)
+      edge_count = load_int(io)
+      id = load_int(io)
+      final = load_bool(io)
+      hash = load_bigint(io)
+      MemoryNode.new(self, index, edge_count, final, hash, edges_pos)
+    end
+
+    def each_edge(index, &block)
+      pos = NODE_START + (NODE_SIZE * index)
+      io = make_io(pos, 8)
+      edges_pos = load_int(io)
+      edge_count = load_int(io)
+      edge_start = NODE_START + NODE_SIZE * @node_count
+      position =  edge_start + edges_pos
+      io = make_io(position, edge_count * EDGE_SIZE)
+
+      edge_count.times do
+        hash = load_bigint(io)
+        char = load_char(io)
+        node_index = load_int(io)
+        yield char, node_index
+      end
+    end
+
+    def self.load(filename)
+      File.open(filename) do |f|
+        slice = StringIO.new(f.read)
+        return MemoryDawg.new(slice)
+      end
+    end
+  end
+
+end

data/lib/dawg/finder.rb

@@ -0,0 +1,52 @@
+module Dawg
+  module Finder
+    def set_the_node(node)
+      @the_node = node
+    end
+
+    def lookup(word)
+      node = @the_node
+      word.each_char do |letter|
+        next_node = node[letter]
+        if next_node != nil
+          node = next_node
+          next
+        else
+          return ['']
+        end
+      end
+      node.final
+    end
+
+    # get all words with given prefix
+    def query(word)
+      node = @the_node
+      results = []
+      word.split("").each do |letter|
+        next_node = node[letter]
+        if next_node != nil
+          node = next_node
+          next
+        else
+          return ['']
+        end
+      end
+      results << Word.new(word, node.final)
+      results += get_childs(node).map{|s| Word.new(word) + s}
+      results.select{|r| r.final}.map{|r| r}
+    end
+
+    def get_childs(node)
+      results = []
+      node.each_edge do |letter|
+        next_node = node[letter]
+        if next_node != nil
+          results += get_childs(next_node).map{|s| Word.new(letter) + s}
+          results << Word.new(letter, next_node.final)
+        end
+      end
+      results
+    end
+  end
+
+end

data/lib/dawg/node/memory_node.rb

@@ -0,0 +1,31 @@
+module Dawg
+  class MemoryNode
+    attr_accessor :index, :io, :edge_count, :final, :hash, :edges_pos
+    def initialize(io, index , edge_count , final, hash, edges_pos )
+      @io = io
+      @index = index
+      @edge_count = edge_count
+      @final = final
+      @hash = hash
+      @edges_pos = edges_pos
+    end
+
+    def [](letter)
+      @io.each_edge @index do |char, node_index|
+        if letter == char
+          return @io.get_node_by_index(node_index)
+        end
+      end
+
+      nil
+    end
+
+    def each_edge(&block)
+      @io.each_edge @index do |char, node_index|
+        yield char
+      end
+
+    end
+  end
+
+end

data/lib/dawg/node/node.rb

@@ -0,0 +1,52 @@
+module Dawg
+  class Node
+    @@next_id = 0
+    @id = 0
+    attr_accessor :edges, :id, :edge_count, :index, :final
+
+    def initialize(id: @@next_id, final: false, edge_count: 0, index: -1)
+      @id = id
+      @@next_id += 1
+      @final = final
+      @edge_count = edge_count
+      @index = index
+      @edges = {}
+    end
+
+    def to_s
+      arr = []
+      if @final
+          arr<<'1'
+      else
+          arr<<'0'
+      end
+
+      @edges.each do |label,node|
+          arr << label.to_s
+          arr << node.id.to_s
+      end
+
+      arr.join('_')
+
+    end
+
+    def hash
+      to_s.hash
+    end
+
+    def ==(other)
+      to_s == other.to_s
+    end
+
+    def [](letter)
+      @edges[letter]
+    end
+
+    def each_edge(&block)
+      @edges.each do |letter, node|
+        yield letter
+      end
+    end
+  end
+
+end

data/lib/dawg/serialization.rb

@@ -0,0 +1,37 @@
+module Dawg
+  module Serialization
+    def load_int(io)
+      io.read(4).unpack('l')[0]
+    end
+
+    def write_int(int, io) #32bit signed integer
+      io << [int].pack('l')
+    end
+
+    def load_bigint(io) #64bit signed integer
+      io.read(8).unpack('q')[0]
+    end
+
+    def write_bigint(int, io)
+      io << [int].pack('q')
+    end
+
+    def write_bool(var, io)
+      bool = var ? 1 : 0
+      io << [bool].pack('c')
+    end
+
+    def load_bool(io)
+      bool = io.read(1).unpack('c')[0]
+      bool == 1 ? true : false
+    end
+
+    def write_char(char, io)
+      io << [char].pack('Z4')
+    end
+
+    def load_char(io)
+      io.read(4).unpack('Z4')[0].force_encoding('utf-8')
+    end
+  end
+end

data/lib/dawg/word.rb

@@ -0,0 +1,22 @@
+module Dawg
+  class Word
+    attr_accessor :final, :word
+
+    def initialize( word = '', final = false)
+      @word = word
+      @final = final
+    end
+
+    def +(other)
+      Word.new(@word + other.word, other.final)
+    end
+
+    def to_s
+      @word
+    end
+
+    def inspect
+      @word
+    end
+  end
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: dawg
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - Maksatbek Manurov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
-dependencies: []
+date: 2017-01-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bindata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 description: Basic deterministic acyclic finite state automaton in ruby
 email:
 - maksat.mansurov@gmail.com
@@ -18,6 +32,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/dawg.rb
+- lib/dawg/dawg/dawg.rb
+- lib/dawg/dawg/memory_dawg.rb
+- lib/dawg/finder.rb
+- lib/dawg/node/memory_node.rb
+- lib/dawg/node/node.rb
+- lib/dawg/serialization.rb
+- lib/dawg/word.rb
 homepage: https://github.com/baltavay/dawg
 licenses:
 - MIT
@@ -38,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Deterministic acyclic finite state automaton