rbahocorasick 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rbahocorasick.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Changli Gao
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# RBAhoCorasick
+
+A Ruby implementation of [the Aho-Corasick string matching algorithm](http://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rbahocorasick'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rbahocorasick
+
+## Usage
+
+````ruby
+require 'rubygems'
+require 'rbahocorasick'
+
+nfa = RBAhoCorasick::NFA.new
+%w{he she his hers}.each{|key| nfa.add(key)}
+nfa.finalize
+nfa.match('he and she are friends').each{|m| puts m.key}
+````
+
+Yes, you can use DFA instead of NFA for better performance. See `test/tc_nfa.rb`
+for more examples.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*.rb']
+end
+
+task :default => :build

data/lib/rbahocorasick.rb

@@ -0,0 +1,5 @@
+
+require 'rbahocorasick/dfa'
+require 'rbahocorasick/nfa'
+require 'rbahocorasick/state'
+require 'rbahocorasick/version'

data/lib/rbahocorasick/dfa.rb

@@ -0,0 +1,26 @@
+
+require 'rbahocorasick/nfa'
+
+module RBAhoCorasick
+  class DFA < RBAhoCorasick::NFA
+    def finalize
+      super
+      queue = []
+      (0..255).each do |byte|
+        unless @initial_state[byte] == @initial_state
+          queue << @initial_state[byte] 
+        end
+      end
+      while queue.length > 0
+        r = queue.shift
+        (0..255).each do |byte|
+          if s = r[byte]
+            queue << s
+          else
+            r[byte] = r.default[byte]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rbahocorasick/nfa.rb

@@ -0,0 +1,95 @@
+
+require 'rbahocorasick/state'
+
+module RBAhoCorasick
+  MatchData = Struct.new(:key, :value, :offset)
+
+  class NFA
+    attr_reader :initial_state, :states_by_state_id
+
+    def initialize
+      @least_unused_state_id = 0
+      @states_by_state_id = []
+      @initial_state = create_state
+    end
+
+    # Add a pair of key and value
+    def add(key, value = nil)
+      @finalized and raise 'finalized'
+      state = @initial_state
+      key.each_byte do |byte|
+        next_state = state[byte]
+        unless next_state
+          next_state = state[byte] = create_state
+        end
+        state = next_state
+      end
+      state.add(key, value)
+    end
+
+    # Finalize the state machine
+    def finalize
+      @finalized and raise 'finalize twice'
+      @finalized = true
+      queue = []
+      (0..255).each do |byte|
+        if @initial_state[byte]
+          next_state = @initial_state[byte]
+          next_state.default = @initial_state
+          queue << next_state
+        else
+          @initial_state[byte] = @initial_state
+        end
+      end
+
+      while queue.length > 0
+        r = queue.shift
+        (0..255).each do |byte|
+          if s = r[byte]
+            queue << s
+            state = r.default
+            state = state.default until state[byte]
+            s.default = state[byte]
+            s.data.concat(s.default.data)
+          end
+        end
+      end
+    end
+
+    # Match the data
+    def match(data, length = nil)
+      finalize unless @finalized
+      length ||= data.length
+      matches = [] unless block_given?
+      offset = 0
+      state = @initial_state
+      data.each_byte do |byte|
+        break if offset >= length
+        offset += 1
+        state = state.default until state[byte]
+        state = state[byte]
+        if state.data.length > 0
+          state.data.each do |state_data|
+            match_data = RBAhoCorasick::MatchData.new(state_data.key,
+              state_data.value, offset - state_data.key.length)
+            if block_given?
+              yield match_data
+            else
+              matches << match_data
+            end
+          end
+        end
+      end
+      return matches unless block_given?
+    end
+
+    private
+
+    def create_state
+      state = RBAhoCorasick::State.new(@least_unused_state_id)
+      @least_unused_state_id += 1
+      @states_by_state_id[state.state_id] = state
+      state
+    end
+  end
+end

data/lib/rbahocorasick/state.rb

@@ -0,0 +1,32 @@
+
+module RBAhoCorasick
+  StateData = Struct.new(:key, :value)
+
+  class State
+    attr_accessor :data, :default
+    attr_reader :state_id
+
+    def initialize(state_id)
+      @transition = []
+      @data = []
+      @state_id = state_id
+    end
+
+    def add(key, value)
+      @data << RBAhoCorasick::StateData.new(key, value)
+    end
+
+    def ==(o)
+      @state_id == o.state_id
+    end
+
+    def [](input)
+      @transition[input]
+    end
+
+    def []=(input, next_state)
+      @transition[input] and raise 'transition exists'
+      @transition[input] = next_state
+    end
+  end
+end

data/lib/rbahocorasick/version.rb

@@ -0,0 +1,4 @@
+
+module RBAhoCorasick
+  VERSION = '0.0.1'
+end

data/rbahocorasick.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rbahocorasick/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "rbahocorasick"
+  gem.version       = RBAhoCorasick::VERSION
+  gem.authors       = ["Changli Gao"]
+  gem.email         = ["xiaosuo@gmail.com"]
+  gem.description   = %q{A Ruby implementation of the Aho-Corasick string matching algorithm}
+  gem.summary       = %q{A Ruby implementation of the Aho-Corasick string matching algorithm}
+  gem.homepage      = "https://github.com/xiaosuo/rbahocorasick"
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.license       = 'MIT'
+  gem.add_development_dependency 'rake'
+end

data/test/tc_dfa.rb

@@ -0,0 +1,31 @@
+
+require 'test/unit'
+require 'rbahocorasick'
+
+class TC_DFA < Test::Unit::TestCase
+  # verify the state machine with the original paper
+  def test_paper
+    dfa = RBAhoCorasick::DFA.new
+    %w{he she his hers}.each{|key| dfa.add(key)}
+    dfa.finalize
+    delta = {
+      0 => {?h.ord => 1, ?s.ord => 3},
+      1 => {?e.ord => 2, ?i.ord => 6, ?h.ord => 1, ?s.ord => 3},
+      3 => {?h.ord => 4, ?s.ord => 3},
+      7 => {?h.ord => 4, ?s.ord => 3},
+      9 => {?h.ord => 4, ?s.ord => 3},
+      5 => {?r.ord => 8, ?h.ord => 1, ?s.ord => 3},
+      2 => {?r.ord => 8, ?h.ord => 1, ?s.ord => 3},
+      6 => {?s.ord => 7, ?h.ord => 1},
+      4 => {?e.ord => 5, ?i.ord => 6, ?h.ord=> 1, ?s.ord => 3},
+      8 => {?s.ord => 9, ?h.ord => 1}
+    }
+    (0..9).each do |i|
+      state = dfa.states_by_state_id[i]
+      (0..255).each do |byte|
+        expect = (delta[i][byte] || 0)
+        assert_equal(expect, state[byte].state_id)
+      end
+    end
+  end
+end

data/test/tc_nfa.rb

@@ -0,0 +1,103 @@
+
+require 'test/unit'
+require 'rbahocorasick'
+
+class TC_NFA < Test::Unit::TestCase
+  def setup
+    @nfa = RBAhoCorasick::NFA.new
+    %w{he she his hers}.each{|key| @nfa.add(key)}
+  end
+
+  # verify the state machine with the original paper
+  def test_paper
+    @nfa.finalize
+    assert_equal(10, @nfa.states_by_state_id.length)
+    assert_equal(0, @nfa.initial_state.state_id)
+    goto = [
+      {?h.ord => 1, ?s.ord => 3},
+      {?e.ord => 2, ?i.ord => 6},
+      {?r.ord => 8},
+      {?h.ord => 4},
+      {?e.ord => 5},
+      {},
+      {?s.ord => 7},
+      {},
+      {?s.ord => 9},
+      {}
+    ]
+    (0...10).each{|i| assert_equal(i, @nfa.states_by_state_id[i].state_id)}
+    f = [0, 0, 0, 0, 1, 2, 0, 3, 0, 3]
+    output = {2 => ['he'], 5 => ['she', 'he'], 7 => ['his'], 9 => ['hers']}
+    (0..9).each do |i|
+      state = @nfa.states_by_state_id[i]
+      # goto function
+      (0..255).each do |byte|
+        if goto[i][byte]
+          assert_equal(goto[i][byte], state[byte].state_id)
+        elsif i == 0
+          assert_equal(0, state[byte].state_id)
+        else
+          assert_equal(nil, state[byte])
+        end
+      end
+      # failure function
+      if i == 0
+        assert_equal(nil, state.default)
+      else
+        assert_equal(f[i], state.default.state_id)
+      end
+      # output function
+      if output[i]
+        assert_equal(output[i].length, state.data.length)
+        keys = state.data.map{|data| data.key}
+        output[i].each{|o| assert(keys.include?(o))}
+      else
+        assert_equal(0, @nfa.states_by_state_id[i].data.length)
+      end
+    end
+  end
+
+  # finalize should be optional
+  def test_finalize
+    matches = @nfa.match('he and she are friends')
+    assert_equal(3, matches.length)
+    assert_equal('he', matches[0].key)
+    assert_equal(0, matches[0].offset)
+    assert_equal('she', matches[1].key)
+    assert_equal(7, matches[1].offset)
+    assert_equal('he', matches[2].key)
+    assert_equal(8, matches[2].offset)
+    assert_raise(RuntimeError){@nfa.finalize}
+    assert_raise(RuntimeError){@nfa.add('test')}
+  end
+
+  def test_match
+    str = 'he and she are friends'
+    # with length
+    matches = @nfa.match(str, 9)
+    assert_equal(1, matches.length)
+    assert_equal('he', matches[0].key)
+    assert_equal(0, matches[0].offset)
+    # with block
+    matches = []
+    @nfa.match(str) do |match_data|
+      matches << match_data.key
+    end
+    assert_equal(%w{he she he}, matches)
+    # block return
+    matches = []
+    @nfa.match(str) do |match_data|
+      matches << match_data.key
+      break if match_data.key == 'she'
+    end
+    assert_equal(%w{he she}, matches)
+  end
+
+  def test_value_retrieve
+    @nfa.add('key', 'value')
+    matches = @nfa.match('she has his key')
+    assert_equal(%w{she he his key}, matches.map{|m| m.key})
+    (0...3).each{|i| assert_equal(nil, matches[i].value)}
+    assert_equal('value', matches[3].value)
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: rbahocorasick
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Changli Gao
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-14 00:00:00 +08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: A Ruby implementation of the Aho-Corasick string matching algorithm
+email: 
+- xiaosuo@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rbahocorasick.rb
+- lib/rbahocorasick/dfa.rb
+- lib/rbahocorasick/nfa.rb
+- lib/rbahocorasick/state.rb
+- lib/rbahocorasick/version.rb
+- rbahocorasick.gemspec
+- test/tc_dfa.rb
+- test/tc_nfa.rb
+has_rdoc: true
+homepage: https://github.com/xiaosuo/rbahocorasick
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A Ruby implementation of the Aho-Corasick string matching algorithm
+test_files: 
+- test/tc_dfa.rb
+- test/tc_nfa.rb