aho_corasick_matcher 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc70023665fc36d0b86508ef0d7c3e330beda705
+  data.tar.gz: e22d0d38e7305f540a6aa3a6f07ca259931eba3a
+SHA512:
+  metadata.gz: 62ba80e800c76fd5bc0db9157746413659b9141f63e17e5d0d7b5c7a6736a13dee2e4aee20adcd41cbc8f57257fbd87646a3f6b93620ef0d2c27010d0548a8c3
+  data.tar.gz: 693edb06c4dfd18e8bcbeac19c723bfcf5ec75eaa752388e0cbb55a306f2cb1206581ca151067252646ac3c8c59eb19bf722678c700658703ab6f3b3a1e5b9f2

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Altmetric LLP
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# AhoCorasickMatcher [![Build Status](https://travis-ci.org/altmetric/aho_corasick_matcher.svg?branch=master)](https://travis-ci.org/altmetric/aho_corasick_matcher)
+
+A Ruby gem for finding strings in text using the [Aho-Corasick string matching search](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.96.4671&rep=rep1&type=pdf).
+
+Aho-Corasick is `O(n + m)` where `n` is the size of the string to be searched
+and `m` is the size of the dictionary. This means it's particularly suited for
+searching for occurrences of words using large dictionaries, as the runtime
+increases only linearly.
+
+It's quite memory-intensive, and building a matcher is expensive – but once it's
+been built, matching terms is very fast.
+
+**Current version:** 1.0.0  
+**Supported Ruby versions:** 1.9.2, 1.9.3, 2.0, 2.1, 2.2, jruby-1.7, rbx-2.2, rbx-2.4
+
+## Usage
+
+```ruby
+require 'aho_corasick_matcher'
+
+matcher = AhoCorasickMatcher.new(['a', 'b', 'ab'])
+matcher.match('aba')
+#=> ['a', 'ab', 'b', 'a']
+
+matcher = AhoCorasickMatcher.new(["thistle", "sift", "thistles"])
+matcher.match("Theophilus thistle, the successful thistle sifter, in sifting a sieve full of un-sifted thistles, thrust three thousand thistles through the thick of his thumb.")
+#=> ["thistle", "thistle", "sift", "sift", "sift", "thistle", "thistles", "thistle", "thistles"]
+```
+
+## Thanks
+
+Loosely based on Tim Cowlishaw's implementation of the same algorithm https://github.com/timcowlishaw/aho_corasick
+
+## License
+
+Copyright © 2015 Altmetric LLP
+
+Distributed under the MIT License.

data/lib/aho_corasick_matcher.rb

@@ -0,0 +1,95 @@
+require 'thread'
+
+class AhoCorasickMatcher
+  attr_reader :root
+  private :root
+
+  def initialize(dictionary)
+    @root = Node.new
+
+    build_trie(dictionary)
+    build_suffix_map
+  end
+
+  def match(string)
+    [].tap do |matches|
+      string.each_char.reduce(root) do |node, char|
+        (node || root).search(char.intern).tap do |child|
+          matches.push(*child.matches) if child
+        end
+      end
+    end
+  end
+
+  private
+
+  def build_trie(dictionary)
+    dictionary.each do |string|
+      string.each_char.reduce(root) do |node, char|
+        node.child_or_create(char.intern)
+      end.matches << string
+    end
+  end
+
+  def build_suffix_map
+    queue = Queue.new
+
+    root.children.each do |child|
+      child.suffix = root
+      queue << child
+    end
+
+    until queue.empty?
+      node = queue.pop
+      node.children.each { |child| queue << child }
+      node.build_child_suffixes
+    end
+  end
+
+  class Node
+    attr_reader :matches, :child_map, :suffix, :parent
+    attr_writer :suffix
+
+    def initialize(parent = nil)
+      @matches = []
+      @child_map = {}
+      @parent = parent
+    end
+
+    def search(char)
+      @child_map[char] || suffix && suffix.search(char)
+    end
+
+    def child_or_create(char)
+      @child_map[char] ||= self.class.new(self)
+    end
+
+    def children
+      @child_map.values
+    end
+
+    def root?
+      parent.nil?
+    end
+
+    def build_child_suffixes
+      child_map.each do |char, child|
+        failure = find_failure_node(char)
+        child_suffix = failure.search(char)
+
+        if child_suffix
+          child.suffix = child_suffix
+          child.matches.push(*child_suffix.matches)
+        elsif failure.root?
+          child.suffix = failure
+        end
+      end
+    end
+
+    def find_failure_node(char)
+      failure = suffix
+      failure = failure.suffix until failure.search(char) || failure.root?
+      failure
+    end
+  end
+end

data/spec/aho_corasick_matcher_spec.rb

@@ -0,0 +1,81 @@
+require 'aho_corasick_matcher'
+
+RSpec.describe AhoCorasickMatcher do
+  subject(:matcher) { described_class.new(dict) }
+
+  context 'with an empty dictionary' do
+    let(:dict) { [] }
+
+    it 'finds no strings' do
+      expect(matcher.match('I am a test string')).to be_empty
+    end
+  end
+
+  context 'with a single-entry dictionary' do
+    let(:dict) { %w(TestString) }
+
+    it 'finds matching strings' do
+      expect(matcher.match('I am a TestString')).to eq(%w(TestString))
+    end
+
+    it 'does not find non-matching strings' do
+      expect(matcher.match('I am a different string')).to be_empty
+    end
+
+    it 'finds all occurrences of strings' do
+      expect(
+        matcher.match('I am a TestString and I say TestString twice')
+      ).to eq(%w(TestString TestString))
+    end
+  end
+
+  context 'with a multiple-entry dictionary' do
+    let(:dict) { %w(TestString1 TestString2) }
+
+    it 'finds all matching strings' do
+      expect(
+        matcher.match('I am both a TestString1 and a TestString2')
+      ).to eq(%w(TestString1 TestString2))
+    end
+
+    it 'finds partial matching strings' do
+      expect(
+        matcher.match('I am a TestString1 but do not contain the other one')
+      ).to eq(%w(TestString1))
+    end
+
+    it 'does not find non-matching strings' do
+      expect(matcher.match('I am a different string')).to be_empty
+    end
+  end
+
+  context 'with a multiple-entry dictionary including prefixes' do
+    let(:dict) { %w(TestString TestStringExtended) }
+
+    it 'finds all matching strings' do
+      expect(
+        matcher.match('I contain TestStringExtended')
+      ).to eq(%w(TestString TestStringExtended))
+    end
+
+    it 'finds partial matching strings' do
+      expect(
+        matcher.match('I am a TestString but do not contain the other one')
+      ).to eq(%w(TestString))
+    end
+
+    it 'does not find non-matching strings' do
+      expect(matcher.match('I am a different string')).to be_empty
+    end
+  end
+
+  context 'with an overlapping dictionary' do
+    let(:dict) { %w(Test String TestString) }
+
+    it 'finds all matching strings' do
+      expect(
+        matcher.match('TestStringTest')
+      ).to eq(%w(Test TestString String Test))
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+RSpec.configure do |config|
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+  config.disable_monkey_patching!
+  config.warnings = true
+  config.order = :random
+  Kernel.srand config.seed
+
+  config.default_formatter = 'doc' if config.files_to_run.one?
+
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: aho_corasick_matcher
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Matthew MacLeod
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: |2
+      Uses the fast Aho-Corasick text search system to find occurrences of any of
+      a dictionary of strings across an input string.
+email: support@altmetric.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/aho_corasick_matcher.rb
+- spec/aho_corasick_matcher_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/altmetric/aho_corasick_matcher
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A library to search text for occurrences of a list of strings
+test_files:
+- spec/aho_corasick_matcher_spec.rb
+- spec/spec_helper.rb