n_grams_generator 0.0.1

data/lib/n_grams_generator.rb

@@ -0,0 +1,124 @@
+#encoding:utf-8
+
+class NGramsGenerator
+
+  attr_reader :all_ngrams, :ns, :auto_clear, :no_count
+
+  # Creates NGramsGenerator object.
+  #
+  # @param [Number, Array<Number>] ns the lengths of generated n-grams
+  # @param [Hash] opts the additional options
+  # @option [true, false] :no_count enables storing of full ngrams instead of counting them
+  # @option [true, false] :auto_clear disables storing of n-grams
+  def initialize(ns, options={})
+    @ns = ns.kind_of?(Enumerable) ? ns : [ns]
+
+    @auto_clear = options[:auto_clear] || false
+    @no_count = options[:no_count] || false
+
+    @all_ngrams = Hash.new([])
+  end
+
+  # Generates n-grams according to lengths specified during construction.
+  #
+  #   NGramsGenerator.new(3).ngrams(['Alice', 'has', 'a', 'cat']) 
+  #     # => {
+  #           3 => { ['Alice', 'has', 'a'] => 1, ['has', 'a', 'cat'] => 1 }
+  #         }
+  #   NGramsGenerator.new([2, 3]).ngrams(['Alice', 'has', 'a', 'cat'], :no_count => true) 
+  #     # => { 
+  #        2 => [ ['Alice', 'has'], ['has', 'a'], ['a', 'cat'] ]
+  #        3 => [ ['Alice', 'has', 'a'], ['has', 'a', 'cat'] ]
+  #      }
+  #   NGramsGenerator.new(2).grams([['very', :adv], ['nice', :adj], ['job', :noun]])
+  #     # => { 
+  #        2 => { 
+  #          ["very", "nice"] => 1, 
+  #          [:adv, "nice"] => 1, 
+  #          ["very", :adj] => 1, 
+  #          [:adv, :adj] => 1, 
+  #          ["nice", "job"] => 1, 
+  #          [:adj, "job"] => 1, 
+  #          ["nice", :noun] => 1, 
+  #          [:adj, :noun] => 1
+  #        }
+  #      }
+  #
+  # @param [Array<Object>, Array<Array<Object>>] data
+  # @return [Hash] the generated ngrams by n-number
+  def ngrams(data)
+    return @all_ngrams if data.nil? or data.empty?
+
+    clear_ngrams if @auto_clear
+
+    temp_ngrams = flat_ngrams(data.map{ |item| item.kind_of?(Enumerable) ? item : [item] })
+    new_ngrams = Hash.new([])
+
+    temp_ngrams.keys.each do |n|
+      temp_ngrams[n].each { |ngram| new_ngrams[n] += multiply_ngram(ngram) }
+    end
+
+    @all_ngrams = add_new_ngrams(new_ngrams)
+    return @all_ngrams
+  end
+
+  def count_ngrams(ngrams=@all_ngrams)
+    @ns.inject({}) { |hsh, n| hsh[n] = count_array_elements(ngrams[n]); hsh }
+  end
+
+  def clear_ngrams
+    @all_ngrams.clear
+  end
+
+  private
+
+  def flat_ngrams(data)
+    temp_ngrams = {}
+
+    @ns.each do |n| 
+      data.each_cons(n) do |ngram| 
+        temp_ngrams[n] = [] unless temp_ngrams.has_key?(n)
+        temp_ngrams[n] << ngram
+      end
+    end
+
+    temp_ngrams
+  end
+
+  def multiply_ngram(ngram)
+    all_ngrams = [[]]
+    
+    ngram.each_with_index do |token, t|
+      token.each_with_index do |item, i|
+        all_ngrams = extend_ngram(all_ngrams, item, t)
+      end
+    end
+
+    all_ngrams
+  end
+
+  def extend_ngram(all_ngrams, item, t)
+    if all_ngrams.first[t].nil?
+      all_ngrams.each { |e| e << item }
+    else
+      all_ngrams += all_ngrams.map { |e1| e2 = e1.clone; e2[t] = item; e2 }
+    end
+
+    all_ngrams
+  end
+
+  def add_new_ngrams(ngrams)
+    if @no_count
+      @all_ngrams.merge(ngrams) { |ngram, c1, c2| c1 + c2 }
+    else
+      @all_ngrams.merge(count_ngrams(ngrams)) do |ngram, h1, h2|
+        h1.merge(h2) { |n, c1, c2| c1 + c2 }
+      end
+    end
+  end
+
+  def count_array_elements(ary)
+    ary.inject(Hash.new(0)) { |hsh, elem| hsh[elem] += 1; hsh }
+  end
+
+end

data/test/test_n_grams_generator.rb

@@ -0,0 +1,114 @@
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/n_grams_generator'
+require 'test/unit'
+
+class TestNGramsGenerator < Test::Unit::TestCase
+
+  def test_initialization_with_number
+    n = 2
+    ngram_generator = NGramsGenerator.new(n)
+
+    assert_kind_of Array, ngram_generator.ns
+    assert_equal ngram_generator.ns, [n]
+  end
+
+  def test_initialization_with_array
+    n = 3
+    ngram_generator = nil
+
+    assert_nothing_raised { ngram_generator = NGramsGenerator.new([n]) }
+    assert_equal ngram_generator.ns, [n]
+
+    ns = [1, 2, 3]
+    
+    assert_nothing_raised { ngram_generator = NGramsGenerator.new(ns) }
+    assert_equal ngram_generator.ns, ns
+  end
+
+  def test_flat_input
+    bigram_generator = NGramsGenerator.new(2, :no_count => true)
+
+    expected_output = [['Ala', 'ma'], ['ma', 'kota']]
+    input = %w{Ala ma kota}
+    ngrams = bigram_generator.ngrams(input)
+    
+    assert_kind_of Hash, ngrams
+    assert_equal expected_output, ngrams[2]
+  end
+
+  def test_nested_input
+    bigram_generator = NGramsGenerator.new(2, :no_count => true)
+
+    expected_output = [[1, 2], [2, 3], [3, 4]]
+    input = [[1], [2], [3], [4]]
+    ngrams = bigram_generator.ngrams(input)
+    
+    assert_equal expected_output, ngrams[2]
+  end
+
+  def test_simple_and_nested_input
+    bigram_generator = NGramsGenerator.new(2, :no_count => true)
+    
+    expected_output = { 2 => [['Ala', 'ma'], [:subst, 'ma']] }
+    input = [['Ala', :subst], 'ma']
+    ngrams = bigram_generator.ngrams(input)
+
+    assert_equal expected_output, ngrams
+  end
+
+  def test_n_number_is_too_high_for_input_length
+    ngram_generator = NGramsGenerator.new(5)
+    
+    expected_output = { 5 => {} }
+    input = [1, 2, 3]
+    ngrams = ngram_generator.ngrams(input)
+
+    assert_equal expected_output, ngrams
+  end
+
+  def test_multiple_n_values
+    ns = [1, 2, 3]
+    ngrams_generator = NGramsGenerator.new(ns, :no_count => true)
+    
+    expected_output = { 1 => [[1], [2], [3]], 2 => [[1, 2], [2, 3]], 3 => [[1, 2, 3]] }
+    input = [1, 2, 3] 
+    ngrams = ngrams_generator.ngrams(input)
+
+    assert_equal ns, ngrams.keys
+    assert_equal expected_output, ngrams 
+  end
+
+  def test_counting_ngrams
+    n = 2
+    input = [1, 2, 2, 2, 3, 3, 3, 3, 3]
+    ngrams = NGramsGenerator.new(n).ngrams(input)
+    
+    assert_equal 1, ngrams[n][[1, 2]]
+    assert_equal 2, ngrams[n][[2, 2]]
+    assert_equal 1, ngrams[n][[2, 3]]
+    assert_equal 4, ngrams[n][[3, 3]]
+  end
+
+  def test_adding_ngrams
+    n = 3
+    ngram_generator = NGramsGenerator.new(n)
+    input = [1, 2, 3, 4]
+
+    3.times do |i|
+      expected_output = { n => { [1, 2, 3] => (i + 1), [2, 3, 4] => (i + 1) } }
+      ngrams = ngram_generator.ngrams(input)
+      assert_equal expected_output, ngrams
+    end
+  end
+
+  def test_auto_clear_option
+    trigram_generator = NGramsGenerator.new(3, :auto_clear => true)
+    input = [:a, :b, :c, :d]
+
+    3.times do |i|
+      expected_output = { 3 => { [:a, :b, :c] => 1, [:b, :c, :d] => 1 } }
+      ngrams = trigram_generator.ngrams(input)
+      assert_equal expected_output, ngrams
+    end
+  end
+
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: n_grams_generator
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- snukky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-22 00:00:00.000000000 Z
+dependencies: []
+description: N-grams generator for multi-level strings, for example words and their
+  morphosyntactic descriptions.
+email:
+- snk987@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/n_grams_generator.rb
+- test/test_n_grams_generator.rb
+homepage: https://github.com/snukky/n_grams_generator
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: N-grams generator for multi-level strings.
+test_files:
+- test/test_n_grams_generator.rb