scylla 0.1.0

data/test/generator_test.rb

@@ -0,0 +1,72 @@
+require 'test/helper'
+
+class GeneratorTest < Test::Unit::TestCase
+  context "create_lm ngrams" do
+    setup do
+      @text = "hello"
+      @ngram = ["_", "l", "lo_", "ello", "lo", "o", "llo", "hel", "o_", "ell", "e", "ello_",
+       "_he", "el", "hello", "hell", "he", "_hel", "h", "_hell", "llo_", "_h", "ll"]
+      @ngram_frequencies =  [["_", 2], ["l", 2], ["lo_", 1], ["ello", 1], ["lo", 1], ["o", 1],
+       ["llo", 1], ["hel", 1], ["o_", 1], ["ell", 1], ["e", 1], ["ello_", 1], ["_he", 1], 
+       ["el", 1], ["hello", 1], ["hell", 1], ["he", 1], ["_hel", 1], ["h", 1], ["_hell", 1],
+       ["llo_", 1], ["_h", 1], ["ll", 1]] 
+    end
+
+    should "create an array of ngrams for a given text input" do
+      sg = Scylla::Generator.new
+      ngram_result = sg.create_lm(@text)
+      ngram_result.each do |res|
+        assert @ngram.include?(res)
+      end
+    end
+
+    should "create an array of ngrams with their associated frequencies for a given text input" do
+      sg = Scylla::Generator.new
+      ngram_result = sg.create_lm(@text, true)
+      ngram_result.each do |res|
+        assert @ngram_frequencies.include?(res)
+      end
+    end
+  end
+
+  context "create .lm files out of text files" do
+    setup do
+      @engtext = 'test/fixtures/source_texts/english.txt'
+      @englm = 'test/fixtures/lms/english.lm'
+      @sg = Scylla::Generator.new('test/fixtures/source_texts', 'test/fixtures/lms')
+      languages = Dir.glob("**/*.lm")
+      text = ""
+      File.readlines(@engtext).each {|line| text += line }
+      @map = @sg.create_lm(text, true)
+    end
+
+    should "create lm file out of text file" do
+      path = 'test/fixtures/source_texts/english.txt'
+      @sg.write_lm(path)
+      i = 0
+      File.readlines(@englm).each do |line|
+        break if i > 400
+        set = line.split("\t")
+        key = set.first
+        value = set.last.strip.to_i
+        assert_equal value, @map[i][1]
+        i += 1
+      end
+    end
+
+    should "create .lm files in bulk" do
+      @sg.train
+      languages = Dir.glob("**/*.lm")
+      assert_equal 8, languages.size
+      i = 0
+      File.readlines(@englm).each do |line|
+        break if i > 400
+        set = line.split("\t")
+        key = set.first
+        value = set.last.strip.to_i
+        assert_equal value, @map[i][1]
+        i += 1
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'scylla'
+require 'mocha'
+
+class Test::Unit::TestCase
+end

data/test/loader_test.rb

@@ -0,0 +1,31 @@
+require 'test/helper'
+
+class LoaderTest < Test::Unit::TestCase
+  context "#languages" do
+    setup do
+      Scylla::Loader.clear
+      @englm = 'test/fixtures/lms/english.lm'
+    end
+
+    context "when being read" do
+      should_eventually "only load from disk once" do
+       Scylla::Loader.expects(:load_language_maps).once.returns([])
+        Scylla::Loader.languages
+        Scylla::Loader.languages
+        Scylla::Loader.unstub(:load_language_maps)
+      end
+    end
+    
+    should "load the correct map for a language" do
+      map = Scylla::Loader.language_map(@englm)
+      rank = 1
+      File.readlines(@englm).each do |line|
+        set = line.split("\t")
+        key = set.first
+        value = set.last.strip.to_i
+        assert_equal rank, map[key]
+        rank += 1
+      end
+    end
+  end
+end

data/test/scylla_test.rb

@@ -0,0 +1,20 @@
+require 'test/helper'
+
+class ScyllaTest < Test::Unit::TestCase
+  context "String methods" do
+    setup do
+      text = "Hello? Is there anybody in there?"
+      @language = text.language
+      @languages = text.guess
+    end
+
+    should "load language results for strings" do
+      assert_not_nil @language
+      assert_not_nil @languages
+      assert String, @language.class
+      assert Array, @languages.class
+      assert_equal "english", @language
+      assert_equal "english", @languages.first
+    end
+  end
+end

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification 
+name: scylla
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ashwin Hegde
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-25 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 6
+        - 4
+        version: 1.6.4
+  type: :development
+  version_requirements: *id002
+description: Allows for text categorization by guessing the language of a given text using n-grams
+email: ahegde@zendesk.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/scylla.rb
+- lib/scylla/classifier.rb
+- lib/scylla/generator.rb
+- lib/scylla/loader.rb
+- lib/scylla/string.rb
+- lib/scylla/tasks.rb
+- scylla.gemspec
+- source_texts/13375P33K.txt
+- source_texts/afrikaans.txt
+- source_texts/arabic.txt
+- source_texts/armenian.txt
+- source_texts/bulgarian.txt
+- source_texts/catalan.txt
+- source_texts/chinese.txt
+- source_texts/danish.txt
+- source_texts/english.txt
+- source_texts/esperanto.txt
+- source_texts/finnish.txt
+- source_texts/french.txt
+- source_texts/german.txt
+- source_texts/greek-iso8859-7.txt
+- source_texts/hebrew.txt
+- source_texts/hindi.txt
+- source_texts/hungarian.txt
+- source_texts/icelandic.txt
+- source_texts/indonesian.txt
+- source_texts/irish.txt
+- source_texts/italian.txt
+- source_texts/japanese.txt
+- source_texts/korean.txt
+- source_texts/latin.txt
+- source_texts/malay.txt
+- source_texts/marathi.txt
+- source_texts/mingo.txt
+- source_texts/nepali.txt
+- source_texts/norwegian.txt
+- source_texts/polish.txt
+- source_texts/portuguese.txt
+- source_texts/quechua.txt
+- source_texts/romanian.txt
+- source_texts/rumantsch.txt
+- source_texts/russian.txt
+- source_texts/sanskrit.txt
+- source_texts/scots_gaelic.txt
+- source_texts/serbian-ascii.txt
+- source_texts/slovak-ascii.txt
+- source_texts/slovenian-ascii.txt
+- source_texts/spanish.txt
+- source_texts/swahili.txt
+- source_texts/swedish.txt
+- source_texts/tagalog.txt
+- source_texts/tamil.txt
+- source_texts/thai.txt
+- source_texts/turkish.txt
+- source_texts/ukrainian-koi8_u.txt
+- source_texts/vietnamese.txt
+- source_texts/welsh.txt
+- source_texts/yiddish-utf.txt
+- test/classifier_test.rb
+- test/fixtures/source_texts/danish.txt
+- test/fixtures/source_texts/english.txt
+- test/fixtures/source_texts/french.txt
+- test/fixtures/source_texts/german.txt
+- test/fixtures/source_texts/spanish.txt
+- test/generator_test.rb
+- test/helper.rb
+- test/loader_test.rb
+- test/scylla_test.rb
+has_rdoc: true
+homepage: http://github.com/hashwin/scylla
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Ruby port of Textcat language guesser
+test_files: []
+