scylla 0.1.0 → 0.2.0

data/lib/scylla/lms/yiddish-utf.lm

@@ -0,0 +1,400 @@
+�	1822
+_	734
+_�	343
+�	273
+�	222
+א	222
+�	203
+ע	203
+ע�	178
+��	178
+��	167
+א�	167
+�	158
+י	158
+�	133
+ר	133
+י�	126
+��	126
+ַ	122
+�	122
+�	108
+ט	108
+��	101
+ַ�	101
+אַ	100
+�ַ	100
+ן	98
+�	98
+��	91
+ר�	91
+_א	86
+�	82
+ל	82
+�ַ�	81
+אַ�	81
+ו	81
+�	81
+�_	77
+ן_	77
+ו�	73
+��	73
+נ	68
+�	68
+�	67
+�ָ	67
+אָ	67
+ָ	67
+ס	65
+�	65
+��	64
+ָ�	64
+��	64
+ל�	64
+אָ�	64
+�ָ�	64
+ער	63
+�ר	63
+נ�	62
+��	62
+��	62
+ט�	62
+פ	61
+��	61
+�	61
+ד�	61
+פ�	61
+��	61
+�	61
+ד	61
+��	55
+א�	55
+�	51
+מ	51
+�	50
+ק	50
+ש	48
+�	48
+_א�	46
+מ�	46
+��	46
+�	43
+��	43
+ֿ	43
+ק�	43
+�	42
+�ֿ	42
+ֿ�	42
+ז	42
+פֿ	42
+��	42
+�ֿ�	41
+װ	41
+�	41
+ײ	41
+פֿ�	41
+�	41
+װ�	40
+_אַ	40
+�	40
+��	40
+_א�	40
+ג	40
+��	38
+ש�	38
+ג�	37
+ס�	37
+��	37
+��	37
+,	36
+�_	35
+ט_	35
+ער�	35
+�ר�	35
+ר_	34
+�_	34
+��	34
+ז�	34
+�	32
+ב	32
+ב�	30
+��	30
+ּ	28
+_פ	28
+_פ�	28
+�	28
+ה	26
+_ד�	26
+,_	26
+�ע	26
+�	26
+דע	26
+_ד	26
+��	25
+ּ�	25
+�ע�	24
+�ע�	24
+�ר_	24
+ון	24
+ער_	24
+_װ	24
+�ע	24
+גע�	24
+ה�	24
+��	24
+גע	24
+�ן	24
+דע�	24
+�ן_	23
+_װ�	23
+ון_	23
+�ל	22
+��	22
+�	22
+�ע	22
+ײַ	22
+צ	22
+��	22
+�ַ	22
+צ�	22
+לע	22
+ַל	22
+ײ�	22
+�_	21
+ס_	21
+או	20
+_מ	20
+�ו	20
+�ר	20
+�ַר	20
+_פֿ	20
+�ַ�	20
+�_	20
+ײַ�	20
+�ו�	20
+או�	20
+ַר	20
+ַ_	20
+_מ�	19
+אי�	19
+�ַל	19
+פּ	19
+�י	19
+ױ	19
+�	19
+אי	19
+�י�	19
+�ע�	19
+�י	19
+לע�	19
+�ּ	19
+די	19
+�	18
+�א�	18
+_ז�	18
+אַ_	18
+"	18
+_ז	18
+רא	18
+�א	18
+רא�	18
+ם	18
+�ַ_	18
+ַר�	18
+ך	18
+�	18
+ַל�	18
+�ר�	18
+�ל�	18
+��	17
+�ער	17
+פּ�	17
+�_	17
+י_	17
+ײ�	17
+�ּ�	17
+כ	16
+ע_	16
+�_	16
+-	16
+_אי	16
+�	16
+�ון	16
+�ע	15
+נע	15
+_ש	15
+מע	15
+ױ�	15
+��	15
+.	15
+�_	15
+ך_	15
+�ע	15
+_או	15
+�ש	14
+�ק	14
+יק	14
+זי	14
+�י	14
+יש	14
+װא	13
+�א�	13
+�ע�	13
+װע	13
+יִ	13
+יט	13
+יִ�	13
+_י�	13
+הא	13
+ִ�	13
+��	13
+י�	13
+װא�	13
+�	13
+�ע	13
+��	13
+�ע	13
+ִ	13
+טע	13
+�א	13
+�ִ	13
+הא�	13
+�ן	13
+_י	13
+װע�	13
+�א�	13
+�ע�	13
+ען	13
+_דע	13
+מע�	13
+�ִ�	13
+�א	13
+�ט	13
+ענ�	12
+�ֿו	12
+�נ�	12
+�נ	12
+ענ	12
+_ק	12
+ני	12
+ַז	12
+ַט	12
+_ק�	12
+ַנ�	12
+ֿא�	12
+�ֿא	12
+�י	12
+�א	12
+ֿו�	12
+ַנ	12
+�ו�	12
+�י	12
+�ו	12
+�נ	12
+לי	12
+�נ�	12
+ֿא	12
+�ז	12
+_ש�	12
+�ט	12
+ֿו	12
+�א�	12
+��	11
+�ע�	11
+רע	11
+�ע	11
+_ה�	11
+�אָ	11
+ן�	11
+_צ�	11
+�ס	11
+�י�	11
+�ָט	11
+על	11
+טא	11
+_ה	11
+�ט	11
+�ל	11
+_ג	11
+_זי	11
+_ב	11
+�ן	11
+לי�	11
+ין	11
+�א	11
+_ב�	11
+ָט	11
+�ך	11
+_צ	11
+יך	11
+עס	11
+טע�	11
+__	11
+"�	10
+�ל�	10
+�י�	10
+�י�	10
+�_	10
+שע	10
+ומ	10
+-�	10
+�ע	10
+�ַז	10
+יש�	10
+�ן_	10
+_ל	10
+די_	10
+ם_	10
+�ש�	10
+ען_	10
+�מ	10
+יק�	10
+ני�	10
+�ל	10
+�י_	10
+_ג�	10
+ל_	10
+�ק�	10
+�א�	10
+�_	10
+_װע	10
+זי�	10
+טא�	10
+קל	10
+קל�	10
+מי	9
+מי�	9
+�ק	9
+�ס	9
+_הא	9
+�ין	9
+די�	9
+יס	9
+�אָ	9
+יך_	9
+,�	9
+�י	9
+�י�	9
+שט	9
+ָס	9
+�ן_	9
+_די	9
+�ס	9
+בע	9
+בע�	9
+נע�	9
+�י�	9
+�ע�	9
+_ל�	9
+�ך_	9
+�ע�	9
+�אָ	9
+�ָס	9
+ין_	9
+עק	9
+_װא	9
+�ט	9
+�אַ	9
+�ע	9
+�ע�	9
+רע�	9

data/lib/scylla/loader.rb

@@ -1,16 +1,20 @@
 module Scylla
   class Loader
+    @@dir = DEFAULT_TARGET_DIR
     # Loads all the language maps once into memory using the .lm files located
     # in lib/scylla/lm
     def self.load_language_maps
       languages = Hash.new
-      Dir.glob("**/*.lm").each do |filepath|
+      Dir.glob(File.join(@@dir, "*.lm")).each do |filepath|
         language = File.basename(filepath, ".lm")
         languages[language] = language_map(filepath)
       end
       return languages
     end
 
+    def self.dir
+      return @@dir
+    end
     # Returns a single language map from a specified .lm file
     def self.language_map(path)
       rank, ngram = 1, Hash.new
@@ -24,6 +28,9 @@ module Scylla
       return ngram
     end
 
+    def self.set_dir(dir)
+      @@dir = dir
+    end
     # Loads all maps from the .lm files, or loads them from memory if the 
     # files have already been read and loaded. 
     def self.languages

data/scylla.gemspec

@@ -5,13 +5,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scylla}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ashwin Hegde"]
-  s.date = %q{2011-08-25}
+  s.date = %q{2011-08-26}
+  s.default_executable = %q{scylla}
   s.description = %q{Allows for text categorization by guessing the language of a given text using n-grams}
   s.email = %q{ahegde@zendesk.com}
+  s.executables = ["scylla"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -24,17 +26,69 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "bin/scylla",
     "lib/scylla.rb",
     "lib/scylla/classifier.rb",
     "lib/scylla/generator.rb",
+    "lib/scylla/lms/13375P33K.lm",
+    "lib/scylla/lms/afrikaans.lm",
+    "lib/scylla/lms/arabic.lm",
+    "lib/scylla/lms/bulgarian.lm",
+    "lib/scylla/lms/catalan.lm",
+    "lib/scylla/lms/chinese.lm",
+    "lib/scylla/lms/danish.lm",
+    "lib/scylla/lms/english.lm",
+    "lib/scylla/lms/esperanto.lm",
+    "lib/scylla/lms/finnish.lm",
+    "lib/scylla/lms/french.lm",
+    "lib/scylla/lms/german.lm",
+    "lib/scylla/lms/greek-iso8859-7.lm",
+    "lib/scylla/lms/hebrew.lm",
+    "lib/scylla/lms/hindi.lm",
+    "lib/scylla/lms/hungarian.lm",
+    "lib/scylla/lms/icelandic.lm",
+    "lib/scylla/lms/indonesian.lm",
+    "lib/scylla/lms/irish.lm",
+    "lib/scylla/lms/italian.lm",
+    "lib/scylla/lms/japanese.lm",
+    "lib/scylla/lms/kannada.lm",
+    "lib/scylla/lms/korean.lm",
+    "lib/scylla/lms/latin.lm",
+    "lib/scylla/lms/malay.lm",
+    "lib/scylla/lms/marathi.lm",
+    "lib/scylla/lms/mingo.lm",
+    "lib/scylla/lms/nepali.lm",
+    "lib/scylla/lms/norwegian.lm",
+    "lib/scylla/lms/polish.lm",
+    "lib/scylla/lms/portuguese.lm",
+    "lib/scylla/lms/quechua.lm",
+    "lib/scylla/lms/romanian.lm",
+    "lib/scylla/lms/rumantsch.lm",
+    "lib/scylla/lms/russian.lm",
+    "lib/scylla/lms/sanskrit.lm",
+    "lib/scylla/lms/scots_gaelic.lm",
+    "lib/scylla/lms/serbian-ascii.lm",
+    "lib/scylla/lms/slovak-ascii.lm",
+    "lib/scylla/lms/slovenian-ascii.lm",
+    "lib/scylla/lms/spanish.lm",
+    "lib/scylla/lms/swahili.lm",
+    "lib/scylla/lms/swedish.lm",
+    "lib/scylla/lms/tagalog.lm",
+    "lib/scylla/lms/tamil.lm",
+    "lib/scylla/lms/thai.lm",
+    "lib/scylla/lms/turkish.lm",
+    "lib/scylla/lms/ukrainian-koi8_u.lm",
+    "lib/scylla/lms/vietnamese.lm",
+    "lib/scylla/lms/welsh.lm",
+    "lib/scylla/lms/yiddish-utf.lm",
     "lib/scylla/loader.rb",
     "lib/scylla/string.rb",
     "lib/scylla/tasks.rb",
+    "scylla-0.1.0.gem",
     "scylla.gemspec",
     "source_texts/13375P33K.txt",
     "source_texts/afrikaans.txt",
     "source_texts/arabic.txt",
-    "source_texts/armenian.txt",
     "source_texts/bulgarian.txt",
     "source_texts/catalan.txt",
     "source_texts/chinese.txt",
@@ -53,6 +107,7 @@ Gem::Specification.new do |s|
     "source_texts/irish.txt",
     "source_texts/italian.txt",
     "source_texts/japanese.txt",
+    "source_texts/kannada.txt",
     "source_texts/korean.txt",
     "source_texts/latin.txt",
     "source_texts/malay.txt",
@@ -83,10 +138,21 @@ Gem::Specification.new do |s|
     "source_texts/welsh.txt",
     "source_texts/yiddish-utf.txt",
     "test/classifier_test.rb",
+    "test/fixtures/lms/13375p33k.lm",
+    "test/fixtures/lms/danish.lm",
+    "test/fixtures/lms/english.lm",
+    "test/fixtures/lms/french.lm",
+    "test/fixtures/lms/german.lm",
+    "test/fixtures/lms/japanese.lm",
+    "test/fixtures/lms/kannada.lm",
+    "test/fixtures/lms/spanish.lm",
+    "test/fixtures/source_texts/13375P33K.txt",
     "test/fixtures/source_texts/danish.txt",
     "test/fixtures/source_texts/english.txt",
     "test/fixtures/source_texts/french.txt",
     "test/fixtures/source_texts/german.txt",
+    "test/fixtures/source_texts/japanese.txt",
+    "test/fixtures/source_texts/kannada.txt",
     "test/fixtures/source_texts/spanish.txt",
     "test/generator_test.rb",
     "test/helper.rb",