word-bloom 0.1

data/lib/bitfield.rb

@@ -0,0 +1,60 @@
+#        NAME: BitField
+#      AUTHOR: Peter Cooper
+#     LICENSE: MIT ( http://www.opensource.org/licenses/mit-license.php )
+#   COPYRIGHT: (c) 2007 Peter Cooper (http://www.petercooper.co.uk/)
+
+class BitField
+  attr_reader :size
+  attr_accessor :field
+  include Enumerable
+
+  ELEMENT_WIDTH = 32
+
+  def initialize(size)
+    @size = size
+    @field = Array.new(((size - 1) / ELEMENT_WIDTH) + 1, 0)
+  end
+
+  # Set a bit (1/0)
+  def []=(position, value)
+    value == 1 ? @field[position / ELEMENT_WIDTH] |= 1 << (position % ELEMENT_WIDTH) : @field[position / ELEMENT_WIDTH] ^= 1 << (position % ELEMENT_WIDTH)
+  end
+
+  # Read a bit (1/0)
+  def [](position)
+    @field[position / ELEMENT_WIDTH] & 1 << (position % ELEMENT_WIDTH) > 0 ? 1 : 0
+  end
+
+  # Iterate over each bit
+  def each(&block)
+    @size.times { |position| yield self[position] }
+  end
+
+  # Returns the field as a string like "0101010100111100," etc.
+  def to_s
+    inject("") { |a, b| a + b.to_s }
+  end
+
+  # Returns the bitwise intersection with another BitField - pads smaller with
+  # 0s
+  def &(other)
+    if self.size < other.size
+      return other & self
+    end
+
+    skip =  self.size - other.size
+    result = BitField.new(self.size)
+    prefix = [0] * skip
+    rest = (self.field[skip..-1]).zip(other.field).map do |left, right|
+      left & right
+    end
+    result.field = prefix + rest
+    return result
+  end
+
+  # Returns the total number of bits that are set
+  # (The technique used here is about 6 times faster than using each or inject direct on the bitfield)
+  def total_set
+    @field.inject(0) { |a, byte| a += byte & 1 and byte >>= 1 until byte == 0; a }
+  end
+end

data/lib/bloominsimple.rb

@@ -0,0 +1,48 @@
+#        NAME: BloominSimple
+#      AUTHOR: Peter Cooper
+#     LICENSE: MIT ( http://www.opensource.org/licenses/mit-license.php )
+#   COPYRIGHT: (c) 2007 Peter Cooper
+
+require 'bitfield'
+
+class BloominSimple
+  attr_accessor :bitfield, :hasher
+
+  def initialize(bitsize, &block)
+    @bitfield = BitField.new(bitsize)
+    @size = bitsize
+    @hasher = block || lambda do |word|
+      word = word.downcase.strip
+      [h1 = word.sum, h2 = word.hash, h2 + h1 ** 3]
+    end
+  end
+
+  # Add item to the filter
+  def add(item)
+    @hasher[item].each { |hi| @bitfield[hi % @size] = 1 }
+  end
+
+  # Find out if the filter possibly contains the supplied item
+  def includes?(item)
+    @hasher[item].each { |hi| return false unless @bitfield[hi % @size] == 1 } and true
+  end
+
+  # Allows comparison between two filters. Returns number of same bits.
+  def &(other)
+    raise "Wrong sizes" if self.bitfield.size != other.bitfield.size
+    return (self.bitfield & other.bitfield).total_set
+  end
+
+  # Dumps the bitfield for a bloom filter for storage
+  def dump
+    [@size, *@bitfield.field].pack("I*")
+  end
+
+  # Creates a new bloom filter object from a stored dump (hasher has to be resent though for additions)
+  def self.from_dump(data, &block)
+    data = data.unpack("I*")
+    temp = new(data[0], &block)
+    temp.bitfield.field = data[1..-1]
+    temp
+  end
+end

data/lib/word-bloom.rb

@@ -0,0 +1,16 @@
+require 'bloominsimple'
+require 'digest/sha1'
+
+class WordBloom
+  HASHER = lambda do |item|
+    begin
+      item = item.encode("UTF-16LE", :invalid => :replace, :undef => :replace, :replace => "").encode("UTF-8")
+      Digest::SHA1.digest(item.downcase.strip).unpack("VV")
+    rescue ArgumentError => ex
+      p __ENCODING__
+      p ex.message, item
+      raise
+    end
+  end
+  LANGUAGE_DIR_PATH = File.expand_path("../../lang", __FILE__)
+end

data/lib/word-bloom/filter-builder.rb

@@ -0,0 +1,15 @@
+class WordBloom
+  class FilterBuilder
+    BITFIELD_WIDTH = 2_000_000
+
+    def initialize(source_path)
+      @filename = source_path
+    end
+
+    def filter_from_dictionary
+      filter = BloominSimple.new(BITFIELD_WIDTH, &HASHER)
+      File.open(@filename).each { |word| filter.add(word) }
+      filter
+    end
+  end
+end

data/lib/word-bloom/quality.rb

@@ -0,0 +1,52 @@
+require 'word-bloom/scorer'
+
+class WordBloom
+  class Quality
+    def self.build_all
+      qual = self.new
+      qual.languages = Scorer::all_languages
+      qual.build_metrics
+      qual
+    end
+
+    def initialize
+      @correlations = {}
+      @languages = []
+    end
+
+    attr_accessor :languages
+    attr_reader :correlations
+
+    def build_metrics
+      require 'word-bloom/filter-builder'
+
+      languages.each do |lang|
+        Scorer::load_filter(lang)
+      end
+
+      languages.each_with_index do |lang, index|
+        languages.drop(index + 1).each do |other_lang|
+          @correlations[[lang, other_lang]] =
+            (Scorer::filter_for(lang) & Scorer::filter_for(other_lang)).to_f / FilterBuilder::BITFIELD_WIDTH
+        end
+      end
+    end
+
+    def to_s
+      width = languages.map{|lang| lang.to_s.length}.max + 2
+      col_sep = "  "
+      ([([" " * width] + languages.map{|lang| lang.to_s.rjust(width)}).join(col_sep)] +
+      languages.map do |left|
+        ([left.to_s.ljust(width)] +
+          languages.map do |right|
+          if num = @correlations[[left, right]] || @correlations[[right, left]]
+            "%#{width}f" % num
+          else
+            "-" * width
+          end
+          end
+        ).join(col_sep)
+      end).join("\n")
+    end
+  end
+end

data/lib/word-bloom/scorer.rb

@@ -0,0 +1,108 @@
+require 'word-bloom'
+
+class WordBloom
+  class Scorer
+    @@filters = {}
+    @@all_languages = nil
+
+    def self.load_filter(name)
+      @@filters[name] ||=
+        begin
+          File.open(File.join(LANGUAGE_DIR_PATH, "#{name}.lang"), 'rb') do |file|
+            BloominSimple.from_dump(file.read, &HASHER)
+          end
+        end
+    end
+
+    def self.filter_for(language)
+      @@filters[language]
+    end
+
+    def self.all_languages
+      @@all_languages ||= Dir.entries(LANGUAGE_DIR_PATH).grep(/\.lang$/).map do |filename|
+        filename.sub(/\.lang$/,'').to_sym
+      end
+    end
+
+    def self.loaded_with(*languages)
+      scorer = self.new
+      if [:all] == languages
+        scorer.add_all_languages
+      else
+        languages.each do |language|
+          scorer.add_language(language)
+        end
+      end
+      return scorer
+    end
+
+    def initialize()
+      @languages = {}
+      @language_weights = Hash.new(1.0)
+      @language_weights[:russian] = 0.8
+    end
+
+    def add_language(name, weight = nil)
+      self.class.load_filter(name)
+      @languages[name] = true
+      @language_weights[name] = weight unless weight.nil?
+    end
+
+    def add_all_languages
+      self.class.all_languages.each do |language|
+        add_language(language)
+      end
+    end
+
+    def confidence(considered, results)
+      top_results = results.values.sort
+      best = top_results[-1]
+      rest = top_results[0..-2].inject{|number, sum| sum + number}
+
+      return OPTIMISM * best - rest
+    end
+
+    OPTIMISM = 3.5
+    MIN_CONFIDENCE = 15
+
+    def apply_weights(results)
+      results.keys.each do |lang|
+        results[lang] *= @language_weights[lang]
+      end
+      results
+    end
+
+    # Very inefficient method for now.. but still beats the non-Bloom
+    # alternatives.
+    # Change to better bit comparison technique later..
+    def process_text(text)
+      results = Hash.new(0)
+      word_count = 0
+      text.split(/\s+/).each do |word|
+        word = word.downcase
+        next if /^\d*$/ =~ word
+        @languages.keys.each do |lang|
+          if @@filters[lang].includes?(word)
+            results[lang] += 1
+          end
+        end
+
+        # Every now and then check to see if we have a really convincing result.. if so, exit early.
+        if word_count % 4 == 0 && results.size > 1
+          #break if confidence(word_count + 1, results) > MIN_CONFIDENCE
+        end
+
+        word_count += 1
+        #break if word_count > 100
+      end
+      apply_weights(results)
+    rescue => ex
+      p ex, ex.backtrace
+      nil
+    end
+
+    def language(text)
+      process_text(text).max { |a,b| a[1] <=> b[1] }.first rescue nil
+    end
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: word-bloom
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Peter Cooper
+- Judson Lester
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: corundum
+  requirement: &71753810 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :development
+  prerelease: false
+  version_requirements: *71753810
+description: ! '  Guesses the natural language of a text sample based on bloom filter
+  matches of words in the text.  Fast, reasonably accurate.
+
+'
+email:
+- ''
+- nyarly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/bloominsimple.rb
+- lib/word-bloom.rb
+- lib/word-bloom/filter-builder.rb
+- lib/word-bloom/scorer.rb
+- lib/word-bloom/quality.rb
+- lib/bitfield.rb
+- lang/pinyin.lang
+- lang/dutch.lang
+- lang/french.lang
+- lang/swedish.lang
+- lang/russian.lang
+- lang/german.lang
+- lang/farsi.lang
+- lang/italian.lang
+- lang/portuguese.lang
+- lang/english.lang
+- lang/spanish.lang
+homepage: http://nyarly.github.com/word-bloom/
+licenses:
+- MIT
+post_install_message: Thanks again to Peter Cooper - JL
+rdoc_options:
+- --inline-source
+- --main
+- doc/README
+- --title
+- word-bloom-0.1 RDoc
+require_paths:
+- lib/
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 379118287
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: word-bloom
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Natural language guessing for text samples
+test_files: []