segment_ruby 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d59d60fbd2e0bd1d80132e2e0f18bd1241cc4bb
-  data.tar.gz: 5025e686e618bf0b54a3ebec6ee97e46f4d769f5
+  metadata.gz: 26ffd5e24afa505dec8c31cac0cad0cecafc27bb
+  data.tar.gz: 4171510ada22f97967654a976da3c014b5fab121
 SHA512:
-  metadata.gz: 85c15f6b79b7be0cb6c3fb3ef8800256889dc85beca996592181a9e5b136c8dae5cf5168f19c1c14fe45b7a2d86063e3c6717c65e8c4eac993a14a991e5b79a4
-  data.tar.gz: 27bf285191eab61d8fc88acaa65574e6ef2ec32f0d4336665dd8b3799fcc84703e91872d8e9336c08846d94a4656a6ec97412d260bdd8628e0c639671b7299cc
+  metadata.gz: fcf6a77d8778367c64563d09be67080b935282250fc625cdb2169ea9f461a0dea2a04ef47a400b7f70ed4f2745c534cd5600b3eb13b9b94285aaf6eb65609348
+  data.tar.gz: c7868ead403e68c2154016150456676269376b136dce0de73c30370ce168e27abf839335a46c4854af037a155238241c79ef89a172163055dc16651ef95e5bfa

data/README.md

@@ -23,17 +23,17 @@ Or install it yourself as:
 
 ```
 require 'segment_ruby'
-t = SegmentRuby::Analyzer.new('twitter'); true
+t = SegmentRuby::Analyzer.new(:twitter)
 t.segment("theboywholived")
 => ["the", "boy", "who", "lived"]
 ```
 Models include:
 
-- `norvig`: based on Google web data
-- `google_books`: based on Google books data
-- `anchor`: based on Web anchor text
-- `twitter`: based on Twitter data
-- `small`: smaller version of the Google books data
+- `:norvig`: based on Google web data
+- `:google_books`: based on Google books data
+- `:anchor`: based on Web anchor text
+- `:twitter`: based on Twitter data
+- `:small`: smaller version of the Google books data
 
 The default model is `small`. Use it if is seems to work for you.
 

data/lib/segment_ruby/version.rb

@@ -1,3 +1,3 @@
 module SegmentRuby
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/segment_ruby.rb

@@ -1,4 +1,4 @@
-require "segment_ruby/version"
+require_relative "./segment_ruby/version"
 require 'pathname'
 
 # Based on "Natural Language Corpus Data"
@@ -13,126 +13,117 @@ module SegmentRuby
     def initialize(total_file_name, data_file_name)
       @total_file_name = total_file_name
       @data_file_name = total_file_name
-      begin
+
+      @log_total = begin
         total = File.read(total_file_name).to_i
-        @log_total= Math.log2(total)
+        Math.log2(total)
       rescue
-        @log_total= Math.log2(10**1000)
+        Math.log2(10**1000)
       end
 
-      @table = Hash.new{|w| -Float::INFINITY}
+      @table = Hash.new { |w| -Float::INFINITY }
+
       File.open(data_file_name).each_line do |line|
         data = line.split(/\s/)
         freq = data[-1].to_i
         keys = data[0..-2]
         key = keys.join(' ')
-        log_p = Math.log2(freq) - @log_total
-        @table[key] = log_p
+        log_p = Math.log2(freq) - log_total
+
+        table[key] = log_p
       end
-      true
     end
 
-    def table
-      @table
-    end
+    attr_reader :log_total, :table
 
     def files
       [@total_file_name, @data_file_name]
     end
 
     def log_prob(w)
-      @table[w]
+      table[w]
     end
 
     def prob(w)
-      2**@table[w]
-    end
-
-    def log_total
-      @log_total
+      2**table[w]
     end
 
     def total
-      2**@log_total
+      2**log_total
     end
 
     def has_key?(w)
-      @table.has_key?(w)
+      table.has_key?(w)
     end
   end
 
   class Analyzer
-
-    def initialize(model='small', max_word_length=20)
-      @model = model
+    def initialize(model_name=:small, max_word_length=20)
+      @model_name = model_name
       @max_word_length = max_word_length
+
       # unigram log probabilities
-      @ulp = ProbabilityDistribution.new(total_file_name(''), freq_file_name(''))
+      @ulp = ProbabilityDistribution.new(total_file_name, freq_file_name)
+
       # bigram log probabilities
       btf = total_file_name('2_')
       bff = freq_file_name('2_')
-      @blp = ((File.exists?(btf) and File.exists?(bff)) ? ProbabilityDistribution.new(btf, bff) : false)
-      true
-    end
-
-    def model
-      @model
+      @blp = (File.exists?(btf) and File.exists?(bff) ? ProbabilityDistribution.new(btf, bff) : false)
     end
 
-    def max_word_length
-      @max_word_length
-    end
-
-    def ulp
-      @ulp
-    end
-
-    def blp
-      @blp
-    end
+    attr_reader :blp, :max_word_length, :model_name, :ulp
 
     def log_Pr(w)
-      @ulp.log_prob(w)
+      ulp.log_prob(w)
     end
 
     def log_CPr(w, prev)
       key = [prev, w].join(' ')
-      (@blp and @blp.has_key?(key)) ? @blp.log_prob(key) : @ulp.log_prob(w)
+
+      blp and blp.has_key?(key) ? blp.log_prob(key) : ulp.log_prob(w)
+    end
+
+    def total_file_name(prefix='')
+      File.join(model_path, prefix + 'total.tsv')
     end
 
-    def total_file_name(prefix)
-      File.join(__dir__, "..", "data", "segment_ruby", @model, prefix + 'total.tsv')
+    def freq_file_name(prefix='')
+      File.join(model_path, prefix + 'frequencies.tsv')
     end
 
-    def freq_file_name(prefix)
-      File.join(__dir__, "..", "data", "segment_ruby", @model, prefix + 'frequencies.tsv')
+    def model_path
+      @model_path ||= File.join(__dir__, "..", "data", "segment_ruby", model_name.to_s)
     end
 
     # Returns all the splits of a string up to a given length
     def splits(text)
-      (0..[@max_word_length,text.size-1].min).map{|i| [text[0..i], text[i+1..text.size]  ] }
+      (0..[max_word_length, text.size-1].min).map { |i| [text[0..i], text[i+1..text.size]] }
     end
 
     def combine(pFirst, first, segmented)
       pRem,rem = segmented
+
       [pFirst+pRem, [first]+rem]
     end
 
     def segment_r(text, prev, n, memo)
       return [0.0, []] if not text or (text.size == 0)
       return memo[text] if memo.has_key?(text)
+
       log_p_segment = splits(text).map do |first, rem|
          log_p = log_CPr(first, prev)
          combine(log_p, first, segment_r(rem, first, n+1, memo))
       end.max
+
       memo[text] = log_p_segment
+
       log_p_segment
     end
 
     def segment(text, prev='<S>')
-      p, segmentation = segment_r(text, prev, 0, Hash.new)
+      _, segmentation = segment_r(text, prev, 0, Hash.new)
+
       segmentation
     end
-
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: segment_ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Will Fitzgerald
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2016-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler