segment_ruby 0.1.0

data/data/segment_ruby/twitter/total.tsv

@@ -0,0 +1 @@
+1174595007

data/lib/segment_ruby.rb

@@ -0,0 +1,138 @@
+require "segment_ruby/version"
+require 'pathname'
+
+# Based on "Natural Language Corpus Data"
+# from the book "Beautiful Data" (Segaran and Hammerbacher, 2009)
+# by Peter Novig
+
+module SegmentRuby
+
+  LOG_1 = Math.log2(1)
+
+  class ProbabilityDistribution
+    def initialize(total_file_name, data_file_name)
+      @total_file_name = total_file_name
+      @data_file_name = total_file_name
+      begin
+        total = File.read(total_file_name).to_i
+        @log_total= Math.log2(total)
+      rescue
+        @log_total= Math.log2(10**1000)
+      end
+
+      @table = Hash.new{|w| -Float::INFINITY}
+      File.open(data_file_name).each_line do |line|
+        data = line.split(/\s/)
+        freq = data[-1].to_i
+        keys = data[0..-2]
+        key = keys.join(' ')
+        log_p = Math.log2(freq) - @log_total
+        @table[key] = log_p
+      end
+      true
+    end
+
+    def table
+      @table
+    end
+
+    def files
+      [@total_file_name, @data_file_name]
+    end
+
+    def log_prob(w)
+      @table[w]
+    end
+
+    def prob(w)
+      2**@table[w]
+    end
+
+    def log_total
+      @log_total
+    end
+
+    def total
+      2**@log_total
+    end
+
+    def has_key?(w)
+      @table.has_key?(w)
+    end
+  end
+
+  class Analyzer
+
+    def initialize(model='small', max_word_length=20)
+      @model = model
+      @max_word_length = max_word_length
+      # unigram log probabilities
+      @ulp = ProbabilityDistribution.new(total_file_name(''), freq_file_name(''))
+      # bigram log probabilities
+      btf = total_file_name('2_')
+      bff = freq_file_name('2_')
+      @blp = ((File.exists?(btf) and File.exists?(bff)) ? ProbabilityDistribution.new(btf, bff) : false)
+      true
+    end
+
+    def model
+      @model
+    end
+
+    def max_word_length
+      @max_word_length
+    end
+
+    def ulp
+      @ulp
+    end
+
+    def blp
+      @blp
+    end
+
+    def log_Pr(w)
+      @ulp.log_prob(w)
+    end
+
+    def log_CPr(w, prev)
+      key = [prev, w].join(' ')
+      (@blp and @blp.has_key?(key)) ? @blp.log_prob(key) : @ulp.log_prob(w)
+    end
+
+    def total_file_name(prefix)
+      File.join(__dir__, "..", "data", "segment_ruby", @model, prefix + 'total.tsv')
+    end
+
+    def freq_file_name(prefix)
+      File.join(__dir__, "..", "data", "segment_ruby", @model, prefix + 'frequencies.tsv')
+    end
+
+    # Returns all the splits of a string up to a given length
+    def splits(text)
+      (0..[@max_word_length,text.size-1].min).map{|i| [text[0..i], text[i+1..text.size]  ] }
+    end
+
+    def combine(pFirst, first, segmented)
+      pRem,rem = segmented
+      [pFirst+pRem, [first]+rem]
+    end
+
+    def segment_r(text, prev, n, memo)
+      return [0.0, []] if not text or (text.size == 0)
+      return memo[text] if memo.has_key?(text)
+      log_p_segment = splits(text).map do |first, rem|
+         log_p = log_CPr(first, prev)
+         combine(log_p, first, segment_r(rem, first, n+1, memo))
+      end.max
+      memo[text] = log_p_segment
+      log_p_segment
+    end
+
+    def segment(text, prev='<S>')
+      p, segmentation = segment_r(text, prev, 0, Hash.new)
+      segmentation
+    end
+
+  end
+end

data/lib/segment_ruby/version.rb

@@ -0,0 +1,3 @@
+module SegmentRuby
+  VERSION = "0.1.0"
+end

data/segment_ruby.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'segment_ruby/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "segment_ruby"
+  spec.version       = SegmentRuby::VERSION
+  spec.authors       = ["Will Fitzgerald"]
+  spec.email         = ["will.fitzgerald@pobox.com"]
+
+  spec.summary       = %q{segments text according word frequency using the Viterbi algorithm.}
+  spec.homepage      = "https://github.com/willf/segment_ruby"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'pry', '~> 0.9.12.2'
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: segment_ruby
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Will Fitzgerald
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-10-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.12.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.12.2
+description: 
+email:
+- will.fitzgerald@pobox.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- data/segment_ruby/anchor/README.md
+- data/segment_ruby/anchor/frequencies.tsv
+- data/segment_ruby/anchor/total.tsv
+- data/segment_ruby/google_books/2_frequencies.tsv
+- data/segment_ruby/google_books/2_total.tsv
+- data/segment_ruby/google_books/README.md
+- data/segment_ruby/google_books/frequencies.tsv
+- data/segment_ruby/google_books/total.tsv
+- data/segment_ruby/norvig/2_frequencies.tsv
+- data/segment_ruby/norvig/2_total.tsv
+- data/segment_ruby/norvig/README.md
+- data/segment_ruby/norvig/frequencies.tsv
+- data/segment_ruby/norvig/total.tsv
+- data/segment_ruby/small/2_frequencies.tsv
+- data/segment_ruby/small/2_total.tsv
+- data/segment_ruby/small/README.md
+- data/segment_ruby/small/frequencies.tsv
+- data/segment_ruby/small/total.tsv
+- data/segment_ruby/twitter/2_frequencies.tsv
+- data/segment_ruby/twitter/2_total.tsv
+- data/segment_ruby/twitter/README.md
+- data/segment_ruby/twitter/frequencies.tsv
+- data/segment_ruby/twitter/total.tsv
+- lib/segment_ruby.rb
+- lib/segment_ruby/version.rb
+- segment_ruby.gemspec
+homepage: https://github.com/willf/segment_ruby
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: segments text according word frequency using the Viterbi algorithm.
+test_files: []