microsoft_ngram 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in microsoft_ngram.gemspec
+gemspec

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2010-08-31
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,8 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/microsoft_ngram
+lib/microsoft_ngram.rb
+spec/microsoft_ngram_spec.rb
+examples/segment.rb

data/README.md

@@ -0,0 +1,117 @@
+microsoft_ngram
+===============
+
+This is a simple ruby gem to access the Bing Ngram data. It's loosely based on Microsoft's Python library.
+Source code at [github.com/willf/microsoft_ngram](http://github.com/willf/microsoft_ngram).
+
+Installation
+------------
+
+Email [webngram@microsoft.com](mailto:webngram@microsoft.com?subject=Token%20Request) and request a token.
+When you get your token, add it to your .bashrc or .bash_profile:
+
+    export NGRAM_TOKEN="YOUR-TOKEN-HERE"
+    
+Then install the gem:
+
+    gem install microsoft_ngram
+
+Usage
+-----
+
+To get a list of currently available models:
+ 
+    > MicrosoftNgram.models
+    => ["bing-anchor/jun09/1", "bing-anchor/jun09/2", "bing-anchor/jun09/3", "bing-anchor/jun09/4", "bing-body/jun09/1", "bing-body/jun09/2", "bing-body/jun09/3", "bing-title/jun09/1", "bing-title/jun09/2", "bing-title/jun09/3", "bing-title/jun09/4", "bing-query/jun09/1", "bing-query/jun09/2", "bing-query/jun09/3"] 
+ 
+To see the default model:
+
+    > MicrosoftNgram.new.model            
+    => "bing-body/jun09/3" 
+
+Parameters to the initializer are:
+
+    :model => <i>string</i> (sets model)
+    :user_token => <i>string</i> (sets user token)
+    :debug => <i>boolean</i> (will show GET/POST calls)
+ 
+So, to use the 2-gram title model:
+
+    > model = MicrosoftNgram.new(:model => "bing-title/jun09/2")
+
+To get a single joint probability, or multiple joint probabilities (If
+you know you want multiple joint probabilities, it is better to ask
+for several at once):
+
+    > MicrosoftNgram.new.jps(['fish sticks', 'frog sticks'])
+    => [["fish sticks", -6.853792], ["frog sticks", -9.91852]] 
+    > MicrosoftNgram.new.jp("fish sticks")
+    => -6.853792 
+
+To get a single conditional probability, or multiple conditional probabilities (If you know you want multiple conditional probabilities, it is better to ask for several at once):
+
+    > MicrosoftNgram.new.cp("fish sticks")
+    => -2.712575 
+    > MicrosoftNgram.new.cps(['fish sticks', 'frog sticks'])
+    => [["fish sticks", -2.712575], ["frog sticks", -4.788582]] 
+
+To yield the most probable next token using the default model:
+
+    > MicrosoftNgram.new.generate("Microsoft Windows",5)  {|x| puts x.join(' ')}
+    xp -0.6964428
+    vista -0.9242383
+    server -1.106876
+    2000 -1.145312
+    currentversion -1.168404
+
+To use the query model for the same thing:
+
+    > MicrosoftNgram.new(:model => 'bing-query/jun09/3').generate("Microsoft Windows",5)  {|x| puts x.join(' ')}
+    xp -0.5429792
+    </s> -1.062959
+    update -1.08291
+    vista -1.199022
+    installer -1.248958
+    
+Sample Script
+-------------
+
+```ruby
+require 'rubygems'
+require 'microsoft_ngram'
+l = []
+Bing::Ngram.new(:model => "bing-body/apr10/5").generate('a bum',50){ |w,_| l << w }
+l.join("; ")
+```
+
+More Info
+---------
+
+See the [REST API](http://web-ngram.research.microsoft.com/info/rest.html) and the 
+[terms of use](http://web-ngram.research.microsoft.com/info/TermsOfUse.htm) for accessing the Microsoft data.
+
+License
+-------
+
+(The MIT License)
+
+Copyright (c) 2010/2011
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler'
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+require 'microsoft_ngram'
+
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new('spec')
+
+# If you want to make this the default task
+task :default => :spec

data/examples/segment.rb

@@ -0,0 +1,77 @@
+require File.dirname(__FILE__) + '/../lib/microsoft_ngram'
+require 'rubygems'
+require 'memoize'
+
+# This code based on Peter Novig's chapter on "Natural Language Corpus Data" in
+# Beautiful Data. 
+
+include Memoize
+
+$bi_body_model = MicrosoftNgram.new(:model => "bing-body/jun09/2", :debug=>false)
+$uni_body_model = MicrosoftNgram.new(:model => "bing-body/jun09/1", :debug=>false)
+$magic_pr =  -13.419954 # twice as uncommon as "kraig" last word in Bing 100k list
+
+# Returns all the splits of a string up to a given length
+def splits(text,max_len=text.size)
+  Range.new(0,[text.size,max_len].min-1).map{|i| [text[0..i],text[i+1..-1]]}
+end
+
+# This keeps just those splits whose first item is above the magic unigram
+# log probability
+def reasonable_splits(text,max_len=text.size)
+  splits(text,max_len).find_all{|pre,suf| Pr(pre)>=$magic_pr}
+end
+
+# Get the unigram log probability of a token
+def Pr(str)
+  $uni_body_model.cp(str)
+end 
+
+# Get the conditional probability of a word, given a prior
+def cPw(word, prev)
+  $bi_body_model.cp([prev,word].join(' '))
+end
+
+# combine data
+def combine(pfirst, first, pr)
+  prem, rem = pr
+  return [pfirst+prem, [first]+rem]
+end
+
+# segment a text, assuming it is at the beginning of a sentence
+# return a pair: the log probability, and the most probable segmentation
+def segment2(text, prev="<s>")
+  #puts "segment2: #{text.inspect} prev: #{prev}"
+  return [0.0,[]] if (!text or text.size==0)
+  reasonable_splits(text).map{|first,rem| combine(cPw(first,prev), first, segment2(rem, first))}.max
+end
+
+# just return the best segmentation
+def segment(text)
+  segment2(text)[1]
+end
+
+# We want to memoize a lot of things.
+memoize :splits
+memoize :reasonable_splits
+memoize :Pr 
+memoize :cPw 
+memoize :segment2 
+
+# These are some Twitter hash tags which I segmented.
+#  > segment("bpcares")
+#  => ["bp", "cares"] 
+#  > segment("Twitter")
+#  => ["Twitter"] 
+#  > segment("writers")
+#  => ["writers"] 
+#  > segment("iamwriting")
+#  => ["i", "am", "writing"] 
+#  > segment("backchannel")
+#  => ["back", "channel"] 
+#  > segment("tcot")
+#  => ["tcot"] 
+#  > segment("vacationfallout")
+#  => ["vacation", "fall", "out"] 
+
+

data/lib/microsoft_ngram.rb

@@ -0,0 +1,98 @@
+require "microsoft_ngram/version"
+require "rest-client"
+
+module Bing
+
+  class Ngram
+
+    @@endpoint = "http://web-ngram.research.microsoft.com/rest/lookup.svc/"
+    @@models = nil
+  
+    def self.models()
+      @@models=RestClient.get(@@endpoint).split(/\s+/)
+    end
+  
+    def self.defined_model?(model)
+      Bing::Ngram.models() if @@models == nil # cache the current models
+      @@models.include?(model)
+    end 
+  
+    attr_accessor :user_token
+    # The model is the current model. Query this.models() for available models
+    attr_accessor :model
+    # Simple debug mode. If non-false, GET calls are display
+    attr_accessor :debug
+    # Ngram size based on model
+    attr_accessor :ngram_size
+  
+    def initialize(args = {}) 
+      @user_token = args["user_token"] || args[:user_token] || ENV["NGRAM_TOKEN"]
+      unless @user_token
+        raise "Must provide user token as NGRAM_TOKEN env variable or as :user_token => token. To get a token, see http://web-ngram.research.microsoft.com/info/ "
+      end
+      # probably shouldn't change
+      @model = args["model"] || args[:model] || Bing::Ngram.models().find_all{|x| x =~ /body/}.max
+      unless Bing::Ngram.defined_model?(@model)
+        raise "Invalid model: #{@model}. Valid models are #{@@models.join('; ')}"
+      end
+      @debug = (args["debug"] || args[:debug] || nil)
+      @ngram_size = @model.split(/\//)[-1].to_i
+    end
+  
+    def get(op,phrase,args)
+      model = args["model"] || args[:model] || @model 
+      RestClient.get(@@endpoint + model + '/' + op, {:params => {:u => @user_token, :p => phrase}.merge(args)}) do |res,req,result|
+        $stderr.puts req.inspect if @debug
+       res
+      end
+    end
+  
+    def post(op,phrases,args)
+      model = args["model"] || args[:model] || @model 
+      RestClient.post(@@endpoint + model + '/' + op + "?u=#{@user_token}", phrases.join("\n")) do |res,req,result|
+        $stderr.puts req.inspect if @debug
+       res
+      end
+    end
+  
+    def cp(phrase,args={})
+      get('cp',phrase,args).to_f
+    end
+  
+    def cps(phrases,args={})
+      phrases.zip(post('cp',phrases,args).split(/\s+/).map{|r| r.strip.to_f})
+    end
+  
+    def jp(phrase,args={})
+      get('jp',phrase,args).to_f
+    end
+  
+    def jps(phrases,args={})
+      phrases.zip(post('jp',phrases,args).split(/\s+/).map{|r| r.strip.to_f})
+    end
+  
+    # Yield up to nstop token, log-prob pairs given the tokens in the phrase. 
+
+    def generate(phrase,nstop=2**32)
+      arg = {}
+      while true do
+        break if nstop <= 0 
+        arg['n']=[1000,[0,nstop].max].min
+        result = get("gen",phrase,arg).split("\r\n")
+        break if result.size <= 2
+        nstop -= (result.size - 2)
+        arg['cookie']=result[0]
+        backoff = result[1]
+        result[2..-1].each do |x|
+          pair = x.split(';')
+          yield [pair[0], pair[1].to_f]
+        end
+      end
+    end
+  
+    # spell-checking 
+    # Bing::Ngram.new(:debug=>nil,:model=>'bing-body/jun09/1').jps(edits1("appresiate").uniq).sort{|a,b| b[1] <=> a[1]}[0..30]
+  
+  end
+  
+end

data/lib/microsoft_ngram/version.rb

@@ -0,0 +1,3 @@
+module MicrosoftNgram
+  VERSION = "0.0.2"
+end

data/microsoft_ngram.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "microsoft_ngram/version"
+
+Gem::Specification.new do |s|
+  s.name        = "microsoft_ngram"
+  s.version     = MicrosoftNgram::VERSION
+  s.authors     = ["Will Fitzgerald", "Zeke Sikelianos"]
+  s.email       = ["will@wordnik.com", "zeke@sikelianos.com"]
+  s.homepage    = "http://developer.wordnik.com"
+  s.summary     = %q{A simple wrapper for Bing's ngram API}
+  s.description = %q{A simple wrapper for Bing's ngram API}
+
+  s.rubyforge_project = "microsoft_ngram"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "hoe"
+  s.add_development_dependency 'rspec', '~> 2.8.0'
+  s.add_development_dependency 'autotest'
+
+  s.add_runtime_dependency "rest-client"
+end

data/spec/microsoft_ngram_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+describe Bing::Ngram do  
+  
+  it "should return a list of models" do
+    Bing::Ngram.models.size.should > 0
+  end 
+  
+  it "should have a user token, or you're hosed" do
+    Bing::Ngram.new.user_token.should_not == nil 
+  end 
+  
+  it "should have a default model" do
+    Bing::Ngram.new.model.should_not == nil 
+  end
+  
+  it "should retrieve a joint probability" do
+    Bing::Ngram.new.jp("fish sticks").should < 0 
+  end 
+  
+  it 'should retrieve a list of joint probabilities' do
+    fish, frog = ["fish sticks", "frog sticks"]
+    fish_results, frog_results = Bing::Ngram.new.jps([fish, frog])
+    fish_results[0].should == fish
+    fish_results[1].should < 0
+    frog_results[0].should == frog
+    frog_results[1].should < 0
+    fish_results[1].should > frog_results[1] 
+  end
+  
+  it "should retrieve a conditional probability" do
+     Bing::Ngram.new.cp("fish sticks").should < 0 
+   end
+  
+   it 'should retrieve a list of conditional probabilities' do
+     fish, frog = ["fish sticks", "frog sticks"]
+     fish_results, frog_results = Bing::Ngram.new.cps([fish, frog])
+     fish_results[0].should == fish
+     fish_results[1].should < 0
+     frog_results[0].should == frog
+     frog_results[1].should < 0
+     fish_results[1].should > frog_results[1] 
+   end
+   
+   it 'should yield most probable next tokens' do
+     two_gram_stream = Bing::Ngram.models.find do |model|
+       name, date, size = model.split(/\//)
+       name.include?("body") && size=="2"
+     end
+     two_gram_stream.should_not == nil
+     m = Bing::Ngram.new(:model => two_gram_stream)
+     count = 0
+     m.generate("the",10) do |word, log_prob|
+       count += 1
+       word.should_not == nil
+       log_prob.should < 0
+     end
+     count.should == 10
+   end
+end 
+

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rspec'
+require 'microsoft_ngram'
+require 'rest-client'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: microsoft_ngram
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Will Fitzgerald
+- Zeke Sikelianos
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-03 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: &70093440050860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70093440050860
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70093440048640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: *70093440048640
+- !ruby/object:Gem::Dependency
+  name: autotest
+  requirement: &70093440048220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70093440048220
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: &70093440047740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70093440047740
+description: A simple wrapper for Bing's ngram API
+email:
+- will@wordnik.com
+- zeke@sikelianos.com
+executables:
+- microsoft_ngram
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- History.txt
+- Manifest.txt
+- README.md
+- Rakefile
+- bin/microsoft_ngram
+- examples/segment.rb
+- lib/microsoft_ngram.rb
+- lib/microsoft_ngram/version.rb
+- microsoft_ngram.gemspec
+- spec/microsoft_ngram_spec.rb
+- spec/spec_helper.rb
+homepage: http://developer.wordnik.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: microsoft_ngram
+rubygems_version: 1.8.12
+signing_key: 
+specification_version: 3
+summary: A simple wrapper for Bing's ngram API
+test_files: []