microsoft_ngram 0.0.3 → 0.0.4

data/README.md

@@ -21,12 +21,12 @@ Usage
 
 To get a list of currently available models:
  
-    > MicrosoftNgram.models
-    => ["bing-anchor/jun09/1", "bing-anchor/jun09/2", "bing-anchor/jun09/3", "bing-anchor/jun09/4", "bing-body/jun09/1", "bing-body/jun09/2", "bing-body/jun09/3", "bing-title/jun09/1", "bing-title/jun09/2", "bing-title/jun09/3", "bing-title/jun09/4", "bing-query/jun09/1", "bing-query/jun09/2", "bing-query/jun09/3"] 
+    >> Bing::Ngram.models
+    => ["bing-anchor/jun09/1", "bing-anchor/jun09/2", "bing-anchor/jun09/3", "bing-anchor/jun09/4", "bing-body/jun09/1", "bing-body/jun09/2", "bing-body/jun09/3", "bing-title/jun09/1", "bing-title/jun09/2", "bing-title/jun09/3", "bing-title/jun09/4", "bing-query/jun09/1", "bing-query/jun09/2", "bing-query/jun09/3", "bing-title/apr10/1", "bing-title/apr10/2", "bing-title/apr10/3", "bing-title/apr10/4", "bing-title/apr10/5", "bing-anchor/apr10/1", "bing-anchor/apr10/2", "bing-anchor/apr10/3", "bing-anchor/apr10/4", "bing-anchor/apr10/5", "bing-body/apr10/1", "bing-body/apr10/2", "bing-body/apr10/3", "bing-body/apr10/4", "bing-body/apr10/5"] 
  
 To see the default model:
 
-    > MicrosoftNgram.new.model            
+    > MicrosoftNgram.default_model            
     => "bing-body/jun09/3" 
 
 Parameters to the initializer are:
@@ -73,6 +73,15 @@ To use the query model for the same thing:
     vista -1.199022
     installer -1.248958
     
+You can also get a list of the N most likely candidates (could be slower for long lists):
+
+    > MicrosoftNgram.new(:model => 'bing-query/jun09/3').generate_list("Microsoft Windows",5).each  {|x| puts x.join(' ')}
+    xp -0.5429792
+    </s> -1.062959
+    update -1.08291
+    vista -1.199022
+    installer -1.248958 
+       
 Sample Script
 -------------
 

data/examples/segment.rb

@@ -7,8 +7,8 @@ require 'memoize'
 
 include Memoize
 
-$bi_body_model = MicrosoftNgram.new(:model => "bing-body/jun09/2", :debug=>false)
-$uni_body_model = MicrosoftNgram.new(:model => "bing-body/jun09/1", :debug=>false)
+$bi_body_model = MicrosoftNgram.new(:model => "bing-body/apr10/2", :debug=>false)
+$uni_body_model = MicrosoftNgram.new(:model => "bing-body/apr10/1", :debug=>false)
 $magic_pr =  -13.419954 # twice as uncommon as "kraig" last word in Bing 100k list
 
 # Returns all the splits of a string up to a given length

data/examples/synthese.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + '/synthese_lib.rb'
+require 'rubygems'
+require 'trollop'
+
+def weighted_random_index l
+    tot = l.inject(0.0){|a,b| a+ b}
+    ns = l.map{|i| i/tot}
+    r = rand
+    c = 0.0
+    ns.each_with_index{|n,i| return i if r < (c+= n)}
+    l.size-1 # PRACTICE SAFETY 
+end 
+
+def random_punctuation
+  p = [".", "?", "!"]
+  f = [0.9, 0.05, 0.05]
+  p[weighted_random_index(f)]
+end
+
+def rand_btw(min,max)
+  rand(max-min)+min
+end
+
+def tcase(w)
+  return w.upcase if w.size < 2
+  w[0..0].upcase + w[1..-1]
+end
+
+def scribble(opts)
+  puts "\n== " + opts[:title].split(/ /).map{|w| tcase(w)}.join(" ") + " ==\n\n" if opts[:title] && opts[:title].size > 0
+  len = rand_btw(opts[:max_sentence_length],opts[:max_sentence_length])
+  wc = -1
+  s = opts[:s]
+  s.generate_n(opts[:max_wc],opts[:starter]){ |w| 
+    ((wc+=1) == 0) ? print(tcase(w)) : (((w=="<s>") || (w=="</s>")) ? print("#{w}\n") : print(w))
+    print(" ")
+    $stdout.flush
+  }
+end
+
+def main
+  opts = Trollop::options do
+    banner "Synthetic writer"
+    opt :title, "Title for story", :default => ""
+    opt :starter, "Starter text for story", :default => "It was the best of times, it was the worst of times"
+    opt :max_wc, "Max number of words to generate", :default => 1000
+    opt :model, "Corpus model to use", :default => Bing::Ngram.default_model
+    opt :depth, "How deep to look for the next token", :depth => 100
+    opt :max_sentence_length, "Maximum length of sentences to generate", :default => 16
+    opt :min_sentence_length, "Minimum length of sentences to generate", :default => 8
+    opt :debug, "Debug errors", :default=> true
+  end
+  
+  opts[:s] = Synthese.new(opts)
+  trap("INT","EXIT")
+  scribble(opts)
+end
+
+main
+  

data/examples/synthese_lib.rb

@@ -0,0 +1,75 @@
+require File.dirname(__FILE__) + '/../lib/microsoft_ngram'
+require 'rubygems'
+
+class BackoffGenerator
+  attr_accessor :model_type, :models, :debug
+  
+  def initialize(model_type="body",max_models=5,debug=false)
+    @model_type = model_type
+    @debug=debug
+    @models = Bing::Ngram::models_find_all(model_type).take(max_models).map{|m|Bing::Ngram.new(:model => m, :debug=>debug)}
+  end
+  
+  def generate_list(text, n, initial_text)
+    @models.each do |model|
+      l = model.generate_list(text,n)
+      return l if l.size > 0
+    end
+    if (text != initial_text) 
+      generate_list(initial_text, n, initial_text)
+    else
+      []
+    end
+  end
+  
+end
+
+class Synthese
+  attr_accessor :generator, :depth, :n, :debug
+  
+  def initialize(args = {})
+    #puts args.inspect
+    mtype = args[:model_type] || "body"
+    @debug = args[:debug] || false
+    @generator = BackoffGenerator.new(mtype,5,false)
+    @depth = (args[:depth] ? args[:depth].to_i : 100)
+    @n = Bing::ModelSpec.new(@generator.models[0].model).size
+  end
+  
+  def self.weighted_random_index l
+      tot = l.inject(0.0){|a,b| a+ b}
+      ns = l.map{|i| i/tot}
+      r = rand
+      c = 0.0
+      ns.each_with_index{|n,i| return i if r < (c+= n)}
+      l.size-1 # PRACTICE SAFETY 
+  end
+
+  def self.random_word l
+    l[Synthese.weighted_random_index(l.map{|w,lp| Math.exp(lp)})][0]
+  end
+  
+  
+  def generate(text)
+    tokens = text.split(/\s/)
+    tokens.each{|token| yield token}
+    tokens = tokens[-@n..-1] if tokens.size > @n
+    initial_text = text
+    text = tokens.join(' ')
+    while true
+      #puts "OUTER. Generating from : #{text}"
+      l = @generator.generate_list(text,@depth,initial_text)
+      break if l.size == 0
+      w = Synthese.random_word(l)
+      yield w
+      tokens << w
+      tokens = tokens[-@n..-1] if tokens.size > @n
+      text = tokens.join(' ')
+    end
+  end
+  
+  def generate_n(n,text)
+    counter = 0
+    generate(text){|w| yield w; counter += 1; break if counter > n }  
+  end
+end

data/examples/veritable.rb

@@ -0,0 +1,29 @@
+require File.dirname(__FILE__) + '/veritable_lib.rb'
+require 'rubygems'
+require 'trollop'
+
+def main
+  opts = Trollop::options do
+    banner "Generate phrases from Bing's Ngram server based on patterns— specify pattern on command line as alternating strings and ints"
+    opt :max_wild_children, "Max number of wildcard tokens to generate", :default => 20
+    opt :max_literal_children, "Max number of tokens to generate after literal strings", :default => 100
+    opt :model, "Corpus model to use", :default => Bing::Ngram.default_model
+    opt :max_length, "Maximum length of phrases to generate", :default => 5
+    opt :verbose, "Send logging messages to $STDERR", :default => false
+  end
+  
+  pattern = []
+  ARGV.each_with_index{|item, i| pattern << ((i % 2 == 0) ? item : [item.to_i])}
+  if opts[:verbose]
+    $stderr.puts opts.inspect
+    $stderr.puts pattern.inspect
+  end
+  R.new(pattern, opts).generate{|t| $stdout.puts(t.join("\t"))}
+end
+
+trap("INT","EXIT")
+main
+
+# examples 
+#  veritable.rb "veritable" 2 "of" 2 
+#  veritable.rb  --max-literal-children 50 "a taste of" 2 

data/examples/veritable_lib.rb

@@ -0,0 +1,121 @@
+require File.dirname(__FILE__) + '/../lib/microsoft_ngram'
+
+# S comtains the state we use to move query the model.
+# 
+class S
+  attr_accessor :prefix, :pattern, :max_wild_children, :max_literal_children, :max_length, :cp, :model
+  def initialize(prefix, pattern, max_wild_children, max_literal_children, max_length, cp, model)
+    @prefix = prefix
+    @pattern = pattern
+    @max_wild_children = max_wild_children
+    @max_literal_children = max_literal_children
+    @cp = cp
+    @model = model
+    @max_length = max_length
+  end
+  
+  def to_s
+    "S(#{@prefix.inspect}:#{@pattern.inspect}:#{cp})"
+  end
+  
+  def children
+    return [] if @pattern.size == 0
+    return [] if @prefix.split.size >= @max_length
+    hd = @pattern.first
+    tail = @pattern[1..-1]
+    # puts "pattern: #{@pattern.inspect}; hd: #{hd.inspect}; tail: #{tail.inspect}"
+    if hd.class == String
+      string_children(hd, tail)
+    else
+      wildcard_children(hd, tail)
+    end
+  end
+  
+  def string_children(hd, tail)
+    query = @prefix.size > 0 ? "#{@prefix} #{@pattern[0]}" : @pattern[0]
+    pairs = @model.generate_list(query,@max_literal_children)
+    pairs.map{|w,cp| S.new("#{query} #{w}", tail, @max_wild_children, @max_literal_children, @max_length, cp, @model)}
+  end
+  
+  def wildcard_children(hd, tail)
+    n = @pattern[0][0]
+    return [] if n<1 # the 'one' was generated by the string children or the previous iteration
+    pairs = @model.generate_list(@prefix,@max_wild_children)
+    pairs.map{|w,cp| S.new("#{@prefix} #{w}",([[n-1]] + tail),@max_wild_children, @max_literal_children, @max_length, cp, @model)}
+  end
+end
+
+# R is the machine that generates most likely phrases with wildcards
+#  it is intialized with
+#   pattern = an array of alternating string and size requirements specified as [i]
+#      eg ["veritable", [2], "of" [2]] -> look for 'veritable', then up to 2 tokens then 'of', then up to two tokens
+#         it has to be at least [1]
+#   max_wild_children : for wildcards (size requirement), the maximum number to inspect at this point
+#   max_literal_children: for strings, the max number of tokens to inspect
+#   model: the corpus in bing format, eg "bing-body/apr10/5". Leave nil to use the 'best' (latest, longest, body)
+#   verbose: if true, a few messages printed to $stderr
+class R
+  attr_accessor :pattern, :regex, :max_wild_children, :max_literal_children, :max_length, :model, :verbose
+  
+  def initialize(pattern, args = {})
+    @pattern = pattern
+    raise "Pattern must be array" unless @pattern.class == Array
+    raise "First item #{@pattern[0].inspect} must be a string" unless @pattern[0].class == String
+    unless pattern.all?{|el| el.class==String || (el.class==Array  && el[0].class==Fixnum)}
+      raise "Pattern items must be strings or arrays of numbers"
+    end 
+    @model = !args[:model] ? Bing::Ngram.new : (args[:model].class==String ? Bing::Ngram.new(args[:model]) : args[:model])
+    @max_wild_children = args[:max_wild_children] || 20
+    @max_literal_children = args[:max_literal_children] || 100
+    @max_length = args[:max_length] || Bing::CorpusSpec.new(@model.model).size
+    @verbose = args[:verbose] || false
+    @regex = pattern_to_regex(pattern)
+  end
+
+  # converts the pattern to a reg expression
+  def pattern_to_regex(pattern)
+    def array_to_regex(n)
+      case n
+      when 1 
+        "\\S+"
+      else
+        "(?:\\S+ ){0,#{n-1}}\\S+"
+      end
+    end
+    str = pattern.map do |el|
+      el.class==String ? el : array_to_regex(el[0])
+    end.join(" ")
+    Regexp.new(str)
+  end
+  
+  # does the generated phrase match?
+  def matches(str)
+    (@regex =~ str) && true
+  end
+  
+  # generate  -- yields tuples of prefix, cp, and jp
+  def generate
+    stack = []
+    s = S.new("",@pattern, @max_wild_children, @max_literal_children, @max_length, 0.0, @model)
+    $stderr.puts "Generate:Initializing.. #{s.inspect}." if @verbose
+    stack.push(s)
+    until stack.empty?
+     current = stack.pop
+     $stderr.puts("Current state: #{current}; stack size: #{stack.size}") if @verbose
+     if matches(current.prefix)
+       jp = @model.jp(current.prefix)
+       yield [current.prefix, current.cp, jp]
+     else
+       $stderr.puts("Not a match: #{current.prefix.inspect}") if @verbose
+     end
+     current.children.each{|child|  stack.push(child);  $stderr.puts "Added child: #{child}" if @verbse}
+    end
+  end
+end
+
+
+
+
+# examples 
+# #x = R.new([ "veritable", [2], "of", [2] ], 20, 100, 5)
+

data/lib/microsoft_ngram.rb

@@ -1,8 +1,41 @@
-require "microsoft_ngram/version"
+require 'rubygems'
 require "rest-client"
 
 module Bing
-
+  
+  # this class is only used to find the best default model, 
+  # that is, for the default_model call
+  class ModelSpec
+    
+    attr_accessor :model_type, :date, :size
+    
+    def initialize (spec)
+      def parse_month(m)
+        case m
+          when 'jan': '01'
+          when 'feb': '02'
+          when 'mar': '03'        
+          when 'apr': '04'        
+          when 'may': '05'        
+          when 'jun': '06'        
+          when 'jul': '07'        
+          when 'aug': '08'        
+          when 'sep': '09'        
+          when 'oct': '10'        
+          when 'nov': '11'
+          when 'dec': '12'
+          else '??'
+        end
+      end
+      parts = spec.split('/')
+      @model_type = parts[0].split('-')[1] # 'bing-body'
+      yr = parts[1].split(/\D/)[-1].to_i + 2000
+      month = parse_month(parts[1].split(/\d/)[0])
+      @date = "#{yr}-#{month}"
+      @size = parts[2].to_i
+    end
+  end
+  
   class Ngram
 
     @@endpoint = "http://web-ngram.research.microsoft.com/rest/lookup.svc/"
@@ -17,6 +50,26 @@ module Bing
       @@models.include?(model)
     end 
   
+    def self.default_model(model_type='body') #most recent, longest
+      Bing::Ngram.models() if @@models == nil # cache the current models
+      ms = @@models.
+        map{|x| [ModelSpec.new(x),x]}.
+        find_all{|c| c[0].model_type == model_type}.
+        sort_by{|c| [c[0].date, c[0].size]}
+        @@default_model = ms.size > 0 ? ms[-1][1] : nil
+    end
+    
+    def self.models_find_all(model_type='body',min_size=1)
+      Bing::Ngram.models() if @@models == nil # cache the current models
+      ms = @@models.
+        map{|x| [ModelSpec.new(x),x]}.
+        find_all{|c| c[0].model_type == model_type}.
+        find_all{|c| c[0].size >= min_size}.
+        sort_by{|c| [c[0].date, c[0].size]}.
+        map{|spec,m| m}.
+        reverse
+    end
+    
     attr_accessor :user_token
     # The model is the current model. Query this.models() for available models
     attr_accessor :model
@@ -24,25 +77,28 @@ module Bing
     attr_accessor :debug
     # Ngram size based on model
     attr_accessor :ngram_size
-  
+    
     def initialize(args = {}) 
       @user_token = args["user_token"] || args[:user_token] || ENV["NGRAM_TOKEN"]
       unless @user_token
         raise "Must provide user token as NGRAM_TOKEN env variable or as :user_token => token. To get a token, see http://web-ngram.research.microsoft.com/info/ "
       end
       # probably shouldn't change
-      @model = args["model"] || args[:model] || Bing::Ngram.models().find_all{|x| x =~ /body/}.max
+      @model = args["model"] || args[:model] || Bing::Ngram.default_model() 
       unless Bing::Ngram.defined_model?(@model)
         raise "Invalid model: #{@model}. Valid models are #{@@models.join('; ')}"
       end
       @debug = (args["debug"] || args[:debug] || nil)
+      #puts "Creating #{@model.inspect} with debug=#{@debug}"
       @ngram_size = @model.split(/\//)[-1].to_i
     end
   
+    
+    
     def get(op,phrase,args)
       model = args["model"] || args[:model] || @model 
       RestClient.get(@@endpoint + model + '/' + op, {:params => {:u => @user_token, :p => phrase}.merge(args)}) do |res,req,result|
-        $stderr.puts req.inspect if @debug
+        $stderr.puts res.inspect if @debug
        res
       end
     end
@@ -50,7 +106,7 @@ module Bing
     def post(op,phrases,args)
       model = args["model"] || args[:model] || @model 
       RestClient.post(@@endpoint + model + '/' + op + "?u=#{@user_token}", phrases.join("\n")) do |res,req,result|
-        $stderr.puts req.inspect if @debug
+        $stderr.puts res.inspect if @debug
        res
       end
     end
@@ -89,6 +145,13 @@ module Bing
         end
       end
     end
+    
+    # get a list of the next most popular tokens with log-freq
+    def generate_list(phrase, max_length)
+      l = []
+      generate(phrase,max_length){|p| l << p}
+      l
+    end
   
     # spell-checking 
     # Bing::Ngram.new(:debug=>nil,:model=>'bing-body/jun09/1').jps(edits1("appresiate").uniq).sort{|a,b| b[1] <=> a[1]}[0..30]

data/lib/microsoft_ngram/version.rb

@@ -1,3 +1,3 @@
 module MicrosoftNgram
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/microsoft_ngram.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'autotest'
 
   s.add_runtime_dependency "rest-client"
+  s.add_runtime_dependency "trollop"
 end

metadata

@@ -1,70 +1,106 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: microsoft_ngram
-version: !ruby/object:Gem::Version
-  version: 0.0.3
+version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 4
+  version: 0.0.4
 platform: ruby
-authors:
+authors: 
 - Will Fitzgerald
 - Zeke Sikelianos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-03 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-02-09 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: hoe
-  requirement: &70357835216420 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
-  prerelease: false
-  version_requirements: *70357835216420
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &70357835211620 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 47
+        segments: 
+        - 2
+        - 8
+        - 0
         version: 2.8.0
   type: :development
-  prerelease: false
-  version_requirements: *70357835211620
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: autotest
-  requirement: &70357835210020 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
-  prerelease: false
-  version_requirements: *70357835210020
-- !ruby/object:Gem::Dependency
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
   name: rest-client
-  requirement: &70357835207140 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: trollop
   prerelease: false
-  version_requirements: *70357835207140
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: A simple wrapper for Bing's ngram API
-email:
+email: 
 - will@wordnik.com
 - zeke@sikelianos.com
-executables:
+executables: 
 - microsoft_ngram
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - .rspec
 - Gemfile
@@ -74,6 +110,10 @@ files:
 - Rakefile
 - bin/microsoft_ngram
 - examples/segment.rb
+- examples/synthese.rb
+- examples/synthese_lib.rb
+- examples/veritable.rb
+- examples/veritable_lib.rb
 - lib/microsoft_ngram.rb
 - lib/microsoft_ngram/version.rb
 - microsoft_ngram.gemspec
@@ -81,26 +121,36 @@ files:
 - spec/spec_helper.rb
 homepage: http://developer.wordnik.com
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: microsoft_ngram
-rubygems_version: 1.8.12
+rubygems_version: 1.8.13
 signing_key: 
 specification_version: 3
 summary: A simple wrapper for Bing's ngram API
 test_files: []
+