words 0.2.2 → 0.3.0

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

data/bin/build_wordnet

@@ -7,6 +7,8 @@ require 'pathname'
 require 'rubygems'
 require 'trollop'
 require 'rufus-tokyo'
+require 'zlib'
+require 'net/http'
 
 POS_FILE_TYPES = %w{ adj adv noun verb }
 POS_FILE_TYPE_TO_SHORT = { 'adj' => 'a', 'adv' => 'r', 'noun' => 'n', 'verb' => 'v' }
@@ -26,6 +28,7 @@ opts = Trollop::options do
   opt :wordnet, "Location of the wordnet dictionary directory", :default => "Search..."
   opt :build_tokyo, "Build the tokyo dataset?", :default => false
   opt :build_pure, "Build the pure ruby dataset?", :default => false
+  opt :build_evocations, "Build the similarity dataset based on the wordnet evocation project (requires internet connection)", :default => false
 end
 Trollop::die :build_tokyo, "You need to specify whether tokyo dataset or pure ruby index building is required" if !opts[:build_tokyo] && !opts[:build_pure]
 puts "Verbose mode enabled" if (VERBOSE = opts[:verbose])
@@ -104,10 +107,36 @@ POS_FILE_TYPES.each do |file_pos|
   
 end
 
+score_hash = Hash.new
+if opts[:build_evocations]
+  puts "Downlaoding score data..." if VERBOSE
+  scores_file = data_path + "scores.txt.gz"
+  scores_file.delete if scores_file.exist?
+  File.open(scores_file,'w') do |file|
+    file.write Net::HTTP.get(URI.parse('http://cloud.github.com/downloads/roja/words/scores.txt.gz'))
+  end
+  abort( "Unable to gather similarities information from http://cloud.github.com/downloads/roja/words/scores.txt.gz... Try again later." ) unless scores_file.exist?
+  
+  puts "Compiling score data..." if VERBOSE
+  Zlib::GzipReader.open(scores_file) do |gz|
+    gz.each_line do |line|
+      mean, median, sense1, sense2 = line.split(',')
+      senses = [sense1, sense2].map! { |sense| sense.strip.split('.') }.map! { |sense| index_hash[sense[0]]["synset_ids"].select { |synset_id| synset_id[0,1] == sense[1].gsub("s", "a") }[sense[2].to_i-1] }
+      senses.each do |sense|
+        relation = (senses - [sense]).first.nil? ? sense : (senses - [sense]).first
+        score_name = sense + "s"
+        score_hash[score_name] = { "relations" => [], "means" => [], "medians" => [] } if score_hash[score_name].nil?
+        score_hash[score_name] = { "relations" => score_hash[score_name]["relations"] << relation, "means" => score_hash[score_name]["means"] << mean, "medians" => score_hash[score_name]["medians"] << median }
+      end unless senses.include? nil
+    end
+  end
+end
+
 if opts[:build_tokyo]
   tokyo_hash = Rufus::Tokyo::Table.new((data_path + "wordnet.tct").to_s)
   index_hash.each { |k,v| tokyo_hash[k] = { "lemma" => v["lemma"], "synset_ids" => v["synset_ids"].join('|'), "tagsense_counts" => v["tagsense_counts"].join('|') } }
   data_hash.each { |k,v| tokyo_hash[k] = v }
+  score_hash.each { |k,v| tokyo_hash[k] = { "relations" => v["relations"].join('|'), "means" => v["means"].join('|'), "medians" => v["medians"].join('|') } } if opts[:build_evocations]
   tokyo_hash.close
 end
 
@@ -117,4 +146,11 @@ if opts[:build_pure]
   File.open(data_path + "index.dmp",'w') do |file|
     file.write Marshal.dump(index)
   end
+  if opts[:build_evocations]
+    score = Hash.new
+    score_hash.each { |k,v| score[k] = [v["relations"].join('|'), v["means"].join('|'), v["medians"].join('|')] }
+    File.open(data_path + "evocations.dmp",'w') do |file|
+      file.write Marshal.dump(score)
+    end
+  end
 end

data/examples.rb

@@ -26,11 +26,16 @@ if __FILE__ == $0
   puts wordnet.find('bat').synsets('noun').last.relations("~")
   puts wordnet.find('bat').synsets('verb').last.inspect
   puts wordnet.find('bat').synsets('verb').last.words
-  puts wordnet.find('bat').synsets('verb').last.words_with_num.inspect
+  puts wordnet.find('bat').synsets('verb').last.words_with_lexical_ids.inspect
   
   puts wordnet.find('bat').synsets('verb').first.lexical.inspect
   puts wordnet.find('bat').synsets('verb').first.lexical_description
   
+  puts wordnet.find('jkashdfajkshfksjdhf')
+  
+  puts wordnet.find("broadcast").senses.first.evocations
+  puts wordnet.find("broadcast").senses.first.evocations[1].inspect
+  
   wordnet.close
   
 end

data/lib/words.rb

@@ -25,9 +25,13 @@ module Words
           @connected = true
         elsif @connection_type == :pure
           # open the index is there
-          File.open(@data_path,'r') do |file|
+          File.open(@data_path, 'r') do |file|
             @connection = Marshal.load file.read
           end
+          evocation_path = Pathname.new("#{File.dirname(__FILE__)}/../data/evocations.dmp")
+          File.open(evocation_path, 'r') do |file|
+            @evocations = Marshal.load file.read
+          end if evocation_path.exist?
           # search for the wordnet files
           if locate_wordnet?(wordnet_path)
             @connected = true
@@ -56,12 +60,25 @@ module Words
     def homographs(term)
       if connection_type == :pure
         raw_homographs = @connection[term]
-        { 'lemma' => raw_homographs[0], 'tagsense_counts' => raw_homographs[1], 'synset_ids' => raw_homographs[2]}
+        { 'lemma' => raw_homographs[0], 'tagsense_counts' => raw_homographs[1], 'synset_ids' => raw_homographs[2]} unless raw_homographs.nil?
       else
         @connection[term]
       end
     end
     
+    def evocations(senset_id)
+      if connection_type == :pure
+        if defined? @evocations
+          raw_evocations = @evocations[senset_id + "s"]
+          { 'relations' => raw_evocations[0], 'means' => raw_evocations[1], 'medians' => raw_evocations[2]} unless raw_evocations.nil?
+        else
+          nil
+        end
+      else
+        @connection[senset_id + "s"]
+      end
+    end
+    
     def synset(synset_id)
       if connection_type == :pure
         pos = synset_id[0,1]      
@@ -102,6 +119,62 @@ module Words
     
   end
   
+  class Evocations
+    
+    def initialize(evocation_construct, source_synset, wordnet_connection)
+      @wordnet_connection = wordnet_connection
+      @source = source_synset
+      @evocation_construct = evocation_construct
+    end
+    
+    def means
+      @means = @evocation_construct["means"].split('|') unless defined? @means
+      @means
+    end
+    
+    def medians
+      @medians = @evocation_construct["medians"].split('|') unless defined? @medians
+      @medians
+    end
+    
+    def size
+      means.size
+    end
+    
+    def first
+      self[0]
+    end
+    
+    def last
+      self[size-1]
+    end
+    
+    def [] (index)
+      { :destination => destinations[index], :mean => means[index], :median => medians[index] }
+    end
+    
+    def destinations(pos = :all)
+      destination_ids(pos).map { |synset_id| Synset.new synset_id, @wordnet_connection, @source.homographs }
+    end
+    
+    def destination_ids(pos = :all)
+      @destination_ids = @evocation_construct["relations"].split('|') unless defined? @destination_ids
+      case 
+        when Homographs::SYMBOL_TO_POS.include?(pos.to_sym)
+        @destination_ids.select { |synset_id| synset_id[0,1] == Homographs::SYMBOL_TO_POS[pos.to_sym] }
+        when Homographs::POS_TO_SYMBOL.include?(pos.to_s)
+        @destination_ids.select { |synset_id| synset_id[0,1] == pos.to_s }
+      else
+        @destination_ids
+      end
+    end
+    
+    def to_s
+      "#{size} evocations from #{@source}"
+    end
+    
+  end
+  
   class Relation
     
     RELATION_TO_SYMBOL = { "-c" => :member_of_this_domain_topic, "+" => :derivationally_related_form, "%p" => :part_meronym, "~i" => :instance_hyponym, "@" => :hypernym, 
@@ -149,7 +222,7 @@ module Words
     end
     
     def destination
-      @destination = Synset.new @dest_synset_id, @wordnet_connection unless defined? @destination
+      @destination = Synset.new(@dest_synset_id, @wordnet_connection, nil) unless defined? @destination
       @destination
     end
     
@@ -235,12 +308,12 @@ module Words
     end
     
     def words
-      @words = words_with_num.map { |word_with_num| word_with_num[:word] } unless defined? @words
+      @words = words_with_lexical_ids.map { |word_with_num| word_with_num[:word] } unless defined? @words
       @words
     end
     
     def lexical_ids
-      @words = words_with_num.map { |word_with_num| word_with_num[:lexical_id] } unless defined? @words
+      @words = words_with_lexical_ids.map { |word_with_num| word_with_num[:lexical_id] } unless defined? @words
       @words
     end
     
@@ -301,6 +374,11 @@ module Words
       end
     end
     
+    def evocations
+      evocations_arr = @wordnet_connection.evocations(synset_id)
+      Evocations.new evocations_arr, self, @wordnet_connection unless evocations_arr.nil?
+    end
+    
     def to_s
       @to_s = "#{synset_type.to_s.capitalize} including word(s): #{words.map { |word| '"' + word + '"' }.join(', ')} meaning: #{gloss}" unless defined? @to_s
       @to_s
@@ -315,7 +393,7 @@ module Words
     
     def initialize(raw_homographs, wordnet_connection)
       @wordnet_connection = wordnet_connection
-      @lemma_hash = raw_homographs
+      @raw_homographs = raw_homographs
       # construct some conveniance menthods for relation type access
       SYMBOL_TO_POS.keys.each do |pos|
         self.class.send(:define_method, "#{pos}s?") do 
@@ -358,7 +436,7 @@ module Words
     end
     
     def synsets(pos = :all)
-      synset_ids(pos).map { |synset_id| Synset.new synset_id, self, @wordnet_connection }
+      synset_ids(pos).map { |synset_id| Synset.new synset_id, @wordnet_connection, self }
     end
     
     def synset_ids(pos = :all)
@@ -393,7 +471,8 @@ module Words
     end
     
     def find(word)
-      Homographs.new  @wordnet_connection.homographs(word), @wordnet_connection
+      homographs = @wordnet_connection.homographs(word)
+      Homographs.new  homographs, @wordnet_connection unless homographs.nil?
     end
     
     def connection_type

data/words.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{words}
-  s.version = "0.2.2"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roja Buck"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: words
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors: 
 - Roja Buck