wordnet 0.0.5 → 1.0.0.pre.126

data/README

@@ -1,93 +0,0 @@
-= Ruby-WordNet
-== General Information
-
-This library is a Ruby interface to WordNet�. WordNet� is an online lexical
-reference system whose design is inspired by current psycholinguistic theories
-of human lexical memory. English nouns, verbs, adjectives and adverbs are
-organized into synonym sets, each representing one underlying lexical
-concept. Different relations link the synonym sets.
-
-You can find out more about WordNet� at <http://wordnet.princeton.edu/>.
-
-This code was loosely based on the Lingua::Wordnet Perl module by Dan Brian, and
-uses a similar strategy of converting the WordNet data files into a BerkeleyDB
-database. The 'convertdb.rb' script in this directory can be used to build these
-databases from the WordNet dictionaries.
-
-This module is intended to offer basically the same functionality as
-Lingua::Wordnet, and you should be able to use either (or both) to access and
-modify the lexical database interchangeably. This module attempts to remain
-fairly close in API to Lingua::Wordnet, so if you're familiar with it already,
-you should be able to port things from one to the other with relatively little
-trouble. This module, however, uses BerkeleyDB's transaction subsystem to allow
-safe concurrent access to the databases.
-
-Many thanks to Dan Brian, who did most of the hard work. His efforts made my job
-mostly a matter of playing around.
-
-
-== Caveats
-
-The database-writing portions of this code have not been extensively tested, and
-there are almost certainly bugs which will cause data to be lost or
-miswritten. You should make backups of changes you make periodically.
-
-I would greatly appreciate feedback on any aspect of this software. Suggestions,
-feature requests, questions, design critiques, and bug reports are most
-welcome. Relevant patches are particularly helpful. I may be reached at
-<ged@FaerieMUD.org>.
-
-
-== Installation
-
-If you use RubyGems, you can install via:
-
-  $ sudo gem install wordnet
-
-You can also install as a site library via the Rakefile:
-
-  $ wget http://deveiate.org/code/wordnet-x.y.z.tar.gz
-  $ tar xzvf wordnet-x.y.z.tar.gz
-  $ cd wordnet-x.y.z
-  $ sudo rake install
-
-
-== More Information
-
-There is a project page for Ruby-WordNet which can be found at:
-<http://deveiate.org/projects/Ruby-WordNet/>.
-
-This library was developed as part of the FaerieMUD Project. For more
-information about the FaerieMUD project see <http://www.FaerieMUD.org/>.
-
-You may also check out the latest development source for this module (which may or
-may not be different than the release) using Subversion from the following URL:
-
-  svn://deveiate.org/Ruby-WordNet/trunk
-
-
-== Authors
-
-* Michael Granger <ged@FaerieMUD.org>
-
-
-== Legal
-
-Ruby-WordNet is Open Source Software which is Copyright � 2001-2008 by The
-FaerieMUD Consortium.
-
-It is licensed under the modified BSD license. See the LICENSE file for details.
-
-WordNet� is a registered trademark of Princeton University. 
-
-Lingua::Wordnet is code licensed under the following terms:
-
-  Lingua::Wordnet
-  Copyright 1999,2000,2001 by Dan Brian.
-
-  This program is free software; you can redistribute it and/or modify
-  it under the same terms as Perl itself.
-
-
-$Id: README 95 2008-09-05 18:49:25Z deveiant $
-

data/Rakefile.local

@@ -1,46 +0,0 @@
-#!rake
-# 
-# Project-local tasks for Ruby-WordNet
-# $Id: Rakefile.local 95 2008-09-05 18:49:25Z deveiant $
-# 
-# Authors:
-# * Michael Granger <ged@FaerieMUD.org>
-# 
-
-unless defined?( BASEDIR )
-	fail "This is meant to be loaded from the main Rakefile, not run directly."
-end
-
-
-require 'wordnet'
-require 'rake'
-
-CONVERT_UTIL = BASEDIR + 'convertdb.rb'
-DATA_BUILD_DIR = BASEDIR + File.basename( WordNet::Lexicon::DEFAULT_DB_ENV )
-DATA_DATABASE_FILE = DATA_BUILD_DIR + 'data'
-
-
-### Tasks
-
-# Add 'convert' to the default task, and the testing tasks
-Rake::Task[:default].prerequisites << :convert
-Rake::Task[:spec].prerequisites << :convert
-
-### Task: convert
-desc "Convert WordNet dict files to a database"
-task :convert => DATA_DATABASE_FILE
-
-# Conversion utility
-file CONVERT_UTIL.to_s
-
-# Build directory for the database files
-directory DATA_BUILD_DIR.to_s
-CLOBBER.include( DATA_BUILD_DIR.to_s )
-
-# BerkeleyDB main database file
-file DATA_DATABASE_FILE.to_s
-task DATA_DATABASE_FILE.to_s => CONVERT_UTIL do
-	load CONVERT_UTIL
-	WordNetConverter.new( DATA_BUILD_DIR ).convertdb
-end
-

data/convertdb.rb

@@ -1,417 +0,0 @@
-#!/usr/bin/ruby
-# 
-# Conversion script for Ruby-WordNet
-# 
-# == Synopsis
-# 
-#   ./convertdb.rb [DATADIR]
-# 
-# == Authors
-#
-# This is a port of Dan Brian's convertdb.pl in the Lingua::Wordnet
-# distribution. It requires the 'strscan' library, which is in the standard
-# library of Ruby 1.8.
-# 
-# * Michael Granger <ged@FaerieMUD.org>
-# 
-# == Copyright
-#
-# Copyright (c) 2003-2008 The FaerieMUD Consortium. All rights reserved.
-# 
-# This module is free software. You may use, modify, and/or redistribute this
-# software under the terms of the Perl Artistic License. (See
-# http://language.perl.com/misc/Artistic.html)
-# 
-# == Version
-#
-#  $Id: convertdb.rb 94 2008-07-25 02:47:42Z deveiant $
-# 
-
-begin
-	base = File::dirname( File::expand_path(__FILE__) )
-	$LOAD_PATH.unshift "#{base}/lib" unless $LOAD_PATH.include?( "#{base}/lib" )
-	$LOAD_PATH.unshift base
-
-    unless defined?( UtilityFunctions )
-        require "#{base}/utils.rb"
-        include UtilityFunctions
-    end
-end
-
-require 'pathname'
-require 'strscan'
-require 'wordnet'
-require 'optparse'
-require 'fileutils'
-
-
-# Globals: Index of words => senses, StringScanner for parsing.
-$senseIndex = {}
-$scanner = StringScanner::new( "" )
-
-class WordNetConverter
-
-	# Source WordNet files
-	IndexFiles = %w[ index.noun index.verb index.adj index.adv ]
-	MorphFiles = {
-		'adj.exc'		=> WordNet::Adjective,
-		'adv.exc'		=> WordNet::Adverb,
-		'noun.exc'		=> WordNet::Noun,
-		'verb.exc'		=> WordNet::Verb,
-		'cousin.exc'	=> '',
-	}
-	DataFiles =  {
-		'data.adj'		=> WordNet::Adjective,
-		'data.adv'		=> WordNet::Adverb,
-		'data.noun'		=> WordNet::Noun,
-		'data.verb'		=> WordNet::Verb,
-	}
-
-	# Struct which represents a list of files, a database, and a processor function
-	# for moving records from each of the files into the database.
-	Fileset = Struct::new( "WordNetFileset", :files, :name, :db, :processor )
-
-	# How many records to insert between commits
-	CommitThreshold = 2000
-
-	# Temporary location for the lexicon data files
-	BuildDir = Pathname.new( __FILE__ ).expand_path.dirname + 
-	           Pathname.new( WordNet::Lexicon::DEFAULT_DB_ENV ).basename
-
-
-	### Create a new converter that will dump WordNet dictionary files into a BerkeleyDB 
-	### in the given +builddir+ 
-	def initialize( builddir=BuildDir )
-		@builddir = Pathname.new( builddir )
-	end
-	
-
-	### Convert the various dict files from the WordNet project into a BerkeleyDB database
-	def convertdb( errorLimit=0 )
-		$stderr.sync = $stdout.sync = true
-		header "WordNet Lexicon Converter"
-
-		# Make sure the user knows what they're in for
-		message "This program will convert WordNet data files into databases\n"\
-			"used by Ruby-WordNet. This will not affect existing WordNet files,\n"\
-			"but will require up to 40Mb of disk space.\n"
-		exit unless /^y/i =~ prompt_with_default("Continue?", "y")
-
-		# Open the database and check to be sure it's empty. Confirm overwrite if
-		# not. Checkpoint and set up logging proc if debugging.
-		if @builddir.exist? && ( @builddir + 'data' ).exist?
-			message ">>> Warning: Existing data in the Ruby-WordNet databases\n"\
-				"will be overwritten.\n"
-			abort( "user cancelled." ) unless 
-				/^y/i =~ prompt_with_default( "Continue?", "n" )
-			@builddir.rmtree
-		end
-
-		# Find the source data files
-		default = nil
-		wndirs = Pathname.glob( Pathname.getwd + 'WordNet-*' )
-		localdict = Pathname.getwd + 'dict'
-		if !wndirs.empty?
-			default = wndirs.first + 'dict'
-		elsif localdict.exist?
-			default = localdict
-		else
-			default = '/usr/local/WordNet-3.0/dict'
-		end
-
-		message "Where can I find the WordNet data files?\n"
-		datadir = prompt_with_default( "Data directory", default )
-		datadir = Pathname.new( datadir )
-
-		abort( "Directory '#{datadir}' does not exist" ) unless datadir.exist?
-		abort( "'#{datadir}' is not a directory" ) unless datadir.directory?
-		testfile = datadir + "data.noun"
-		abort( "'#{datadir}' doesn't seem to contain the necessary files.") unless testfile.exist?
-
-		# Open the lexicon readwrite into the temporary datadir
-		@builddir.mkpath
-		lexicon = WordNet::Lexicon::new( @builddir.to_s, 0666 )
-
-		# Process each fileset
-		[	  # Fileset,  name,    database handle, processor
-			Fileset::new( IndexFiles, "index", lexicon.index_db, method(:parse_index_line) ),
-			Fileset::new( MorphFiles, "morph", lexicon.morph_db, method(:parse_morph_line) ),
-			Fileset::new( DataFiles,  "data",  lexicon.data_db,  method(:parse_synset_line) ),
-		].each do |set|
-			message "Converting %s files...\n" % set.name
-			set.db.truncate
-
-			# Process each file in the set with the appropriate processor method and
-			# insert results into the corresponding table.
-			set.files.each do |file,pos|
-				message "    #{file}..."
-
-				filepath = File::join( datadir, file )
-				if !File::exists?( filepath )
-					message "missing: skipped\n"
-					next
-				end
-
-				txn, dbh = lexicon.env.txn_begin( 0, set.db )
-				entries = lineNumber = errors = 0
-				File::readlines( filepath ).each do |line|
-					lineNumber += 1
-					next if /^\s/ =~ line
-
-					key, value = set.processor.call( line.chomp, lineNumber, pos )
-					unless key
-						errors += 1
-						if errorLimit.nonzero? && errors >= errorLimit
-							abort( "Too many errors" )
-						end
-					end
-
-					dbh[ key ] = value
-					entries += 1
-					print "%d%s" % [ entries, "\x08" * entries.to_s.length ]
-
-					# Commit and start a new transaction every 1000 records
-					if (entries % CommitThreshold).zero?
-						print "."
-						txn.commit( BDB::TXN_NOSYNC )
-						txn, dbh = lexicon.env.txn_begin( 0, set.db )
-					end
-				end
-			
-				message "committing..."
-				txn.commit( BDB::TXN_SYNC )
-				message "done (%d entries, %d errors).\n" %
-					[ entries, errors ]
-			end
-
-			lock_stats = lexicon.env.lock_stat
-			message "Lock statistics:\n"
-			puts "  Lock objects: #{lock_stats['st_nobjects']}/#{lock_stats['st_maxnobjects']}",
-				 "  Locks: #{lock_stats['st_nlocks']}/#{lock_stats['st_maxnlocks']}",
-				 "  Lockers: #{lock_stats['st_nlockers']}/#{lock_stats['st_maxnlockers']}"
-		
-
-			message "Checkpointing DB and cleaning logs..."
-			lexicon.checkpoint
-			lexicon.clean_logs
-			puts "done."
-		end
-
-		message "done.\n\n"
-	end
-
-
-	#######
-	private
-	#######
-
-	# Index entry patterns
-	IndexEntry		= /^(\S+)\s(\w)\s(\d+)\s(\d+)\s/
-	PointerSymbol	= /(\S{1,2})\s/
-	SenseCounts		= /(\d+)\s(\d+)\s/
-	SynsetId		= /(\d{8})\s*/
-
-	### Parse an entry from one of the index files and return the key and
-	### data. Returns +nil+ if any part of the netry isn't able to be parsed. The
-	### +pos+ argument is not used -- it's just to make the interface between all
-	### three processor methods the same.
-	def parse_index_line( string, lineNumber, pos=nil )
-		$scanner.string = string
-		synsets = []
-		lemma, pos, polycnt = nil, nil, nil
-
-		raise "whole error" unless $scanner.scan( IndexEntry )
-		lemma, pos, polycnt, pcnt = $scanner[1], $scanner[2], $scanner[3], $scanner[4]
-
-		# Discard pointer symbols
-		pcnt.to_i.times do |i|
-			$scanner.skip( PointerSymbol ) or raise "couldn't skip pointer #{i}"
-		end
-
-		# Parse sense and tagsense counts
-		$scanner.scan( SenseCounts ) or raise "couldn't parse sense counts"
-		senseCount, tagSenseCount = $scanner[1], $scanner[2]
-
-		# Find synsets
-		senseCount.to_i.times do |i|
-			$scanner.scan( SynsetId ) or raise "couldn't parse synset #{i}"
-			synset = $scanner[1]
-			synsets.push( synset )
-			$senseIndex[ synset + "%" + pos + "%" + lemma ] = i.to_s
-		end
-
-		# Make the index entry and return it
-		key = lemma + "%" + pos
-		data = synsets.join(WordNet::SUB_DELIM)
-
-		return key, data
-	rescue => err
-		message "Index entry did not parse: %s at '%s...' (line %d)\n\t%s\n" % [
-			err.message,
-			$scanner.rest[0,20],
-			lineNumber,
-			err.backtrace[0]
-		]
-		return nil
-	end
-
-
-	### "Parse" a morph line and return it as a key and value.
-	def parse_morph_line( string, lineNumber, pos )
-		key, value = string.split
-		return "#{key}%#{pos}", value
-	rescue => err
-		message "Morph entry did not parse: %s for %s (pos = %s, line %d)\n\t%s\n" % [
-			err.message,
-			string.inspect,
-			pos.inspect,
-			lineNumber,
-			err.backtrace[0]
-		]
-		return nil
-	end
-
-
-	# Synset data patterns
-	Synset		= /(\d+)\s(\d{2})\s(\w)\s(\w{2})\s/
-	SynWord		= /(\S+)\s(\w)*\s*/
-	SynPtrCnt	= /(\d{3})\s/
-	SynPtr		= /(\S{1,2})\s(\d+)\s(\w)\s(\w{4})\s/
-	SynFrameCnt	= /\s*(\d{2})\s/
-	SynFrame	= /\+\s(\d{2})\s(\w{2})\s/
-	SynGloss	= /\s*\|\s*(.+)?/
-
-	### Parse an entry from a data file and return the key and data. Returns +nil+
-	### if any part of the entry isn't able to be parsed.
-	def parse_synset_line( string, lineNumber, pos )
-		$scanner.string = string
-	
-		filenum, synsetType, gloss = nil, nil, nil
-		words = []
-		ptrs = []
-		frames = []
-
-		# Parse the first part of the synset
-		$scanner.scan( Synset ) or raise "unable to parse synset"
-		offset, filenum, synsetType, wordCount =
-			$scanner[1], $scanner[2], $scanner[3], $scanner[4]
-
-		# Parse the words
-		wordCount.to_i(16).times do |i|
-			$scanner.scan( SynWord ) or raise "unable to parse word #{i}"
-			word, lexid = $scanner[1], $scanner[2]
-			senseKey = (offset + "%" + pos + "%" + word).downcase
-			if !$senseIndex.key?( senseKey )
-				newKey = senseKey.sub( /\(\w+\)$/, '' )
-				if !$senseIndex.key?( newKey )
-					raise "Sense index does not contain sense '#{senseKey}' "\
-						"(tried #{newKey}, too)."
-				end
-				senseKey = newKey
-			end
-
-			words.push( word + "%" + $senseIndex[senseKey].to_s )
-		end
-	
-		# Parse pointers
-		if $scanner.scan( SynPtrCnt )
-			$scanner[1].to_i.times do |i|
-				$scanner.scan( SynPtr ) or raise "unable to parse synptr #{i}"
-				ptrs.push "%s %s%%%s %s" % [
-					$scanner[1],
-					$scanner[2],
-					$scanner[3],
-					$scanner[4],
-				]
-			end
-		else
-			raise "Couldn't parse pointer count"
-		end
-
-		# Parse frames if this synset is a verb
-		if synsetType == WordNet::Verb
-			if $scanner.scan( SynFrameCnt )
-				$scanner[1].to_i.times do |i|
-					$scanner.scan( SynFrame ) or raise "unable to parse frame #{i}"
-					frames.push "#{$scanner[1]} #{$scanner[2]}"
-				end
-			else
-				raise "Couldn't parse frame count"
-			end
-		end
-
-		# Find the gloss
-		if $scanner.scan( SynGloss )
-			gloss = $scanner[1].strip
-		end
-
-		# This should never happen, as the gloss matches pretty much anything to
-		# the end of line.
-		if !$scanner.empty?
-			raise "Trailing miscellaneous found at end of entry"
-		end
-
-		# Build the synset entry and return it
-		synsetType = WordNet::Adjective if synsetType == WordNet::Other
-		key = [ offset, synsetType ].join("%")
-		data = [
-			filenum,
-			words.join( WordNet::SUB_DELIM ),
-			ptrs.join( WordNet::SUB_DELIM ),
-			frames.join( WordNet::SUB_DELIM ),
-			gloss,
-		].join( WordNet::DELIM )
-
-		return key, data
-	rescue => err
-		message "Synset did not parse: %s at '%s...' (pos = %s, line %d)\n\t%s\n" % [
-			err.message,
-			$scanner.rest[0,20],
-			pos.inspect,
-			lineNumber,
-			err.backtrace[0]
-		]
-		return nil
-	end
-
-end # class WordNetConverter
-
-
-# Start the program if it's run directly
-if $0 == __FILE__
-    errorLimit = 0
-    
-	ARGV.options {|oparser|
-		oparser.banner = "Usage: #{File::basename($0)} -dv\n"
-
-		# Debugging on/off
-		oparser.on( "--debug", "-d", TrueClass, "Turn debugging on" ) {
-			$DEBUG = true
-			debugMsg "Turned debugging on."
-		}
-
-		# Verbose
-		oparser.on( "--verbose", "-v", TrueClass, "Verbose progress messages" ) {
-			$VERBOSE = true
-			debugMsg "Turned verbose on."
-		}
-
-		# Error-limit
-		oparser.on( "--error-limit=COUNT", "-eCOUNT", Integer,
-			"Error limit -- quit after COUNT errors" ) {|arg|
-			errorLimit = arg.to_i
-			debugMsg "Set error limit to #{errorLimit}"
-		}
-
-		# Handle the 'help' option
-		oparser.on( "--help", "-h", "Display this text." ) {
-			$stderr.puts oparser
-			exit!(0)
-		}
-
-		oparser.parse!
-	}
-
-    WordNetConverter.new.convertdb( errorLimit )
-end
-

data/examples/addLacedBoots.rb

@@ -1,27 +0,0 @@
-#!/usr/bin/ruby -w
-#
-# Add a synset for laced boots
-#
-
-$: << "lib"
-require "WordNet"
-
-lex = WordNet::Lexicon.new( "ruby-wordnet" )
-
-boot = lex.lookup_synsets( "boot", "n", 1 )
-laced_boot = lex.create_synset( "laced boot", "n" )
-tongue = lex.lookup_synsets( "tongue", "n", 6 )
-
-laced_boot.add_hypernyms( boot )
-laced_boot.add_component_meronyms( tongue )
-
-lex.unlock {
-	laced_boot.write
-	boot.write
-	tongue.write
-}
-
-
-
-
-

data/examples/clothesWithCollars.rb

@@ -1,36 +0,0 @@
-#!/usr/bin/ruby -w
-#
-#	Find all articles of clothing that have collars (Adapted from the synopsis
-#	of Lingua::Wordnet::Analysis)
-#
-
-$LOAD_PATH.unshift "lib"
-require "wordnet"
-
-# Create the lexicon
-lex = WordNet::Lexicon.new
-
-# Look up the clothing synset as the origin
-clothing = lex.lookup_synsets( "clothing", WordNet::Noun, 1 )
-
-part_word = ARGV.shift || "collar"
-part = lex.lookup_synsets( part_word, WordNet::Noun, 1 ) or
-	abort( "Couldn't find synset for #{part_word}" )
-
-
-puts "Looking for instances of:",
-	"  #{part}",
-	"in the hyponyms of",
-	"  #{clothing}",
-	""
-
-# Now traverse all hyponyms of the clothing synset, and check for "part" among
-# each one's meronyms, printing any we find
-clothing.traverse( :hyponyms ) do |syn,depth|
-	if syn.search( :meronyms, part )
-		puts "Has a #{part_word}: #{syn}"
-	else
-		puts "Doesn't have a #{part_word}: #{syn}" if $DEBUG
-	end
-end
-

data/rake/dependencies.rb

@@ -1,76 +0,0 @@
-# 
-# Dependency-checking and Installation Rake Tasks
-# $Id: dependencies.rb 43 2008-09-05 18:19:16Z deveiant $
-# 
-
-require 'rubygems/dependency_installer'
-require 'rubygems/source_index'
-require 'rubygems/requirement'
-require 'rubygems/doc_manager'
-
-### Install the specified +gems+ if they aren't already installed.
-def install_gems( gems )
-	
-	defaults = Gem::DependencyInstaller::DEFAULT_OPTIONS.merge({
-		:generate_rdoc     => true,
-		:generate_ri       => true,
-		:install_dir       => Gem.dir,
-		:format_executable => false,
-		:test              => false,
-		:version           => Gem::Requirement.default,
-	  })
-    
-	# Check for root
-	if Process.euid != 0
-		$stderr.puts "This probably won't work, as you aren't root, but I'll try anyway"
-	end
-
-	gemindex = Gem::SourceIndex.from_installed_gems
-
-	gems.each do |gemname, reqstring|
-		requirement = Gem::Requirement.new( reqstring )
-		trace "requirement is: %p" % [ requirement ]
-		
-		trace "Searching for an installed #{gemname}..."
-		specs = gemindex.find_name( gemname )
-		trace "...found %d specs: %s" %
-			[ specs.length, specs.collect {|s| "%s %s" % [s.name, s.version] }.join(', ') ]
-		
-		if spec = specs.find {|spec| requirement.satisfied_by?(spec.version) }
-			log "Version %s of %s is already installed (needs %s); skipping..." % 
-				[ spec.version, spec.name, requirement ]
-			next
-		end
-
-		rgv = Gem::Version.new( Gem::RubyGemsVersion )
-		installer = nil
-		
-		log "Trying to install #{gemname.inspect} #{requirement}..."
-		if rgv >= Gem::Version.new( '1.1.1' )
-			installer = Gem::DependencyInstaller.new
-			installer.install( gemname, requirement )
-		else
-			installer = Gem::DependencyInstaller.new( gemname )
-			installer.install
-		end
-
-		installer.installed_gems.each do |spec|
-			log "Installed: %s" % [ spec.full_name ]
-		end
-
-	end
-end
-
-
-### Task: install runtime dependencies
-desc "Install runtime dependencies as gems"
-task :install_dependencies do
-	install_gems( DEPENDENCIES )
-end
-
-### Task: install gems for development tasks
-desc "Install development dependencies as gems"
-task :install_dev_dependencies do
-	install_gems( DEVELOPMENT_DEPENDENCIES )
-end
-