josephwilk-semantic 0.1.0

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 
+
+* Changed internal representation of vector space. Using columns as documents and rows as terms. This is more consistent which LSA research papers.
+* Wrap DMatrix in VectorSpace::Model, allowing us to store keywords with the matrix and get pretty output

data/Manifest.txt

@@ -0,0 +1,38 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+TODO.txt
+config/hoe.rb
+config/requirements.rb
+gem_tasks/deployment.rake
+gem_tasks/environment.rake
+gem_tasks/examples.rake
+gem_tasks/fix_cr_lf.rake
+gem_tasks/gemspec.rake
+gem_tasks/rspec.rake
+gem_tasks/website.rake
+lib/semantic.rb
+lib/semantic/compare.rb
+lib/semantic/matrix_transformer.rb
+lib/semantic/parser.rb
+lib/semantic/search.rb
+lib/semantic/transform.rb
+lib/semantic/transform/lsa_transform.rb
+lib/semantic/transform/tf_idf_transform.rb
+lib/semantic/vector_space.rb
+lib/semantic/vector_space/builder.rb
+lib/semantic/vector_space/model.rb
+lib/semantic/version.rb
+resources/english.stop
+rsemantic.gemspec
+spec/semantic/compare_spec.rb
+spec/semantic/matrix_transformer_spec.rb
+spec/semantic/parser_spec.rb
+spec/semantic/search_spec.rb
+spec/semantic/transform/lsa_transform_spec.rb
+spec/semantic/transform/tf_idf_transform_spec.rb
+spec/semantic/vector_space/builder_spec.rb
+spec/semantic/vector_space/model_spec.rb
+spec/spec.opts
+spec/spec_helper.rb

data/README.txt

@@ -0,0 +1,48 @@
+= Rsemantic
+
+* http://github.com/JoeSniff/rsemantic
+
+== DESCRIPTION:
+
+A Ruby document vector search with flexible matrix transforms. Current supported transforms: 
+
+* Latent semantic analysis
+* Term frequency - inverse document frequency
+
+Documentation: http://github.com/josephwilk/rsemantic/wikis/home
+
+== REQUIREMENTS:
+
+* Linalg - http://rubyforge.org/projects/linalg/
+* stemmer - http://rubyforge.org/projects/stemmer/
+
+== INSTALL:
+
+* git clone git://github.com/josephwilk/rsemantic.git
+
+== LICENSE
+
+(The MIT License)
+
+Copyright (c) 2008 Joseph Wilk
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

data/Rakefile

@@ -0,0 +1,9 @@
+ENV['NODOT'] = 'true' # We don't want class diagrams in RDoc
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['gem_tasks/**/*.rake'].each { |rake| load rake }
+
+# Hoe gives us :default => :test, but we don't have Test::Unit tests.
+Rake::Task[:default].clear_prerequisites
+task :default => [:spec]

data/TODO.txt

@@ -0,0 +1,9 @@
+== FEATURES/PROBLEMS:
+
+* Applying transforms to query vectors
+* Detect the optimal dimension reduction in LSA.
+* Allow objects to be passed in as transforms.
+* Implement Probabilistic latent semantic analysis
+* Implement Latent Dirichlet Allocation
+
+* Matrix transformer has to popout the matrix of VectorSpace::Model and reassign it, get rid of this.

data/config/hoe.rb

@@ -0,0 +1,69 @@
+require 'semantic/version'
+
+AUTHOR = 'Joseph Wilk'  # can also be an array of Authors
+EMAIL = "josephwilk@joesniff.co.uk"
+DESCRIPTION = "A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency"
+GEM_NAME = 'semantic' # what ppl will type to install your gem
+HOMEPATH = "http://github.com/josephwilk/rsemantic"
+RUBYFORGE_PROJECT = 'rsemantic'
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "joseph_wilk"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = YAML.load(`svn info`)['Revision']
+VERS = Semantic::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'Rsemantic documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README.textile",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.developer(AUTHOR, EMAIL)
+  p.description = DESCRIPTION
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store', '**/*.class', '**/*.jar']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  p.extra_deps = [ ['term-ansicolor', '>= 1.0.3'], ['rspec', '>= 1.1.5'], ['diff-lcs', '>= 1.1.2'] ]
+
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+$hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/gem_tasks/deployment.rake

@@ -0,0 +1,34 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end
+
+namespace :manifest do
+  desc 'Recreate Manifest.txt to include ALL files'
+  task :refresh do
+    `rake check_manifest | patch -p0 > Manifest.txt`
+  end
+end

data/gem_tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end

data/gem_tasks/examples.rake

@@ -0,0 +1,29 @@
+require 'lib/semantic'
+
+namespace :example do
+
+  documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
+
+  desc "run main LSA example"
+  task :lsa do
+    search = Semantic::Search.new(documents, :verbose => true)
+  end
+
+  desc "run main Vector space example"
+  task :vector_space do
+    search = Semantic::Search.new(documents)
+
+    puts "Documents:"
+    documents.each_with_index { |document, index| puts "#{index}: #{document}"  }
+    puts
+
+    puts "Documents related to first document: #{documents[0]}"
+    puts search.related(0)
+    puts
+
+    puts "Searching for the word cat:"
+    puts search.search(["cat"])
+    puts
+  end
+
+end

data/gem_tasks/fix_cr_lf.rake

@@ -0,0 +1,10 @@
+desc 'Make all files use UNIX (\n) line endings'
+task :fix_cr_lf do
+  files = FileList['**/*']
+  files.each do |f|
+    next if File.directory?(f)
+    s = IO.read(f)
+    s.gsub!(/\r?\n/, "\n")
+    File.open(f, "w") { |io| io.write(s) }
+  end
+end

data/gem_tasks/gemspec.rake

@@ -0,0 +1,6 @@
+namespace :gemspec do
+  desc 'Refresh rsemantic.gemspec to include ALL files'
+  task :refresh => 'manifest:refresh' do
+    File.open('rsemantic.gemspec', 'w') {|io| io.write($hoe.spec.to_ruby)}
+  end
+end

data/gem_tasks/rspec.rake

@@ -0,0 +1,33 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+  require 'spec/rake/verify_rcov'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  
+    unless ENV['NO_RCOV']
+      t.rcov = true
+      t.rcov_dir = 'coverage'
+      t.rcov_opts = ['--exclude', '_helper\.rb,_spec\.rb,spec\/boss,\/var\/lib\/gems,\/Library\/Ruby,\.autotest']
+    end
+end
+
+RCov::VerifyTask.new(:verify_rcov => :spec) do |t|
+  t.threshold = 99.7 # Make sure you have rcov 0.9 or higher!
+  t.index_html = 'coverage/index.html'
+end

data/gem_tasks/website.rake

@@ -0,0 +1,17 @@
+desc 'Generate website files'
+task :website_generate => :ruby_env do
+  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
+    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]

data/lib/semantic.rb

@@ -0,0 +1,29 @@
+$:.unshift(File.dirname(__FILE__)) unless
+$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require "semantic/vector_space"
+require "semantic/compare"
+require "semantic/parser"
+require "semantic/matrix_transformer"
+require "semantic/search"
+require "semantic/transform"
+require "semantic/version"
+
+require 'rubygems'
+require 'linalg'
+#http://rubyforge.org/projects/stemmer/
+#A processor for removing the commoner morphological and inflexional endings from words in English
+require 'stemmer'
+require 'logger'
+
+module Semantic
+
+  def self.logger
+    return @logger if @logger
+    @logger = Logger.new(STDOUT)
+    @logger.formatter = proc { |severity, time, progname, msg| "#{msg}\n" }
+    @logger.level = Logger::ERROR
+    @logger
+  end
+  
+end

data/lib/semantic/compare.rb

@@ -0,0 +1,19 @@
+module Semantic
+  class Compare
+
+    class << self
+      
+      def similarity(vector1, vector2)
+        cosine(vector1, vector2)
+      end
+      
+      def cosine(vector1, vector2)
+        unless vector2.nil? or vector1.nil?
+          (vector2.dot(vector1)) / (vector1.norm * vector2.norm)
+        end
+      end
+
+    end
+    
+  end
+end

data/lib/semantic/matrix_transformer.rb

@@ -0,0 +1,25 @@
+module Semantic
+  class MatrixTransformer
+
+    def initialize(options={})
+      @transforms = options[:transforms] || [:TFIDF, :LSA]
+      @options = options
+    end
+
+    def apply_transforms(vector_space_model)
+      @transforms.each do |transform|
+        begin
+          transform_class = Semantic::Transform.const_get(transform)
+          Semantic.logger.info("Applying #{transform} transform")
+          vector_space_model.matrix = transform_class.send(:transform, vector_space_model.matrix) if transform_class.respond_to?(:transform)
+          Semantic.logger.info(vector_space_model)
+        rescue Exception => e
+          Semantic.logger.error("Error: Cannot perform transform: #{transform}")
+          Semantic.logger.error(e)
+        end
+      end
+      vector_space_model
+    end
+
+  end
+end

data/lib/semantic/parser.rb

@@ -0,0 +1,40 @@
+require 'stemmer'
+
+module Semantic
+  class Parser
+
+    def initialize
+      #English stopwords from ftp://ftp.cs.cornell.edu/pub/smart/english.stop
+      #TODO: nicer way to reference stop file location?
+      File.open(File.dirname(__FILE__)+'/../../resources/english.stop', 'r') do |file|
+        @stopwords = file.read().split()
+      end
+    end
+
+    def tokenise_and_filter(string)
+      word_list = tokenise_and_stem(string)
+      remove_stop_words(word_list)
+    end
+
+    #remove any nasty grammar tokens from string
+    def clean(string)
+      string = string.gsub(".","")
+      string = string.gsub(/\s+/," ")
+      string = string.downcase
+      return string
+    end
+
+    #stop words are common words which have no search value
+    def remove_stop_words(list)
+      list.select {|word| word unless @stopwords.include? word }
+    end
+
+    def tokenise_and_stem(string)
+      string = clean(string)
+      words = string.split(" ")
+
+      words.map {|word| word.stem }
+    end
+
+  end
+end