rsemantic 0.1.3 → 0.1.4

data/Rakefile

@@ -1,9 +0,0 @@
-ENV['NODOT'] = 'true' # We don't want class diagrams in RDoc
-require 'config/requirements'
-require 'config/hoe' # setup Hoe + all gem configuration
-
-Dir['gem_tasks/**/*.rake'].each { |rake| load rake }
-
-# Hoe gives us :default => :test, but we don't have Test::Unit tests.
-Rake::Task[:default].clear_prerequisites
-task :default => [:spec]

data/config/hoe.rb

@@ -1,69 +0,0 @@
-require 'semantic/version'
-
-AUTHOR = 'Joseph Wilk'  # can also be an array of Authors
-EMAIL = "josephwilk@joesniff.co.uk"
-DESCRIPTION = "A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency"
-GEM_NAME = 'rsemantic' # what ppl will type to install your gem
-HOMEPATH = "http://github.com/josephwilk/rsemantic"
-RUBYFORGE_PROJECT = 'rsemantic'
-
-@config_file = "~/.rubyforge/user-config.yml"
-@config = nil
-RUBYFORGE_USERNAME = "joseph_wilk"
-def rubyforge_username
-  unless @config
-    begin
-      @config = YAML.load(File.read(File.expand_path(@config_file)))
-    rescue
-      puts <<-EOS
-ERROR: No rubyforge config file found: #{@config_file}
-Run 'rubyforge setup' to prepare your env for access to Rubyforge
- - See http://newgem.rubyforge.org/rubyforge.html for more details
-      EOS
-      exit
-    end
-  end
-  RUBYFORGE_USERNAME.replace @config["username"]
-end
-
-
-REV = nil 
-# UNCOMMENT IF REQUIRED: 
-# REV = YAML.load(`svn info`)['Revision']
-VERS = Semantic::VERSION::STRING + (REV ? ".#{REV}" : "")
-RDOC_OPTS = ['--quiet', '--title', 'Rsemantic documentation',
-    "--opname", "index.html",
-    "--line-numbers", 
-    "--main", "README.textile",
-    "--inline-source"]
-
-class Hoe
-  def extra_deps 
-    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
-    @extra_deps
-  end 
-end
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-$hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.developer(AUTHOR, EMAIL)
-  p.description = DESCRIPTION
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store', '**/*.class', '**/*.jar']  #An array of file patterns to delete on clean.
-  
-  # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  p.extra_deps = [ ['term-ansicolor', '>= 1.0.3'], ['rspec', '>= 1.1.5'], ['diff-lcs', '>= 1.1.2'] ]
-
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
-  
-end
-
-CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
-PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
-$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
-$hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -1,15 +0,0 @@
-require 'fileutils'
-include FileUtils
-
-require 'rubygems'
-%w[rake hoe].each do |req_gem|
-  begin
-    require req_gem
-  rescue LoadError
-    puts "This Rakefile requires the '#{req_gem}' RubyGem."
-    puts "Installation: gem install #{req_gem} -y"
-    exit
-  end
-end
-
-$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/gem_tasks/deployment.rake

@@ -1,34 +0,0 @@
-desc 'Release the website and new gem version'
-task :deploy => [:check_version, :website, :release] do
-  puts "Remember to create SVN tag:"
-  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
-    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
-  puts "Suggested comment:"
-  puts "Tagging release #{CHANGES}"
-end
-
-desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
-task :local_deploy => [:website_generate, :install_gem]
-
-task :check_version do
-  unless ENV['VERSION']
-    puts 'Must pass a VERSION=x.y.z release version'
-    exit
-  end
-  unless ENV['VERSION'] == VERS
-    puts "Please update your version.rb to match the release version, currently #{VERS}"
-    exit
-  end
-end
-
-desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
-task :install_gem_no_doc => [:clean, :package] do
-  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
-end
-
-namespace :manifest do
-  desc 'Recreate Manifest.txt to include ALL files'
-  task :refresh do
-    `rake check_manifest | patch -p0 > Manifest.txt`
-  end
-end

data/gem_tasks/environment.rake

@@ -1,7 +0,0 @@
-task :ruby_env do
-  RUBY_APP = if RUBY_PLATFORM =~ /java/
-    "jruby"
-  else
-    "ruby"
-  end unless defined? RUBY_APP
-end

data/gem_tasks/examples.rake

@@ -1,29 +0,0 @@
-require 'lib/semantic'
-
-namespace :example do
-
-  documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
-
-  desc "run main LSA example"
-  task :lsa do
-    search = Semantic::Search.new(documents, :verbose => true)
-  end
-
-  desc "run main Vector space example"
-  task :vector_space do
-    search = Semantic::Search.new(documents)
-
-    puts "Documents:"
-    documents.each_with_index { |document, index| puts "#{index}: #{document}"  }
-    puts
-
-    puts "Documents related to first document: #{documents[0]}"
-    puts search.related(0)
-    puts
-
-    puts "Searching for the word cat:"
-    puts search.search(["cat"])
-    puts
-  end
-
-end

data/gem_tasks/fix_cr_lf.rake

@@ -1,10 +0,0 @@
-desc 'Make all files use UNIX (\n) line endings'
-task :fix_cr_lf do
-  files = FileList['**/*']
-  files.each do |f|
-    next if File.directory?(f)
-    s = IO.read(f)
-    s.gsub!(/\r?\n/, "\n")
-    File.open(f, "w") { |io| io.write(s) }
-  end
-end

data/gem_tasks/gemspec.rake

@@ -1,6 +0,0 @@
-namespace :gemspec do
-  desc 'Refresh rsemantic.gemspec to include ALL files'
-  task :refresh => 'manifest:refresh' do
-    File.open('rsemantic.gemspec', 'w') {|io| io.write($hoe.spec.to_ruby)}
-  end
-end

data/gem_tasks/rspec.rake

@@ -1,33 +0,0 @@
-begin
-  require 'spec'
-rescue LoadError
-  require 'rubygems'
-  require 'spec'
-end
-begin
-  require 'spec/rake/spectask'
-  require 'spec/rake/verify_rcov'
-rescue LoadError
-  puts <<-EOS
-To use rspec for testing you must install rspec gem:
-    gem install rspec
-EOS
-  exit(0)
-end
-
-desc "Run the specs under spec/models"
-Spec::Rake::SpecTask.new do |t|
-  t.spec_opts = ['--options', "spec/spec.opts"]
-  t.spec_files = FileList['spec/**/*_spec.rb']
-  
-    unless ENV['NO_RCOV']
-      t.rcov = true
-      t.rcov_dir = 'coverage'
-      t.rcov_opts = ['--exclude', '_helper\.rb,_spec\.rb,spec\/boss,\/var\/lib\/gems,\/Library\/Ruby,\.autotest']
-    end
-end
-
-RCov::VerifyTask.new(:verify_rcov => :spec) do |t|
-  t.threshold = 99.7 # Make sure you have rcov 0.9 or higher!
-  t.index_html = 'coverage/index.html'
-end

data/gem_tasks/website.rake

@@ -1,17 +0,0 @@
-desc 'Generate website files'
-task :website_generate => :ruby_env do
-  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
-    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
-  end
-end
-
-desc 'Upload website files to rubyforge'
-task :website_upload do
-  host = "#{rubyforge_username}@rubyforge.org"
-  remote_dir = "/var/www/gforge-projects/#{PATH}/"
-  local_dir = 'website'
-  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
-end
-
-desc 'Generate and upload website files'
-task :website => [:website_generate, :website_upload, :publish_docs]

data/rsemantic.gemspec

@@ -1,41 +0,0 @@
-Gem::Specification.new do |s|
-  s.name = %q{rsemantic}
-  s.version = "0.1.3"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Joseph Wilk"]
-  s.date = %q{2009-08-01}
-  s.description = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
-  s.email = ["joe@josephwilk.net"]
-  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.txt", "TODO.txt"]
-  s.files = ["History.txt", "Manifest.txt", "README.txt", "Rakefile", "TODO.txt", "config/hoe.rb", "config/requirements.rb", "gem_tasks/deployment.rake", "gem_tasks/environment.rake", "gem_tasks/examples.rake", "gem_tasks/fix_cr_lf.rake", "gem_tasks/gemspec.rake", "gem_tasks/rspec.rake", "gem_tasks/website.rake", "lib/semantic.rb", "lib/semantic/compare.rb", "lib/semantic/matrix_transformer.rb", "lib/semantic/parser.rb", "lib/semantic/search.rb", "lib/semantic/transform.rb", "lib/semantic/transform/lsa_transform.rb", "lib/semantic/transform/tf_idf_transform.rb", "lib/semantic/vector_space.rb", "lib/semantic/vector_space/builder.rb", "lib/semantic/vector_space/model.rb", "lib/semantic/version.rb", "resources/english.stop", "rsemantic.gemspec", "spec/semantic/compare_spec.rb", "spec/semantic/matrix_transformer_spec.rb", "spec/semantic/parser_spec.rb", "spec/semantic/search_spec.rb", "spec/semantic/transform/lsa_transform_spec.rb", "spec/semantic/transform/tf_idf_transform_spec.rb", "spec/semantic/vector_space/builder_spec.rb", "spec/semantic/vector_space/model_spec.rb", "spec/spec.opts", "spec/spec_helper.rb"]
-  s.has_rdoc = true
-  s.homepage = %q{http://github.com/josephwilk/rsemantic}
-  s.rdoc_options = ["--main", "README.txt"]
-  s.require_paths = ["lib"]
-  s.rubyforge_project = %q{rsemantic}
-  s.rubygems_version = %q{1.3.1}
-  s.summary = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 2
-
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<term-ansicolor>, [">= 1.0.3"])
-      s.add_runtime_dependency(%q<rspec>, [">= 1.1.5"])
-      s.add_runtime_dependency(%q<diff-lcs>, [">= 1.1.2"])
-      s.add_development_dependency(%q<hoe>, [">= 2.3.2"])
-    else
-      s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
-      s.add_dependency(%q<rspec>, [">= 1.1.5"])
-      s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
-      s.add_dependency(%q<hoe>, [">= 2.3.2"])
-    end
-  else
-    s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
-    s.add_dependency(%q<rspec>, [">= 1.1.5"])
-    s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
-    s.add_dependency(%q<hoe>, [">= 2.3.2"])
-  end
-end

data/spec/semantic/compare_spec.rb

@@ -1,16 +0,0 @@
-require File.dirname(__FILE__) + '/../spec_helper'
-
-module Semantic
-  describe Compare do
-
-    def vector(values)
-      Linalg::DMatrix.columns([values])
-    end
-
-    it "should calculate cosine" do
-      cosine =  Compare.cosine( vector([0.1,0.5]), vector([0.9, 0.3]) )
-      cosine.should be_close(0.4961, 0.0001)
-    end
-
-  end
-end

data/spec/semantic/matrix_transformer_spec.rb

@@ -1,51 +0,0 @@
-require File.dirname(__FILE__) + '/../spec_helper'
-
-module Semantic
-  describe MatrixTransformer do
-
-    def mock_transform
-      @transform ||= mock(Transform)
-    end
-
-    def mock_vector_space
-      mock("vector space", :matrix => Linalg::DMatrix.rows([[1,0],[0,1]]), :matrix= => nil )
-    end
-
-
-    describe "transforming matrix" do
-
-      it "should ignore invalid transform class" do
-        matrix_transformer = MatrixTransformer.new(:transforms => [:FAKE])
-        lambda {
-          matrix_transformer.apply_transforms(mock_vector_space)
-        }.should_not raise_error
-      end
-
-      it "should use defaults transforms in none are specified" do
-        matrix_transformer = MatrixTransformer.new
-        Transform.should_receive(:const_get).with(:LSA).and_return(mock_transform)
-        Transform.should_receive(:const_get).with(:TFIDF).and_return(mock_transform)
-
-        matrix_transformer.apply_transforms(mock_vector_space)
-      end
-
-      it "should send transform message to class to transform matrix" do
-        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
-        Transform.stub!(:const_get).and_return(mock_transform)
-
-        mock_transform.should_receive(:transform)
-
-        matrix_transformer.apply_transforms(mock_vector_space)
-      end
-
-      it "should check that transform class is capable of transforming" do
-        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
-        Transform.stub!(:const_get).and_return(mock_transform)
-        mock_transform.should_receive(:respond_to?).with(:transform)
-
-        matrix_transformer.apply_transforms(mock_vector_space)
-      end
-
-    end
-  end
-end

data/spec/semantic/parser_spec.rb

@@ -1,34 +0,0 @@
-require File.dirname(__FILE__) + '/../spec_helper'
-
-module Semantic
-  describe Parser do
-
-    it "should remove stop words" do
-      file = mock("file")
-      file.stub!(:read).and_return("a to be")
-      File.stub!(:open).and_yield(file)
-      parser = Parser.new
-
-      parser.remove_stop_words(['a','house']).should == ['house']
-    end
-
-    it "should remove any non characters" do
-      file = mock("file")
-      file.stub!(:read).and_return("a to be")
-      File.stub!(:open).and_yield(file)
-
-      parser = Parser.new
-      parser.tokenise_and_stem("dragon.").should == ["dragon"]
-    end
-
-    it "should tokenise the string" do
-      parser = Parser.new
-
-      parser.stub!(:remove_stop_words).and_return(['mouse','trap'])
-      parser.should_receive(:tokenise_and_stem).and_return(['mouse','trap'])
-
-      parser.tokenise_and_filter(['the mouse trap'])
-    end
-
-  end
-end

data/spec/semantic/search_spec.rb

@@ -1,129 +0,0 @@
-require File.dirname(__FILE__) + '/../spec_helper'
-
-module Semantic
-  describe Search do
-
-    documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
-
-    def mock_builder
-      @builder ||= mock(VectorSpace::Builder)
-    end
-
-    def mock_matrix_transformer
-      @matrix_transformer ||= mock(MatrixTransformer)
-    end
-
-    def query_vector
-      @query_vector ||= Linalg::DMatrix.columns([[1,0]])
-    end
-
-    def vector_space_model(stubs = {})
-      @vector_space_model ||= VectorSpace::Model.new(Linalg::DMatrix.rows([[0,1],[1,0]]), {})
-    end
-
-    def matrix(array)
-      Linalg::DMatrix.rows(array)
-    end
-
-    def vector(vector)
-      matrix([vector])
-    end
-      
-    describe "setting up" do
-
-      it "should build the vector space" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.should_receive(:build_document_matrix).with(['test']).and_return(vector_space_model)
-
-        Search.new(['test'])
-      end
-
-      it "should transform matrices" do
-        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-        
-        #FIXME: with will not match vector_space_model, requests class Data. Think this is related to Delegate and Rspec
-        mock_matrix_transformer.should_receive(:apply_transforms).with(anything).and_return(vector_space_model)
-
-        Search.new(['test'])
-      end
-
-    end
-
-    describe "searching" do
-
-      it "should map search term to vector space" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-
-        mock_builder.should_receive(:build_query_vector).with("cat").and_return(query_vector)
-
-        vector_search = Search.new(documents)
-        vector_search.search("cat")
-      end
-
-      it "should compare the documents using cosine" do
-        pending
-      end
-
-    end
-
-    describe "relating" do
-
-      it "should find related documents by comparing cosine" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-      
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-
-        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
-        mock_matrix_transformer.stub!(:apply_transforms).and_return(vector_space_model)
-
-        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[0],[1]]))
-        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[1],[0]]))
-
-        vector_search = Search.new(documents)
-
-        vector_search.related(0)
-      end
-
-    end
-
-    describe "logging" do
-
-      before(:each) do
-        @out = StringIO.new
-        Semantic.logger = Logger.new(@out)
-      end
-
-      it "should set info level if in verbose mode" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-
-        Search.new(['test'], :verbose => true)
-        
-        Semantic.logger.level.should == Logger::INFO
-      end
-      
-      it "should set error level if not in verbose mode" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-
-        Search.new(['test'], :verbose => false)
-        
-        Semantic.logger.level.should == Logger::ERROR
-      end
-
-      it "should default to error level if verbose is not specified" do
-        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
-        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
-
-        Search.new(['test'])
-        
-        Semantic.logger.level.should == Logger::ERROR
-      end
-
-    end
-
-  end
-end