josephwilk-rsemantic 0.1.0

data/lib/semantic.rb

@@ -0,0 +1,29 @@
+$:.unshift(File.dirname(__FILE__)) unless
+$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require "semantic/vector_space"
+require "semantic/compare"
+require "semantic/parser"
+require "semantic/matrix_transformer"
+require "semantic/search"
+require "semantic/transform"
+require "semantic/version"
+
+require 'rubygems'
+require 'linalg'
+#http://rubyforge.org/projects/stemmer/
+#A processor for removing the commoner morphological and inflexional endings from words in English
+require 'stemmer'
+require 'logger'
+
+module Semantic
+
+  def self.logger
+    return @logger if @logger
+    @logger = Logger.new(STDOUT)
+    @logger.formatter = proc { |severity, time, progname, msg| "#{msg}\n" }
+    @logger.level = Logger::ERROR
+    @logger
+  end
+  
+end

data/resources/english.stop

@@ -0,0 +1,571 @@
+a
+a's
+able
+about
+above
+according
+accordingly
+across
+actually
+after
+afterwards
+again
+against
+ain't
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+b
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+c
+c'mon
+c's
+came
+can
+can't
+cannot
+cant
+cause
+causes
+certain
+certainly
+changes
+clearly
+co
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+currently
+d
+definitely
+described
+despite
+did
+didn't
+different
+do
+does
+doesn't
+doing
+don't
+done
+down
+downwards
+during
+e
+each
+edu
+eg
+eight
+either
+else
+elsewhere
+enough
+entirely
+especially
+et
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+f
+far
+few
+fifth
+first
+five
+followed
+following
+follows
+for
+former
+formerly
+forth
+four
+from
+further
+furthermore
+g
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+h
+had
+hadn't
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+he's
+hello
+help
+hence
+her
+here
+here's
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+i
+i'd
+i'll
+i'm
+i've
+ie
+if
+ignored
+immediate
+in
+inasmuch
+inc
+indeed
+indicate
+indicated
+indicates
+inner
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+it's
+its
+itself
+j
+just
+k
+keep
+keeps
+kept
+know
+knows
+known
+l
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+little
+look
+looking
+looks
+ltd
+m
+mainly
+many
+may
+maybe
+me
+mean
+meanwhile
+merely
+might
+more
+moreover
+most
+mostly
+much
+must
+my
+myself
+n
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+no
+nobody
+non
+none
+noone
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+p
+particular
+particularly
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provides
+q
+que
+quite
+qv
+r
+rather
+rd
+re
+really
+reasonably
+regarding
+regardless
+regards
+relatively
+respectively
+right
+s
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+she
+should
+shouldn't
+since
+six
+so
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+t
+t's
+take
+taken
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that's
+thats
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+there's
+thereafter
+thereby
+therefore
+therein
+theres
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+think
+third
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+twice
+two
+u
+un
+under
+unfortunately
+unless
+unlikely
+until
+unto
+up
+upon
+us
+use
+used
+useful
+uses
+using
+usually
+uucp
+v
+value
+various
+very
+via
+viz
+vs
+w
+want
+wants
+was
+wasn't
+way
+we
+we'd
+we'll
+we're
+we've
+welcome
+well
+went
+were
+weren't
+what
+what's
+whatever
+when
+whence
+whenever
+where
+where's
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+who's
+whoever
+whole
+whom
+whose
+why
+will
+willing
+wish
+with
+within
+without
+won't
+wonder
+would
+would
+wouldn't
+x
+y
+yes
+yet
+you
+you'd
+you'll
+you're
+you've
+your
+yours
+yourself
+yourselves
+z
+zero

data/rsemantic.gemspec

@@ -0,0 +1,41 @@
+Gem::Specification.new do |s|
+  s.name = %q{rsemantic}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Joseph Wilk"]
+  s.date = %q{2008-11-13}
+  s.description = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
+  s.email = ["josephwilk@joesniff.co.uk"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.txt", "TODO.txt"]
+  s.files = ["History.txt", "Manifest.txt", "README.txt", "Rakefile", "TODO.txt", "config/hoe.rb", "config/requirements.rb", "gem_tasks/deployment.rake", "gem_tasks/environment.rake", "gem_tasks/examples.rake", "gem_tasks/fix_cr_lf.rake", "gem_tasks/gemspec.rake", "gem_tasks/rspec.rake", "gem_tasks/website.rake", "lib/semantic.rb", "lib/semantic/compare.rb", "lib/semantic/matrix_transformer.rb", "lib/semantic/parser.rb", "lib/semantic/search.rb", "lib/semantic/transform.rb", "lib/semantic/transform/lsa_transform.rb", "lib/semantic/transform/tf_idf_transform.rb", "lib/semantic/vector_space.rb", "lib/semantic/vector_space/builder.rb", "lib/semantic/vector_space/model.rb", "lib/semantic/version.rb", "resources/english.stop", "rsemantic.gemspec", "spec/semantic/compare_spec.rb", "spec/semantic/matrix_transformer_spec.rb", "spec/semantic/parser_spec.rb", "spec/semantic/search_spec.rb", "spec/semantic/transform/lsa_transform_spec.rb", "spec/semantic/transform/tf_idf_transform_spec.rb", "spec/semantic/vector_space/builder_spec.rb", "spec/semantic/vector_space/model_spec.rb", "spec/spec.opts", "spec/spec_helper.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/josephwilk/rsemantic}
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{rsemantic}
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if current_version >= 3 then
+      s.add_runtime_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+      s.add_runtime_dependency(%q<rspec>, [">= 1.1.5"])
+      s.add_runtime_dependency(%q<diff-lcs>, [">= 1.1.2"])
+      s.add_development_dependency(%q<hoe>, [">= 1.8.2"])
+    else
+      s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+      s.add_dependency(%q<rspec>, [">= 1.1.5"])
+      s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
+      s.add_dependency(%q<hoe>, [">= 1.8.2"])
+    end
+  else
+    s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+    s.add_dependency(%q<rspec>, [">= 1.1.5"])
+    s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
+    s.add_dependency(%q<hoe>, [">= 1.8.2"])
+  end
+end

data/spec/semantic/compare_spec.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Compare do
+
+    def vector(values)
+      Linalg::DMatrix.columns([values])
+    end
+
+    it "should calculate cosine" do
+      cosine =  Compare.cosine( vector([0.1,0.5]), vector([0.9, 0.3]) )
+      cosine.should be_close(0.4961, 0.0001)
+    end
+
+  end
+end

data/spec/semantic/matrix_transformer_spec.rb

@@ -0,0 +1,51 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe MatrixTransformer do
+
+    def mock_transform
+      @transform ||= mock(Transform)
+    end
+
+    def mock_vector_space
+      mock("vector space", :matrix => Linalg::DMatrix.rows([[1,0],[0,1]]), :matrix= => nil )
+    end
+
+
+    describe "transforming matrix" do
+
+      it "should ignore invalid transform class" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:FAKE])
+        lambda {
+          matrix_transformer.apply_transforms(mock_vector_space)
+        }.should_not raise_error
+      end
+
+      it "should use defaults transforms in none are specified" do
+        matrix_transformer = MatrixTransformer.new
+        Transform.should_receive(:const_get).with(:LSA).and_return(mock_transform)
+        Transform.should_receive(:const_get).with(:TFIDF).and_return(mock_transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+      it "should send transform message to class to transform matrix" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
+        Transform.stub!(:const_get).and_return(mock_transform)
+
+        mock_transform.should_receive(:transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+      it "should check that transform class is capable of transforming" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
+        Transform.stub!(:const_get).and_return(mock_transform)
+        mock_transform.should_receive(:respond_to?).with(:transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+    end
+  end
+end

data/spec/semantic/parser_spec.rb

@@ -0,0 +1,34 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Parser do
+
+    it "should remove stop words" do
+      file = mock("file")
+      file.stub!(:read).and_return("a to be")
+      File.stub!(:open).and_yield(file)
+      parser = Parser.new
+
+      parser.remove_stop_words(['a','house']).should == ['house']
+    end
+
+    it "should remove any non characters" do
+      file = mock("file")
+      file.stub!(:read).and_return("a to be")
+      File.stub!(:open).and_yield(file)
+
+      parser = Parser.new
+      parser.tokenise_and_stem("dragon.").should == ["dragon"]
+    end
+
+    it "should tokenise the string" do
+      parser = Parser.new
+
+      parser.stub!(:remove_stop_words).and_return(['mouse','trap'])
+      parser.should_receive(:tokenise_and_stem).and_return(['mouse','trap'])
+
+      parser.tokenise_and_filter(['the mouse trap'])
+    end
+
+  end
+end