josephwilk-semantic 0.1.0

data/lib/semantic/search.rb

@@ -0,0 +1,35 @@
+module Semantic
+  class Search
+
+    def initialize(documents, options={})
+      Semantic.logger.level = Logger::INFO if options[:verbose]
+      
+      @builder = VectorSpace::Builder.new(options)
+      @matrix_transformer = MatrixTransformer.new(options)
+
+      @vector_space_model = @builder.build_document_matrix(documents)
+      
+      Semantic.logger.info(@vector_space_model)
+      
+      @vector_space_model = @matrix_transformer.apply_transforms(@vector_space_model)
+    end
+  
+    def related(documentId)
+      ratings = []
+      for index in (0...@vector_space_model.ncol)
+        ratings << Compare.similarity(@vector_space_model.column(documentId), @vector_space_model.column(index))
+      end
+      ratings
+    end
+
+    def search(searchList)
+      ratings = []
+      query_vector = @builder.build_query_vector(searchList)
+      for index in (0...@vector_space_model.ncol)
+        ratings << Compare.similarity(query_vector, @vector_space_model.column(index))
+      end
+      ratings
+    end
+  
+  end
+end

data/lib/semantic/transform.rb

@@ -0,0 +1 @@
+%w{tf_idf lsa}.each{|f| require "semantic/transform/#{f}_transform.rb"}

data/lib/semantic/transform/lsa_transform.rb

@@ -0,0 +1,34 @@
+module Semantic
+  module Transform
+    class LSA
+
+      class << self
+
+        # Calculate SVD of objects matrix: U . SIGMA . VT = MATRIX
+        # Reduce the dimension of sigma by specified factor producing sigma'.
+        # Then dot product the matrices:  U . SIGMA' . VT = MATRIX'
+        def transform(matrix, dimensions=1)
+          columns = matrix.num_columns
+
+          if dimensions <= columns: #Its a valid reduction
+
+            u, sigma, vt = matrix.singular_value_decomposition
+
+            #Dimension reduction, build SIGMA'
+            for index in ((columns-dimensions)...columns)
+              sigma[index,index]=0
+            end
+
+            #Reconstruct MATRIX' and Save transform
+            matrix = u * sigma * vt
+
+          else
+            raise Exception, "dimension reduction cannot be greater than %s" % rows
+          end
+          
+          matrix
+        end
+      end
+    end
+  end
+end

data/lib/semantic/transform/tf_idf_transform.rb

@@ -0,0 +1,42 @@
+module Semantic
+  module Transform
+    class TFIDF
+
+      @@number_of_documents_with_term = []
+
+      def self.transform(matrix)
+        number_of_documents = matrix.num_columns
+        @@number_of_documents_with_term = []
+
+        matrix.columns.each_with_index do |document, column_index|
+          document_term_total = document.rows.inject(0.0) {|word_sum, word_count| word_sum + word_count.to_f }
+
+          document.rows.each_with_index do |term_weight, row_index|
+            unless term_weight.to_f == 0.0
+              matrix[row_index, column_index] = (term_weight / document_term_total) *
+              Math.log((number_of_documents / number_of_documents_with_term(row_index, matrix).to_f).abs)
+            end
+          end
+        end
+        matrix
+      end
+
+      def self.number_of_documents_with_term(row_index, matrix)
+        return @@number_of_documents_with_term[row_index] unless @@number_of_documents_with_term[row_index].nil?
+
+        term_document_occurences = 0
+
+        rows,cols = matrix.dimensions
+
+        for n in (0...cols)
+          if matrix[row_index, n] > 0 #Term appears in document
+            term_document_occurences += 1
+          end
+        end
+        @@number_of_documents_with_term[row_index] = term_document_occurences
+        @@number_of_documents_with_term[row_index]
+      end
+
+    end
+  end
+end

data/lib/semantic/vector_space.rb

@@ -0,0 +1 @@
+%w{model builder}.each{|f| require "semantic/vector_space/#{f}"}

data/lib/semantic/vector_space/builder.rb

@@ -0,0 +1,69 @@
+module Semantic
+  module VectorSpace
+    #A algebraic model for representing text documents as vectors of identifiers.
+    #A document is represented as a vector. Each dimension of the vector corresponds to a
+    #separate term. If a term occurs in the document, then the value in the vector is non-zero.
+    class Builder
+
+      def initialize(options={})
+        @parser = Parser.new
+        @options = options
+        @parsed_document_cache = []
+      end
+
+      def build_document_matrix(documents)
+        @vector_keyword_index = build_vector_keyword_index(documents)
+       
+        document_vectors = documents.enum_for(:each_with_index).map{|document,document_id| build_vector(document, document_id)}
+        document_matrix = Linalg::DMatrix.join_columns(document_vectors)
+        
+        Model.new(document_matrix, @vector_keyword_index)
+      end
+
+      def build_query_vector(term_list)
+        build_vector(term_list.join(" "))
+      end
+
+      private
+      def build_vector_keyword_index(documents)
+        parse_and_cache(documents)
+        vocabulary_list = find_unique_vocabulary
+	      map_vocabulary_to_vector_positions(vocabulary_list)
+      end
+
+      def parse_and_cache(documents)
+        documents.each_with_index do |document, index|
+          @parsed_document_cache[index] = @parser.tokenise_and_filter(document)
+        end
+      end
+
+      def find_unique_vocabulary
+        vocabulary_list = @parsed_document_cache.inject([]) { |parsed_document, vocabulary_list| vocabulary_list + parsed_document  }
+        vocabulary_list.uniq
+      end
+      
+      def map_vocabulary_to_vector_positions(vocabulary_list)
+        vector_index={}
+        column = 0
+	      vocabulary_list.each do |word|
+          vector_index[word] = column
+          column += 1
+        end
+        vector_index
+      end 
+    
+      def build_vector(word_string, document_id=nil)
+        if document_id.nil?
+          word_list = @parser.tokenise_and_filter(word_string)
+        else
+          word_list = @parsed_document_cache[document_id]
+        end
+        
+        vector = Linalg::DMatrix.new(@vector_keyword_index.length, 1)
+        word_list.each { |word| vector[@vector_keyword_index[word] , 0] += 1 if @vector_keyword_index.has_key?(word)  }
+        vector
+      end
+
+    end
+  end
+end

data/lib/semantic/vector_space/model.rb

@@ -0,0 +1,45 @@
+require 'linalg'
+require 'delegate'
+
+module Semantic
+  module VectorSpace
+
+    class Model < DelegateClass(::Linalg::DMatrix)
+
+      def initialize(matrix, keywords)
+        @keywords = keywords || {}
+        super(matrix)
+      end
+      
+      def matrix=(matrix)
+        @_dc_obj = matrix
+      end
+      
+      def matrix
+        @_dc_obj
+      end
+            
+      def to_s
+        out = StringIO.new
+        out.print " " * 9
+
+        matrix.ncol.times do |id|
+          out.print "  D#{id+1}  " 
+        end
+        out.puts
+
+        matrix.rows.each_with_index do |terms, index|
+          out.print "#{@keywords.index(index).ljust(6)}" if @keywords.has_value?(index)
+          out.print "[ "
+          terms.columns.each do |document|
+            out.print "%+0.2f " % document
+          end
+          out.print "]"
+          out.puts
+        end
+        out.string
+      end
+      
+    end
+  end
+end

data/lib/semantic/version.rb

@@ -0,0 +1,9 @@
+module Semantic #:nodoc:
+  class VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/resources/english.stop

@@ -0,0 +1,571 @@
+a
+a's
+able
+about
+above
+according
+accordingly
+across
+actually
+after
+afterwards
+again
+against
+ain't
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+b
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+c
+c'mon
+c's
+came
+can
+can't
+cannot
+cant
+cause
+causes
+certain
+certainly
+changes
+clearly
+co
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+currently
+d
+definitely
+described
+despite
+did
+didn't
+different
+do
+does
+doesn't
+doing
+don't
+done
+down
+downwards
+during
+e
+each
+edu
+eg
+eight
+either
+else
+elsewhere
+enough
+entirely
+especially
+et
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+f
+far
+few
+fifth
+first
+five
+followed
+following
+follows
+for
+former
+formerly
+forth
+four
+from
+further
+furthermore
+g
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+h
+had
+hadn't
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+he's
+hello
+help
+hence
+her
+here
+here's
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+i
+i'd
+i'll
+i'm
+i've
+ie
+if
+ignored
+immediate
+in
+inasmuch
+inc
+indeed
+indicate
+indicated
+indicates
+inner
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+it's
+its
+itself
+j
+just
+k
+keep
+keeps
+kept
+know
+knows
+known
+l
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+little
+look
+looking
+looks
+ltd
+m
+mainly
+many
+may
+maybe
+me
+mean
+meanwhile
+merely
+might
+more
+moreover
+most
+mostly
+much
+must
+my
+myself
+n
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+no
+nobody
+non
+none
+noone
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+p
+particular
+particularly
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provides
+q
+que
+quite
+qv
+r
+rather
+rd
+re
+really
+reasonably
+regarding
+regardless
+regards
+relatively
+respectively
+right
+s
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+she
+should
+shouldn't
+since
+six
+so
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+t
+t's
+take
+taken
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that's
+thats
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+there's
+thereafter
+thereby
+therefore
+therein
+theres
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+think
+third
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+twice
+two
+u
+un
+under
+unfortunately
+unless
+unlikely
+until
+unto
+up
+upon
+us
+use
+used
+useful
+uses
+using
+usually
+uucp
+v
+value
+various
+very
+via
+viz
+vs
+w
+want
+wants
+was
+wasn't
+way
+we
+we'd
+we'll
+we're
+we've
+welcome
+well
+went
+were
+weren't
+what
+what's
+whatever
+when
+whence
+whenever
+where
+where's
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+who's
+whoever
+whole
+whom
+whose
+why
+will
+willing
+wish
+with
+within
+without
+won't
+wonder
+would
+would
+wouldn't
+x
+y
+yes
+yet
+you
+you'd
+you'll
+you're
+you've
+your
+yours
+yourself
+yourselves
+z
+zero