inverted_index 0.0.3

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+gem "nokogiri", ">= 1.5.0"
+gem "hpricot", ">= 0.8.6"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.8.3"
+  #gem "rcov", ">= 0"
+end
+

data/Gemfile.lock

@@ -0,0 +1,31 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    hpricot (0.8.6)
+    jeweler (1.8.3)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.6.5)
+    nokogiri (1.5.0)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    shoulda (3.0.1)
+      shoulda-context (~> 1.0.0)
+      shoulda-matchers (~> 1.0.0)
+    shoulda-context (1.0.0)
+    shoulda-matchers (1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  hpricot (>= 0.8.6)
+  jeweler (~> 1.8.3)
+  nokogiri (>= 1.5.0)
+  rdoc (~> 3.12)
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 sfigart
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,4 @@
+= inverted_index
+
+This is a set of classes to that allow an inverted index to be created
+This is a project for a course and is not intended to be used in production system

data/Rakefile

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "inverted_index"
+  gem.homepage = "http://github.com/sfigart/inverted_index"
+  gem.license = "MIT"
+  gem.summary = %Q{Inverted Index}
+  gem.description = %Q{Inverted Index implementation}
+  gem.email = "sfigart@gmail.com"
+  gem.authors = ["sfigart"]
+  # dependencies defined in Gemfile
+  gem.add_dependency 'nokogiri', '>=1.5.0'
+  gem.add_dependency 'hpricot', '>=0.8.6'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+=begin
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+=end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "inverted_index #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.3

data/inverted_index.gemspec

@@ -0,0 +1,74 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "inverted_index"
+  s.version = "0.0.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["sfigart"]
+  s.date = "2012-03-17"
+  s.description = "Inverted Index implementation"
+  s.email = "sfigart@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "inverted_index.gemspec",
+    "lib/inverted_index.rb",
+    "lib/inverted_index/cleaner.rb",
+    "lib/inverted_index/parse.rb",
+    "lib/inverted_index/stopwords.rb",
+    "test/helper.rb",
+    "test/test_inverted_index.rb"
+  ]
+  s.homepage = "http://github.com/sfigart/inverted_index"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.17"
+  s.summary = "Inverted Index"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0.8.6"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0.8.6"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_dependency(%q<hpricot>, [">= 0.8.6"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_dependency(%q<hpricot>, [">= 0.8.6"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+    s.add_dependency(%q<hpricot>, [">= 0.8.6"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+    s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+    s.add_dependency(%q<hpricot>, [">= 0.8.6"])
+  end
+end
+

data/lib/inverted_index.rb

@@ -0,0 +1,3 @@
+require 'inverted_index/cleaner'
+require 'inverted_index/parse'
+require 'inverted_index/stopwords'

data/lib/inverted_index/cleaner.rb

@@ -0,0 +1,36 @@
+module InvertedIndex
+  class Cleaner
+    def self.clean(tokens, text='')
+      # To lowercase
+      tokens = tokens.each {|token| token.downcase}
+     
+      # Remove stopwords
+      tokens = tokens - InvertedIndex::Stopwords.words
+
+      # Remove all non-word characters
+      words = []
+      tokens = tokens.each do |token|
+        word = token.gsub(/\W/,'')
+        words << word if !word.empty?
+      end
+      tokens = words
+
+      # TODO: Scan text for special text (e.g. dates, time)
+      # A date looks like /((january|february|march)\s\d,\s\d\d\d\d)/i
+      # A time looks like
+      # 00:00 # 00:00:00 # 00:00:00 a.m. # 00:00:00 p.m. # 00:00:00 pm
+      matches = text.scan(/(\d\d:\d\d(:\d\d)?(\s(a|p)\.?m\.?)?)/i)
+      matches.each {|match| tokens << match[0].downcase.strip}
+   
+      # Remove all non-ascii words
+      ascii_terms = []
+      tokens.each {|token| ascii_terms << token if token.ascii_only?}
+      tokens = ascii_terms
+
+      # Stem
+      stemmed_terms = []
+      tokens.each {|token| stemmed_terms << token.stem.downcase if !token.stem.empty?}
+      tokens = stemmed_terms
+    end
+  end
+end

data/lib/inverted_index/parse.rb

@@ -0,0 +1,42 @@
+require 'hpricot'
+
+module InvertedIndex
+  class Parse
+    attr_accessor :html, :doc, :body, :text, :tokens
+    def initialize(html)
+      @html = html
+    end
+
+    def parse
+      @doc = Hpricot(@html)
+
+      # Remove sections that shouldn't be indexed
+      @doc.search('head').remove
+      @doc.search('script').remove
+      @doc.search('style').remove
+      @doc.search('iframe').remove
+      @doc.search('embed').remove
+
+      # Get all text nodes
+      @text_nodes = (@doc/"body//*/text()")
+      @tokens = []
+
+      # Clean up each text node
+      @text_nodes.each do |node|
+        text = node.to_plain_text.strip
+        words = clean(text).split(' ')
+        words.each do |word|
+          @tokens << word unless word.empty?
+        end
+      end
+      
+      # Return text separated by spaces
+      @text = @tokens.join(' ')
+    end
+
+    def clean(text)
+      # Replace new line and tabs with space
+      return text.gsub(/(\n|\t)/,' ').strip
+    end
+  end
+end

data/lib/inverted_index/stopwords.rb

@@ -0,0 +1,49 @@
+module InvertedIndex
+  class Stopwords
+    def self.words
+      [
+      "a", "about", "above", "across", "after", "again", "against", "all", "almost", "alone", "along", 
+      "already", "also", "although", "always", "among", "an", "and", "another", "any", "anybody", 
+      "anyone", "anything", "anywhere", "are", "area", "areas", "around", "as", "ask", "asked",
+      "asking", "asks", "at", "away", "b", "back", "backed", "backing", "backs", "be", "became",
+      "because", "become", "becomes", "been", "before", "began", "behind", "being", "beings", 
+      "best", "better", "between", "big", "both", "but", "by", "c", "came", "can", "cannot", "case", 
+      "cases", "certain", "certainly", "clear", "clearly", "come", "could", "d", "did", "differ", 
+      "different", "differently", "do", "does", "done", "down", "down", "downed", "downing", 
+      "downs", "during", "e", "each", "early", "either", "end", "ended", "ending", "ends", "enough", 
+      "even", "evenly", "ever", "every", "everybody", "everyone", "everything", "everywhere", "f", 
+      "face", "faces", "fact", "facts", "far", "felt", "few", "find", "finds", "first", "for", "four", "from", 
+      "full", "fully", "further", "furthered", "furthering", "furthers", "g", "gave", "general", 
+      "generally", "get", "gets", "give", "given", "gives", "go", "going", "good", "goods", "got", 
+      "great", "greater", "greatest", "group", "grouped", "grouping", "groups", "h", "had", "has", 
+      "have", "having", "he", "her", "here", "herself", "high", "high", "high", "higher", "highest", 
+      "him", "himself", "his", "how", "however", "i", "if", "important", "in", "interest", "interested", 
+      "interesting", "interests", "into", "is", "it", "its", "itself", "j", "just", "k", "keep", "keeps", "kind", 
+      "knew", "know", "known", "knows", "l", "large", "largely", "last", "later", "latest", "least", 
+      "less", "let", "lets", "like", "likely", "long", "longer", "longest", "m", "made", "make", "making", 
+      "man", "many", "may", "me", "member", "members", "men", "might", "more", "most", 
+      "mostly", "mr", "mrs", "much", "must", "my", "myself", "n", "necessary", "need", "needed", 
+      "needing", "needs", "never", "new", "new", "newer", "newest", "next", "no", "nobody", "non", 
+      "noone", "not", "nothing", "now", "nowhere", "number", "numbers", "o", "of", "off", "often", 
+      "old", "older", "oldest", "on", "once", "one", "only", "open", "opened", "opening", "opens", 
+      "or", "order", "ordered", "ordering", "orders", "other", "others", "our", "out", "over", "p", 
+      "part", "parted", "parting", "parts", "per", "perhaps", "place", "places", "point", "pointed", 
+      "pointing", "points", "possible", "present", "presented", "presenting", "presents", "problem", 
+      "problems", "put", "puts", "q", "quite", "r", "rather", "really", "right", "right", "room", 
+      "rooms", "s", "said", "same", "saw", "say", "says", "second", "seconds", "see", "seem", 
+      "seemed", "seeming", "seems", "sees", "several", "shall", "she", "should", "show", "showed", 
+      "showing", "shows", "side", "sides", "since", "small", "smaller", "smallest", "so", "some", 
+      "somebody", "someone", "something", "somewhere", "state", "states", "still", "still", "such", 
+      "sure", "t", "take", "taken", "than", "that", "the", "their", "them", "then", "there", "therefore", 
+      "these", "they", "thing", "things", "think", "thinks", "this", "those", "though", "thought", 
+      "thoughts", "three", "through", "thus", "to", "today", "together", "too", "took", "toward", 
+      "turn", "turned", "turning", "turns", "two", "u", "under", "until", "up", "upon", "us", "use", 
+      "used", "uses", "v", "very", "w", "want", "wanted", "wanting", "wants", "was", "way", "ways", 
+      "we", "well", "wells", "went", "were", "what", "when", "where", "whether", "which", "while", 
+      "who", "whole", "whose", "why", "will", "with", "within", "without", "work", "worked", 
+      "working", "works", "would", "x", "y", "year", "years", "yet", "you", "young", "younger", 
+      "youngest", "your", "yours", "z"
+      ]
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'inverted_index'
+
+class Test::Unit::TestCase
+end

data/test/test_inverted_index.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestInvertedIndex < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: inverted_index
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+  prerelease: 
+platform: ruby
+authors:
+- sfigart
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70366066585320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366066585320
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: &70366066588780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366066588780
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &70366065868620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70366065868620
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &70366065870080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: *70366065870080
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &70366065871520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *70366065871520
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &70366065873460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :development
+  prerelease: false
+  version_requirements: *70366065873460
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70366065857600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366065857600
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: &70366065766900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366065766900
+description: Inverted Index implementation
+email: sfigart@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- inverted_index.gemspec
+- lib/inverted_index.rb
+- lib/inverted_index/cleaner.rb
+- lib/inverted_index/parse.rb
+- lib/inverted_index/stopwords.rb
+- test/helper.rb
+- test/test_inverted_index.rb
+homepage: http://github.com/sfigart/inverted_index
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -747465224356148538
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Inverted Index
+test_files: []