xapian-indexer 1.2.3.4 → 1.2.19.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72b813e2b0019183e68679aebf13532f607f4adb
+  data.tar.gz: 3806c150659093f02f1c394b227715790bfbe218
+SHA512:
+  metadata.gz: 2583e40e2e9eeecf3247822595e8dbedc09d07fd1bc18ba64c95b72a5ff0413cc0bba51434a69b6ccb61e60669a4f05616d615e3e229f4105512e93315316e12
+  data.tar.gz: f77f2f725d2fb549b6c0eea955ef709390090916c090e8259c0d94474bbaaa686d340a56d7987711e755591b5e40aa757e9e2e1b88fe0651b753d0304be4b664

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in xapian-indexer.gemspec
+gemspec

data/README.md

@@ -0,0 +1,109 @@
+# Xapian::Indexer
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+	gem 'xapian-indexer'
+
+And then execute:
+
+	$ bundle
+
+Or install it yourself as:
+
+	$ gem install xapian-indexer
+
+## Usage
+
+The following example shows how to create a spider and index some resources:
+
+	#!/usr/bin/env ruby
+
+	require 'rubygems'
+
+	require 'uri'
+
+	require 'xapian/indexer'
+	require 'xapian/indexer/loaders/http'
+	require 'xapian/indexer/extractors/html'
+
+	$stderr.sync = true
+
+	database = Xapian::WritableDatabase.new("/tmp/xapian-oriontransfer", Xapian::DB_CREATE_OR_OPEN)
+	generator = Xapian::TermGenerator.new()
+
+	# Setup the controller
+	controller = Xapian::Indexer::Controller.new
+
+	controller.loaders << Xapian::Indexer::Loaders::HTTP.new()
+	controller.extractors['text/html'] = Xapian::Indexer::Extractors::HTML.new()
+
+	spider = Xapian::Indexer::Spider.new(database, generator, controller)
+
+	spider.add("http://www.oriontransfer.co.nz/welcome/index")
+
+	spider.process(:depth => 2, :count => 50) do |link|
+		uri = URI.parse(link)
+	
+		case uri.host
+		when "www.oriontransfer.co.nz"
+			link
+		else
+			$stderr.puts "Skipping #{link}"
+			nil
+		end
+	end
+
+	# Start an enquire session.
+	enquire = Xapian::Enquire.new(database)
+
+	# Setup the query parser
+	qp = Xapian::QueryParser.new()
+	query = qp.parse_query("services")
+
+	enquire.query = query
+	matchset = enquire.mset(0, 10)
+
+	# Display the results.
+	puts "#{matchset.matches_estimated()} results found."
+	puts "Matches 1-#{matchset.size}:\n"
+
+	matchset.matches.each do |m|
+		resource = YAML::load(m.document.data)
+		puts "#{m.rank + 1}: #{m.percent}% docid=#{m.docid} [#{resource[:name]}]"
+	end
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Released under the MIT license.
+
+Copyright, 2015, by [Samuel G. D. Williams](http://www.codeotaku.com/samuel-williams).
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/xapian/indexer.rb

@@ -1,7 +1,4 @@
 
-require 'logger'
-
-require 'xapian/indexer/extensions'
 require 'xapian/indexer/version'
 require 'xapian/indexer/resource'
 require 'xapian/indexer/spider'

data/lib/xapian/indexer/extractors/html.rb

@@ -98,7 +98,6 @@ module Xapian
 					html.search("form").remove
 					html.css('.noindex').remove
 					
-					
 					body = html.at('html/body')
 					
 					if body

data/lib/xapian/indexer/loaders/http.rb

@@ -18,10 +18,9 @@ require 'xapian/indexer/version'
 
 module Xapian
 	module Indexer
-		
 		module Loaders
 			class HTTP
-				UserAgent = "Xapian-Spider #{Xapian::Indexer::VERSION::STRING}"
+				UserAgent = "Xapian-Spider #{Xapian::Indexer::VERSION}"
 				
 				def initialize(options = {})
 					@options = options
@@ -53,10 +52,7 @@ module Xapian
 					return false
 				end
 			end
-			
-			
 		end
-		
 	end
 end
 

data/lib/xapian/indexer/resource.rb

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'digest/md5'
+require 'yaml'
 
 module Xapian
 	module Indexer
@@ -76,6 +77,7 @@ module Xapian
 			
 			def recreate(data)
 				values = YAML::load(data)
+				
 				Resource.new(values[:name], self, values)
 			end
 		end

data/lib/xapian/indexer/spider.rb

@@ -16,6 +16,8 @@
 require 'xapian'
 require 'set'
 
+require 'logger'
+
 module Xapian
 	module Indexer
 		# Represents a process which consumes resources into the database

data/lib/xapian/indexer/version.rb

@@ -15,13 +15,6 @@
 
 module Xapian
 	module Indexer
-		module VERSION #:nodoc:
-			MAJOR = 1
-			MINOR = 2
-			TINY  = 3
-			REV   = 4
-			
-			STRING = [MAJOR, MINOR, TINY, REV].join('.')
-		end
+		VERSION = "1.2.19.1"
 	end
 end

data/test/oriontransfer.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+
+require 'uri'
+
+require 'xapian/indexer'
+require 'xapian/indexer/loaders/http'
+require 'xapian/indexer/extractors/html'
+
+$stderr.sync = true
+
+database = Xapian::WritableDatabase.new("/tmp/xapian-oriontransfer", Xapian::DB_CREATE_OR_OPEN)
+generator = Xapian::TermGenerator.new()
+
+# Setup the controller
+controller = Xapian::Indexer::Controller.new
+
+controller.loaders << Xapian::Indexer::Loaders::HTTP.new()
+controller.extractors['text/html'] = Xapian::Indexer::Extractors::HTML.new()
+
+spider = Xapian::Indexer::Spider.new(database, generator, controller)
+
+spider.add("http://www.oriontransfer.co.nz/welcome/index")
+
+spider.process(:depth => 2, :count => 50) do |link|
+	uri = URI.parse(link)
+	
+	case uri.host
+	when "www.oriontransfer.co.nz"
+		link
+	else
+		$stderr.puts "Skipping #{link}"
+		nil
+	end
+end
+
+# Start an enquire session.
+enquire = Xapian::Enquire.new(database)
+
+# Setup the query parser
+qp = Xapian::QueryParser.new()
+query = qp.parse_query("services")
+
+enquire.query = query
+matchset = enquire.mset(0, 10)
+
+# Display the results.
+puts "#{matchset.matches_estimated()} results found."
+puts "Matches 1-#{matchset.size}:\n"
+
+matchset.matches.each do |m|
+	resource = YAML::load(m.document.data)
+	puts "#{m.rank + 1}: #{m.percent}% docid=#{m.docid} [#{resource[:name]}]"
+end

data/xapian-indexer.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'xapian/indexer/version'
+
+Gem::Specification.new do |spec|
+	spec.name          = "xapian-indexer"
+	spec.version       = Xapian::Indexer::VERSION
+	spec.authors       = ["Samuel Williams"]
+	spec.email         = ["samuel.williams@oriontransfer.co.nz"]
+	spec.summary       = %q{Xapian::Indexer provides a flexible spider for indexing resources.}
+	spec.homepage      = ""
+	spec.license       = "MIT"
+
+	spec.files         = `git ls-files -z`.split("\x0")
+	spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+	spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+	spec.require_paths = ["lib"]
+
+	spec.add_dependency 'xapian-core'
+	spec.add_dependency 'nokogiri'
+	
+	spec.add_development_dependency "bundler", "~> 1.6"
+	spec.add_development_dependency "rake"
+end

metadata

@@ -1,101 +1,113 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: xapian-indexer
-version: !ruby/object:Gem::Version 
-  hash: 75
-  prerelease: false
-  segments: 
-  - 1
-  - 2
-  - 3
-  - 4
-  version: 1.2.3.4
+version: !ruby/object:Gem::Version
+  version: 1.2.19.1
 platform: ruby
-authors: 
+authors:
 - Samuel Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-01-24 00:00:00 +13:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: xapian-core
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
-email: samuel.williams@oriontransfer.co.nz
+email:
+- samuel.williams@oriontransfer.co.nz
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- lib/xapian/indexer/extensions.rb
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- lib/xapian/indexer.rb
 - lib/xapian/indexer/extractors/html.rb
 - lib/xapian/indexer/loaders/http.rb
 - lib/xapian/indexer/resource.rb
 - lib/xapian/indexer/spider.rb
 - lib/xapian/indexer/version.rb
-- lib/xapian/indexer.rb
-has_rdoc: true
-homepage: http://www.oriontransfer.co.nz/software/xapian
-licenses: []
-
+- test/oriontransfer.rb
+- xapian-indexer.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Xapian::Indexer provides a flexible spider for indexing resources.
-test_files: []
-
+test_files:
+- test/oriontransfer.rb

data/lib/xapian/indexer/extensions.rb

@@ -1,45 +0,0 @@
-# Copyright (c) 2010 Samuel Williams. Released under the GNU GPLv3.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-require 'uri'
-
-class URI::Generic
-	def absolute_path?
-		path.match('^/')
-	end
-	
-	def relative_path?
-		!absolute_path?
-	end
-	
-	# Behavior in 1.8.7 seems to be broken...?
-	def merge0(oth)
-		case oth
-		when Generic
-		when String
-			oth = URI.parse(oth)
-		else
-			raise ArgumentError, "bad argument(expected URI object or URI string)"
-		end
-		
-		if oth.absolute?
-			return oth, oth
-		else
-			return self.dup, oth
-		end
-	end
-end
-
-# puts URI.parse("/bob/dole") + URI.parse("http://www.lucidsystems.org")