xapian-rack 1.2.3.1

data/lib/xapian/rack/search.rb

@@ -0,0 +1,169 @@
+#	This file is part of the "Utopia Framework" project, and is licensed under the GNU AGPLv3.
+#	Copyright 2010 Samuel Williams. All rights reserved.
+#	See <utopia.rb> for licensing details.
+
+require 'uri'
+require 'pathname'
+
+require 'logger'
+
+require 'xapian/indexer'
+require 'xapian/indexer/loaders/http'
+require 'xapian/indexer/extractors/html'
+
+module Xapian
+	module Rack
+		
+		def self.find(env, query, options = {})
+			search = env['xapian.search']
+			return search.find(query, options)
+		end
+		
+		def self.get(env)
+			return env['xapian.search']
+		end
+		
+		class RelativeLoader
+			def initialize(app, options = {})
+				@app = app
+				
+				@logger = options[:logger] || Logger.new($stdout)
+			end
+			
+			def call(name, &block)
+				# Handle relative URIs
+				if URI.parse(name).relative?
+					env = {
+						'HTTP_ACCEPT' => '*/*',
+						'QUERY_STRING' => '',
+						'REQUEST_METHOD' => 'GET',
+						'PATH_INFO' => name,
+						'rack.input' => StringIO.new,
+					}
+					
+					begin
+						status, header, content = @app.call(env)
+					rescue
+						@logger.error "Error requesting resource #{name}: #{$!}"
+						$!.backtrace.each{|line| @logger.error(line)}
+						
+						yield 500, {}, nil
+					end
+				
+					body = lambda do 
+						buffer = ""
+						content.each{|segment| buffer += segment}
+
+						buffer
+					end
+					
+					downcase_header = {}
+					header.each{|k,v| downcase_header[k.downcase] = v}
+					
+					yield status, downcase_header, body
+					
+					return true
+				end
+				
+				return false
+			end
+		end
+		
+		class Search
+			def initialize(app, options = {})
+				@app = app
+				@database_path = options[:database]
+				@database = nil
+				
+				# Setup the controller
+				@controller = Xapian::Indexer::Controller.new
+				
+				@controller.loaders << RelativeLoader.new(@app)
+				@controller.loaders << Xapian::Indexer::Loaders::HTTP.new
+				@controller.extractors['text/html'] = Xapian::Indexer::Extractors::HTML.new
+				
+				# Setup the generator
+				@generator = Xapian::TermGenerator.new()
+				
+				@logger = options[:logger] || Logger.new($stderr)
+				
+				unless options[:indexer] == :disabled
+					@indexer = Thread.new do
+						while true
+							begin
+								@logger.info "Updating index in background..."
+								index(options[:roots], options)
+							rescue
+								@logger.error "Error while updating index: #{$!}"
+								$!.backtrace.each{|line| @logger.error(line)}
+							ensure
+								@logger.info "Index update finished."
+							end
+							
+							sleep options[:refresh] || 3600
+						end
+					end
+				end
+			end
+
+			def find(query, options = {})
+				if @database
+					@database.reopen
+				else
+					@database = Xapian::Database.new(@database_path)
+				end
+				
+				# Start an enquire session.
+				enquire = Xapian::Enquire.new(@database)
+
+				# Setup the query parser
+				qp = Xapian::QueryParser.new()
+				qp.database = @database
+				query = qp.parse_query(query, options[:flags] || 0)
+
+				enquire.query = query
+				
+				start = options[:start] || 0
+				count = options[:count] || 10
+				
+				matchset = enquire.mset(start, count)
+				
+				return matchset
+			end
+			
+			def index(roots, options = {})
+				writable_database = Xapian::WritableDatabase.new(@database_path, Xapian::DB_CREATE_OR_OPEN)
+				
+				begin
+					spider = Xapian::Indexer::Spider.new(writable_database, @generator, @controller)
+				
+					spider.add(roots)
+				
+					spider.process(options) do |link|
+						uri = URI.parse(link)
+					
+						if uri.relative?
+							link
+						elsif options[:domains] && options[:domains].include?(uri.host)
+							link
+						else
+							nil
+						end
+					end
+				
+					spider.remove_old!
+				ensure
+					writable_database.close
+				end
+			end
+
+			attr :database
+
+			def call(env)
+				env['xapian.search'] = self
+				return @app.call(env)
+			end
+		end
+
+	end
+end

data/lib/xapian/rack/version.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2010 Samuel Williams. Released under the GNU GPLv3.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Xapian
+	module Rack
+		module VERSION #:nodoc:
+			MAJOR = 1
+			MINOR = 2
+			TINY  = 3
+			REV   = 1
+			
+			STRING = [MAJOR, MINOR, TINY, REV].join('.')
+		end
+	end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: xapian-rack
+version: !ruby/object:Gem::Version 
+  hash: 65
+  prerelease: false
+  segments: 
+  - 1
+  - 2
+  - 3
+  - 1
+  version: 1.2.3.1
+platform: ruby
+authors: 
+- Samuel Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-19 00:00:00 +13:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: xapian-indexer
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: 
+email: samuel.williams@oriontransfer.co.nz
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/xapian/rack/search.rb
+- lib/xapian/rack/version.rb
+has_rdoc: true
+homepage: http://www.oriontransfer.co.nz/software/xapian
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Xapian-rack provides indexing and searching integration with Rack.
+test_files: []
+