picolena 0.1.5 → 0.1.6

data/History.txt

@@ -1,11 +1,19 @@
-== 0.1.5  2008-04-
+== 0.1.6  2008-04-25
+
+* 1 minor enhancement:
+  * replaced index key by Document#probably_unique_id
+
+* bug fixes:
+  * Added forgotten public/images/flags to generator file.
+
+== 0.1.5  2008-04-25
 
 * 1 major enhancement:
   * yet another Indexer & Index rewrite
 
 * 1 minor enhancement:
   * flags to indicate found language
-  
+
 * bug fixes:
   * No more (or just less?) index lock errors
 
@@ -25,14 +33,13 @@
   * new DSL syntax
   * multi-threaded Indexer
 
-
 == 0.1.1  2008-04-12
 
 * 1 major enhancement:
   * cache à la Google
 
 * minor enhancements:
-  * easier configuration. Every picolena specific conf files in config/custom 
+  * easier configuration. Every picolena specific conf files in config/custom
   * lots of refactoring
 
 * bug fixes

data/Manifest.txt

@@ -89,6 +89,7 @@ lib/picolena/templates/public/images/flags/pl.png
 lib/picolena/templates/public/images/flags/pt-br.png
 lib/picolena/templates/public/images/flags/pt-pt.png
 lib/picolena/templates/public/images/flags/readme.txt
+lib/picolena/templates/public/images/flags/sv.png
 lib/picolena/templates/public/images/icons/cad.png
 lib/picolena/templates/public/images/icons/code.png
 lib/picolena/templates/public/images/icons/doc.png

data/README.txt

@@ -8,7 +8,7 @@
 Picolena is a lightweight ferret-powered documents search engine written in Ruby on rails:
 
    1. Just let Ferret index any directory you want.
-   2. Enter queries on your browser to get corresponding documents in a few milliseconds. 
+   2. Enter queries on your browser to get corresponding documents in a few milliseconds.
 
 == FEATURES:
 
@@ -33,7 +33,7 @@ web-server just like you would with any Rails application.
     picolena ~/shared_documents /media/literature
     cd picolena
     ruby script/server
-    
+
 This would create the picolena file structure, index every file inside ~/shared_documents and /media/literature, and launch a web-server available at http://localhost:3000
 
 == REQUIREMENTS:

data/bin/picolena

@@ -14,7 +14,7 @@ end
 action= ARGV.include?("--spec-only") ? "testing" : "installing"
 
 require 'rubigen/scripts/generate'
-source = RubiGen::PathSource.new(:application, 
+source = RubiGen::PathSource.new(:application,
   File.join(File.dirname(__FILE__), "../lib"))
 RubiGen::Base.reset_sources
 RubiGen::Base.append_sources source

data/config/hoe.rb

@@ -28,21 +28,21 @@ Run 'rubyforge setup' to prepare your env for access to Rubyforge
 end
 
 
-REV = nil 
-# UNCOMMENT IF REQUIRED: 
+REV = nil
+# UNCOMMENT IF REQUIRED:
 # REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
 VERS = Picolena::VERSION::STRING + (REV ? ".#{REV}" : "")
 RDOC_OPTS = ['--quiet', '--title', 'picolena documentation',
     "--opname", "index.html",
-    "--line-numbers", 
+    "--line-numbers",
     "--main", "README",
     "--inline-source"]
 
 class Hoe
-  def extra_deps 
-    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+  def extra_deps
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' }
     @extra_deps
-  end 
+  end
 end
 
 # Generate all the Rake tasks
@@ -54,10 +54,10 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
   p.url = HOMEPATH
   p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
   p.test_globs = ["spec/picolena_spec.rb"]
-  
+
   files_to_clean = File.readlines(File.join(File.dirname(__FILE__),'files_to_clean')).collect{|line| line.chomp}
   p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']+files_to_clean  #An array of file patterns to delete on clean.
-  
+
   # == Optional
   p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
   p.extra_deps = [ ['rails', '>= 2.0.2'],
@@ -70,9 +70,9 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
                    ['paginator', '>= 1.1.0'],
                    ['rspec', '>= 1.1.3']
                  ]
-  
+
   #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
-  
+
   p.rdoc_pattern = /txt$/
 end
 

data/lib/picolena/USAGE

@@ -9,7 +9,7 @@ Example:
     picolena ~/shared_documents /media/literature
     cd picolena
     ruby script/server
-    
+
     This would create the picolena file structure, index every file
     inside ~/shared_documents and /media/literature, and launch a
     web-server available at http://localhost:3000

data/lib/picolena/config/basic.rb

@@ -2,16 +2,16 @@ module Picolena
   # Specify indexes path.
   # Storage should be sufficient in order to store all indexed data.
   IndexesSavePath=File.join(RAILS_ROOT, 'tmp/ferret_indexes/')
-  
-  
+
+
   # Which language should be used?
   # English (:en), German (:de), French (:fr) and Spanish (:es) are currently supported
   # English is chosen by default.
   # If you'd like to use another language, you can find templates in #{RAILS_ROOT}/lang/ui,
   # then add your own language in this directory, and modify this line:
   Globalite.language = :en
-  
-  
+
+
   # Is more than one language used in indexed documents?
   # Picolena can try to recognise the language used, and save it in the index.
   # It is then possible to look for documents according to their language.
@@ -19,15 +19,15 @@ module Picolena
   # If every document is written in the same language, turning UseLanguageRecognition to false
   # will speed up the indexing process
   UseLanguageRecognition = true
-  
+
   # Specify which locale should be used by Ferret
   Ferret.locale = "en_US.UTF-8"
-  
-  
+
+
   # Results per page
   ResultsPerPage = 10
-  
-  
+
+
   # Length of "probably unique id" 's
   # Those id's are used to characterize every document, thus allowing tiny URLs in Controllers
   #  HashLength = 10
@@ -37,8 +37,8 @@ module Picolena
   # The more documents you have, the bigger HashLength should be in order to avoid collisions.
   # It would not be wise (and specs won't pass) to specify HashLength smaller than 10.
   HashLength = 10
-  
-  
+
+
   # Specify the default Levenshtein distance when using FuzzyQuery
   # see http://ferret.davebalmain.com/api/classes/Ferret/QueryParser.html for more information.
   Ferret::Search::FuzzyQuery.default_min_similarity=0.6

data/lib/picolena/picolena_generator.rb

@@ -3,14 +3,14 @@ require 'fileutils'
 require 'pathname'
 
 class PicolenaGenerator < RubiGen::Base #:nodoc:
-  
+
   DEFAULT_SHEBANG = File.join(Config::CONFIG['bindir'],
                               Config::CONFIG['ruby_install_name'])
-  
+
   default_options :destination => 'picolena'
-  
+
   attr_reader :name
-  
+
   def initialize(runtime_args, runtime_options = {})
     super
     usage if args.empty? and !options[:spec_only]
@@ -20,17 +20,17 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       abs_dir=Pathname.new(relative_path).realpath.to_s
       "\"#{abs_dir}\" : \"#{abs_dir}\""
     }.join("\n  ")
-    
+
     extract_options
   end
 
   def manifest
     script_options     = { :chmod => 0755, :shebang => options[:shebang] == DEFAULT_SHEBANG ? nil : options[:shebang] }
-    
+
     record do |m|
       #Create base dir
       m.directory ''
-      
+
       # Picolena file structure, without any plugin.
       BASEDIRS.each { |path|
         # Ensure appropriate folder exists
@@ -38,7 +38,7 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
         # Copy every file included in BASEDIRS
         m.folder path, path
       }
-      
+
       # Moved plugins away so they don't get parsed by rdoc/ri.
       RAILS_PLUGINS.each{ |path|
         plugin_source = '../../../rails_plugins/'+path
@@ -56,7 +56,7 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       %w( about breakpointer console destroy generate performance/benchmarker performance/profiler performance/request process/reaper process/spawner process/inspector runner server plugin spec spec_server).each do |file|
         m.file "script/#{file}", "script/#{file}", script_options
       end
-      
+
       # Picolena configuration files
       m.file '../config/white_list_ip.yml', 'config/custom/white_list_ip.yml'
       m.file '../config/basic.rb', 'config/custom/picolena.rb'
@@ -69,10 +69,10 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       m.file '../../../README.txt', 'README'
       m.file '../../../README.txt', 'doc/README_FOR_APP'
       m.file 'Rakefile', 'Rakefile'
-     
+
       unless options[:no_index]
         # Indexing documents for development environment
-        m.rake 'index:create' 
+        m.rake 'index:create'
         # Mirroring Ferret development index instead of indexing documents again for production.
         m.mirror 'tmp/ferret_indexes/development', 'tmp/ferret_indexes/production'
       end
@@ -109,10 +109,10 @@ EOS
       opts.on(nil, "--spec-only", "Test picolena framework without installing it."){
         options[:spec_only]=true
         options[:no_index]=true
-        options[:destination]=File.join(Dir::tmpdir,"picolena_test_#{Time.now.to_i}") 
+        options[:destination]=File.join(Dir::tmpdir,"picolena_test_#{Time.now.to_i}")
       }
     end
-    
+
     def extract_options
       # for each option, extract it into a local variable (and create an "attr_reader :author" at the top)
       # Templates can access these value via the attr_reader-generated methods, but not the
@@ -143,6 +143,7 @@ EOS
     public/help
     public/images
     public/images/icons
+    public/images/flags
     public/javascripts
     public/stylesheets
     spec
@@ -169,7 +170,7 @@ EOS
     tmp/sessions
     tmp/sockets
     )
-    
+
     RAILS_PLUGINS=%w(
     globalite
     globalite/data

data/lib/picolena/templates/app/controllers/application.rb

@@ -3,23 +3,23 @@
 # The request is sent to DocumentsController only if the IP is included in the white list.
 # Otherwise, it returns "Access denied" 403.
 
-class ApplicationController < ActionController::Base  
+class ApplicationController < ActionController::Base
   session :disabled => true
   before_filter :should_only_be_available_for_white_list_IPs, :except=> :access_denied
-  
+
   # Returns 403 status in case of an unknown remote IP address
   def access_denied
     render :text=>"Access denied", :status => 403
   end
-  
+
   # Redirects to documents_url in case route hasn't been recognised
   def unknown_request
     flash[:warning]="Unknown URL"
     redirect_to documents_url
   end
-  
+
   private
-  
+
   # Tries to match remote IP address with the white list defined in config/custom/white_list_ip.yml
   # Redirects to :access_denied if the remote IP is not white listed.
   def should_only_be_available_for_white_list_IPs

data/lib/picolena/templates/app/controllers/documents_controller.rb

@@ -6,17 +6,18 @@
 #  - displays document content
 #  - displays cached content.
 
-class DocumentsController < ApplicationController  
+class DocumentsController < ApplicationController
+  before_filter :ensure_index_is_created, :except=> :index
   before_filter :check_if_valid_link, :only=> [:download, :content, :cached]
-  
+
   # Actually doesn't check anything at all. Just a redirect to show_document(query)
   #
   # FIXME: remove this useless action, and go directly from submit button to GET :action => show
   def check_query
     redirect_to :action=>'show', :id=>params[:query]
   end
-  
-  
+
+
   # Show the matching documents for a given query
   def show
     start=Time.now
@@ -31,14 +32,14 @@ class DocumentsController < ApplicationController
       @total_hits=finder.total_hits
     @time_needed=Time.now-start
   end
-  
-  
+
+
   # Download the file whose probably_unique_id is given.
   # If the checksum is incorrect, redirect to documents_url via no_valid_link
   def download
     send_file @document.complete_path
   end
-  
+
   # Returns the content of the document identified by probably_unique_id, as it is *now*.
   def content
   end
@@ -47,9 +48,9 @@ class DocumentsController < ApplicationController
   # similar to Google cache.
   def cached
   end
-  
+
   private
-  
+
   # Returns corresponding document for any given "probably unique id"
   # Redirects to no_valid_link if:
   #  there are more than one matching document (hash collision)
@@ -59,6 +60,13 @@ class DocumentsController < ApplicationController
     @document=Document.find_by_unique_id(@probably_unique_id) rescue no_valid_link
   end
   
+  def ensure_index_is_created
+    Indexer.ensure_index_existence
+    while Indexer.do_not_disturb_while_indexing do
+      sleep 1
+    end
+  end
+
   # Flashes a warning and redirects to documents_url.
   def no_valid_link
     flash[:warning]="no valid link"

data/lib/picolena/templates/app/helpers/documents_helper.rb

@@ -3,7 +3,7 @@ module DocumentsHelper
   def nothing_found?
     @matching_documents.nil? or @matching_documents.entries.empty?
   end
-  
+
   # Very basic pagination.
   # Provides liks to Next, Prev and FirstPage when needed.
   def should_paginate(page,query)
@@ -11,7 +11,7 @@ module DocumentsHelper
      (link_to("←", :action => :show, :id => query, :page => page.prev.number) if page.prev?),
      (link_to("→", :action => :show, :id => query, :page => page.next.number) if page.next?)].compact.join(" | ")
   end
-  
+
   # Returns a localized sentence like "Results 1-10 of 12 for Zimbabwe (0.472s)" or
   # "Résultats 1-2 parmi 2 pour whatever (0.012s)"
   def describe_results(page, total_hits, dt, query)
@@ -24,45 +24,46 @@ module DocumentsHelper
     show_time_needed(dt)
     ].join(' ')
   end
-  
+
   # Returns the time needed to treat the query and launch the search, with a ms precision : (0.472s)
   def show_time_needed(dt)
     content_tag(:small,'('<<number_with_precision(dt,3)<<'s)')
   end
-  
+
   # When possible, highlights content of the document that matches the query.
   def highlight_matching_content(document)
     content_tag(:ul,document.matching_content.collect{|sentence|
       content_tag(:li,h(sentence).gsub(/&lt;&lt;(.*?)&gt;&gt;/,'<strong>\1</strong>').gsub(/\v|\f/,''))
     }) if document.matching_content
   end
-  
+
   def language_icon_for(document)
-    (lang=document.language) && image_tag("flags/#{lang}.png")
+    lang=document.language
+    link_to(image_tag("flags/#{lang}.png"), document_url('lang:'<<lang)) unless lang.blank?
   end
-  
+
   # Returns icon and filename for any given document.
   def icon_and_filename_for(document)
-    [icon_for(document.extname),document.filename].join("&nbsp;")
+    [icon_for(document.ext_as_sym),document.filename].join("&nbsp;")
   end
-  
+
   # Returns the location (if avaible) of the filetype icon.
-  def icon_for(filetype)    
-    icon_symbol=FiletypeToIconSymbol[filetype.downcase.sub(/^\./,'')]
+  def icon_for(ext)
+    icon_symbol=Picolena::FiletypeToIconSymbol[ext]
     image_tag("icons/#{icon_symbol}.png") if icon_symbol
   end
-  
+
   # Returns a link to a backup search engine that could maybe find more results for the same query.
   def link_to_backup_search_engine(query)
     link_to :backup_search_engine_name.l, :backup_search_engine_url.l<<query
   end
-  
+
   # For any indexed document, returns a link to its containing directory.
   def link_to_containing_directory(document)
     link_name=image_tag('icons/remote_folder.png')<<'&nbsp;'<<content_tag(:small,document.alias_path)
     link_to link_name, document.alias_path, :target=>'_blank'
   end
-  
+
   # For any indexed document, returns a link to show its content.
   def link_to_plain_text_content(document)
     link_name=image_tag('icons/plain_text_small.png')<<'&nbsp;'<<content_tag(:small,:text_content.l)

data/lib/picolena/templates/app/models/document.rb

@@ -1,36 +1,33 @@
 # Document class retrieves information from filesystem and the index for any given document.
 class Document
   attr_reader :complete_path
-  attr_writer :index_id
-  attr_accessor :user, :score, :matching_content
-  
+  attr_accessor :score, :matching_content
+
   def initialize(path)
     #To ensure @complete_path is an absolute direction.
     @complete_path=File.expand_path(path)
     validate_existence_of_file
     validate_in_indexed_directory
   end
-  
+
   #Delegating properties to File::method_name(complete_path)
-  [:dirname, :basename, :extname, :size?, :file?, :read, :ext_as_sym].each{|method_name|
+  [:dirname, :basename, :extname, :ext_as_sym, :file?, :ext_as_sym].each{|method_name|
     define_method(method_name){File.send(method_name,complete_path)}
   }
-  alias_method :size, :size?
   alias_method :filename, :basename
-  alias_method :to_s, :basename
-  
+
   # Returns filename without extension
   #   "buildings.odt" => "buildings"
   def basename
     filename.chomp(extname)
   end
-  
+
   # End users should not always know where documents are stored internally.
   # An alias path can be specified in config/indexed_directories.yml
-  # 
+  #
   # For example, with:
   #   "/media/wiki_dump/" : "http://www.mycompany.com/wiki/"
-  # 
+  #
   # The documents
   #   "/media/wiki_dump/organigram.odp"
   # will be displayed as being:
@@ -40,7 +37,7 @@ class Document
     alias_dir=Picolena::IndexedDirectories[original_dir]
     dirname.sub(original_dir,alias_dir)
   end
-  
+
   # Returns an id for this document.
   # This id will be used in Controllers in order to get tiny urls.
   # Since it's a base26 hash of the absolute filename, it can only be "probably unique".
@@ -48,46 +45,41 @@ class Document
   def probably_unique_id
     @probably_unique_id||=complete_path.base26_hash
   end
-  
+
   # Returns true iff some PlainTextExtractor has been defined to convert it to plain text.
   #  Document.new("presentation.pdf").supported? => true
   #  Document.new("presentation.some_weird_extension").supported? => false
   def supported?
     PlainTextExtractor.supported_extensions.include?(self.ext_as_sym)
   end
-  
+
   # Retrieves content as it is *now*.
   def content
     PlainTextExtractor.extract_content_from(complete_path)
   end
-  
+
   # Cache à la Google.
   # Returns content as it was at the time it was indexed.
   def cached
     from_index[:content]
   end
-  
+
   # FIXME: Not just date anymore.
   # Returns the last modification date before the document got indexed.
   # Useful to know how old a document is, and to which version the cache corresponds.
   def date
     from_index[:modified].sub(/(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/,'\1-\2-\3 \4:\5:\6')
   end
-  
+
   def mtime
     from_index[:modified].to_i
   end
-  
+
   # Returns language.
   def language
     from_index[:language]
   end
-  
-  # Returns the id with which the document is indexed.
-  def index_id
-    @index_id ||= Finder.term_search(:complete_path, complete_path).doc
-  end
-  
+
   # Fields that are shared between every document.
   def self.default_fields_for(complete_path)
     {
@@ -97,36 +89,35 @@ class Document
       :basename           => File.basename(complete_path, File.extname(complete_path)).gsub(/_/,' '),
       :filetype           => File.extname(complete_path),
       :modified           => File.mtime(complete_path).strftime("%Y%m%d%H%M%S")
-    }             
+    }
   end
-  
+
   private
-  
+
   # Retrieves the document from the index.
   # Useful to get meta-info about it.
   def from_index
-    Indexer.index[index_id]
+    Indexer.index[probably_unique_id]
   end
-  
-  def self.find_by_unique_id(some_id)
-    doc_id=Finder.term_search(:probably_unique_id, some_id).doc
-    new(Indexer.index[doc_id][:complete_path])
+
+  def self.find_by_unique_id(probably_unique_id)
+    new(Indexer.index[probably_unique_id][:complete_path])
   end
- 
+
   def in_indexed_directory?
     !indexed_directory.nil?
   end
-  
+
   def indexed_directory
     Picolena::IndexedDirectories.keys.find{|indexed_dir|
       dirname.starts_with?(indexed_dir)
-    }    
+    }
   end
-  
+
   def validate_existence_of_file
     raise Errno::ENOENT, @complete_path unless file?
   end
-  
+
   def validate_in_indexed_directory
     raise ArgumentError, "required document is not in indexed directory" unless in_indexed_directory?
   end