picolena 0.1.8 → 0.2.0

data/README.txt

@@ -1,7 +1,7 @@
 = Picolena
 
 * http://picolena.devjavu.com
-* http://svn.devjavu.com/picolena
+* http://github.com/EricDuminil/picolena/
 
 == DESCRIPTION:
 
@@ -38,8 +38,8 @@ This would create the picolena file structure, index every file inside ~/shared_
 
 == REQUIREMENTS:
 
-* packages : antiword poppler-utils odt2txt html2text catdoc unrtf
-* gems     : rails ferret paginator haml rubyzip rubigen
+* packages : antiword catppt exiftool grep html2text iconv pdftotext sed unrtf xls2csv
+* gems     : rails ferret paginator haml rubyzip rubigen rspec rspec-rails
 
 == INSTALL:
 
@@ -47,7 +47,7 @@ This would create the picolena file structure, index every file inside ~/shared_
 
 == LICENSE:
 
-Copyright (c) 2008 Eric Duminil
+Copyright (c) 2009 Eric Duminil
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/config/files_to_clean

@@ -1,5 +1,6 @@
 lib/picolena/templates/LICENSE
 lib/picolena/templates/README
+lib/picolena/templates/config/environment.rb
 lib/picolena/templates/config/custom/picolena.rb
 lib/picolena/templates/config/custom/indexed_directories.yml
 lib/picolena/templates/config/custom/white_list_ip.yml

data/config/hoe.rb

@@ -60,15 +60,15 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
 
   # == Optional
   p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  p.extra_deps = [ ['rails', '>= 2.0.2'],
-                   # Ferret 0.11.6 is not yet available for win32 (and will never be, it is a fix release for *nix)
-                   # FIXME: How to require 0.11.6 for *nix and 0.11.5 for win32?
+  p.extra_deps = [ ['rails', '= 2.2.2'],
                    ['ferret', '>= 0.11.5'],
-                   ['haml', '>= 1.8.2'],
+                   ['haml', '>= 2.0.7'],
                    ['rubyzip', '>= 0.9.1'],
+                  # TODO: upgrade to 1.5.2
                    ['rubigen', '>= 1.2.4'],
-                   ['paginator', '>= 1.1.0'],
-                   ['rspec', '>= 1.1.3']
+                   ['paginator', '>= 1.1.1'],
+                   ['rspec', '>= 1.1.12'],
+                   ['rspec-rails', '>= 1.1.12']
                  ]
 
   #p.spec_extras = {}    # A hash of extra values to set in the gemspec.

data/lib/picolena/{templates/config → config}/environment.rb

@@ -5,7 +5,7 @@
 # ENV['RAILS_ENV'] ||= 'production'
 
 # Specifies gem version of Rails to use when vendor/rails is not present
-RAILS_GEM_VERSION = '2.0.2' unless defined? RAILS_GEM_VERSION
+RAILS_GEM_VERSION = '2.2.2' unless defined? RAILS_GEM_VERSION
 
 # Bootstrap the Rails environment, frameworks, and default configuration
 require File.join(File.dirname(__FILE__), 'boot')
@@ -15,3 +15,8 @@ Rails::Initializer.run do |config|
   # Ferret backs us up.
   config.frameworks -= [ :active_record ]
 end
+
+#Initialises Picolena module.
+module Picolena
+  VERSION='<%= version %>'
+end

data/lib/picolena/config/icons_and_filetypes.yml

@@ -56,11 +56,19 @@ xls:
 picture:
   psd
   jpg
+  jpeg
+  tga
   png
   gif
   eps
   bmp
   ico
+  tiff
+  tif
+  svg
+  raw
+  nef
+  crw
 cad:
   dwg
   dxf

data/lib/picolena/config/indexed_directories.yml

@@ -8,3 +8,10 @@ test:
   "spec/test_dirs/indexed": "http://picolena.devjavu.com/browser/trunk/lib/picolena/templates/spec/test_dirs/indexed"
 production:
   <%= directories_to_index %>
+
+## Defines which files should be ignored by the indexer. 
+ignore:
+  Thumbs.db
+#  *.bak
+#  *.log
+#  *.tmp

data/lib/picolena/config/indexing_performance.yml

@@ -5,6 +5,9 @@
 
 ## Main performance parameters
 
+## Number of threads created by the Indexer
+threads_number: 8 
+
 # Allowed memory for indexing process.
 # 128MB by default, or 2^27
 max_buffer_memory: 134_217_728
@@ -27,4 +30,4 @@ max_buffered_docs: 10_000
 # max_merge_docs: -1
 use_compound_file: true
 index_skip_interval: 128
-doc_skip_interval: 16
+doc_skip_interval: 16

data/lib/picolena/picolena_generator.rb

@@ -62,6 +62,7 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       end
 
       # Picolena configuration files
+      m.template '../config/environment.rb', 'config/environment.rb', :assigns => {:version => Picolena::VERSION::STRING}
       m.file '../config/white_list_ip.yml', 'config/custom/white_list_ip.yml'
       m.file '../config/basic.rb', 'config/custom/picolena.rb'
       m.template '../config/indexed_directories.yml', 'config/custom/indexed_directories.yml', :assigns => {:directories_to_index => @directories_to_index}
@@ -86,7 +87,7 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       m.rake 'spec' unless options[:no_spec]
 
       # Cleaning up temp folder if --spec-only
-      m.clean if options[:spec_only]
+      m.clean if false
     end
   end
 
@@ -164,6 +165,7 @@ EOS
     spec/test_dirs/indexed/just_one_doc
     spec/test_dirs/indexed/lang
     spec/test_dirs/indexed/literature
+    spec/test_dirs/indexed/media
     spec/test_dirs/indexed/others
     spec/test_dirs/indexed/others/nested
     spec/test_dirs/indexed/yet_another_dir
@@ -201,133 +203,5 @@ EOS
     globalite/spec/lang/ui
     globalite/tasks
     haml
-    rspec
-    rspec/autotest
-    rspec/bin
-    rspec/examples
-    rspec/examples/pure
-    rspec/examples/stories
-    rspec/examples/stories/game-of-life
-    rspec/examples/stories/game-of-life/behaviour
-    rspec/examples/stories/game-of-life/behaviour/examples
-    rspec/examples/stories/game-of-life/behaviour/stories
-    rspec/examples/stories/game-of-life/life
-    rspec/examples/stories/steps
-    rspec/failing_examples
-    rspec/lib
-    rspec/lib/autotest
-    rspec/lib/spec
-    rspec/lib/spec/example
-    rspec/lib/spec/expectations
-    rspec/lib/spec/expectations/differs
-    rspec/lib/spec/expectations/extensions
-    rspec/lib/spec/extensions
-    rspec/lib/spec/interop
-    rspec/lib/spec/interop/test
-    rspec/lib/spec/interop/test/unit
-    rspec/lib/spec/interop/test/unit/ui
-    rspec/lib/spec/interop/test/unit/ui/console
-    rspec/lib/spec/matchers
-    rspec/lib/spec/mocks
-    rspec/lib/spec/mocks/extensions
-    rspec/lib/spec/rake
-    rspec/lib/spec/runner
-    rspec/lib/spec/runner/formatter
-    rspec/lib/spec/runner/formatter/story
-    rspec/lib/spec/story
-    rspec/lib/spec/story/extensions
-    rspec/lib/spec/story/runner
-    rspec/plugins
-    rspec/plugins/mock_frameworks
-    rspec/pre_commit
-    rspec/pre_commit/lib
-    rspec/pre_commit/lib/pre_commit
-    rspec/pre_commit/spec
-    rspec/pre_commit/spec/pre_commit
-    rspec/rake_tasks
-    rspec/spec
-    rspec/spec/autotest
-    rspec/spec/spec
-    rspec/spec/spec/example
-    rspec/spec/spec/expectations
-    rspec/spec/spec/expectations/differs
-    rspec/spec/spec/expectations/extensions
-    rspec/spec/spec/extensions
-    rspec/spec/spec/interop
-    rspec/spec/spec/interop/test
-    rspec/spec/spec/interop/test/unit
-    rspec/spec/spec/interop/test/unit/resources
-    rspec/spec/spec/matchers
-    rspec/spec/spec/mocks
-    rspec/spec/spec/package
-    rspec/spec/spec/runner
-    rspec/spec/spec/runner/formatter
-    rspec/spec/spec/runner/formatter/story
-    rspec/spec/spec/runner/resources
-    rspec/spec/spec/runner/spec_parser
-    rspec/spec/spec/story
-    rspec/spec/spec/story/extensions
-    rspec/spec/spec/story/runner
-    rspec/stories
-    rspec/stories/example_groups
-    rspec/stories/interop
-    rspec/stories/pending_stories
-    rspec/stories/resources
-    rspec/stories/resources/helpers
-    rspec/stories/resources/matchers
-    rspec/stories/resources/spec
-    rspec/stories/resources/steps
-    rspec/stories/resources/stories
-    rspec/stories/resources/test
-    rspec/story_server
-    rspec/story_server/prototype
-    rspec/story_server/prototype/javascripts
-    rspec/story_server/prototype/lib
-    rspec/story_server/prototype/stylesheets
-    rspec_on_rails
-    rspec_on_rails/generators
-    rspec_on_rails/generators/helpers
-    rspec_on_rails/generators/rspec
-    rspec_on_rails/generators/rspec/templates
-    rspec_on_rails/generators/rspec/templates/script
-    rspec_on_rails/generators/rspec_controller
-    rspec_on_rails/generators/rspec_controller/templates
-    rspec_on_rails/generators/rspec_model
-    rspec_on_rails/generators/rspec_model/templates
-    rspec_on_rails/generators/rspec_scaffold
-    rspec_on_rails/generators/rspec_scaffold/templates
-    rspec_on_rails/lib
-    rspec_on_rails/lib/autotest
-    rspec_on_rails/lib/spec
-    rspec_on_rails/lib/spec/rails
-    rspec_on_rails/lib/spec/rails/example
-    rspec_on_rails/lib/spec/rails/extensions
-    rspec_on_rails/lib/spec/rails/extensions/action_controller
-    rspec_on_rails/lib/spec/rails/extensions/action_view
-    rspec_on_rails/lib/spec/rails/extensions/active_record
-    rspec_on_rails/lib/spec/rails/extensions/spec
-    rspec_on_rails/lib/spec/rails/extensions/spec/example
-    rspec_on_rails/lib/spec/rails/extensions/spec/matchers
-    rspec_on_rails/lib/spec/rails/matchers
-    rspec_on_rails/spec
-    rspec_on_rails/spec/rails
-    rspec_on_rails/spec/rails/autotest
-    rspec_on_rails/spec/rails/example
-    rspec_on_rails/spec/rails/extensions
-    rspec_on_rails/spec/rails/matchers
-    rspec_on_rails/spec/rails/mocks
-    rspec_on_rails/spec_resources
-    rspec_on_rails/spec_resources/controllers
-    rspec_on_rails/spec_resources/helpers
-    rspec_on_rails/spec_resources/views
-    rspec_on_rails/spec_resources/views/controller_spec
-    rspec_on_rails/spec_resources/views/render_spec
-    rspec_on_rails/spec_resources/views/rjs_spec
-    rspec_on_rails/spec_resources/views/tag_spec
-    rspec_on_rails/spec_resources/views/view_spec
-    rspec_on_rails/spec_resources/views/view_spec/foo
-    rspec_on_rails/stories
-    rspec_on_rails/stories/steps
-    rspec_on_rails/tasks
 )
 end

data/lib/picolena/templates/app/controllers/documents_controller.rb

@@ -21,7 +21,7 @@ class DocumentsController < ApplicationController
   # Show the matching documents for a given query
   def show
     start=Time.now
-      @query=[params[:id],params.delete(:format)].compact.join('.')
+      @query=params[:id]
       @sort_by=params[:sort_by]
       page=params[:page]||1
       finder=Finder.new(@query,@sort_by,page)
@@ -38,7 +38,7 @@ class DocumentsController < ApplicationController
   # Download the file whose probably_unique_id is given.
   # If the checksum is incorrect, redirect to documents_url via no_valid_link
   def download
-    send_file @document.complete_path
+    send_file @document.complete_path, :type => @document.mime, :disposition => 'inline'
   end
 
   # Returns the content of the document identified by probably_unique_id, as it is *now*.

data/lib/picolena/templates/app/helpers/documents_helper.rb

@@ -27,7 +27,7 @@ module DocumentsHelper
   
   # Returns the time needed to treat the query and launch the search, with a ms precision : (0.472s)
   def show_time_needed(dt)
-    content_tag(:small,'('<<number_with_precision(dt,3)<<'s)')
+    content_tag(:small,'('<<number_with_precision(dt,:precision => 3)<<'s)')
   end
 
   # When possible, highlights content of the document that matches the query.
@@ -37,9 +37,11 @@ module DocumentsHelper
     }) if document.matching_content
   end
 
-  def language_icon_for(document)
+  def language_icon_for(document,query)
     lang=document.language
-    link_to(image_tag("flags/#{lang}.png"), document_url('lang:'<<lang)) unless lang.blank?
+    return if lang.blank?
+    query_with_lang= query=~/#{:language.l}/ ? query : "#{query} lang:#{lang}"
+    link_to(image_tag("flags/#{lang}.png"), document_url(query_with_lang))
   end
 
   # Returns icon and filename for any given document.
@@ -76,12 +78,21 @@ module DocumentsHelper
     link_to link_name, cached_document_path(:id => document.probably_unique_id, :query => query)
   end
   
+  # Returns multicolor cache à la Google.
   def highlighted_cache(document, query)
-    h(document.highlighted_cache(query)).gsub(/\n/,'<br/>').gsub(/&lt;&lt;(.*?)&gt;&gt;/,content_tag(:span, '\1', :class=>"matching_content"))
+    # What are the terms related to :content field?
+    content_terms=Query.content_terms(query)
+    # Parses cache for found terms
+    # Replaces linebreak by <br/>
+    # < and > by &lt; and &gt;
+    h(document.highlighted_cache(query)).gsub(/\n/,'<br/>').gsub(/&lt;&lt;(.*?)&gt;&gt;/){|c| term=$1
+      # and affects a span class to each term
+      content_tag(:span, term, :class=>"matching_content_#{content_terms.index(term.downcase)}")
+    }
   end
   
-  def sort_by_date_or_relevance(query)
-    [link_to_unless_current('By date', document_path(query, :sort_by=>'date')),
-     link_to_unless_current('By relevance', document_path(query))].join("&nbsp;")
+  def sort_by_date_or_relevance(query,params)
+    [link_to_if(params[:sort_by]!='date',:by_date.l, document_path(query, :sort_by=>'date')),
+     link_to_if(params[:sort_by],:by_relevance.l, document_path(query))].join("&nbsp;")
   end
 end

data/lib/picolena/templates/app/models/document.rb

@@ -3,14 +3,17 @@ class Document
   attr_reader :complete_path
   attr_accessor :score, :matching_content
 
+  # Instantiates a new Document, and ensure that the given path exists and
+  # is included in an indexed directory.
+  # Raises otherwise.
   def initialize(path)
-    #To ensure @complete_path is an absolute direction.
+    # To ensure @complete_path is an absolute direction.
     @complete_path=File.expand_path(path)
     validate_existence_of_file
     validate_in_indexed_directory
   end
 
-  #Delegating properties to File::method_name(complete_path)
+  # Delegating properties to File::method_name(complete_path)
   [:dirname, :basename, :extname, :ext_as_sym, :file?, :plain_text?, :size, :ext_as_sym].each{|method_name|
     define_method(method_name){File.send(method_name,complete_path)}
   }
@@ -18,12 +21,14 @@ class Document
   alias_method :to_s, :complete_path
 
 
+  # Returns complete path as well as matching score and language if available.
+  #  ../spec/test_dirs/indexed/just_one_doc/for_test.txt (56.3%) (language:en)
+  # Used for example by
+  #  rake index:search query="some query"
   def inspect
     [self,("(#{pretty_score})" if @score),("(language:#{language})" if language)].compact.join(" ")
   end
 
-
-
   # Returns filename without extension
   #   "buildings.odt" => "buildings"
   def basename
@@ -61,6 +66,14 @@ class Document
     PlainTextExtractor.supported_extensions.include?(self.ext_as_sym) unless ext_as_sym==:no_extension and !plain_text?
   end
 
+  def extractor
+    PlainTextExtractor.find_by_extension(self.ext_as_sym) rescue nil
+  end
+
+  def mime
+    extractor.mime_name rescue 'application/octet-stream'
+  end
+
   # Retrieves content as it is *now*.
   def content
     PlainTextExtractor.extract_content_from(complete_path)
@@ -72,8 +85,8 @@ class Document
     from_index[:content]
   end
   
+  # Returns cached content with matching terms between '<<' '>>'.
   def highlighted_cache(raw_query)
-    #TODO: Report to Ferret. Highlight should accept :key and not only :doc_id.
     Indexer.index.highlight(Query.extract_from(raw_query), doc_id,
                             :field => :content, :excerpt_length => :all,
                             :pre_tag => "<<", :post_tag => ">>"
@@ -82,28 +95,37 @@ class Document
 
   # Returns the last modification date before the document got indexed.
   # Useful to know how old a document is, and to which version the cache corresponds.
+  #   >> doc.pretty_date
+  #   => "2008-05-09"
   def pretty_date
     from_index[:modified].sub(/(\d{4})(\d{2})(\d{2})\d{6}/,'\1-\2-\3')
   end
   
+  # Returns the last modification time before the document got indexed.
+  #   >> doc.pretty_mtime
+  #   => "2008-05-09 09:39:51"
   def pretty_mtime
     from_index[:modified].sub(/(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/,'\1-\2-\3 \4:\5:\6')
   end
 
+  # Returns the last modification time before the document got indexed, as YYYYMMDDHHMMSS integer.
+  #   >> doc.mtime
+  #   => 20080509093951
   def mtime
     from_index[:modified].to_i
   end
 
-  # Returns language.
+  # Returns found language, if any.
   def language
     from_index[:language]
   end
 
+  # Returns matching score as a percentage, e.g. 56.3%
   def pretty_score
     "%3.1f%" % (@score*100)
   end
 
-  # Fields that are shared between every document.
+  # Indexing fields that are shared between every document.
   def self.default_fields_for(complete_path)
     {
       :complete_path      => complete_path,
@@ -138,16 +160,21 @@ class Document
     !indexed_directory.nil?
   end
 
+  # Returns the IndexedDirectory in which the Document is included.
+  # Returns nil if no corresponding dir is found.
   def indexed_directory
     Picolena::IndexedDirectories.keys.find{|indexed_dir|
       dirname.starts_with?(indexed_dir)
     }
   end
 
+  # Raises unless @complete_path is the path of an existing file
   def validate_existence_of_file
     raise Errno::ENOENT, @complete_path unless file?
   end
 
+  # Raises unless @complete_path is included in an indexed_directory.
+  # It prevents end user to get information about non-indexed sensitive files.
   def validate_in_indexed_directory
     raise ArgumentError, "required document is not in indexed directory" unless in_indexed_directory?
   end

data/lib/picolena/templates/app/models/finder.rb

@@ -1,10 +1,20 @@
+# Finder is the class that is in charge to ensure that an index is instantiated and
+# that the raw query is converted into a Ferret::Query.
+# It then launches a search and returns found documents with corresponding matching scores.
 class Finder
   attr_reader :query
 
+  # No need to instantiate a new index for every search, so it gets cached.
   def index
     @@index ||= Indexer.index 
   end
 
+  # Instantiates a new Finder
+  # extracts a Ferret::Query from plain text raw query
+  # ensures that an Index has been instantiated
+  # reloads the index if needed
+  # prepares matching_documents pagination
+  # and ensures that the index contains at least one document.
   def initialize(raw_query,sort_by='relevance', page=1,results_per_page=Picolena::ResultsPerPage)
     @query = Query.extract_from(raw_query)
     @raw_query= raw_query
@@ -16,6 +26,13 @@ class Finder
     index_should_have_documents
   end
 
+  # Launches the search and writes correspondings
+  #  @matching_documents
+  #  @total_hits
+  #  @time_needed
+  #
+  # If a document is found in the index but not on the filesystem, it doesn't appear on
+  # the @matching_documents list.
   def execute!
     @matching_documents=[]
     start=Time.now
@@ -54,25 +71,34 @@ class Finder
   }
 
   private
-  
+  # Closes Index and clears the cache.
+  # This is used when the index has been modified by an external process
+  # and needs to get reloaded.
+  # Processes sharing the same index can force each other to reload by touching
+  # the "reload" file in Picolena::MetaIndexPath.
   def reload_index!
     Indexer.close
     @@index = nil
     @@last_reload = Time.now
   end
 
+  # Returns true if reload file has been touched since last reload.
   def should_be_reloaded?
     Indexer.reload_file_mtime > last_reload
   end
- 
+
+  # Returns the last time the index was reloaded.
+  # Returns Time.at(0) if undefined.
   def last_reload
     @@last_reload ||= Time.at(0)
   end
-  
+ 
+  # Returns a SortField that sorts documents by reversed modified time.
   def sort_by_date
     Ferret::Search::SortField.new(:modified, :type => :byte, :reverse => true)
   end
 
+  # Raises if index does not contain any document.
   def index_should_have_documents
     raise IndexError, "no document found" unless index.size > 0
   end

data/lib/picolena/templates/app/models/indexer.rb

@@ -2,13 +2,16 @@
 # It can create, update, delete and prune the index, and take care that only
 # one IndexWriter exists at any given time, even when used in a multi-threaded
 # way.
+#
+# Related configuration files are written as .yml files in /config/custom/:
+#   indexed_directories.yml
+#   indexing_performance.yml
+#   picolena.rb
+# They control the indexing performance, which directories should be indexed
+# , which files should be ignored and if language recognition is to be used.
+
 require 'indexer_logger'
 class Indexer
-  # This regexp defines which files should *not* be indexed.
-  @@exclude          = /(Thumbs\.db)/
-  # Number of threads that will be used during indexing process
-  @@threads_number = 8
-
   class << self
     # Finds every document included in IndexedDirectories, parses them with
     # PlainTextExtractor and adds them to the index.
@@ -34,12 +37,12 @@ class Indexer
     # To do so, it retrieves a list of every included document, cuts it in
     # @@threads_number chunks, and create a new indexing thread for every chunk.
     def index_directory_with_multithreads(dir)
-      logger.debug "Indexing #{dir}, #{@@threads_number} threads"
+      logger.debug "Indexing #{dir}, #{Picolena::IndexingConfiguration[:threads_number]} threads"
       indexing_list=Dir[File.join(dir,"**/*")].select{|filename|
-        File.file?(filename) && filename !~ @@exclude
+        File.file?(filename) && File.basename(filename) !~ Picolena::ToIgnore
       }
 
-      indexing_list_chunks=indexing_list.in_transposed_slices(@@threads_number)
+      indexing_list_chunks=indexing_list.in_transposed_slices(Picolena::IndexingConfiguration[:threads_number])
       prepare_multi_threads_environment
 
       indexing_list_chunks.each_with_thread{|chunk|

data/lib/picolena/templates/app/models/plain_text_extractor.rb

@@ -1,5 +1,8 @@
 require 'plain_text_extractor_DSL'
 
+# PlainTextExtractor is the class responsible for extracting plain text contents from
+# different documents filetypes (.doc, .html, .pdf, .od?), as defined in
+#   lib/plain_text_extractors/*.rb
 class PlainTextExtractor
   include PlainTextExtractorDSL
   class<<self
@@ -68,19 +71,6 @@ class PlainTextExtractor
   end
 
   ## Conversion part
-
-  # destination method can be used by some conversion command that cannot output to stdout (example?)
-  # a file containing plain text result will first be written by command, and then be read by extract_content.
-  def destination
-    require 'tmpdir'
-    @@temp_file_as_destination ||= File.join(Dir::tmpdir,"ferret_#{Time.now.to_i}")
-  end
-
-  # Replaces generic command with specific source and destination (if specified) files
-  def specific_command
-    command.sub('SOURCE','"'<<source<<'"').sub('DESTINATION','"'<<destination<<'"')
-  end
-
   # Returns plain text content of source file
   def extract_content
     if command.is_a?(String) then
@@ -89,11 +79,11 @@ class PlainTextExtractor
         # If command includes 'DESTINATION' keyword,
         # launches the command and returns the content of
         # DESTINATION file.
-        system(specific_command)
+        IO.popen(specific_command){}
         File.read_and_remove(destination)
       else
         # Otherwise, launches the command and returns STDOUT.
-        IO.popen(specific_command){|io| io.read}
+        Open3.popen3(specific_command){|stdin,stdout,stderr| stdout.read}
       end
     else
       # command is a Block.
@@ -127,4 +117,18 @@ class PlainTextExtractor
     }
     {:content => content, :language => language}
   end
-end
+
+  private
+
+  # destination method can be used by some conversion command that cannot output to stdout (example?)
+  # a file containing plain text result will first be written by command, and then be read by extract_content.
+  def destination
+    require 'tmpdir'
+    @@temp_file_as_destination ||= File.join(Dir::tmpdir,"ferret_#{Time.now.to_i}")
+  end
+
+  # Replaces generic command with specific source and destination (if specified) files
+  def specific_command
+    command.sub('SOURCE','"'<<source<<'"').sub('DESTINATION','"'<<destination<<'"')
+  end
+end

data/lib/picolena/templates/app/views/documents/_document.html.haml

@@ -1,6 +1,6 @@
 %h2
   =link_to icon_and_filename_for(document), download_document_path(document.probably_unique_id)
-  =language_icon_for(document)
+  =language_icon_for(document,query)
   %small=number_to_percentage(document.score*100, :precision=>1)
 =highlight_matching_content(document)
 %p