picolena 0.2.0 → 0.2.2

data/History.txt

@@ -1,3 +1,17 @@
+== 0.2.2 2009-02-13
+
+* 3 major enhancements :
+  * Thumbnails created for pictures & videos
+  * Support for .zip  & .rar archives
+  * Alias_path is now LetterTokenized
+
+* 2 minor enhancements:
+  * More specs
+  * Some aesthetical changes
+
+* 1 bug fix :
+  * Displaying filetypes without any icon would raise an Exception
+
 == 0.2.0  2009-02-02
 
 * 1 major enhancement :

data/Manifest.txt

@@ -37,22 +37,25 @@ lib/picolena/templates/config/boot.rb
 lib/picolena/templates/config/environments/development.rb
 lib/picolena/templates/config/environments/production.rb
 lib/picolena/templates/config/environments/test.rb
-lib/picolena/templates/config/initializers/001_load_custom_config.rb
-lib/picolena/templates/config/initializers/002_load_indexed_dirs.rb
-lib/picolena/templates/config/initializers/003_load_white_list_IPs.rb
-lib/picolena/templates/config/initializers/004_load_plain_text_extractors.rb
-lib/picolena/templates/config/initializers/005_load_custom_title_and_names_and_links.rb
-lib/picolena/templates/config/initializers/006_load_icons.rb
-lib/picolena/templates/config/initializers/007_load_performance_tweaks.rb
+lib/picolena/templates/config/initializers/001_load_ferret.rb
+lib/picolena/templates/config/initializers/002_load_custom_config.rb
+lib/picolena/templates/config/initializers/003_load_indexed_dirs.rb
+lib/picolena/templates/config/initializers/004_load_white_list_IPs.rb
+lib/picolena/templates/config/initializers/005_load_plain_text_extractors.rb
+lib/picolena/templates/config/initializers/006_load_custom_title_and_names_and_links.rb
+lib/picolena/templates/config/initializers/007_load_icons.rb
+lib/picolena/templates/config/initializers/008_load_performance_tweaks.rb
 lib/picolena/templates/config/routes.rb
 lib/picolena/templates/lang/ui/de.yml
 lib/picolena/templates/lang/ui/en.yml
 lib/picolena/templates/lang/ui/es.yml
 lib/picolena/templates/lang/ui/fr.yml
 lib/picolena/templates/lib/core_exts.rb
+lib/picolena/templates/lib/development_helpers.rb
 lib/picolena/templates/lib/indexer_logger.rb
-lib/picolena/templates/lib/plain_text_extractor_DSL.rb
+lib/picolena/templates/lib/plain_text_extractor_dsl.rb
 lib/picolena/templates/lib/plain_text_extractors/adobe.pdf.rb
+lib/picolena/templates/lib/plain_text_extractors/adobe.photoshop.rb
 lib/picolena/templates/lib/plain_text_extractors/html.rb
 lib/picolena/templates/lib/plain_text_extractors/ms.excel.rb
 lib/picolena/templates/lib/plain_text_extractors/ms.powerpoint.rb
@@ -63,7 +66,11 @@ lib/picolena/templates/lib/plain_text_extractors/opendocument.spreadsheet.rb
 lib/picolena/templates/lib/plain_text_extractors/opendocument.text.rb
 lib/picolena/templates/lib/plain_text_extractors/pictures.rb
 lib/picolena/templates/lib/plain_text_extractors/plain_text.rb
+lib/picolena/templates/lib/plain_text_extractors/rar.rb
+lib/picolena/templates/lib/plain_text_extractors/videos.rb
+lib/picolena/templates/lib/plain_text_extractors/zip.rb
 lib/picolena/templates/lib/tasks/annotations.rake
+lib/picolena/templates/lib/tasks/extract.rake
 lib/picolena/templates/lib/tasks/index.rake
 lib/picolena/templates/lib/tasks/install_dependencies.rake
 lib/picolena/templates/lib/tasks/log.rake
@@ -112,6 +119,7 @@ lib/picolena/templates/public/images/icons/txt.png
 lib/picolena/templates/public/images/icons/video.png
 lib/picolena/templates/public/images/icons/xls.png
 lib/picolena/templates/public/images/main_img.jpg
+lib/picolena/templates/public/images/thumbnails/NOTE
 lib/picolena/templates/public/images/zafh_net.png
 lib/picolena/templates/public/robots.txt
 lib/picolena/templates/public/stylesheets/style.css
@@ -147,12 +155,15 @@ lib/picolena/templates/spec/rcov.opts
 lib/picolena/templates/spec/spec.opts
 lib/picolena/templates/spec/spec_helper.rb
 lib/picolena/templates/spec/test_dirs/indexed/README
+lib/picolena/templates/spec/test_dirs/indexed/archives/dumb_file.rar
+lib/picolena/templates/spec/test_dirs/indexed/archives/some_test_files.zip
 lib/picolena/templates/spec/test_dirs/indexed/basic/another_plain.text
 lib/picolena/templates/spec/test_dirs/indexed/basic/basic.odt
 lib/picolena/templates/spec/test_dirs/indexed/basic/basic.pdf
 lib/picolena/templates/spec/test_dirs/indexed/basic/basic.tex
 lib/picolena/templates/spec/test_dirs/indexed/basic/crossed.text
 lib/picolena/templates/spec/test_dirs/indexed/basic/crossed.txt
+lib/picolena/templates/spec/test_dirs/indexed/basic/fake_thumbnailer
 lib/picolena/templates/spec/test_dirs/indexed/basic/hello.rb
 lib/picolena/templates/spec/test_dirs/indexed/basic/myfirstjavaprog.java
 lib/picolena/templates/spec/test_dirs/indexed/basic/one_page.ppt
@@ -173,7 +184,16 @@ lib/picolena/templates/spec/test_dirs/indexed/lang/lorca
 lib/picolena/templates/spec/test_dirs/indexed/lang/shakespeare
 lib/picolena/templates/spec/test_dirs/indexed/literature/Simulation of district heating systems for evaluation of real-time control strategies.pdf
 lib/picolena/templates/spec/test_dirs/indexed/literature/Types of malfunction in DH substations.doc
+lib/picolena/templates/spec/test_dirs/indexed/media/badminton.avi
+lib/picolena/templates/spec/test_dirs/indexed/media/caution.tif
 lib/picolena/templates/spec/test_dirs/indexed/media/crow.jpg
+lib/picolena/templates/spec/test_dirs/indexed/media/cygnus.jpeg
+lib/picolena/templates/spec/test_dirs/indexed/media/diceface.eps
+lib/picolena/templates/spec/test_dirs/indexed/media/glass.png
+lib/picolena/templates/spec/test_dirs/indexed/media/gnu.bmp
+lib/picolena/templates/spec/test_dirs/indexed/media/picolena.psd
+lib/picolena/templates/spec/test_dirs/indexed/media/rails_logo_remix.gif
+lib/picolena/templates/spec/test_dirs/indexed/media/warning.tiff
 lib/picolena/templates/spec/test_dirs/indexed/others/'weird'filename.txt
 lib/picolena/templates/spec/test_dirs/indexed/others/7.html
 lib/picolena/templates/spec/test_dirs/indexed/others/BIN_FILE_WITHOUT_EXTENSION

data/config/files_to_clean

@@ -13,3 +13,4 @@ lib/picolena/templates/tmp
 lib/picolena/templates/vendor
 lib/picolena/templates/coverage
 lib/picolena/templates/doc
+lib/picolena/templates/public/images/thumbnails/*.jpg

data/config/requirements.rb

@@ -2,7 +2,7 @@ require 'fileutils'
 include FileUtils
 
 require 'rubygems'
-%w[rake hoe newgem rubigen].each do |req_gem|
+%w[rake newgem rubigen].each do |req_gem|
   begin
     require req_gem
   rescue LoadError

data/lib/picolena/config/basic.rb

@@ -46,5 +46,6 @@ module Picolena
   # PerFieldAnalyzer is used to prevent queries like "language:it" to be broken by StopFilter.
   per_field_analyzer=Ferret::Analysis::PerFieldAnalyzer.new(Ferret::Analysis::StandardAnalyzer.new)
   per_field_analyzer[:language]=Ferret::Analysis::WhiteSpaceAnalyzer.new
+  per_field_analyzer[:alias_path]=Ferret::Analysis::LetterAnalyzerWithStopFilter.new
   Analyzer=per_field_analyzer
-end
+end

data/lib/picolena/config/icons_and_filetypes.yml

@@ -3,6 +3,7 @@ video:
   wmv
   mpg
   mpeg
+  mov
 ogg:
   mp3
   ogg
@@ -17,6 +18,10 @@ txt:
   bib
   log
   ini
+  sub
+  srt
+  vcf
+  vcard
   no_extension
 doc:
   doc

data/lib/picolena/picolena_generator.rb

@@ -87,7 +87,7 @@ class PicolenaGenerator < RubiGen::Base #:nodoc:
       m.rake 'spec' unless options[:no_spec]
 
       # Cleaning up temp folder if --spec-only
-      m.clean if false
+      m.clean if options[:spec_only]
     end
   end
 
@@ -150,6 +150,7 @@ EOS
     public/images
     public/images/icons
     public/images/flags
+    public/images/thumbnails
     public/javascripts
     public/stylesheets
     spec
@@ -160,6 +161,7 @@ EOS
     spec/test_dirs
     spec/test_dirs/empty_folder
     spec/test_dirs/indexed
+    spec/test_dirs/indexed/archives
     spec/test_dirs/indexed/basic
     spec/test_dirs/indexed/different_encodings
     spec/test_dirs/indexed/just_one_doc

data/lib/picolena/templates/app/helpers/documents_helper.rb

@@ -46,13 +46,13 @@ module DocumentsHelper
 
   # Returns icon and filename for any given document.
   def icon_and_filename_for(document)
-    [icon_for(document.ext_as_sym),document.filename].join(" ")
+    [icon_for(document),document.filename].join(" ")
   end
 
   # Returns the location (if avaible) of the filetype icon.
-  def icon_for(ext)
-    icon_symbol=Picolena::FiletypeToIconSymbol[ext]
-    image_tag("icons/#{icon_symbol}.png") if icon_symbol
+  def icon_for(document)
+    path=document.icon_path
+    image_tag(document.icon_path) if path
   end
 
   # Returns a link to a backup search engine that could maybe find more results for the same query.

data/lib/picolena/templates/app/models/document.rb

@@ -87,10 +87,11 @@ class Document
   
   # Returns cached content with matching terms between '<<' '>>'.
   def highlighted_cache(raw_query)
-    Indexer.index.highlight(Query.extract_from(raw_query), doc_id,
+    excerpts=Indexer.index.highlight(Query.extract_from(raw_query), doc_id,
                             :field => :content, :excerpt_length => :all,
                             :pre_tag => "<<", :post_tag => ">>"
-    ).first
+             )
+    excerpts.is_an?(Array) ? excerpts.first : ""
   end
 
   # Returns the last modification date before the document got indexed.
@@ -127,18 +128,40 @@ class Document
 
   # Indexing fields that are shared between every document.
   def self.default_fields_for(complete_path)
+    doc=Document.new(complete_path)
     {
       :complete_path      => complete_path,
       :probably_unique_id => complete_path.base26_hash,
+      :alias_path         => doc.alias_path,
       :filename           => File.basename(complete_path),
       :basename           => File.basename(complete_path, File.extname(complete_path)).gsub(/_/,' '),
       :filetype           => File.extname(complete_path),
       :modified           => File.mtime(complete_path).strftime("%Y%m%d%H%M%S")
     }
   end
-
-
+  
+  # Returns thumbnail if available, mime icon otherwise
+  def icon_path
+    if File.exists?(thumbnail_path) then
+      thumbnail_path(:public_dir)
+    else
+      icon_symbol=Picolena::FiletypeToIconSymbol[ext_as_sym]
+      "icons/#{icon_symbol}.png" if icon_symbol
+    end
+  end
+
+  # Did at least one letter got extracted from the document?
+  # This boolean is used in views to know if a link should be
+  # displayed to show the content
+  def has_content?
+    cached =~ /\w/
+  end
+  
   private
+ 
+  def thumbnail_path(public_dir=false)
+    File.thumbnail_path(complete_path,public_dir)
+  end
   
   # FIXME: Is there a way to easily retrieve doc_id for a given document?
   # Better yet, fix Index#highlight to accept :probably_unique_id and stop using :doc_id.

data/lib/picolena/templates/app/models/indexer.rb

@@ -67,7 +67,8 @@ class Indexer
     def add_or_update_file(complete_path)
       document = Document.default_fields_for(complete_path)
       begin
-        document.merge! PlainTextExtractor.extract_content_and_language_from(complete_path)
+        PlainTextExtractor.extract_thumbnail_from(complete_path)
+        document.merge! PlainTextExtractor.extract_information_from(complete_path)
         raise "empty document #{complete_path}" if document[:content].strip.empty?
         logger.add_document document
       rescue => e
@@ -177,6 +178,8 @@ class Indexer
     end
     
     # Copied from Ferret book, By David Balmain
+    # FIXME : Find an alternative that doesn't need any more dependency.
+    # NOTE: Not supported on windows.
     def index_time_dbm_file
       @@dbm_file ||= DBM.open(File.join(Picolena::MetaIndexPath, 'added_at'))
     end
@@ -201,13 +204,14 @@ class Indexer
 
     def default_field_infos
       returning Ferret::Index::FieldInfos.new do |field_infos|
+        field_infos.add_field(:probably_unique_id, :store => :no,  :index => :untokenized)
         field_infos.add_field(:complete_path,      :store => :yes, :index => :untokenized)
         field_infos.add_field(:content,            :store => :yes, :index => :yes)
+        field_infos.add_field(:alias_path,         :store => :no,  :index => :yes, :boost => 0.5)
         field_infos.add_field(:basename,           :store => :no,  :index => :yes, :boost => 1.5)
         field_infos.add_field(:filename,           :store => :no,  :index => :yes, :boost => 1.5)
         field_infos.add_field(:filetype,           :store => :no,  :index => :yes, :boost => 1.5)
         field_infos.add_field(:modified,           :store => :yes, :index => :untokenized)
-        field_infos.add_field(:probably_unique_id, :store => :no,  :index => :untokenized)
         field_infos.add_field(:language,           :store => :yes, :index => :untokenized)
       end
     end

data/lib/picolena/templates/app/models/plain_text_extractor.rb

@@ -1,4 +1,4 @@
-require 'plain_text_extractor_DSL'
+require 'plain_text_extractor_dsl'
 
 # PlainTextExtractor is the class responsible for extracting plain text contents from
 # different documents filetypes (.doc, .html, .pdf, .od?), as defined in
@@ -47,27 +47,29 @@ class PlainTextExtractor
     end
 
     # Launches extractor on given file and outputs plain text result and language (if found)
-    def extract_content_and_language_from(source)
-      find_by_filename(source).extract_content_and_language
+    def extract_information_from(source)
+      find_by_filename(source).extract_information
+    end
+
+    # Tries to extract a thumbnail from source.
+    # Doesn't do anything if thumbnail_command isn't defined for the corresponding filetype.
+    def extract_thumbnail_from(source)
+      find_by_filename(source).extract_thumbnail
     end
 
     # Returns which language guesser should be used by the system.
     # Returns nil if none is found.
     def language_guesser
-      @@language_guesser||=('mguesser -n1' unless IO.popen("which mguesser"){|i| i.read}.empty?)
+      @@language_guesser||=('mguesser -n1' if 'mguesser'.installed?)
     end
   end
 
   attr_accessor :source
 
-  # Parses command in order to know which programs are needed.
+  # Parses commands in order to know which programs are needed.
   # rspec will then check that every dependecy is installed on the system
   def dependencies
-    if command.is_a?(String) then
-      command.split(/\|\s*/).collect{|command_part| command_part.split(/ /).first}
-    else
-      @dependencies
-    end
+      [@dependencies, command.dependencies, thumbnail_command.dependencies].flatten
   end
 
   ## Conversion part
@@ -79,11 +81,11 @@ class PlainTextExtractor
         # If command includes 'DESTINATION' keyword,
         # launches the command and returns the content of
         # DESTINATION file.
-        IO.popen(specific_command){}
+        silently_execute(specific_command)
         File.read_and_remove(destination)
       else
         # Otherwise, launches the command and returns STDOUT.
-        Open3.popen3(specific_command){|stdin,stdout,stderr| stdout.read}
+        silently_execute(specific_command)
       end
     else
       # command is a Block.
@@ -97,14 +99,16 @@ class PlainTextExtractor
   # using mguesser to guess used language.
   # This method only returns probable language if the content is bigger than 500 chars
   # and if probability score is higher than 90%.
-  def extract_content_and_language
+  def extract_information
     content=extract_content
+
     return {:content => content} unless [# Is LanguageRecognition turned on? (cf config/custom/picolena.rb)
                                          Picolena::UseLanguageRecognition,
                                          # Is a language guesser already installed?
                                          PlainTextExtractor.language_guesser,
                                          # Language recognition is too unreliable for small files.
                                          content.size > 500].all?
+
     language=IO.popen(PlainTextExtractor.language_guesser,'w+'){|lang_guesser|
       lang_guesser.write content
       lang_guesser.close_write
@@ -115,9 +119,14 @@ class PlainTextExtractor
         lang unless score<0.9
       end
     }
+
     {:content => content, :language => language}
   end
 
+  def extract_thumbnail
+    silently_execute(specific_thumbnail_command) if thumbnail_command
+  end
+
   private
 
   # destination method can be used by some conversion command that cannot output to stdout (example?)
@@ -131,4 +140,9 @@ class PlainTextExtractor
   def specific_command
     command.sub('SOURCE','"'<<source<<'"').sub('DESTINATION','"'<<destination<<'"')
   end
+
+  # Replaces generic command with specific source and thumbnail (if specified) files
+  def specific_thumbnail_command
+    thumbnail_command.sub('SOURCE','"'<<source<<'"').sub('THUMBNAIL','"'<<File.thumbnail_path(source)<<'"')
+  end
 end

data/lib/picolena/templates/app/models/query.rb

@@ -32,7 +32,7 @@ class Query
 
     # Instantiates a QueryParser once, and keeps it in cache.
     def parser
-      @@parser ||= Ferret::QueryParser.new(:fields => [:content, :filename, :basename, :filetype, :modified], :or_default => false, :analyzer=>Picolena::Analyzer)
+      @@parser ||= Ferret::QueryParser.new(:fields => [:content, :filename, :basename, :alias_path, :filetype, :modified], :or_default => false, :analyzer=>Picolena::Analyzer)
     end
   end
-end
+end

data/lib/picolena/templates/app/views/documents/_document.html.haml

@@ -6,7 +6,7 @@
 %p
   =link_to_containing_directory(document)
   %br/
-  -if document.supported?
+  -if document.supported? && document.has_content?
     =link_to_plain_text_content(document)
     -
     =link_to_cached_content(document,query)

data/lib/picolena/templates/config/environments/development.rb

@@ -19,3 +19,5 @@ config.action_mailer.raise_delivery_errors = false
 module Picolena
   LOGLEVEL = Logger::DEBUG
 end
+
+require 'development_helpers'

data/lib/picolena/templates/config/initializers/001_load_ferret.rb

@@ -0,0 +1,17 @@
+require 'ferret'
+module Ferret
+  module Analysis
+   # Used for alias_path queries
+   class LetterAnalyzerWithStopFilter
+     def initialize(stop_words = FULL_ENGLISH_STOP_WORDS, lower = true)
+      @lower = lower
+      @stop_words = stop_words
+     end
+
+    def token_stream(field, str)
+      ts = LetterTokenizer.new(str, @lower)
+      StopFilter.new(ts, @stop_words)
+    end
+   end
+  end
+end

data/lib/picolena/templates/config/initializers/{001_load_custom_config.rb → 002_load_custom_config.rb}

@@ -1,3 +1,2 @@
 custom_config_file = File.join(RAILS_ROOT, 'config/custom/picolena.rb')
-require 'ferret'
-require custom_config_file
+require custom_config_file

data/lib/picolena/templates/config/initializers/{004_load_plain_text_extractors.rb → 005_load_plain_text_extractors.rb}

@@ -1,5 +1,5 @@
 require 'core_exts'
-require 'plain_text_extractor_DSL'
+require 'plain_text_extractor_dsl'
 Picolena::Extractors=[]
 Dir.glob(File.join(RAILS_ROOT,'lib/plain_text_extractors/*.rb')).each{|extractor|
   require extractor