picolena 0.2.0 → 0.2.2

data/lib/picolena/templates/lib/plain_text_extractors/videos.rb

@@ -0,0 +1,13 @@
+# Basic support for different videos
+# It only has been tested with .avi so far
+
+PlainTextExtractor.new {
+  every :avi, :mpg, :mpeg, :mov, :wmv
+  as "video/*"
+  aka "some video"
+
+  extract_thumbnail_with           'ffmpegthumbnailer -i SOURCE -o THUMBNAIL'
+
+  extract_content_with             'exiftool SOURCE'
+  which_should_for_example_extract 'Image Size 320x200 Duration 1.96s', :from => 'badminton.avi'
+}

data/lib/picolena/templates/lib/plain_text_extractors/zip.rb

@@ -0,0 +1,17 @@
+PlainTextExtractor.new {
+  every :zip
+  as "archive/zip"
+  aka "ZIP Archive"
+
+  # TODO: Transpose this structure to support .tgz, .rar, .deb, 7z, ar, .cab, .ace, .lzma, .bz2
+  # NOTE: What to do when the archive is too big?
+
+  extract_content_from_archive_with "unzip SOURCE -d TEMPDIR"
+
+  which_should_for_example_extract 'some_test_files some_dir dumb.rb', :from => 'some_test_files.zip'
+  or_extract                       'puts 2+2',                         :from => 'some_test_files.zip'
+  # Now *that* is some cool spec!
+  # It relies on .jpg and .rar Extractors
+  or_extract                       '"This sentence is hidden in the EXIF metadata of a .jpg file archived in a .rar file archived in a .zip file!"',
+                                                                       :from => 'some_test_files.zip'
+}

data/lib/picolena/templates/lib/tasks/extract.rake

@@ -0,0 +1,16 @@
+desc 'Extract information from given file'
+namespace :extract do
+  desc 'Extract plain text content from given file'
+  task :content => :environment do
+    ARGV.shift
+    ARGV.select{|fn| File.file?(fn)}.each{|filename|
+      begin
+        content=PlainTextExtractor.extract_content_from(filename)
+        puts "### Content extracted from : #{filename}"
+        puts content
+      rescue => e
+        puts "### No content extracted from #{filename} (#{e.message})"
+      end
+    }
+  end
+end

data/lib/picolena/templates/lib/tasks/install_dependencies.rake

@@ -30,7 +30,7 @@ namespace :install_dependencies do
   task :deb_packages do
     root_privileges_required!
     #TODO: Should load this list from defined PlainTextExtractor's
-    packages=%w{antiword poppler-utils odt2txt html2text catdoc unrtf mguesser libdbm-ruby1.8}.join(" ")
+    packages=%w{antiword poppler-utils odt2txt html2text catdoc unrtf mguesser libdbm-ruby1.8 libimage-exiftool-perl ffmpegthumbnailer imagemagick}.join(" ")
     puts "Installing "<<packages
     system("apt-get install "<<packages)
   end

data/lib/picolena/templates/public/images/thumbnails/NOTE

@@ -0,0 +1,2 @@
+## TODO: Find a way to manage thumbnails for different environments.
+Accessorily, leaving this file here prevents rake manifest:refresh to remove thumbnails dir

data/lib/picolena/templates/spec/controllers/documents_controller_spec.rb

@@ -91,6 +91,14 @@ describe DocumentsController do
     orig_assigns['matching_documents'].any?{|doc| doc.matching_content.join.starts_with?("just another content test in a pdf file")}.should be_true
   end
 
+  it "GET 'show' should accept * as query" do
+    params_from(:get, '/documents/*').should == {:controller => 'documents', :action => 'show', :id => '*'}
+    lambda{get 'show', :id=>'*'}.should_not raise_error(ActionController::RoutingError)
+    response.should be_success
+    orig_assigns['matching_documents'].entries.should_not be_empty
+    orig_assigns['total_hits'].should == Indexer.size
+  end
+
   it "GET 'show' should accept combined queries" do
     get 'show', :id=>'test filetype:pdf'
     response.should be_success

data/lib/picolena/templates/spec/helpers/documents_helper_spec.rb

@@ -3,7 +3,18 @@ require File.dirname(__FILE__) + '/../spec_helper'
 describe DocumentsHelper do
   PlainTextExtractor.supported_extensions.each{|ext|
     it "should have an icon for .#{ext} filetype" do
-      helper.icon_for(ext).should_not be_nil
+      Picolena::FiletypeToIconSymbol[ext].should_not be_nil
     end
   }
+
+  it "should not raise for filetypes without any associated icon when calling icon_for" do
+     Picolena::FiletypeToIconSymbol[:xyz].should be_nil
+     document=Spec::Mocks::Mock.new('Document', :ext_as_sym => :xyz, :icon_path => nil)
+     lambda {helper.icon_for(document)}.should_not raise_error
+  end
+
+  it "should not raise for files without content when calling highlighted_cache" do
+     document=Spec::Mocks::Mock.new('Document', :highlighted_cache => "\f")
+     lambda {helper.highlighted_cache(document, 'some_query')}.should_not raise_error
+  end
 end

data/lib/picolena/templates/spec/models/basic_finder_spec.rb

@@ -17,7 +17,8 @@ describe "Finder without index on disk" do
   end
 
   it "should create index" do
-    Picolena::IndexedDirectories.replace({'spec/test_dirs/indexed/just_one_doc'=>'//justonedoc/'})
+    new_dir=File.expand_path(File.join(RAILS_ROOT, 'spec/test_dirs/indexed/just_one_doc'))
+    Picolena::IndexedDirectories.replace(new_dir => '//justonedoc/')
     lambda {@finder_with_new_index=Finder.new("test moi")}.should change(Indexer, :index_exists?).from(false).to(true)
     File.exists?(File.join(@new_index_path,'_0.cfs')).should be_true
     Indexer.index.size.should >0
@@ -33,12 +34,13 @@ describe "Finder without index on disk" do
     Picolena::IndexedDirectories.replace(@original_indexed_dirs)
     Picolena::IndexSavePath.replace(@original_index_path)
     Picolena::MetaIndexPath.replace(File.join(@original_index_path,'meta'))
+    FileUtils.remove_entry_secure(@new_index_path)
   end
 end
 
 
 fields={
-  # description => key
+  # description       => key
   :content            => :content,
   :complete_path      => :complete_path,
   :basename           => :basename,
@@ -95,9 +97,9 @@ describe "Basic Finder" do
 
   it "should know how much time was needed for execution" do
     finder=Finder.new("yet another stupid query")
-    finder.executed?.should be_nil
+    finder.should_not be_executed
     finder.time_needed.should > 0
-    finder.executed?.should be_true
+    finder.should be_executed
   end
 
   it "should not need more than 100ms to find documents" do

data/lib/picolena/templates/spec/models/document_spec.rb

@@ -47,8 +47,6 @@ describe Document do
     another_doc.content.should == "just a content test\nin a txt file"
   end
   
-  #FIXME: Check if content has been cached before trying to display cached content. extension check is not enough
-  #(e.g. unreadable pdf file)
   it "should know its cached content" do
     another_doc=Document.new("spec/test_dirs/indexed/basic/plain.txt")
     another_doc.cached.should == "just a content test\nin a txt file"
@@ -104,8 +102,6 @@ describe Document do
   it "should not be considered supported if binary" do
     Document.new("spec/test_dirs/indexed/others/BIN_FILE_WITHOUT_EXTENSION").should_not be_supported
   end
-
-
   
   it "should know its language when enough content is available" do
     Document.new("spec/test_dirs/indexed/lang/goethe").language.should == "de"
@@ -133,6 +129,30 @@ describe Document do
     @valid_document.matching_content.should include("thermal cooling")
   end
 
+  it "should know its icon_path if a thumbnail if available" do
+    doc=Document.new("spec/test_dirs/indexed/media/badminton.avi")
+    doc.icon_path.should_not be_nil
+    doc.icon_path.should == "thumbnails/#{doc.probably_unique_id}.jpg"
+  end
+
+  it "should know its icon_path if an icon  if available for its mimetype" do
+    doc=Document.new("spec/test_dirs/indexed/others/xor.vee")
+    doc.icon_path.should_not be_nil
+    doc.icon_path.should == 'icons/insel.png'
+  end
+
+  it "should return nil as icon_path if no icon or thumbnail is available" do
+    doc=Document.new("spec/test_dirs/indexed/others/ghjopdfg.xyz")
+    doc.icon_path.should be_nil
+  end
+
+  it "should not raise when asked for highlighted_cache even though content is empty" do
+    doc=Document.new("spec/test_dirs/indexed/others/nested/unreadable.pdf")
+    doc.should_not have_content
+    lambda {doc.highlighted_cache('unreadable')}.should_not raise_error
+    doc.highlighted_cache('unreadable').should ~ /^\s*$/
+  end
+
   after(:all) do
     revert_changes!("spec/test_dirs/indexed/others/placeholder.txt","Absorption and Adsorption cooling machines!!!")
   end

data/lib/picolena/templates/spec/models/finder_spec.rb

@@ -48,6 +48,13 @@ describe Finder do
     Finder.new("crossed").total_hits.should >= 2
   end
 
+  it "should find documents according to parts of their alias_path" do
+    media_files=Dir["#{RAILS_ROOT}/spec/test_dirs/indexed/media/*"]
+    Finder.new("media").total_hits.should >= media_files.size
+    Finder.new("nested indexed").matching_documents.collect{|d| d.filename}.should include("ReadMe.rtf")
+    matching_document_for("test_dirs yet_another_dir ext:xlsx").basename.should == "office2007-excel"
+  end
+
   it "should also index unreadable files with known mimetypes" do
     Finder.new("unreadable.pdf").matching_documents.should_not be_empty
     Finder.new("too_small.doc").matching_documents.should_not be_empty
@@ -81,18 +88,18 @@ describe Finder do
     matching_document_for("bäñüßé").content.should == "just to know if files are indexed with utf8 filenames"
   end
 
-  it "should find documents according to their modification date" do
-    matching_document_for("19831209").basename.should == "office2003-word-template"
-    matching_document_for("19820216").basename.should == "basic"
-  end
+  it "should find documents according to their modification date"
+    ## Not implemented yet!
+    # matching_document_for("19831209").basename.should == "office2003-word-template"
+    # matching_document_for("19820216").basename.should == "basic"
 
-  it "should find documents according to their modification year" do
-    Finder.new("date:<1982").matching_documents.should be_empty
-    matching_document_for("date:<1983").filename.should == "basic.pdf"
-    matching_document_for("date:1982").filename.should == "basic.pdf"
-    matching_document_for("year:1983").filename.should == "basic.pdf"
-    matching_document_for("date:>=1989 AND date:<=1992").filename.should == "placeholder.txt"
-  end
+  it "should find documents according to their modification year"
+    ## Not implemented yet!
+    # Finder.new("date:<1982").matching_documents.should be_empty
+    # matching_document_for("date:<1983").filename.should == "basic.pdf"
+    # matching_document_for("date:1982").filename.should == "basic.pdf"
+    # matching_document_for("year:1983").filename.should == "basic.pdf"
+    # matching_document_for("date:>=1989 AND date:<=1992").filename.should == "placeholder.txt"
 
   it "should not concatenate cells from xls file" do
     Finder.new("content:ABC").matching_documents.select{|doc| doc.extname==".xls"}.should be_empty

data/lib/picolena/templates/spec/models/host_indexing_system_spec.rb

@@ -3,7 +3,7 @@ require File.dirname(__FILE__) + '/../spec_helper'
 describe "Host indexing system" do
  PlainTextExtractor.dependencies.each do |dependency|
     it "should have #{dependency} installed" do
-       IO.popen("which #{dependency}"){|i| i.read.should_not be_empty}
+       dependency.should be_installed
     end
   end
 

data/lib/picolena/templates/spec/models/plain_text_extractor_spec.rb

@@ -7,10 +7,16 @@ describe "PlainTextExtractors" do
 
   PlainTextExtractor.all.each{|extractor|
     extractor.exts.each{|ext|
-      should_extract= "should be able to extract content from #{extractor.description} (.#{ext})"
+      should_extract_content  = "should be able to extract content from #{extractor.description} (.#{ext})"
+      should_extract_thumbnail= "should be able to extract thumbnail from #{extractor.description} (.#{ext})"
       content_and_file_examples_for_this_ext=extractor.content_and_file_examples.select{|content,file| File.ext_as_sym(file)==ext}
-      unless content_and_file_examples_for_this_ext.empty? then
-        it should_extract do
+      if content_and_file_examples_for_this_ext.empty? then
+        ## It means that the spec for this extension file is "Not yet implemented"!
+        ## add this line to the corresponding extractor in lib/extractors:
+        # which_should_for_example_extract 'some content', :from => 'a file you could add in spec/test_dirs/indexed/'
+        it should_extract_content
+      else
+        it should_extract_content do
           content_and_file_examples_for_this_ext.each{|content_example,file_example|
             finder=Finder.new(content_example)
             finder.execute!
@@ -19,11 +25,22 @@ describe "PlainTextExtractors" do
             matching_documents_filenames.should include(file_example)
           }
         end
-      else
-        ## It means that the spec for this extension file is "Not yet implemented"!
-        ## add this line to the corresponding extractor in lib/extractors:
-        # which_should_for_example_extract 'some content', :from => 'a file you could add in spec/test_dirs/indexed/'
-        it should_extract
+      end
+
+      if extractor.thumbnail_command then
+        doc=Document.find_by_extension(ext)
+        if doc then
+          it should_extract_thumbnail do
+            thumb_path=doc.send(:thumbnail_path)
+            # NOTE: This doesn't seem to work, why?
+            #  File.should exist(doc.send(:thumbnail_path))
+            File.exist?(thumb_path).should be_true
+            # NOTE: It seems that ffmpegthumbnailer outputs a png file even with -o output.jpg
+            %x(file -ib #{thumb_path}).chomp.should =~ /image\/(jpeg|png)/
+          end
+        else
+          it should_extract_thumbnail
+        end
       end
     }
   }

data/lib/picolena/templates/spec/models/query_spec.rb

@@ -87,10 +87,9 @@ describe Query do
     Query.content_terms("LIKE absorption").include?("adsorption").should be_true
   end
 
-  it "should remove / and - from date queries" do
-    #TODO: Implement corresponding analyzer
-    Query.extract_from("date:1982-02-16").should == Query.extract_from("date:19820216")
-    Query.extract_from("date:1982/02/16").should == Query.extract_from("date:19820216")
-  end
+  it "should remove / and - from date queries"
+    ##TODO: Implement corresponding analyzer
+    # Query.extract_from("date:1982-02-16").should == Query.extract_from("date:19820216")
+    # Query.extract_from("date:1982/02/16").should == Query.extract_from("date:19820216")
 
 end

data/lib/picolena/templates/spec/spec_helper.rb

@@ -11,6 +11,15 @@ def revert_changes!(file,content)
   }
 end
 
+
+class Document
+  def self.find_by_extension(ext)
+    Finder.new("ext:#{ext}").matching_documents.first
+  end
+end
+
+
+## Rspec-rails assume that everybody uses ActiveRecord, but that's not Picolena's case.
 module Spec
   module Matchers
     class Change

data/lib/picolena/templates/spec/test_dirs/indexed/basic/fake_thumbnailer

@@ -0,0 +1,14 @@
+#! /bin/bash
+#
+#   extract_thumbnail_with 'spec/test_dirs/indexed/basic/fake_thumbnailer SOURCE THUMBNAIL'
+# written in a PlainTextExtractpr will write some debug information
+# about the thumbnailing process
+echo "Beginning" >> $2
+echo $1 >> $2
+date >> $2
+for ((i=1; i<=3; i+=1)); do
+	sleep 1
+	echo $i >> $2
+done
+date >> $2
+echo "End" >> $2

data/lib/picolena/templates/spec/test_dirs/indexed/media/diceface.eps

@@ -0,0 +1,79 @@
+%!PS-Adobe-2.0 EPSF-1.2
+%%BoundingBox: 0 0 258 43
+%
+% commented version of EPSDICE.EPS  by Thomas A. Heim 
+%
+% LICENSE: LPPL
+%
+% 2001/02/09 -- thomas.heim@unibas.ch
+%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%
+% face measures 32x32 and fits within a 43x43 bounding box
+%
+% ==> if you change these dimensions, you will have to adjust
+%     the bounding box of the clipped region in the .STY file!
+%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%
+% the frame macro: a simple box with rounded corners
+%
+%     takes one argument off the stack: n (x-offset is 43*(n - 1) )
+%
+/frame {
+  1 sub 43 mul        %  calculate 43*(n-1), given n on the stack
+  /o exch def         %  store result in o
+  gsave
+  newpath 
+  o 0 translate       %  shift coordinate system by offset o in x
+  32 5 moveto         % 
+  32 10 5 -90 0 arc   %  the frame goes from 5 to 37 units both 
+  37 32 lineto        %  in x and y, with rounded corners having
+  32 32 5 0 90 arc    %  radius 5 units, centered 5 units inward
+  10 37 lineto        %  in both directions
+  10 32 5 90 180 arc
+  5 10 lineto 
+  10 10 5 180 270 arc
+  closepath
+  stroke
+  grestore
+} def
+% 
+% dot positions are designated by (x,y) labels from (1,1) to (3,3)
+%
+% change position and radius of filled circles as you like
+%
+% the dot macro: a filled circle 
+%
+%     takes three arguments off the stack: 
+%                            n (x-offset 43*(n - 1) )
+%                            x-label: 1, 2, or 3
+%                            y-label: 1, 2, or 3
+/dot {
+  /y exch def         %  store y-label of dot
+  /x exch def         %  store x-label of dot
+  1 sub 43 mul        %  calculate 43*(n-1), given n on the stack
+  /o exch def         %  store result in offset o
+  gsave
+  newpath 
+  o 0 translate       %  shift coordinate system by offset o in x
+  x 8 mul 5 add       %  x-position of dot: 8*x+5 (-> 13,21,29)
+  y 8 mul 5 add       %  y-position of dot: 8*y+5 (-> 13,21,29) 
+  3.5 0 360 arc       %  I like big dots (radius 3.5 units)
+  closepath 
+  fill
+  grestore
+} def
+%
+2 setlinewidth        %  lines 2 units wide
+0 setgray             %  fill the dots in black
+%
+% now use the macros to draw the dice faces in loops
+%
+1 1 6 { frame } for                      % the six frames
+2 1 6 { dup 1 2 3 { dup dot } for } for  % (1,1), (3,3) on 2, 3, 4, 5, 6
+1 2 5 { 2 2 dot } for                    % (2,2) dot on 1, 3, 5
+4 1 6 { dup 1 3 dot 3 1 dot } for        % (1,3), (3,1) dots on 4, 5, 6
+1 2 3 { 6 exch 2 dot } for               % (1,2), (3,2) dots only on 6
+%%EOF
+

data/lib/picolena/version.rb

@@ -2,7 +2,7 @@ module Picolena #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 0
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end