findler 0.0.3 → 0.0.4

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+script: bundle exec rake
+rvm:
+  - 1.8.7
+  - 1.9.3
+  - rbx-18mode

data/Gemfile

@@ -1,6 +1,3 @@
 source "http://rubygems.org"
 gemspec
 
-gem "rake"
-gem "yard"
-gem "rspec", '~> 2.7.0'

data/README.md

@@ -1,5 +1,7 @@
 # Findler: Filesystem Iteration with Persistable State
 
+[![Build Status](https://secure.travis-ci.org/mceachen/findler.png?branch=master)](http://travis-ci.org/mceachen/findler)
+
 Findler is a Ruby library for iterating over a filtered set of files from a given
 path, written to be suitable with concurrent workers and very large
 filesystem hierarchies.
@@ -8,9 +10,9 @@ filesystem hierarchies.
 
 ```ruby
 f = Findler.new "/Users/mrm"
-f.append_extension ".jpg", ".jpeg"
+f.add_extensions ".jpg", ".jpeg"
 iterator = f.iterator
-iterator.next
+iterator.next_file
 # => "/Users/mrm/Photos/img_1000.jpg"
 ```
 
@@ -31,7 +33,7 @@ To resume iteration:
 
 ```ruby
 Marshal.load(IO.open('iterator.state'))
-iterator.next
+iterator.next_file
 # => "/Users/mrm/Photos/img_1001.jpg"
 ```
 
@@ -39,13 +41,89 @@ To re-check a directory hierarchy for files that you haven't visited yet:
 
 ```ruby
 iterator.rescan!
-iterator.next
+iterator.next_file
 # => "/Users/mrm/Photos/img_1002.jpg"
 ```
 
+External synchronization between the serialized state of the
+iterator and the other processes will have to be done by you, of course.
+The ```load```, ```next_file``` , and ```dump``` should be done while holding
+an iteration mutex of some sort.
+
+## Filtering and ordering
+
+Filters provide custom exclusion and ordering criteria, so you don't
+have to do that logic in the code that consumes from your iterator.
+
+Filters can't be procs or lambdas because those aren't safely serializable.
+
+What you provide to ```add_filter``` is a symbolized name of a class method
+on ```Findler::Filters```:
+
+```ruby
+f = Findler.new(".")
+f.add_filter :order_by_name
+```
+
+Note that the last filter added will be last to order the children, so it will be the
+"primary" sort criterion. Note also that the ordering is only done in
+the context of a given directory.
+
+### Implementing your own filter
+
+Filter methods receive an array of ```Pathname``` instances. Those pathnames will:
+
+1. have the same parent
+2. will not have been enumerated by ```next_file()``` already
+3. will satisfy the settings given to the parent Findler instance, like ```include_hidden```
+   and added patterns.
+
+Note that the last filter added will be last to order the children, so it will be the
+"primary" sort criterion.
+
+The returned values from the class method will be the final set of elements (both files
+and directories) that Findler will return from ```next_file()```.
+
+### Example
+
+To find files that have valid EXIF headers, using the *most* excellent
+[exiftoolr](https://github.com/mceachen/exiftoolr) gem, you'd do this:
+
+```ruby
+require 'findler'
+require 'exiftoolr'
+
+# Monkey-patch Filters to add our custom filter:
+class Findler::Filters
+  def self.exif_only(children)
+    child_files = children.select{|ea|ea.file?}
+    child_dirs = children.select{|ea|ea.directory?}
+    e = Exiftoolr.new(child_files)
+    e.files_with_results + child_dirs
+  end
+end
+
+f = Findler.new "/Users/mrm"
+f.add_extensions ".jpg", ".jpeg", ".cr2", ".nef"
+f.case_insensitive!
+f.add_filter(:exif_only)
+```
+
+### Filter implementation notes
+
+* The array of ```Pathname``` instances can be assumed to be absolute.
+* Only child files that satisfy the ```extension``` and ```pattern``` filters will be seen by the filter class method.
+* If a directory doesn't have any relevant files, the filter method will be called multiple times for a given call to ```next_file()```.
+* if you want to be notified when new directories are walked into, and you want to do a bulk operation within that directory,
+  this gives you that hook–-just remember to return the children array at the end of your block.
+
+### Why can't ```filter_with``` be a proc?
+
+Because procs and lambdas aren't Marshal-able, and I didn't want to use something scary like ruby2ruby and eval.
 
 ## Changelog
 
-* 0.0.1 First `find`
+* 0.0.4 Added custom filters for ```next_file()``` and singular aliases for ```add_extension``` and ```add_pattern```
+* 0.0.3 Fixed gemfile packaging
 * 0.0.2 Added scalable Bloom filter so ```Iterator#rescan``` is possible
-* 0.0.3 Fixed gemfile packaging
+* 0.0.1 First `find`

data/Rakefile

@@ -4,7 +4,13 @@ YARD::Rake::YardocTask.new do |t|
   t.files = ['lib/**/*.rb', 'README.md']
 end
 
-require "rspec/core/rake_task"
-RSpec::Core::RakeTask.new(:spec)
+require 'rake/testtask'
 
-task :default => :spec
+Rake::TestTask.new do |t|
+  t.libs.push "lib"
+  t.libs.push "test"
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+task :default => :test

data/examples/find_exif_files

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'exiftoolr'
+require 'findler'
+
+class Findler::Filters
+  def self.with_exif(children)
+    child_files = children.select { |ea| ea.file? }
+    child_dirs = children.select { |ea| ea.directory? }
+    e = Exiftoolr.new(child_files)
+    good = e.files_with_results
+    bad = child_files - good
+    puts "Files missing EXIF:\n  #{(bad).join("\n  ")}" unless bad.empty?
+    good + child_dirs
+  end
+end
+
+f = Findler.new(ENV['HOME'])
+f.add_extensions ".jpg", ".jpeg", ".cr2", ".nef"
+f.add_filter :with_exif
+f.case_insensitive!
+iter = f.iterator
+
+while nxt = iter.next_file
+  puts "next file: #{nxt}"
+end

data/findler.gemspec

@@ -16,11 +16,12 @@ Gem::Specification.new do |s|
   s.rubyforge_project = "findler"
 
   s.files = `git ls-files`.split("\n")
-  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.test_files = `git ls-files -- {test,test,features}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ["lib"]
   s.add_development_dependency "rake"
   s.add_development_dependency "yard"
-  s.add_development_dependency "rspec", "~> 2.7.0"
+  s.add_development_dependency "minitest"
+  s.add_development_dependency "minitest-reporters"
   s.add_dependency "bloomer"
 end

data/lib/findler.rb

@@ -1,25 +1,46 @@
 class Findler
-  IGNORE_CASE = 1
-  INCLUDE_HIDDEN = 2
 
   autoload :Iterator, "findler/iterator"
+  autoload :Error, "findler/error"
+  require "findler/filters"
 
-  def initialize path
+  IGNORE_CASE = 1
+  INCLUDE_HIDDEN = 2
+
+  def initialize(path)
     @path = path
     @flags = 0
   end
 
-  # These are File.fnmatch patterns. If any pattern matches, it will be returned by Iterator#next.
+  # These are File.fnmatch patterns.
+  # If any pattern matches, it will be returned by Iterator#next.
   # (see File.fnmatch?)
-  def add_pattern *patterns
-    patterns.each { |ea| (@patterns ||= []) << ea }
+  def patterns
+    @patterns ||= []
+  end
+
+  def add_patterns(*patterns)
+    self.patterns += patterns
   end
 
-  def append_extension *extensions
-    extensions.each { |ea| add_pattern "*#{normalize_extension(ea)}" }
+  def add_pattern(pattern)
+    self.patterns << pattern
+  end
+
+  def add_extension(extension)
+    add_pattern "*#{normalize_extension(extension)}"
+  end
+
+  def add_extensions(*extensions)
+    extensions.each { |ea| add_extension(ea) }
+  end
+
+  # Should patterns be interpreted in a case-sensitive manner? The default is case sensitive,
+  # but if your local filesystem is not case sensitive, this flag is a no-op.
+  def case_sensitive!
+    @flags &= ~IGNORE_CASE
   end
 
-  # Should patterns be interpreted in a case-insensitive manor? (default is case sensitive)
   def case_insensitive!
     @flags |= IGNORE_CASE
   end
@@ -30,18 +51,60 @@ class Findler
     @flags |= INCLUDE_HIDDEN
   end
 
+  def exclude_hidden!
+    @flags &= ~INCLUDE_HIDDEN
+  end
+
+  def filter_class
+    (@filter_class ||= Filters)
+  end
+
+  def filter_class=(new_filter_class)
+    raise Error unless new_filter_class.is_a? Class
+    filters.each{|ea|new_filter_class.method(ea)} # verify the filters class has those methods defined
+    @filter_class = new_filter_class
+  end
+
+  # Accepts symbols whose names are class methods on Finder::Filters.
+  #
+  # Filter methods receive an array of Pathname instances, and are in charge of ordering
+  # and filtering the array. The returned array of pathnames will be used by the iterator.
+  #
+  # Those pathnames will:
+  # a) have the same parent
+  # b) will not have been enumerated by next() already
+  # c) will satisfy the hidden flag and patterns preferences
+  #
+  # Note that the last filter added will be last to order the children, so it will be the
+  # "primary" sort criterion.
+  def add_filter(filter_symbol)
+    filter_class.method(filter_symbol)
+    filters << filter_symbol
+  end
+
+  def filters
+    (@filters ||= [])
+  end
+
+  def add_filters(*filter_symbols)
+    filter_symbols.each { |ea| add_filter(ea) }
+  end
+
   def iterator
-    Iterator.new(:path => @path, :patterns => @patterns, :flags => @flags)
+    Iterator.new(:path => @path,
+                 :patterns => @patterns,
+                 :flags => @flags,
+                 :filters => @filters)
   end
 
   private
 
-  def normalize_extension extension
+  def normalize_extension(extension)
     if extension.nil? || extension.empty? || extension.start_with?(".")
       extension
     else
       ".#{extension}"
     end
   end
-
 end
+

data/lib/findler/error.rb

@@ -0,0 +1,2 @@
+class Findler::Error < StandardError
+end

data/lib/findler/filters.rb

@@ -0,0 +1,38 @@
+class Findler::Filters
+  # files first, then directories
+  def self.files_first(paths)
+    preserve_sort_by(paths) { |ea| ea.file? ? -1 : 1 }
+  end
+
+  # directories first, then files
+  def self.directories_first(paths)
+    preserve_sort_by(paths) { |ea| ea.directory? ? -1 : 1 }
+  end
+
+  # order by the mtime of each file. Oldest files first.
+  def self.order_by_mtime_asc(paths)
+    preserve_sort_by(paths)  { |ea| ea.mtime }
+  end
+
+  # reverse order by the mtime of each file. Newest files first.
+  def self.order_by_mtime_desc(paths)
+    preserve_sort_by(paths)  { |ea| -ea.mtime }
+  end
+
+  # order by the name of each file.
+  def self.order_by_name(paths)
+    preserve_sort_by(paths)  { |ea| ea.basename.to_s }
+  end
+
+  # reverse the order of the sort
+  def self.reverse(paths)
+    paths.reverse
+  end
+
+  def self.preserve_sort_by(array, &block)
+    ea_to_index = Hash[array.zip((0..array.size-1).to_a)]
+    array.sort_by do |ea|
+      [yield(ea), ea_to_index[ea]]
+    end
+  end
+end

data/lib/findler/iterator.rb

@@ -1,56 +1,50 @@
 require 'bloomer'
+require 'pathname'
 
 class Findler
+
   class Iterator
 
-    attr_reader :path, :parent, :patterns, :flags, :visited_dirs, :visited_files
+    attr_reader :path, :parent, :patterns, :flags, :filters_class, :filters, :visited_dirs, :visited_files
 
     def initialize(attrs, parent = nil)
       @path = attrs[:path]
       @path = Pathname.new(@path) unless @path.is_a? Pathname
+      @path = @path.expand_path unless @path.absolute?
       @parent = parent
 
-      set_ivar(:visited_dirs, attrs) { Bloomer::Scalable.new(256, 1.0/1_000_000) }
-      set_ivar(:visited_files, attrs) { Bloomer::Scalable.new(256, 1.0/1_000_000) }
-      set_ivar(:patterns, attrs) { nil }
-      set_ivar(:flags, attrs) { 0 }
+      set_inheritable_ivar(:visited_dirs, attrs) { self.class.new_presence_collection }
+      set_inheritable_ivar(:visited_files, attrs) { self.class.new_presence_collection }
+      set_inheritable_ivar(:patterns, attrs) { nil }
+      set_inheritable_ivar(:flags, attrs) { 0 }
+      set_inheritable_ivar(:filters, attrs) { [] }
+      set_inheritable_ivar(:filters_class, attrs) { Filters }
+      set_inheritable_ivar(:sort_with, attrs) { nil }
 
       @sub_iter = self.class.new(attrs[:sub_iter], self) if attrs[:sub_iter]
     end
 
     # Visit this directory and all sub directories, and check for unseen files. Only call on the root iterator.
     def rescan!
-      raise "Only invoke on root" unless @parent.nil?
-      @visited_dirs = Bloomer::Scalable.new(256, 1.0/1_000_000)
+      raise Error, "Only invoke on root" unless @parent.nil?
+      @visited_dirs = self.class.new_presence_collection
       @children = nil
       @sub_iter = nil
     end
 
-    #def to_hash
-    #  {:path => @path, :visited_dirs:patterns => @patterns, :flags => @flags, :sub_iter => @sub_iter && @sub_iter.to_hash}
-    #end
-    #
-    #def _dump(depth)
-    #  Marshal.dump(to_hash)
-    #end
-    #
-    #def self._load(data)
-    #  new(Marshal.load(data))
-    #end
-
-    def case_insensitive?
-      (Findler::IGNORE_CASE | flags) != 0
+    def ignore_case?
+      (Findler::IGNORE_CASE & flags) > 0
     end
 
-    def skip_hidden?
-      (Findler::INCLUDE_HIDDEN | flags) == 0
+    def include_hidden?
+      (Findler::INCLUDE_HIDDEN & flags) > 0
     end
 
     def fnmatch_flags
-      @_fnflags ||= (@parent && @parent.fnmatch_flags) || begin
+      @fnmatch_flags ||= (@parent && @parent.fnmatch_flags) || begin
         f = 0
-        f |= File::FNM_CASEFOLD if case_insensitive?
-        f |= File::FNM_DOTMATCH if !skip_hidden?
+        f |= File::FNM_CASEFOLD if ignore_case?
+        f |= File::FNM_DOTMATCH if include_hidden?
         f
       end
     end
@@ -59,11 +53,11 @@ class Findler
       @path
     end
 
-    def next
+    def next_file
       return nil unless @path.exist?
-      
+
       if @sub_iter
-        nxt = @sub_iter.next
+        nxt = @sub_iter.next_file
         return nxt unless nxt.nil?
         @visited_dirs.add @sub_iter.path.to_s
         @sub_iter = nil
@@ -71,10 +65,13 @@ class Findler
 
       # If someone touches the directory while we iterate, redo the @children.
       @children = nil if @path.ctime != @ctime || @path.mtime != @mtime
-      @children ||= begin
+
+      if @children.nil?
         @mtime = @path.mtime
         @ctime = @path.ctime
-        @path.children.delete_if { |ea| skip?(ea) }
+        children = @path.children.delete_if { |ea| skip?(ea) }
+        filtered_children = @filters.inject(children){ |c, f| filter(c, f) }
+        @children = filtered_children
       end
 
       nxt = @children.shift
@@ -82,7 +79,7 @@ class Findler
 
       if nxt.directory?
         @sub_iter = Iterator.new({:path => nxt}, self)
-        self.next
+        self.next_file
       else
         @visited_files.add nxt.to_s
         nxt
@@ -91,12 +88,30 @@ class Findler
 
     private
 
-    def set_ivar(field, attrs, &block)
-      sym = "@#{field}".to_sym
-      v = attrs[field]
-      v ||= begin
-        (p = instance_variable_get(:@parent)) && p.instance_variable_get(sym)
+    def self.new_presence_collection
+      Bloomer::Scalable.new(256, 1.0/1_000_000)
+    end
+
+    def filter(children, filter_symbol)
+      filtered_children = filters_class.send(filter_symbol, children)
+      unless filtered_children.respond_to? :collect
+        raise Error, "#{path.to_s}: filter_with, must return an Enumerable"
+      end
+      children_as_pathnames = filtered_children.collect { |ea| ea.is_a?(Pathname) ? ea : Pathname.new(ea) }
+      illegal_children = children_as_pathnames - children
+      unless illegal_children.empty?
+        raise Error, "#{path.to_s}: filter_with returned unexpected paths: #{illegal_children.join(",")}"
       end
+      children_as_pathnames
+    end
+
+    # Sets the instance variable to the value in attrs[field].
+    # If attrs is missing a value, pull the value from the parent.
+    # If the parent doesn't have a value, use the block to generate a default.
+    def set_inheritable_ivar(field, attrs, &block)
+      v = attrs[field]
+      sym = "@#{field}".to_sym
+      v ||= parent.instance_variable_get(sym)
       v ||= yield
       instance_variable_set(sym, v)
     end
@@ -107,13 +122,13 @@ class Findler
 
     def skip? pathname
       s = pathname.to_s
-      return true if hidden?(pathname) && skip_hidden?
-      return @visited_dirs.include?(s) if pathname.directory?
-      return true if @visited_files.include?(s)
+      return true if !include_hidden? && hidden?(pathname)
+      return visited_dirs.include?(s) if pathname.directory?
+      return true if visited_files.include?(s)
       unless patterns.nil?
         return true if patterns.none? { |p| pathname.fnmatch(p, fnmatch_flags) }
       end
       return false
     end
   end
-end
+end

data/lib/findler/version.rb

@@ -1,3 +1,3 @@
 class Findler
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/test/filters_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+
+describe Findler::Filters do
+  it "should preserve sort order with no result" do
+    a = %w(h a p p y)
+    b = Findler::Filters.preserve_sort_by(a) { 0 }
+    b.must_equal a
+  end
+
+  it "should preserve sort order" do
+    a = [3.5, 3.1, 1.1, 2.0, 1.5, 1.3]
+    b = Findler::Filters.preserve_sort_by(a) { |ea| ea.to_i }
+    b.must_equal [1.1, 1.5, 1.3, 2.0, 3.5, 3.1]
+  end
+end

data/test/findler_test.rb

@@ -0,0 +1,224 @@
+require 'test_helper'
+
+class Findler::Filters
+  def self.non_empty_files(children)
+    (children.select { |ea| ea.directory? || ea.size > 0 })
+  end
+
+  def self.no_return(children)
+  end
+
+  def self.invalid_return(children)
+    "/invalid/file"
+  end
+
+  def self.files_to_s(children)
+    (children).collect { |ea| ea.to_s }
+  end
+end
+
+describe Findler do
+
+  def touch_secrets
+    `mkdir .hide ; touch .outer-hide dir-0/.hide .hide/normal.txt .hide/.secret`
+  end
+
+  it "should detect hidden files properly" do
+    i = Findler::Iterator.new(:path => "/tmp")
+    i.send(:hidden?, Pathname.new("/a/b")).must_equal false
+    i.send(:hidden?, Pathname.new("/a/.b")).must_equal true
+    i.send(:hidden?, Pathname.new("/.a/.b")).must_equal true
+    i.send(:hidden?, Pathname.new("/.a/b")).must_equal false
+  end
+
+  it "should skip hidden files by default" do
+    i = Findler.new("/tmp").iterator
+    i.send(:skip?, Pathname.new("/tmp/not-hidden")).must_equal false
+    i.send(:skip?, Pathname.new("/tmp/.hidden")).must_equal true
+  end
+
+  it "should not skip hidden files when set" do
+    f = Findler.new("/tmp")
+    f.include_hidden!
+    i = f.iterator
+    i.send(:skip?, Pathname.new("/tmp/not-hidden")).must_equal false
+    i.send(:skip?, Pathname.new("/tmp/.hidden")).must_equal false
+  end
+
+  it "should find all non-hidden files by default" do
+    with_tree(%W(.jpg .txt)) do |dir|
+      touch_secrets
+      f = Findler.new(dir)
+      collect_files(f.iterator).sort.must_equal `find * -type f -not -name '.*'`.split.sort
+      f.exclude_hidden! # should be a no-op
+      collect_files(f.iterator).sort.must_equal `find * -type f -not -name '.*'`.split.sort
+      f.include_hidden!
+      collect_files(f.iterator).sort.must_equal `find . -type f | sed -e 's/^\\.\\///'`.split.sort
+    end
+  end
+
+  it "should find only .jpg files when constrained" do
+    with_tree(%W(.jpg .txt .JPG)) do |dir|
+      f = Findler.new(dir)
+      f.add_extension ".jpg"
+      if fs_case_sensitive?
+        f.case_sensitive!
+        collect_files(f.iterator).sort.must_equal `find * -type f -name \\*.jpg`.split.sort
+      end
+      f.case_insensitive!
+      collect_files(f.iterator).sort.must_equal `find * -type f -iname \\*.jpg`.split.sort
+    end
+  end
+
+  it "should find .jpg or .JPG files when constrained" do
+    with_tree(%W(.jpg .txt .JPG)) do |dir|
+      f = Findler.new(dir)
+      f.add_extension ".jpg"
+      f.case_insensitive!
+      iter = f.iterator
+      collect_files(iter).sort.must_equal `find * -type f -iname \\*.jpg`.split.sort
+    end
+  end
+
+  it "should find files added after iteration started" do
+    with_tree(%W(.txt)) do |dir|
+      f = Findler.new(dir)
+      iter = f.iterator
+      iter.next_file.wont_be_nil
+
+      # cheating with mtime on the touch doesn't properly update the parent directory ctime,
+      # so we have to deal with the second-granularity resolution of the filesystem.
+      sleep(1.1)
+
+      FileUtils.touch(dir + "new.txt")
+      collect_files(iter).must_include("new.txt")
+    end
+  end
+
+  it "should find new files after a rescan" do
+    with_tree([".txt", ".no"]) do |dir|
+      f = Findler.new(dir)
+      f.add_extension ".txt"
+      iter = f.iterator
+      collect_files(iter).sort.must_equal `find * -type f -iname \\*.txt`.split.sort
+      FileUtils.touch(dir + "dir-0" + "dir-1" + "new-0.txt")
+      FileUtils.touch(dir + "dir-1" + "dir-0" + "new-1.txt")
+      FileUtils.touch(dir + "dir-2" + "dir-2" + "new-2.txt")
+      collect_files(iter).must_be_empty
+      iter.rescan!
+      collect_files(iter).sort.must_equal ["dir-0/dir-1/new-0.txt", "dir-1/dir-0/new-1.txt", "dir-2/dir-2/new-2.txt"]
+    end
+  end
+
+  it "should not return files removed after iteration started" do
+    with_tree([".txt"]) do |dir|
+      f = Findler.new(dir)
+      iter = f.iterator
+      iter.next_file.wont_be_nil
+      sleep(1.1) # see above for hand-wringing-defense of this atrocity
+
+      (dir + "tmp-1.txt").unlink
+      collect_files(iter).wont_include("tmp-1.txt")
+    end
+  end
+
+  it "should dump/load in the middle of iterating" do
+    with_tree(%W(.jpg .txt .JPG)) do |dir|
+      all_files = `find * -type f -iname \\*.jpg`.split
+      all_files.size.times do |i|
+        f = Findler.new(dir)
+        f.add_extension ".jpg"
+        f.case_insensitive!
+        iter_a = f.iterator
+        files_a = i.times.collect { relative_path(iter_a.path, iter_a.next_file) }
+        iter_b = Marshal.load(Marshal.dump(iter_a))
+        files_b = collect_files(iter_b)
+
+        iter_c = Marshal.load(Marshal.dump(iter_b))
+        collect_files(iter_c)
+        iter_c.next_file.must_be_nil
+
+        (files_a + files_b).sort.must_equal all_files.sort
+      end
+    end
+  end
+
+  it "should create an iterator even for a non-existent directory" do
+    tmpdir = nil
+    Dir.mktmpdir do |dir|
+      tmpdir = Pathname.new dir
+    end
+    tmpdir.exist?.must_equal false
+    f = Findler.new(tmpdir)
+    collect_files(f.iterator).must_be_empty
+  end
+
+  it "should raise an error if the block given to next_file returns nil" do
+    Dir.mktmpdir do |dir|
+      f = Findler.new(dir)
+      f.add_filter :no_return
+      i = f.iterator
+      lambda { i.next_file }.must_raise(Findler::Error)
+    end
+  end
+
+  it "should raise an error if the block returns non-children" do
+    with_tree(%W(.txt)) do |dir|
+      f = Findler.new(dir)
+      f.add_filter :invalid_return
+      i = f.iterator
+      lambda { i.next_file }.must_raise(Findler::Error)
+    end
+  end
+
+  it "should support filter_with against global/Kernel methods" do
+    with_tree(%W(.txt)) do |dir|
+      f = Findler.new(dir)
+      f.add_filter :files_to_s
+      iter = f.iterator
+      files = collect_files(iter)
+      files.sort.must_equal `find * -type f`.split.sort
+    end
+  end
+
+  it "should support next_file blocks properly" do
+    with_tree(%W(.a .b)) do |dir|
+      Dir["**/*.a"].each { |ea| File.open(ea, 'w') { |f| f.write("hello") } }
+      f = Findler.new(dir)
+      f.add_filter :non_empty_files
+      iter = f.iterator
+      files = collect_files(iter)
+      files.sort.must_equal `find * -type f -name \\*.a`.split.sort
+    end
+  end
+
+  it "should support files_first ordering" do
+    with_tree(%W(.a), {
+        :depth => 2,
+        :files_per_dir => 2,
+        :subdirs_per_dir => 1,
+    }) do |dir|
+      f = Findler.new(dir)
+      f.add_filters :order_by_name, :files_first
+      expected = %W(tmp-0.a tmp-1.a dir-0/tmp-0.a dir-0/tmp-1.a)
+      collect_files(f.iterator).must_equal expected
+      f.add_filter :reverse
+      collect_files(f.iterator).must_equal expected.reverse
+    end
+  end
+
+  it "should support directory_first ordering" do
+    with_tree(%W(.a), {
+        :depth => 2,
+        :files_per_dir => 2,
+        :subdirs_per_dir => 1,
+    }) do |dir|
+      f = Findler.new(dir)
+      f.add_filters :order_by_name, :directories_first
+      expected = %W(dir-0/tmp-0.a dir-0/tmp-1.a tmp-0.a tmp-1.a)
+      collect_files(f.iterator).must_equal expected
+      f.add_filter :reverse
+      collect_files(f.iterator).must_equal expected.reverse
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,83 @@
+require 'minitest/spec'
+require 'minitest/reporters'
+require 'minitest/autorun'
+require 'tmpdir'
+require 'fileutils'
+require 'findler'
+
+MiniTest::Unit.runner = MiniTest::SuiteRunner.new
+if ENV["RM_INFO"] || ENV["TEAMCITY_VERSION"]
+  MiniTest::Unit.runner.reporters << MiniTest::Reporters::RubyMineReporter.new
+elsif ENV['TM_PID']
+  MiniTest::Unit.runner.reporters << MiniTest::Reporters::RubyMateReporter.new
+else
+  MiniTest::Unit.runner.reporters << MiniTest::Reporters::ProgressReporter.new
+end
+
+def with_tmp_dir(&block)
+  cwd = Dir.pwd
+  Dir.mktmpdir do |dir|
+    Dir.chdir(dir)
+    yield(Pathname.new dir)
+    Dir.chdir(cwd) # jruby needs us to cd out of the tmpdir so it can remove it
+  end
+ensure
+  Dir.chdir(cwd)
+end
+
+def with_tree(sufficies, options = {}, &block)
+  with_tmp_dir do |dir|
+    sufficies.each { |suffix| mk_tree dir, options.merge(:suffix => suffix) }
+    yield(dir)
+  end
+end
+
+def mk_tree(target_dir, options)
+  opts = {
+      :depth => 3,
+      :files_per_dir => 3,
+      :subdirs_per_dir => 3,
+      :prefix => "tmp",
+      :suffix => "",
+      :dir_prefix => "dir",
+      :dir_suffix => ""
+  }.merge options
+  p = target_dir.is_a?(Pathname) ? target_dir : Pathname.new(target_dir)
+  p.mkdir unless p.exist?
+
+  opts[:files_per_dir].times do |i|
+    fname = "#{opts[:prefix]}-#{i}#{opts[:suffix]}"
+    FileUtils.touch(p + fname).to_s
+  end
+  return if (opts[:depth] -= 1) <= 0
+  opts[:subdirs_per_dir].times do |i|
+    dir = "#{opts[:dir_prefix]}-#{i}#{opts[:dir_suffix]}"
+    mk_tree(p + dir, opts)
+  end
+end
+
+def expected_files(depth, files_per_dir, subdirs_per_dir)
+  return 0 if depth == 0
+  files_per_dir + (subdirs_per_dir * expected_files(depth - 1, files_per_dir, subdirs_per_dir))
+end
+
+def relative_path(parent, pathname)
+  pathname.relative_path_from(parent).to_s
+end
+
+def collect_files(iter)
+  files = []
+  while nxt = iter.next_file
+    files << relative_path(iter.path, nxt)
+  end
+  files
+end
+
+def fs_case_sensitive?
+  @fs_case_sensitive ||= begin
+    `touch CASETEST`
+    !File.exist?('casetest')
+  ensure
+    `rm CASETEST`
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: findler
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: 
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Matthew McEachen
@@ -15,10 +15,10 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-23 00:00:00 Z
+date: 2012-03-28 00:00:00 -07:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  prerelease: false
   type: :development
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -29,10 +29,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   name: rake
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  prerelease: false
   type: :development
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -43,28 +43,40 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   name: yard
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  prerelease: false
   type: :development
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 3
         segments: 
-        - 2
-        - 7
         - 0
-        version: 2.7.0
-  name: rspec
+        version: "0"
+  prerelease: false
+  name: minitest
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
+  type: :development
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   prerelease: false
+  name: minitest-reporters
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
   type: :runtime
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -73,8 +85,9 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   name: bloomer
-  version_requirements: *id004
+  version_requirements: *id005
 description: |-
   Findler is designed for very large filesystem hierarchies,
     where simple block processing, or returning an array of matches, just isn't feasible.
@@ -89,16 +102,22 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- .travis.yml
 - Gemfile
 - MIT-LICENSE
 - README.md
 - Rakefile
+- examples/find_exif_files
 - findler.gemspec
 - lib/findler.rb
+- lib/findler/error.rb
+- lib/findler/filters.rb
 - lib/findler/iterator.rb
 - lib/findler/version.rb
-- spec/findler_spec.rb
-- spec/spec_helper.rb
+- test/filters_test.rb
+- test/findler_test.rb
+- test/test_helper.rb
+has_rdoc: true
 homepage: https://github.com/mceachen/findler/
 licenses: []
 
@@ -128,11 +147,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: findler
-rubygems_version: 1.8.15
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Findler is a stateful filesystem iterator
 test_files: 
-- spec/findler_spec.rb
-- spec/spec_helper.rb
-has_rdoc: 
+- test/filters_test.rb
+- test/findler_test.rb
+- test/test_helper.rb

data/spec/findler_spec.rb

@@ -1,117 +0,0 @@
-require 'spec_helper'
-
-describe Findler do
-
-  before :each do
-    @opts = {
-      :depth => 3,
-      :files_per_dir => 3,
-      :subdirs_per_dir => 3,
-      :prefix => "tmp",
-      :suffix => "",
-      :dir_prefix => "dir",
-      :dir_suffix => ""
-    }
-  end
-
-  it "should find all files by default" do
-    with_tree([".jpg", ".txt"]) do |dir|
-      f = Findler.new(dir)
-      iter = f.iterator
-      collect_files(iter).should =~ `find * -type f`.split
-    end
-  end
-
-  it "should find only .jpg files when constrained" do
-    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
-      f = Findler.new(dir)
-      f.append_extension ".jpg"
-      iter = f.iterator
-      collect_files(iter).should =~ `find * -type f -name \\*.jpg`.split
-    end
-  end
-
-  it "should find .jpg or .JPG files when constrained" do
-    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
-      f = Findler.new(dir)
-      f.append_extension ".jpg"
-      f.case_insensitive!
-      iter = f.iterator
-      collect_files(iter).should =~ `find * -type f -iname \\*.jpg`.split
-    end
-  end
-
-  it "should find files added after iteration started" do
-    with_tree([".txt"]) do |dir|
-      f = Findler.new(dir)
-      iter = f.iterator
-      iter.next.should_not be_nil
-
-      # cheating with mtime on the touch doesn't properly update the parent directory ctime,
-      # so we have to deal with the second-granularity resolution of the filesystem.
-      sleep(1.1)
-
-      FileUtils.touch(dir + "new.txt")
-      collect_files(iter).should include("new.txt")
-    end
-  end
-
-  it "should find new files after a rescan" do
-    with_tree([".txt", ".no"]) do |dir|
-      f = Findler.new(dir)
-      f.append_extension ".txt"
-      iter = f.iterator
-      collect_files(iter).should =~ `find * -type f -iname \\*.txt`.split
-      FileUtils.touch(dir + "dir-0" + "dir-1" + "new-0.txt")
-      FileUtils.touch(dir + "dir-1" + "dir-0" + "new-1.txt")
-      FileUtils.touch(dir + "dir-2" + "dir-2" + "new-2.txt")
-      collect_files(iter).should be_empty
-      iter.rescan!
-      collect_files(iter).should =~ ["dir-0/dir-1/new-0.txt", "dir-1/dir-0/new-1.txt", "dir-2/dir-2/new-2.txt"]
-    end
-  end
-
-  it "should not return files removed after iteration started" do
-    with_tree([".txt"]) do |dir|
-      f = Findler.new(dir)
-      iter = f.iterator
-      iter.next.should_not be_nil
-      sleep(1.1) # see above for hand-wringing-defense of this atrocity
-
-      (dir + "tmp-1.txt").unlink
-      collect_files(iter).should_not include("tmp-1.txt")
-    end
-  end
-
-  it "should dump/load in the middle of iterating" do
-    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
-      all_files = `find * -type f -iname \\*.jpg`.split
-      all_files.size.times do |i|
-        f = Findler.new(dir)
-        f.append_extension ".jpg"
-        f.case_insensitive!
-        iter_a = f.iterator
-        files_a = i.times.collect { relative_path(iter_a, iter_a.next) }
-        iter_b = Marshal.load(Marshal.dump(iter_a))
-        files_b = collect_files(iter_b)
-
-        iter_c = Marshal.load(Marshal.dump(iter_b))
-        collect_files(iter_c)
-        iter_c.next.should be_nil
-
-        (files_a + files_b).should =~ all_files
-      end
-    end
-  end
-
-  it "should create an iterator even for a non-existent directory" do
-    tmpdir = nil
-    cwd = Dir.pwd
-    Dir.mktmpdir do |dir|
-      tmpdir = Pathname.new dir
-    end
-    tmpdir.should_not exist
-    f = Findler.new(tmpdir)
-    collect_files(f.iterator).should be_empty
-  end
-end

data/spec/spec_helper.rb

@@ -1,66 +0,0 @@
-require 'rspec'
-require 'tmpdir'
-require 'fileutils'
-require 'findler'
-
-RSpec.configure do |config|
-  config.color_enabled = true
-  config.formatter = 'documentation'
-end
-
-def with_tmp_dir(&block)
-  cwd = Dir.pwd
-  Dir.mktmpdir do |dir|
-    Dir.chdir(dir)
-    yield(Pathname.new dir)
-  end
-  Dir.chdir(cwd)
-end
-
-def with_tree(sufficies, &block)
-  with_tmp_dir do |dir|
-    sufficies.each { |suffix| mk_tree dir, @opts.merge(:suffix => suffix) }
-    yield(dir)
-  end
-end
-
-def mk_tree(target_dir, options)
-  opts = {
-    :depth => 3,
-    :files_per_dir => 3,
-    :subdirs_per_dir => 3,
-    :prefix => "tmp",
-    :suffix => "",
-    :dir_prefix => "dir",
-    :dir_suffix => ""
-  }.merge options
-  p = target_dir.is_a?(Pathname) ? target_dir : Pathname.new(target_dir)
-  p.mkdir unless p.exist?
-
-  opts[:files_per_dir].times do |i|
-    fname = "#{opts[:prefix]}-#{i}#{opts[:suffix]}"
-    FileUtils.touch(p + fname).to_s
-  end
-  return if (opts[:depth] -= 1) <= 0
-  opts[:subdirs_per_dir].times do |i|
-    dir = "#{opts[:dir_prefix]}-#{i}#{opts[:dir_suffix]}"
-    mk_tree(p + dir, opts)
-  end
-end
-
-def expected_files(depth, files_per_dir, subdirs_per_dir)
-  return 0 if depth == 0
-  files_per_dir + (subdirs_per_dir * expected_files(depth - 1, files_per_dir, subdirs_per_dir))
-end
-
-def relative_path(parent, pathname)
-  pathname.relative_path_from(parent.path).to_s
-end
-
-def collect_files(iter)
-  files = []
-  while nxt = iter.next
-    files << relative_path(iter, nxt)
-  end
-  files
-end