findler 0.0.2

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.idea/
+.yardoc/
+.bundle/
+Gemfile.lock
+pkg/

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+gemspec
+
+gem "rake"
+gem "yard"
+gem "rspec", '~> 2.7.0'

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2012 Matthew McEachen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Findler: Filesystem Iteration with Persistable State
+
+Findler is a Ruby library for iterating over a filtered set of files from a given
+path, written to be suitable with concurrent workers and very large
+filesystem hierarchies.
+
+## Usage
+
+```ruby
+f = Findler.new "/Users/mrm"
+f.append_extension ".jpg", ".jpeg"
+iterator = f.iterator
+iterator.next
+# => "/Users/mrm/Photos/img_1000.jpg"
+```
+
+## Cross-process continuations
+
+This should smell an awful lot like [hike](https://github.com/sstephenson/hike),
+except for that last bit.
+
+```Findler::Iterator``` instances can be "paused" and "resumed" with ```Marshal```.
+The entire state of the iteration for the filesystem is returned, which can then
+be pushed onto any durable storage, like ActiveRecord or Redis, or just a local file:
+
+```ruby
+File.open('iterator.state', 'w') { |f| Marshal.dump(iterator, f) }
+```
+
+To resume iteration:
+
+```ruby
+Marshal.load(IO.open('iterator.state'))
+iterator.next
+# => "/Users/mrm/Photos/img_1001.jpg"
+```
+
+To re-check a directory hierarchy for files that you haven't visited yet:
+
+```ruby
+iterator.rescan!
+iterator.next
+# => "/Users/mrm/Photos/img_1002.jpg"
+```
+
+
+## Changelog
+
+* 0.0.1 First `find`
+* 0.0.2 Added scalable Bloom filter so ```Iterator#rescan``` is possible

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "yard"
+YARD::Rake::YardocTask.new do |t|
+  t.files = ['lib/**/*.rb', 'README.md']
+end
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/findler.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "findler"
+
+Gem::Specification.new do |s|
+  s.name = "findler"
+  s.version = Findler::VERSION
+  s.authors = ["Matthew McEachen"]
+  s.email = ["matthew+github@mceachen.org"]
+  s.homepage = "https://github.com/mceachen/findler/"
+  s.summary = %q{Findler is a stateful filesystem iterator}
+  s.description = %q{Findler is designed for very large filesystem hierarchies,
+  where simple block processing, or returning an array of matches, just isn't feasible.
+  Usage instructions are available in the README.}
+
+  s.rubyforge_project = "findler"
+
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency "bloomer"
+end

data/lib/findler.rb

@@ -0,0 +1,50 @@
+class Findler
+
+  VERSION = "0.0.2"
+
+  IGNORE_CASE = 1
+  INCLUDE_HIDDEN = 2
+
+  autoload :Iterator, "findler/iterator"
+
+  def initialize path
+    @path = path
+    @flags = 0
+  end
+
+  # These are File.fnmatch patterns. If any pattern matches, it will be returned by Iterator#next.
+  # (see File.fnmatch?)
+  def add_pattern *patterns
+    patterns.each { |ea| (@patterns ||= []) << ea }
+  end
+
+  def append_extension *extensions
+    extensions.each { |ea| add_pattern "*#{normalize_extension(ea)}" }
+  end
+
+  # Should patterns be interpreted in a case-insensitive manor? (default is case sensitive)
+  def case_insensitive!
+    @flags |= IGNORE_CASE
+  end
+
+  # Should we traverse hidden directories and files? (default is to skip files that start
+  # with a '.')
+  def include_hidden!
+    @flags |= INCLUDE_HIDDEN
+  end
+
+  def iterator
+    Iterator.new(:path => @path, :patterns => @patterns, :flags => @flags)
+  end
+
+  private
+
+  def normalize_extension extension
+    if extension.nil? || extension.empty? || extension.start_with?(".")
+      extension
+    else
+      ".#{extension}"
+    end
+  end
+
+end

data/lib/findler/iterator.rb

@@ -0,0 +1,119 @@
+require 'bloomer'
+
+class Findler
+  class Iterator
+
+    attr_reader :path, :parent, :patterns, :flags, :visited_dirs, :visited_files
+
+    def initialize(attrs, parent = nil)
+      @path = attrs[:path]
+      @path = Pathname.new(@path) unless @path.is_a? Pathname
+      @parent = parent
+
+      set_ivar(:visited_dirs, attrs) { Bloomer::Scalable.new(256, 1.0/1_000_000) }
+      set_ivar(:visited_files, attrs) { Bloomer::Scalable.new(256, 1.0/1_000_000) }
+      set_ivar(:patterns, attrs) { nil }
+      set_ivar(:flags, attrs) { 0 }
+
+      @sub_iter = self.class.new(attrs[:sub_iter], self) if attrs[:sub_iter]
+    end
+
+    # Visit this directory and all sub directories, and check for unseen files. Only call on the root iterator.
+    def rescan!
+      raise "Only invoke on root" unless @parent.nil?
+      @visited_dirs = Bloomer::Scalable.new(256, 1.0/1_000_000)
+      @children = nil
+      @sub_iter = nil
+    end
+
+    #def to_hash
+    #  {:path => @path, :visited_dirs:patterns => @patterns, :flags => @flags, :sub_iter => @sub_iter && @sub_iter.to_hash}
+    #end
+    #
+    #def _dump(depth)
+    #  Marshal.dump(to_hash)
+    #end
+    #
+    #def self._load(data)
+    #  new(Marshal.load(data))
+    #end
+
+    def case_insensitive?
+      (Findler::IGNORE_CASE | flags) != 0
+    end
+
+    def skip_hidden?
+      (Findler::INCLUDE_HIDDEN | flags) == 0
+    end
+
+    def fnmatch_flags
+      @_fnflags ||= (@parent && @parent.fnmatch_flags) || begin
+        f = 0
+        f |= File::FNM_CASEFOLD if case_insensitive?
+        f |= File::FNM_DOTMATCH if !skip_hidden?
+        f
+      end
+    end
+
+    def path
+      @path
+    end
+
+    def next
+      return nil unless @path.exist?
+      
+      if @sub_iter
+        nxt = @sub_iter.next
+        return nxt unless nxt.nil?
+        @visited_dirs.add @sub_iter.path.to_s
+        @sub_iter = nil
+      end
+
+      # If someone touches the directory while we iterate, redo the @children.
+      @children = nil if @path.ctime != @ctime || @path.mtime != @mtime
+      @children ||= begin
+        @mtime = @path.mtime
+        @ctime = @path.ctime
+        @path.children.delete_if { |ea| skip?(ea) }
+      end
+
+      nxt = @children.shift
+      return nil if nxt.nil?
+
+      if nxt.directory?
+        @sub_iter = Iterator.new({:path => nxt}, self)
+        self.next
+      else
+        @visited_files.add nxt.to_s
+        nxt
+      end
+    end
+
+    private
+
+    def set_ivar(field, attrs, &block)
+      sym = "@#{field}".to_sym
+      v = attrs[field]
+      v ||= begin
+        (p = instance_variable_get(:@parent)) && p.instance_variable_get(sym)
+      end
+      v ||= yield
+      instance_variable_set(sym, v)
+    end
+
+    def hidden?(pathname)
+      pathname.basename.to_s.start_with?(".")
+    end
+
+    def skip? pathname
+      s = pathname.to_s
+      return true if hidden?(pathname) && skip_hidden?
+      return @visited_dirs.include?(s) if pathname.directory?
+      return true if @visited_files.include?(s)
+      unless patterns.nil?
+        return true if patterns.none? { |p| pathname.fnmatch(p, fnmatch_flags) }
+      end
+      return false
+    end
+  end
+end

data/spec/findler_spec.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+
+describe Findler do
+
+  before :each do
+    @opts = {
+      :depth => 3,
+      :files_per_dir => 3,
+      :subdirs_per_dir => 3,
+      :prefix => "tmp",
+      :suffix => "",
+      :dir_prefix => "dir",
+      :dir_suffix => ""
+    }
+  end
+
+  it "should find all files by default" do
+    with_tree([".jpg", ".txt"]) do |dir|
+      f = Findler.new(dir)
+      iter = f.iterator
+      collect_files(iter).should =~ `find * -type f`.split
+    end
+  end
+
+  it "should find only .jpg files when constrained" do
+    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
+      f = Findler.new(dir)
+      f.append_extension ".jpg"
+      iter = f.iterator
+      collect_files(iter).should =~ `find * -type f -name \\*.jpg`.split
+    end
+  end
+
+  it "should find .jpg or .JPG files when constrained" do
+    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
+      f = Findler.new(dir)
+      f.append_extension ".jpg"
+      f.case_insensitive!
+      iter = f.iterator
+      collect_files(iter).should =~ `find * -type f -iname \\*.jpg`.split
+    end
+  end
+
+  it "should find files added after iteration started" do
+    with_tree([".txt"]) do |dir|
+      f = Findler.new(dir)
+      iter = f.iterator
+      iter.next.should_not be_nil
+
+      # cheating with mtime on the touch doesn't properly update the parent directory ctime,
+      # so we have to deal with the second-granularity resolution of the filesystem.
+      sleep(1.1)
+
+      FileUtils.touch(dir + "new.txt")
+      collect_files(iter).should include("new.txt")
+    end
+  end
+
+  it "should find new files after a rescan" do
+    with_tree([".txt", ".no"]) do |dir|
+      f = Findler.new(dir)
+      f.append_extension ".txt"
+      iter = f.iterator
+      collect_files(iter).should =~ `find * -type f -iname \\*.txt`.split
+      FileUtils.touch(dir + "dir-0" + "dir-1" + "new-0.txt")
+      FileUtils.touch(dir + "dir-1" + "dir-0" + "new-1.txt")
+      FileUtils.touch(dir + "dir-2" + "dir-2" + "new-2.txt")
+      collect_files(iter).should be_empty
+      iter.rescan!
+      collect_files(iter).should =~ ["dir-0/dir-1/new-0.txt", "dir-1/dir-0/new-1.txt", "dir-2/dir-2/new-2.txt"]
+    end
+  end
+
+  it "should not return files removed after iteration started" do
+    with_tree([".txt"]) do |dir|
+      f = Findler.new(dir)
+      iter = f.iterator
+      iter.next.should_not be_nil
+      sleep(1.1) # see above for hand-wringing-defense of this atrocity
+
+      (dir + "tmp-1.txt").unlink
+      collect_files(iter).should_not include("tmp-1.txt")
+    end
+  end
+
+  it "should dump/load in the middle of iterating" do
+    with_tree([".jpg", ".txt", ".JPG"]) do |dir|
+      all_files = `find * -type f -iname \\*.jpg`.split
+      all_files.size.times do |i|
+        f = Findler.new(dir)
+        f.append_extension ".jpg"
+        f.case_insensitive!
+        iter_a = f.iterator
+        files_a = i.times.collect { relative_path(iter_a, iter_a.next) }
+        iter_b = Marshal.load(Marshal.dump(iter_a))
+        files_b = collect_files(iter_b)
+
+        iter_c = Marshal.load(Marshal.dump(iter_b))
+        collect_files(iter_c)
+        iter_c.next.should be_nil
+
+        (files_a + files_b).should =~ all_files
+      end
+    end
+  end
+
+  it "should create an iterator even for a non-existent directory" do
+    tmpdir = nil
+    cwd = Dir.pwd
+    Dir.mktmpdir do |dir|
+      tmpdir = Pathname.new dir
+    end
+    tmpdir.should_not exist
+    f = Findler.new(tmpdir)
+    collect_files(f.iterator).should be_empty
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,65 @@
+require 'rspec'
+require 'tmpdir'
+require 'fileutils'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter = 'documentation'
+end
+
+def with_tmp_dir(&block)
+  cwd = Dir.pwd
+  Dir.mktmpdir do |dir|
+    Dir.chdir(dir)
+    yield(Pathname.new dir)
+  end
+  Dir.chdir(cwd)
+end
+
+def with_tree(sufficies, &block)
+  with_tmp_dir do |dir|
+    sufficies.each { |suffix| mk_tree dir, @opts.merge(:suffix => suffix) }
+    yield(dir)
+  end
+end
+
+def mk_tree(target_dir, options)
+  opts = {
+    :depth => 3,
+    :files_per_dir => 3,
+    :subdirs_per_dir => 3,
+    :prefix => "tmp",
+    :suffix => "",
+    :dir_prefix => "dir",
+    :dir_suffix => ""
+  }.merge options
+  p = target_dir.is_a?(Pathname) ? target_dir : Pathname.new(target_dir)
+  p.mkdir unless p.exist?
+
+  opts[:files_per_dir].times do |i|
+    fname = "#{opts[:prefix]}-#{i}#{opts[:suffix]}"
+    FileUtils.touch(p + fname).to_s
+  end
+  return if (opts[:depth] -= 1) <= 0
+  opts[:subdirs_per_dir].times do |i|
+    dir = "#{opts[:dir_prefix]}-#{i}#{opts[:dir_suffix]}"
+    mk_tree(p + dir, opts)
+  end
+end
+
+def expected_files(depth, files_per_dir, subdirs_per_dir)
+  return 0 if depth == 0
+  files_per_dir + (subdirs_per_dir * expected_files(depth - 1, files_per_dir, subdirs_per_dir))
+end
+
+def relative_path(parent, pathname)
+  pathname.relative_path_from(parent.path).to_s
+end
+
+def collect_files(iter)
+  files = []
+  while nxt = iter.next
+    files << relative_path(iter, nxt)
+  end
+  files
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: findler
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Matthew McEachen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-01-21 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: bloomer
+  version_requirements: *id001
+description: |-
+  Findler is designed for very large filesystem hierarchies,
+    where simple block processing, or returning an array of matches, just isn't feasible.
+    Usage instructions are available in the README.
+email: 
+- matthew+github@mceachen.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- findler.gemspec
+- lib/findler.rb
+- lib/findler/iterator.rb
+- spec/findler_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: https://github.com/mceachen/findler/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: findler
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Findler is a stateful filesystem iterator
+test_files: 
+- spec/findler_spec.rb
+- spec/spec_helper.rb