sonar_connector_filestore 0.2.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,23 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+*.gemspec
+tmp

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Trampoline Systems Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= sonar-connector-filestore
+
+sonar-connector works with sets of files. it pulls some files from a server and stores them locally on disk. it takes those files and pushes them to a sonar server etc
+
+sonar-connector-filestore provides some simple mechanisms for connectors to deal reliably with sets of files, establish ownership of files and transfer ownership between connectors
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Trampoline Systems Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sonar_connector_filestore"
+    gem.summary = %Q{reliable staged processing with the filesystem}
+    gem.description = %Q{support for staged processing using a vanilla filesystem for storage}
+    gem.email = "craig@trampolinesystems.com"
+    gem.homepage = "http://github.com/trampoline/sonar-connector-filestore"
+    gem.authors = ["mccraigmccraig"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "rr", ">= 0.10.11"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sonar-connector-filestore #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/lib/sonar_connector_filestore.rb

@@ -0,0 +1,355 @@
+require 'fileutils'
+require 'uuidtools'
+require 'logger'
+require 'set'
+
+module Sonar
+  module Connector
+
+    # a FileStore has an on-disk directory structure :
+    #
+    # - root, effectively a parent directory
+    # - name : the filestore directory name
+    # - areas : names of acceptable sub-directories in the FileStore directory
+    # so a filestore with (@root=="/foo", @name==:bar, @areas=[:area51, :area52])
+    # would have directories :
+    #
+    #  /foo
+    #  /foo/bar
+    #  /foo/bar/area51
+    #  /foo/bar/area52
+    class FileStore
+      class << self
+        # the default logger...
+        attr_accessor :logger
+      end
+      FileStore.logger = Logger.new($stdout)
+      FileStore.logger.level = Logger::INFO
+
+      attr_reader :root
+      attr_reader :name
+      attr_reader :areas
+      attr_writer :logger
+
+      def self.valid_filestore_name?(f)
+        (f.to_s == File.basename(f.to_s)) && 
+          ordinary_directory_name?(f)
+      end
+
+      def self.valid_area_name?(a)
+        a.to_s != "tmp"
+      end
+
+      def self.ordinary_directory_name?(f)
+        File.basename(f.to_s) !~ /^\./
+      end
+
+      def self.ordinary_directory?(f)
+        ordinary_directory_name?(f.to_s) && File.directory?(f.to_s)
+      end
+
+      def initialize(root, name, areas, opts={})
+        raise "directory '#{root}' does not exist or is not a directory" if !File.directory?(root)
+        @root = root
+
+        raise "#{name} is not a valid filestore name" if !FileStore.valid_filestore_name?(name)
+        @name = name
+        FileUtils.mkdir_p(filestore_path)
+
+        @areas = Set.new([*areas])
+        @areas.each{|area| raise "#{area} is not a valid area name" if !FileStore.valid_area_name?(area)}
+        @areas.each{|area| FileUtils.mkdir_p(area_path(area))}
+
+        @logger = opts[:logger]
+      end
+
+      def logger
+        @logger || FileStore.logger
+      end
+
+      def destroy!
+        FileUtils.rm_r(filestore_path)
+      end
+
+      def filestore_path
+        File.join(root, name.to_s)
+      end
+
+      def check_area(area)
+        raise "no such area: #{area}" if !@areas.include?(area) && area!=:tmp
+      end
+      
+      def area_path(area)
+        check_area(area)
+        File.join(filestore_path, area.to_s)
+      end
+
+      def file_path(area, filename)
+        File.join(area_path(area), filename)
+      end
+
+      # marker exception to tell process and process_batch to 
+      # leave files in the source area
+      class LeaveInSourceArea < RuntimeError
+      end
+
+      # process files from source_area. move it to error_area if the block
+      # raises an exception and to success_area if the block completes. if
+      # LeaveInSourceArea is raised, don't do anything with the files
+      def process(source_area, error_area=nil, success_area=nil)
+        raise "i need a block" if !block_given?
+        
+        files = area_files(source_area)
+        files.each do |f|
+          begin
+            yield f
+            if success_area
+              move(source_area, f, success_area)
+            else
+              delete(source_area, f)
+            end
+          rescue LeaveInSourceArea=>e
+            logger.info("leaving files in #{source_area}")
+            raise
+          rescue Exception=>e
+            logger.warn(FileStore.to_s){[e.class.to_s, e.message, *e.backtrace].join("\n")}
+            if error_area
+              move(source_area, f, error_area)
+            else
+              delete(source_area, f)
+            end
+            raise
+          end
+        end
+      end
+
+      # process a batch of files from source_area. move them to error_area if
+      # the block raises and exception, and to success_area if the block completes,
+      # and leave where they are if LeaveInSourceArea is raised.
+      # returns the number of items processed, 0 if all work is done.
+      def process_batch(batch_size, source_area, error_area=nil, success_area=nil)
+        raise "i need a block" if !block_given?
+
+        batch = area_files(source_area, batch_size)
+        return 0 if batch.size==0
+        begin
+          yield batch
+          if success_area
+            batch.each{|p| move(source_area, p, success_area)}
+          else
+            batch.each{|p| delete(source_area, p)}
+          end
+        rescue LeaveInSourceArea=>e
+          logger.info("leaving files in #{source_area}")
+          raise
+        rescue Exception=>e
+          logger.warn(FileStore.to_s){[e.class.to_s, e.message, *e.backtrace].join("\n")}
+          if error_area
+            batch.each{|p| move(source_area, p, error_area)}
+          else
+            batch.each{|p| delete(source_area, p)}
+          end
+          raise
+        end
+        return batch.size
+      end
+
+      # fetch at most max regular file paths from an area
+      def area_files(area, max=nil)
+        relative_file_paths(area_path(area), max)
+      end
+
+      # number of items in an area
+      def count(area)
+        ap = area_path(area)
+        Dir[File.join(ap, "*")].length
+      end
+
+      # hash of counts keyed by area
+      def area_count
+        @areas.reduce({}){|h,area| h[area]=count(area) ; h}
+      end
+
+      # disk usage of an area in kb
+      def size(area)
+        ap = area_path(area)
+        `du -k #{ap}`.gsub(/\W+tmp\W*$/m,'').to_i
+      end
+
+      # hash of sizes keyed by area
+      def area_size
+        @areas.reduce({}){|h,area| h[area]=size(area) ; h}
+      end
+
+      # iterate over all files in top level of an area, calling a block on each
+      def for_each(area)
+        ap = area_path(area)
+        Dir.foreach(area_path(area)) do |f|
+          fp = File.join(ap,f)
+          yield f if File.file?(fp) || FileStore.ordinary_directory?(fp)
+        end
+      end
+
+      # write a file to an area
+      def write(area, filename, content)
+        ensure_area_directory(area, filename)
+        File.open(file_path(area, filename), "w"){ |io| io << content }
+      end
+
+      # read a file from an area
+      def read(area, filename)
+        File.read(file_path(area, filename))
+      end
+
+      # remove a file from an area
+      def delete(area, filename)
+        FileUtils.rm_r(file_path(area, filename))
+      end
+
+      # move a file from one area to another
+      def move(from_area, filename, to_area)
+        move_file(area_path(from_area), filename, area_path(to_area))
+      end
+
+      # remove any empty directories from an area
+      def scrub!(area)
+        scrub_path(area_path(area), false)
+      end
+
+      # flip files from an area into a sub-directory of an area 
+      # in another
+      # filestore, named by the name of this filestore
+      # thus 
+      # fs1.flip(:complete, fs2, :working ) moves
+      # fs1/complete/* => fs2/working/fs1/*
+      # if unique_names is false, then unique directories
+      # are constructued in the targetfs to flip to, otherwise
+      # identical names are assumed to be identical files
+      # and will overwrite already present files
+      def flip(area, filestore, to_area, unique_names=true)
+        ap = area_path(area)
+        paths = []
+
+        scrub!(area) # only move what we need to
+
+        # collect all moveable paths
+        for_each(area) do |f|
+          paths << File.join(ap, f)
+        end
+        filestore.receive_flip(name, to_area, paths, unique_names) if paths.length>0
+      end
+
+      # receive a flip... move all paths to be flipped 
+      # into a temporary directory, and then move that
+      # directory into place in one atomic move operation
+      def receive_flip(from_filestore_name, to_area, paths, unique_names)
+#        $stderr << "receive_flip(#{from_filestore_name}, #{to_area}, #{paths.inspect}, #{unique_names})\n"
+        tmp_area_path = area_path(:tmp)
+
+        # tmp_uuid
+        tmp_uuid = unique_name
+
+        # first move all moveable paths to a unique named tmp area within the receive area
+        tmp_path = File.join(tmp_area_path, tmp_uuid)
+        if paths.length>0
+          FileUtils.mkdir_p(tmp_path)
+          paths.each do |path|
+            FileUtils.mv(path, tmp_path)
+          end
+        end
+
+        # move everything from the receive area... recovers interrupted receive_flips too
+        to_path = area_path(to_area)
+        Dir.foreach(tmp_area_path) do |path|
+          path_1 = File.join(tmp_area_path, path)
+          if unique_names
+
+            if FileStore.ordinary_directory?(path_1)
+              # names are unique, so don't move the uuid folders
+              Dir.foreach(path_1) do |file_path|
+                path_2 = File.join(path_1, file_path)
+                FileUtils.mv(path_2, to_path, :force=>true) if File.file?(path_2) || FileStore.ordinary_directory?(path_2)              
+              end
+            elsif File.file?(path_1) # names are unique, so ok to move plain files too
+              FileUtils.mv(path_1, to_path, :force=>true)
+            end
+
+          else
+            # move uuid named dirs
+            FileUtils.mv(path_1, to_path, :force=>true) if File.file?(path_1) || FileStore.ordinary_directory?(path_1)
+          end
+        end
+
+        # finally remove any empty tmp dirs
+        scrub!(:tmp)
+      end
+
+      private
+
+      def unique_name
+        UUIDTools::UUID.timestamp_create
+      end
+
+      # depth first search
+      def scrub_path(dir, scrub)
+        empty = scrub
+        Dir.foreach(dir) do |f|
+          path = File.join(dir, f)
+          if File.directory?(path) 
+            # want to descend : so avoid short-cut evaluation
+            empty = scrub_path(path, true) && empty if FileStore.ordinary_directory_name?(f)
+          else
+            empty = false
+          end
+        end
+        FileUtils.rm_rf(dir) if empty
+      end
+
+      # fetch at most max relative regular file paths from a directory hierarchy
+      # rooted at dir
+      def relative_file_paths(dir, max=nil)
+        file_paths(dir, max).map{|p| p.gsub(/^#{dir}#{File::SEPARATOR}/,'')}
+      end
+
+      # fetch at most max regular file paths from a directory hierarchy
+      # rooted at dir
+      def file_paths(dir, max=nil)
+        paths = []
+        Dir.foreach(dir) do |f|
+          return paths if max && paths.size >= max
+          path = File.join(dir, f)
+          if File.directory?(path)
+            paths += file_paths(path, max) if FileStore.ordinary_directory_name?(f)
+          elsif File.file?(path)
+            paths << path
+          end
+        end
+        paths
+      end
+
+      # move a file named relative to filename dir
+      # to the same filename relative to to_dir
+      def move_file(from_dir, filename, to_dir)
+        f1 = File.join(from_dir, filename)
+        f2 = File.join(to_dir, filename)
+        ensure_directory(to_dir, filename)
+        FileUtils.mv(f1, f2)
+      end
+
+      # ensure that the directory of a filename exists in the given area
+      def ensure_area_directory(area, filename)
+        # create a directory in the destination area if necessary
+        ensure_directory(area_path(area), filename)
+      end
+
+      # given a directory, and a filename relative to it, ensure
+      # that the directory containing the actual file exists
+      # e.g. given dir==/a/b/c and filename==d/e/f.txt
+      # then ensure directory /a/b/c/d/e exists
+      def ensure_directory(dir, filename)
+        file_dir = File.expand_path(File.join(dir, File.dirname(filename)))
+        FileUtils.mkdir_p(file_dir)
+      end
+    end
+  end
+end

data/spec/sonar_connector_filestore_spec.rb

@@ -0,0 +1,420 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'set'
+require 'fileutils'
+
+module Sonar
+  module Connector
+    describe "SonarConnectorFilestore" do
+
+      before(:all) do
+        FileStore::logger.level = Logger::FATAL
+      end
+
+      before(:each) do
+        FileUtils.rm_rf(TMP_DIR)
+        FileUtils.mkdir_p(TMP_DIR)
+      end
+
+      after(:each) do
+      end
+
+      def create_testfs(*areas)
+        areas = [:foo, :bar] if areas.empty?
+        FileStore.new(TMP_DIR, :testfs, areas)
+      end
+
+      describe "valid_area_name?" do
+        it "should not permit :tmp as an area name" do
+          FileStore.valid_area_name?(:tmp).should == false
+        end
+
+        it "should permit :foo as an area name" do
+          FileStore.valid_area_name?(:foo).should == true
+        end
+      end
+
+      describe "ordinary_directory_name?" do
+        it "should return false for .." do
+          FileStore.ordinary_directory_name?("..").should == false
+        end
+
+        it "should return false for ." do
+          FileStore.ordinary_directory_name?(".").should == false
+        end
+
+        it "should return false for .foo" do 
+          FileStore.ordinary_directory_name?(".foo").should == false
+       end
+
+        it "should return true for foo" do
+          FileStore.ordinary_directory_name?("foo").should == true
+        end
+
+        it "should return false for foo/." do
+          FileStore.ordinary_directory_name?("foo/.").should == false
+        end
+
+        it "should return false for foo/.foo" do
+          FileStore.ordinary_directory_name?("foo/.foo").should == false
+        end
+      end
+
+      describe "valid_filestore_name?" do
+        it "should not permitted sub-directory names" do
+          FileStore.valid_filestore_name?("foo/bar").should == false
+        end
+
+        it "should permit simple filenames" do
+          FileStore.valid_filestore_name?("foo").should == true
+        end
+
+        it "should not permit dot filenames" do
+          FileStore.valid_filestore_name?(".foo").should == false
+        end
+
+        it "should not permit special dirnames" do
+          FileStore.valid_filestore_name?(".").should == false
+        end
+      end
+
+      describe "ordinary_directory?" do
+        it "should return true for regular real directory" do
+          FileStore.ordinary_directory?(TMP_DIR).should == true
+        end
+
+        it "should return false for nested special directory" do
+          FileStore.ordinary_directory?(File.join(TMP_DIR, '.')).should == false
+        end
+
+        it "should return false for non-existent directory" do
+          FileStore.ordinary_directory?("blahblah").should == false
+        end
+      end
+
+
+      it "should initialize with a root, a name and areas, and create directories" do
+        fs=create_testfs
+        fs.root.should == TMP_DIR
+        fs.name.should == :testfs
+        fs.areas.should == [:foo, :bar].to_set
+        File.directory?(File.join(TMP_DIR, "testfs")).should == true
+        File.directory?(File.join(TMP_DIR, "testfs", "foo")).should == true
+        File.directory?(File.join(TMP_DIR, "testfs", "bar")).should == true
+      end
+
+      it "should destroy itself cleanly" do
+        fs=create_testfs
+        fs.destroy!
+        File.exist?(File.join(TMP_DIR, "testfs")).should == false
+      end
+
+      it "should write files to an area" do
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        File.read(File.join(TMP_DIR, "testfs", "foo", "testfile.txt")).should == "one two three"
+      end
+      
+      it "should count files in an area" do
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        fs.write(:foo, "testfile2.txt", "one two three")
+        fs.count(:foo).should == 2
+      end
+
+      it "should count files across all areas" do
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        fs.write(:bar, "testfile2.txt", "one two three")
+        fs.area_count.should == {:foo=>1, :bar=>1}
+      end
+      
+      it "should give a kb based disk usage for an area" do
+        # don't actually know what disk page size is
+        # so test that each file occupies >0 space, and that two
+        # files the same size occupy twice as much as one
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        sz1 = fs.size(:foo)
+        sz1.should > 0
+        fs.write(:foo, "testfile2.txt", "one two three")
+        fs.size(:foo).should == 2*sz1
+      end
+
+      it "should give kb based disk usage across all areas" do
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        sz1 = fs.size(:foo)
+        sz1.should > 0
+        fs.write(:bar, "testfile.txt", "one two three")
+        fs.write(:bar, "testfile2.txt", "one two three")
+        fs.area_size.should == {:foo=>sz1, :bar=>2*sz1}
+      end
+
+      it "should scrub empty directories from an area" do
+        fs = create_testfs
+        ap = fs.area_path(:foo)
+        FileUtils.mkdir_p(File.join(ap, "bar", "baz"))
+        FileUtils.mkdir_p(File.join(ap, "woo"))
+        FileUtils.mkdir_p(File.join(ap, "waz"))
+        fs.write(:foo, File.join("waz", "testfile.txt"), "one two three")
+
+        fs.scrub!(:foo)
+
+        File.exist?(File.join(ap, "bar")).should == false
+        File.exist?(File.join(ap, "woo")).should == false
+        File.exist?(File.join(ap, "waz", "testfile.txt")).should == true
+      end
+
+
+      it "should iterate over all files in an area" do
+        fs = create_testfs
+        fs.write(:foo, "testfile.txt", "one two three")
+        fs.write(:foo, "testfile2.txt", "four five six")
+
+        texts = Set.new
+        ap = fs.area_path(:foo)
+        fs.for_each(:foo) do |f|
+          texts << File.read(File.join(ap, f))
+        end
+
+        texts.should == ["one two three", "four five six"].to_set
+      end
+
+      it "should ignore . and .. files when iterating" do
+        fs = create_testfs(:foo, :bar, :baz)
+        ap = fs.area_path(:foo)
+
+        # only files "foo" and "bar" exist in area :foo
+        stub(File).file?(File.join(ap,"foo")){true}
+        stub(File).file?(File.join(ap,"bar")){true}
+        stub(File).file?(){false}
+
+        stub(Dir).foreach do |path, proc| 
+          [".", "..", "foo", "bar"].each{ |p| proc.call(p)}
+        end
+        
+        files = Set.new
+        fs.for_each(:foo){|f| files << f}
+        files.should == ["foo", "bar"].to_set
+        
+      end
+
+      describe "process" do
+        before do
+          @fs = create_testfs(:foo, :bar, :baz)
+          @fs.write(:foo, "testfile.txt", "one two three")
+          @fs.write(:foo, "testfile2.txt", "four five six")
+          @fs.write(:foo, "testfile3.txt", "seven eight nine")
+        end
+
+        it "should process all files in an area" do
+          texts = Set.new
+          @fs.process(:foo) do |f|
+            texts << File.read(@fs.file_path(:foo, f))
+          end
+          texts.should == ["one two three", "four five six", "seven eight nine"].to_set
+          @fs.count(:foo).should == 0
+        end
+
+        it "should move failed processings to the error_area" do
+          texts = Set.new
+          lambda {
+            @fs.process(:foo, :bar) do |f|
+              s = File.read(@fs.file_path(:foo, f))
+              raise "five" if s =~ /five/
+              texts << s
+            end
+          }.should raise_error("five")
+          @fs.count(:foo).should == 1
+          @fs.count(:bar).should == 1
+          @fs.read(:bar, "testfile2.txt").should == "four five six"
+        end
+
+        it "should move completed processings to the success_area" do
+          texts = Set.new
+          lambda {
+            @fs.process(:foo, :bar, :baz) do |f|
+              s = File.read(@fs.file_path(:foo, f))
+              raise "five" if s =~ /five/
+              texts << s
+            end
+          }.should raise_error("five")
+          @fs.count(:foo).should == 1
+          @fs.count(:bar).should == 1
+          @fs.count(:baz).should == 1
+          @fs.read(:bar, "testfile2.txt").should == "four five six"
+        end
+      end
+
+      describe "process_batch" do
+        before do
+          @fs = create_testfs(:foo, :bar, :baz)
+          FileUtils.mkdir_p(File.join(@fs.area_path(:foo), "a", "b"))
+          FileUtils.mkdir_p(File.join(@fs.area_path(:foo), "c", "d"))
+          @fs.write(:foo, "a/b/testfile.txt", "one two three")
+          @fs.write(:foo, "a/testfile2.txt", "four five six")
+          @fs.write(:foo, "c/d/testfile3.txt", "seven eight nine")
+          @fs.write(:foo, "c/testfile3.txt", "ten eleven twelve")
+
+          @files = ["a/b/testfile.txt", "a/testfile2.txt", "c/d/testfile3.txt", "c/testfile3.txt"].to_set
+          @processed = Set.new
+        end
+
+        def process_batch(fs, size, source_area, error_area=nil, success_area=nil)
+          fs.process_batch(size, source_area, error_area, success_area) do |batch|
+            @processed += batch
+            batch.each do |f| 
+              raise "#{f} not in @files" if !@files.delete?(f) 
+            end
+            if block_given?
+              batch.each do |f|
+                yield f
+              end
+            end
+          end
+        end
+
+        def check_files(fs, area, paths, test)
+          paths.each{|f| File.exist?(fs.file_path(area, f)).should == test}
+        end
+
+        it "should process and delete a limited batch of files" do
+          process_batch(@fs, 2, :foo).should == 2
+          
+          @processed.size.should == 2
+          @files.size.should == 2
+          check_files(@fs, :foo, @processed, false)
+          check_files(@fs, :foo, @files, true)
+
+          process_batch(@fs, 3, :foo).should == 2
+
+          @processed.size.should == 4
+          @files.size.should == 0
+          check_files(@fs, :foo, @processed, false)
+
+          process_batch(@fs, 2, :foo).should == 0
+        end
+
+        it "should not call the block if there are no files to process" do
+          process_batch(@fs, 4, :foo).should == 4
+          @fs.process_batch(1, :foo) do
+            raise "i should not be called"
+          end
+        end
+
+        it "should move failed batches to error_area if given" do
+          lambda {
+            process_batch(@fs, 2, :foo, :bar){|f| raise "foo"}
+          }.should raise_error("foo")
+          
+          @files.size.should == 2
+          check_files(@fs, :foo, @files, true)
+          check_files(@fs, :bar, @processed, true)
+
+          @ok = Set.new
+          process_batch(@fs, 2, :foo, :bar){|f| @ok << f}.should == 2
+          @files.size.should == 0
+          check_files(@fs, :foo, @ok, false)
+          check_files(@fs, :bar, @ok, false)
+
+          process_batch(@fs, 2, :foo, :bar).should == 0
+        end
+
+        it "should move successful batches to success_area if given" do
+          process_batch(@fs, 2, :foo, :bar, :baz).should == 2
+          @files.size.should == 2
+          check_files(@fs, :foo, @files, true)
+          check_files(@fs, :baz, @processed, true)
+
+          process_batch(@fs, 2, :foo, :bar, :baz).should == 2
+          check_files(@fs, :baz, @processed, true)
+          
+          process_batch(@fs, 2, :foo, :bar, :baz).should == 0
+        end
+
+        it "should leave files in source area if LeaveInSourceArea is thrown" do
+          lambda {
+            process_batch(@fs, 2, :foo, :bar){|f| raise FileStore::LeaveInSourceArea }
+          }.should raise_error(FileStore::LeaveInSourceArea)
+          check_files(@fs, :foo, @files, true)
+          check_files(@fs, :baz, @processed, false)
+          check_files(@fs, :foo, @processed, true)
+        end
+
+      end
+
+      describe "flip" do
+        before do
+          @testfs = create_testfs(:foo, :bar, :baz)
+          @testfs.write(:foo, "testfile.txt", "one two three")
+          
+          @targetfs = FileStore.new(TMP_DIR, :targetfs, [:a, :b])
+        end
+
+
+        it "should flip non-unique names from testfs to uniquely named subdir of targetfs" do
+          stub(@targetfs).unique_name{"some-uuid-string"}
+
+          @testfs.flip(:foo, @targetfs, :a, false)
+
+          File.exists?(File.join(@targetfs.area_path(:a), "some-uuid-string", "testfile.txt")).should == true
+
+          # should recreate area in flipped source, so source is
+          # still valid
+          File.exists?(File.join(@testfs.area_path(:foo)))
+        end
+
+        it "should flip unique names from testfs to targetfs without introducing additional subdirectories" do
+          @testfs.flip(:foo, @targetfs, :a)
+
+          File.exists?(File.join(@targetfs.area_path(:a), "testfile.txt")).should == true
+
+          # should recreate area in flipped source, so source is
+          # still valid
+          File.exists?(File.join(@testfs.area_path(:foo)))
+        end
+
+      end
+
+      describe "area_files" do
+        before do
+          @fs = create_testfs(:foo)
+          FileUtils.mkdir_p(File.join(@fs.area_path(:foo), "a", "b"))
+          FileUtils.mkdir_p(File.join(@fs.area_path(:foo), "c", "d"))
+          @fs.write(:foo, "a/b/testfile.txt", "one two three")
+          @fs.write(:foo, "a/testfile2.txt", "four five six")
+          @fs.write(:foo, "c/d/testfile3.txt", "seven eight nine")
+          @fs.write(:foo, "c/testfile3.txt", "ten eleven twelve")
+
+          @files = ["a/b/testfile.txt", "a/testfile2.txt", "c/d/testfile3.txt", "c/testfile3.txt"].to_set
+        end
+
+        it "should fetch all paths if max not given" do 
+          @fs.area_files(:foo).to_set.should == @files
+        end
+
+        it "should fetch a limited number of paths if max given" do
+          fs = @fs.area_files(:foo, 2).to_set
+          fs.size.should == 2
+          (@files - fs).size.should == 2
+        end
+      end
+
+      describe "logger" do
+        it "should write to the class logger if no instance logger given" do
+          fs = create_testfs(:foo)
+          mock(FileStore.logger).debug("foo")
+          fs.logger.debug("foo")
+        end
+
+        it "should write to the instance logger if given" do
+          fs = create_testfs(:foo)
+          mock(logger = Object.new).debug("foo")
+          fs.logger = logger
+          fs.logger.debug("foo")
+        end
+      end
+
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rubygems'
+require 'sonar_connector_filestore'
+require 'spec'
+require 'spec/autorun'
+require 'rr'
+require 'fileutils'
+
+Spec::Runner.configure do |config|
+  config.mock_with RR::Adapters::Rspec
+end
+
+TMP_DIR = File.expand_path("../../tmp", __FILE__)

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification 
+name: sonar_connector_filestore
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
+platform: ruby
+authors: 
+- mccraigmccraig
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-25 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rr
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 33
+        segments: 
+        - 0
+        - 10
+        - 11
+        version: 0.10.11
+  type: :development
+  version_requirements: *id002
+description: support for staged processing using a vanilla filesystem for storage
+email: craig@trampolinesystems.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/sonar_connector_filestore.rb
+- spec/sonar_connector_filestore_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/trampoline/sonar-connector-filestore
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: reliable staged processing with the filesystem
+test_files: 
+- spec/sonar_connector_filestore_spec.rb
+- spec/spec_helper.rb