swineherd 0.0.1 → 0.0.2

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/swineherd.rb

@@ -8,4 +8,7 @@ module Swineherd
   autoload :FileSystem,   'swineherd/filesystem'
   autoload :Script,       'swineherd/script'
   autoload :Workflow,     'swineherd/workflow'
+  
+  # For rake 0.9 compatibility
+  include Rake::DSL if defined?(Rake::DSL)
 end

data/lib/swineherd/filesystem/basefilesystem.rb

@@ -21,6 +21,12 @@ module Swineherd
     def open path, mode="r", &blk
     end
 
+    #
+    # Recursively measure the size of path. Results in bytes.
+    #
+    def size path
+    end
+    
     #
     # Recursively delete the path and all paths below it.
     #

data/lib/swineherd/filesystem/hadoopfilesystem.rb

@@ -32,7 +32,22 @@ module Swineherd
       HadoopFile.new(path,mode,self,&blk)
     end
 
+    def size path
+      lr(path).inject(0){|sz, f| sz += @hdfs.get_file_status(Path.new(f)).get_len}
+    end
 
+    #
+    # Recursively list paths
+    #
+    def lr path
+      paths = entries(path)
+      if (paths && !paths.empty?)
+        paths.map{|e| lr(e)}.flatten
+      else
+        path
+      end
+    end
+    
     def rm path
       @hdfs.delete(Path.new(path), true)
       [path]

data/lib/swineherd/filesystem/localfilesystem.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'find'
 module Swineherd
 
   class LocalFileSystem
@@ -12,6 +13,12 @@ module Swineherd
       return LocalFile.new path, mode, &blk
     end
 
+    def size path
+      sz = 0
+      Find.find(path){|f| sz += File.size(f)}
+      sz
+    end
+    
     def rm path
       FileUtils.rm_r path
     end

data/lib/swineherd/filesystem/s3filesystem.rb

@@ -22,6 +22,24 @@ module Swineherd
       S3File.new(path,mode,self,&blk)
     end
 
+    def size path
+      sz = 0
+      if type(path) == "directory"
+        lr(path).each do |f|
+          sz += file_size(f)
+        end        
+      else
+        sz += file_size(path)
+      end
+      sz
+    end    
+
+    def file_size path
+      containing_bucket = bucket(path)
+      header            = @s3.interface.head(containing_bucket, key_path(path))
+      header['content-length'].to_i
+    end
+    
     def rm path
       bkt = bucket(path)
       key = key_path(path)

data/lib/swineherd/foo

@@ -0,0 +1 @@
+@('_')@

data/lib/swineherd/script.rb

@@ -6,6 +6,7 @@ module Swineherd
     autoload :RScript,      'swineherd/script/r_script'
 
     module Common
+      
       attr_accessor :input, :output, :options, :attributes
       def initialize(source, input = [], output = [], options = {}, attributes ={})
         @source     = source

data/lib/swineherd/workflow.rb

@@ -1,7 +1,7 @@
 module Swineherd
   class Workflow
     attr_accessor :workdir, :outputs, :output_counts
-
+    
     #
     # Create a new workflow and new namespace for this workflow
     #

data/swineherd.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{swineherd}
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jacob Perkins"]
-  s.date = %q{2011-04-20}
+  s.date = %q{2011-06-01}
   s.description = %q{Swineherd is for running scripts and workflows on filesystems.}
   s.email = %q{jacob.a.perkins@gmail.com}
   s.executables = ["hdp-tree", "hadoop-stream"]
@@ -39,6 +39,7 @@ Gem::Specification.new do |s|
     "lib/swineherd/filesystem/localfilesystem.rb",
     "lib/swineherd/filesystem/localfs.rb",
     "lib/swineherd/filesystem/s3filesystem.rb",
+    "lib/swineherd/foo",
     "lib/swineherd/script.rb",
     "lib/swineherd/script/hadoop_script.rb",
     "lib/swineherd/script/pig_script.rb",
@@ -56,7 +57,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/Ganglion/swineherd}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.4.2}
   s.summary = %q{Flexible data workflow glue.}
   s.test_files = [
     "examples/pagerank/pagerank.rb",
@@ -64,7 +65,6 @@ Gem::Specification.new do |s|
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/tests/test_filesystem.rb

@@ -11,8 +11,9 @@ current_test = options['filesystem_to_test']
 describe "A new filesystem" do
 
   before do
-    @test_path  = "/tmp/rspec/test_path"
-    @test_path2 = "/tmp/rspec/test_path2"
+    @test_path   = "/tmp/rspec/test_path"
+    @test_path2  = "/tmp/rspec/test_path2"
+    @test_string = "@('_')@" 
     @fs = Swineherd::FileSystem.get(current_test)
   end
 
@@ -31,6 +32,16 @@ describe "A new filesystem" do
     @fs.exists?(@test_path).should eql(false)
   end
 
+  it "should implement size" do
+    @fs.mkpath(File.dirname(@test_path))
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.write(@test_string)
+    fileobj.close
+    7.should eql(@fs.size(@test_path))
+    @fs.rm(@test_path)
+    @fs.rm(File.dirname(@test_path))
+  end
+
   it "should be able to copy paths" do
     @fs.mkpath(@test_path)
     @fs.cp(@test_path, @test_path2)

data/tests/test_s3_filesystem.rb

@@ -38,8 +38,9 @@ current_test = 's3'
 describe "A new filesystem" do
 
   before do
-    @test_path  = "#{options['s3_test_bucket']}/tmp/rspec/test_path"
-    @test_path2 = "#{options['s3_test_bucket']}/tmp/rspec/test_path2"
+    @test_path   = "#{options['s3_test_bucket']}/tmp/rspec/test_path"
+    @test_path2  = "#{options['s3_test_bucket']}/tmp/rspec/test_path2"
+    @test_string = "@('_')@"     
     @fs = Swineherd::FileSystem.get(current_test, options['aws_access_key_id'], options['aws_secret_access_key'])
   end
 
@@ -58,6 +59,16 @@ describe "A new filesystem" do
     @fs.exists?(@test_path).should eql(false)
   end
 
+  it "should implement size" do
+    @fs.mkpath(File.dirname(@test_path))
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.write(@test_string)
+    fileobj.close
+    7.should eql(@fs.size(@test_path))
+    @fs.rm(@test_path)
+    @fs.rm(File.dirname(@test_path))
+  end
+
   it "should be able to copy paths" do
     @fs.mkpath(@test_path)
     @fs.cp(@test_path, @test_path2)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: swineherd
 version: !ruby/object:Gem::Version 
-  hash: 29
-  prerelease: false
+  hash: 27
+  prerelease: 
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Jacob Perkins
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-20 00:00:00 -05:00
+date: 2011-06-01 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -152,6 +152,7 @@ files:
 - lib/swineherd/filesystem/localfilesystem.rb
 - lib/swineherd/filesystem/localfs.rb
 - lib/swineherd/filesystem/s3filesystem.rb
+- lib/swineherd/foo
 - lib/swineherd/script.rb
 - lib/swineherd/script/hadoop_script.rb
 - lib/swineherd/script/pig_script.rb
@@ -195,7 +196,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.4.2
 signing_key: 
 specification_version: 3
 summary: Flexible data workflow glue.