swineherd 0.0.1

data/tests/test_filesystem.rb

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << 'lib'
+require 'swineherd/filesystem' ; include Swineherd
+require 'rubygems'
+require 'yaml'
+require 'rspec'
+
+options      = YAML.load(File.read(File.dirname(__FILE__)+'/testcfg.yaml'))
+current_test = options['filesystem_to_test']
+describe "A new filesystem" do
+
+  before do
+    @test_path  = "/tmp/rspec/test_path"
+    @test_path2 = "/tmp/rspec/test_path2"
+    @fs = Swineherd::FileSystem.get(current_test)
+  end
+
+  it "should implement exists?" do
+    [true, false].should include(@fs.exists?(@test_path))
+  end
+
+  it "should be able to create a path" do
+    @fs.mkpath(@test_path)
+    @fs.exists?(@test_path).should eql(true)
+  end
+
+  it "should be able to remove a path" do
+    @fs.mkpath(@test_path)
+    @fs.rm(@test_path)
+    @fs.exists?(@test_path).should eql(false)
+  end
+
+  it "should be able to copy paths" do
+    @fs.mkpath(@test_path)
+    @fs.cp(@test_path, @test_path2)
+    @fs.exists?(@test_path2).should eql(true)
+    @fs.rm(@test_path)
+    @fs.rm(@test_path2)
+  end
+
+  it "should be able to move paths" do
+    @fs.mkpath(@test_path)
+    @fs.mv(@test_path, @test_path2)
+    @fs.exists?(@test_path).should eql(false)
+    @fs.exists?(@test_path2).should eql(true)
+    @fs.rm(@test_path2)
+  end
+
+  it "should return a sane path type" do
+    @fs.mkpath(@test_path)
+    ["file", "directory", "symlink", "unknown"].should include(@fs.type(@test_path))
+    @fs.rm(@test_path)
+  end
+
+  it "can return an array of directory entries" do
+    sub_paths = ["a", "b", "c"]
+    sub_paths.each do |sub_path|
+      @fs.mkpath(File.join(@test_path, sub_path))
+    end
+    @fs.entries(@test_path).class.should eql(Array)
+    @fs.entries(@test_path).map{|path| File.basename(path)}.reject{|x| x =~ /\./}.sort.should eql(sub_paths.sort)
+    @fs.rm(@test_path)
+  end
+
+  it "can answer to open with a writable file object" do
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.should respond_to :write
+    @fs.rm(@test_path)
+  end
+
+end
+
+describe "A new file" do
+  before do
+    @test_path   = "/tmp/rspec/test_path"
+    @test_path2  = "/tmp/rspec/test_path2"
+    @test_string = "@('_')@"
+    @fs = Swineherd::FileSystem.get(current_test)
+  end
+
+  it "should be closeable" do
+    @fs.open(@test_path, 'w').close
+  end
+
+  it "should be writeable" do
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.write(@test_string)
+    fileobj.close
+    @fs.rm(@test_path)
+  end
+
+  it "should be readable" do
+
+    fileobjw = @fs.open(@test_path, 'w')
+    fileobjw.write(@test_string)
+    fileobjw.close
+
+    fileobjr = @fs.open(@test_path, 'r')
+    fileobjr.read.should eql(@test_string)
+
+    @fs.rm(@test_path)
+  end
+
+end

data/tests/test_s3_filesystem.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/env ruby
+
+#
+# These tests cannot possibly pass unless you have an amazon account with proper
+# credentials. Furthermore, you definitely want a test bucket to play with. In
+# this set of mock tests I've called it 'test-bucket' which will certainly get
+# you and 'access-denied' error. Also, despite all that, 4 tests (see below)
+# will fail outright.
+#
+# This one has to break the rules slightly because amazon-s3 is not actually a
+# filesystem implementation. There's no such thing as a 'path' and so the following
+# tests will fail:
+#
+# 1. it "should be able to create a path" (path wont exist but it's ok, thats what
+# we expect)
+#
+# 2. it "should be able to copy paths" (it can't create paths that aren't files
+# and so we expect this to fail, again it's ok.)
+#
+# 3. it "should be able to move paths" (it can't create paths that aren't files
+# and so we expect this to fail, again it's ok.)
+#
+# 4. it "can return an array of directory entries" (ditto)
+#
+# Note: If one were to rewrite the above tests to use existing paths on s3 then the
+# tests will succeed. Try it.
+#
+
+
+$LOAD_PATH << 'lib'
+require 'swineherd/filesystem' ; include Swineherd
+require 'rubygems'
+require 'yaml'
+require 'rspec'
+
+options = YAML.load(File.read(File.dirname(__FILE__)+'/testcfg.yaml'))
+current_test = 's3'
+describe "A new filesystem" do
+
+  before do
+    @test_path  = "#{options['s3_test_bucket']}/tmp/rspec/test_path"
+    @test_path2 = "#{options['s3_test_bucket']}/tmp/rspec/test_path2"
+    @fs = Swineherd::FileSystem.get(current_test, options['aws_access_key_id'], options['aws_secret_access_key'])
+  end
+
+  it "should implement exists?" do
+    [true, false].should include(@fs.exists?(@test_path))
+  end
+
+  it "should be able to create a path" do
+    @fs.mkpath(@test_path)
+    @fs.exists?(@test_path).should eql(true)
+  end
+
+  it "should be able to remove a path" do
+    @fs.mkpath(@test_path)
+    @fs.rm(@test_path)
+    @fs.exists?(@test_path).should eql(false)
+  end
+
+  it "should be able to copy paths" do
+    @fs.mkpath(@test_path)
+    @fs.cp(@test_path, @test_path2)
+    @fs.exists?(@test_path2).should eql(true)
+    @fs.rm(@test_path)
+    @fs.rm(@test_path2)
+  end
+
+  it "should be able to move paths" do
+    @fs.mkpath(@test_path)
+    @fs.mv(@test_path, @test_path2)
+    @fs.exists?(@test_path).should eql(false)
+    @fs.exists?(@test_path2).should eql(true)
+    @fs.rm(@test_path2)
+  end
+
+  it "should return a sane path type" do
+    @fs.mkpath(@test_path)
+    ["file", "directory", "symlink", "unknown"].should include(@fs.type(@test_path))
+    @fs.rm(@test_path)
+  end
+
+  it "can return an array of directory entries" do
+    sub_paths = ["a", "b", "c"]
+    sub_paths.each do |sub_path|
+      @fs.mkpath(File.join(@test_path, sub_path))
+    end
+    @fs.entries(@test_path).class.should eql(Array)
+    @fs.entries(@test_path).map{|path| File.basename(path)}.reject{|x| x =~ /\./}.sort.should eql(sub_paths.sort)
+    @fs.rm(@test_path)
+  end
+
+  it "can answer to open with a writable file object" do
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.should respond_to :write
+    @fs.rm(@test_path)
+  end
+
+end
+
+describe "A new file" do
+  before do
+    @test_path   = "#{options['s3_test_bucket']}/tmp/rspec/test_path"
+    @test_path2  = "#{options['s3_test_bucket']}/test_path2"
+    @test_string = "@('_')@"
+    @fs = Swineherd::FileSystem.get(current_test, options['aws_access_key_id'], options['aws_secret_access_key'])
+  end
+
+  it "should be closeable" do
+    @fs.open(@test_path, 'w').close
+  end
+
+  it "should be writeable" do
+    fileobj = @fs.open(@test_path, 'w')
+    fileobj.write(@test_string)
+    fileobj.close
+    @fs.rm(@test_path)
+  end
+
+  it "should be readable" do
+
+    fileobjw = @fs.open(@test_path, 'w')
+    fileobjw.write(@test_string)
+    fileobjw.close
+
+    fileobjr = @fs.open(@test_path, 'r')
+    fileobjr.read.should eql(@test_string)
+
+    @fs.rm(@test_path)
+  end
+
+end

data/tests/testcfg.yaml

@@ -0,0 +1,7 @@
+---
+filesystem_to_test: file
+s3_test_bucket: infochimps-test
+
+# :) you'll probably want to change these
+aws_access_key_id: myaccessid
+aws_secret_access_key: 1234mysecretaccesskey8q7fh

metadata

@@ -0,0 +1,204 @@
+--- !ruby/object:Gem::Specification 
+name: swineherd
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jacob Perkins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-20 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        - 0
+        version: 0.6.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: configliere
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: gorillib
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: erubis
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+description: Swineherd is for running scripts and workflows on filesystems.
+email: jacob.a.perkins@gmail.com
+executables: 
+- hdp-tree
+- hadoop-stream
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.textile
+files: 
+- LICENSE
+- README.textile
+- Rakefile
+- VERSION
+- bin/hadoop-stream
+- bin/hdp-tree
+- examples/pagerank/data/seinfeld_network.tsv
+- examples/pagerank/pagerank.rb
+- examples/pagerank/scripts/cut_off_list.rb
+- examples/pagerank/scripts/histogram.R
+- examples/pagerank/scripts/pagerank.pig
+- examples/pagerank/scripts/pagerank_initialize.pig
+- lib/swineherd.rb
+- lib/swineherd/filesystem.rb
+- lib/swineherd/filesystem/README_filesystem.textile
+- lib/swineherd/filesystem/basefilesystem.rb
+- lib/swineherd/filesystem/filesystems.rb
+- lib/swineherd/filesystem/hadoopfilesystem.rb
+- lib/swineherd/filesystem/localfilesystem.rb
+- lib/swineherd/filesystem/localfs.rb
+- lib/swineherd/filesystem/s3filesystem.rb
+- lib/swineherd/script.rb
+- lib/swineherd/script/hadoop_script.rb
+- lib/swineherd/script/pig_script.rb
+- lib/swineherd/script/r_script.rb
+- lib/swineherd/script/wukong_script.rb
+- lib/swineherd/template.rb
+- lib/swineherd/workflow.rb
+- lib/swineherd/workflow/job.rb
+- notes.txt
+- swineherd.gemspec
+- tests/test_filesystem.rb
+- tests/test_s3_filesystem.rb
+- tests/testcfg.yaml
+has_rdoc: true
+homepage: http://github.com/Ganglion/swineherd
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Flexible data workflow glue.
+test_files: 
+- examples/pagerank/pagerank.rb
+- examples/pagerank/scripts/cut_off_list.rb