cumulus_csv 0.0.2

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 evizitei
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,44 @@
+= cumulus_csv
+
+CSV Files: I hate them, you probably do too, but sometimes you need to get data into your system and this is the only way it's happening.  
+
+If you're deploying a rails app in a cloud setup, you may have troubles if you're trying to store an uploaded file locally and process it later in a background thread (I know I have).  
+
+cumulus_csv is one way to solve that problem.  You can save your file to your S3 account, and loop over the data inside it at your convenience later.  So it doesn't matter where you're doing the processing, you just need to have the key you used to store the file, and you can process away.   
+
+THIS GEM IS DEPENDANT ON AWS::S3!         
+
+Since this gem uses AWS::S3, it should be no suprise that you'll need similar auth parameters:
+
+  manager = Cumulus::CSV::DataFileManager.new(
+    :access_key_id     => 'abc', 
+    :secret_access_key => '123'
+  )
+
+this manager has 2 main functions: storing your files as they're uploaded, and letting you iterate over them later when you need to.
+
+To store your file on S3 when you upload it, you'd do something like this:
+
+  key = manager.store_uploaded_file!(params[:uploaded_file])  
+
+That will work for your standard multi part form. The key the file is stored under is returned, it's just the basename of the file.  You can pass this key to a rake task or whatever you're using, and it will be made use of later when you want to process this file:
+
+  manager.each_row_of(key) do |row|
+    #...some processing of this CSV row
+  end
+
+in that block, you can load each row into your database, or send an email based on each one, whatever it is you're trying to accomplish by having your app interact with this data file.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 evizitei. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,59 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "cumulus_csv"
+    gem.summary = %Q{Helps you save uploaded csv files containing data to amazon s3, and gives you a way to download and loop through the data in a background process easily}
+    gem.description = %Q{CSV Files: I hate them, you probably do too, but sometimes you need to get data into your system and this is the only way it's happening.  
+
+    If you're deploying a rails app in a cloud setup, you may have troubles if you're trying to store an uploaded file locally and process it later in a background thread (I know I have).  
+
+    cumulus_csv is one way to solve that problem.  You can save your file to your S3 account, and loop over the data inside it at your convenience later.  So it doesn't matter where you're doing the processing, you just need to have the key you used to store the file, and you can process away.}
+    gem.email = "ethan.vizitei@gmail.com"
+    gem.homepage = "http://github.com/evizitei/cumulus_csv"
+    gem.authors = ["evizitei"]
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "mocha", ">= 0.9.8"
+    gem.add_dependency "aws-s3", ">= 0.6.2"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "cumulus_csv #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.2

data/cumulus_csv.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{cumulus_csv}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["evizitei"]
+  s.date = %q{2010-03-09}
+  s.description = %q{CSV Files: I hate them, you probably do too, but sometimes you need to get data into your system and this is the only way it's happening.  
+
+    If you're deploying a rails app in a cloud setup, you may have troubles if you're trying to store an uploaded file locally and process it later in a background thread (I know I have).  
+
+    cumulus_csv is one way to solve that problem.  You can save your file to your S3 account, and loop over the data inside it at your convenience later.  So it doesn't matter where you're doing the processing, you just need to have the key you used to store the file, and you can process away.}
+  s.email = %q{ethan.vizitei@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "cumulus_csv.gemspec",
+     "lib/cumulus_csv.rb",
+     "lib/cumulus_csv/data_file_manager.rb",
+     "test/helper.rb",
+     "test/test_data_file_manager.rb"
+  ]
+  s.homepage = %q{http://github.com/evizitei/cumulus_csv}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Helps you save uploaded csv files containing data to amazon s3, and gives you a way to download and loop through the data in a background process easily}
+  s.test_files = [
+    "test/helper.rb",
+     "test/test_data_file_manager.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0.9.8"])
+      s.add_runtime_dependency(%q<aws-s3>, [">= 0.6.2"])
+    else
+      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0.9.8"])
+      s.add_dependency(%q<aws-s3>, [">= 0.6.2"])
+    end
+  else
+    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0.9.8"])
+    s.add_dependency(%q<aws-s3>, [">= 0.6.2"])
+  end
+end
+

data/lib/cumulus_csv.rb

@@ -0,0 +1,3 @@
+require 'csv'
+require 'aws/s3'
+require 'cumulus_csv/data_file_manager'

data/lib/cumulus_csv/data_file_manager.rb

@@ -0,0 +1,50 @@
+module Cumulus
+  module CSV
+    BUCKET_NAME = "cumuluscsvtmp"     
+    # DataFileManager is the gatekeeper for sending your data files to S3, and for iterating over them later. 
+    #
+    # In the constructor, It takes the same authentication parameters as aws-s3:
+    # 
+    #   DataFileManager.new(:access_key_id => 'abc',:secret_access_key => '123')  
+    #
+    # For storing your csv data file on S3, you need to setup a controller to send your uploaded files through this interface:
+    #
+    #   DataFileManager.new(connection_params).store_uploaded_file!(params[:uploaded_file])
+    #
+    # The file will be posted to S3 in a bucket set aside for this gem (it will be created upon connection if it doesn't exist already)
+    # 
+    # When you're ready to iterate over this csv file later in a background job (or wherever), you'll use this:
+    #  
+    #    DataFileManager.new(connection_params).each_row_of(name) {|row| #...whatever processing you need }
+    #   
+    class DataFileManager        
+      attr_reader :bucket
+      
+      def initialize(connect_params)
+        AWS::S3::Base.establish_connection!(connect_params)
+        cache_bucket
+      end  
+      
+      def store_uploaded_file!(uploaded_file)        
+        name = File.basename(uploaded_file.original_filename)   
+        AWS::S3::S3Object.store(name,uploaded_file.read,BUCKET_NAME)    
+        return name
+      end  
+      
+      def each_row_of(file_name) 
+        data = AWS::S3::S3Object.value(file_name,BUCKET_NAME)
+        ::CSV::Reader.parse(data).each{|row| yield row }
+      end
+      
+    private
+      def cache_bucket
+        begin 
+          @bucket = AWS::S3::Bucket.find(BUCKET_NAME)
+        rescue AWS::S3::S3Exception
+          AWS::S3::Bucket.create(BUCKET_NAME)      
+          @bucket = AWS::S3::Bucket.find(BUCKET_NAME)
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'cumulus_csv'
+
+class Test::Unit::TestCase
+  include Cumulus::CSV
+end

data/test/test_data_file_manager.rb

@@ -0,0 +1,64 @@
+require 'helper'
+
+class TestDataFileManager < Test::Unit::TestCase 
+  context "Test" do       
+    setup do
+      nueter_aws!
+      @auth_hash = {:access_key_id => 'abc',:secret_access_key => '123'}
+    end
+  
+    context "Infrastructure" do
+      should "connect to s3 at creation" do
+        AWS::S3::Base.expects(:establish_connection!).with(@auth_hash)
+        DataFileManager.new(@auth_hash)
+      end
+  
+      should "cache cumulus bucket if it already exists" do
+        AWS::S3::Bucket.expects(:create).times(0)
+        AWS::S3::Bucket.stubs(:find).with(BUCKET_NAME).returns(AWS::S3::Bucket.new(:name=>BUCKET_NAME))
+        manager = DataFileManager.new(@auth_hash)  
+        assert_equal AWS::S3::Bucket, manager.bucket.class 
+      end
+  
+      should "create new cumulus bucket if it does not yet exist" do
+        AWS::S3::Bucket.expects(:create).times(1)
+        AWS::S3::Bucket.stubs(:find).with(BUCKET_NAME).raises(AWS::S3::S3Exception).then.returns(AWS::S3::Bucket.new(:name=>BUCKET_NAME))
+        manager = DataFileManager.new(@auth_hash)
+        assert_equal AWS::S3::Bucket, manager.bucket.class
+      end
+    end    
+  
+    context "Uploaded File" do           
+      setup do         
+        @uploaded_file = stub("uploaded_file")
+        @uploaded_file.stubs(:original_filename).returns("/long/path/to/some_name.csv")
+        @uploaded_file.stubs(:read).returns("file data galore")
+      end
+    
+      should "be stored by key" do   
+        manager = DataFileManager.new(@auth_hash)   
+        AWS::S3::S3Object.expects("store").with("some_name.csv","file data galore",BUCKET_NAME)
+        assert_equal "some_name.csv",manager.store_uploaded_file!(@uploaded_file)
+      end
+    end  
+    
+    context "Stored csv file" do    
+      should "be iterated over row by row" do    
+        AWS::S3::S3Object.expects(:value).with("some_name.csv",BUCKET_NAME).returns("1,2,3\nA,B,C\nx,y,z\n")
+        manager = DataFileManager.new(@auth_hash)
+        results = []
+        manager.each_row_of("some_name.csv") do |row|  
+          results << row 
+        end             
+        assert_equal ["1","2","3"],results.first
+        assert_equal ["x","y","z"],results.last
+      end
+    end
+                          
+  end
+  def nueter_aws!  
+    AWS::S3::Base.stubs(:establish_connection!)
+    AWS::S3::Bucket.stubs(:find)
+    AWS::S3::Bucket.stubs(:create)
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification 
+name: cumulus_csv
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- evizitei
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-09 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.8
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.2
+    version: 
+description: |-
+  CSV Files: I hate them, you probably do too, but sometimes you need to get data into your system and this is the only way it's happening.  
+  
+      If you're deploying a rails app in a cloud setup, you may have troubles if you're trying to store an uploaded file locally and process it later in a background thread (I know I have).  
+  
+      cumulus_csv is one way to solve that problem.  You can save your file to your S3 account, and loop over the data inside it at your convenience later.  So it doesn't matter where you're doing the processing, you just need to have the key you used to store the file, and you can process away.
+email: ethan.vizitei@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- cumulus_csv.gemspec
+- lib/cumulus_csv.rb
+- lib/cumulus_csv/data_file_manager.rb
+- test/helper.rb
+- test/test_data_file_manager.rb
+has_rdoc: true
+homepage: http://github.com/evizitei/cumulus_csv
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Helps you save uploaded csv files containing data to amazon s3, and gives you a way to download and loop through the data in a background process easily
+test_files: 
+- test/helper.rb
+- test/test_data_file_manager.rb