s3up 0.1.0

data/bin/s3up

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 's3up'
+
+options = Trollop::options do
+  opt :verbose, "verbose logging"
+  opt :'generate-manifest', "create a json manifest file"
+  opt :dump, "show the manifest output"
+  opt :folder, "folders to include", :default => '**'
+  opt :types, "file types to include", :default => 'jpg,mp3'
+  opt :manifest, 'specify file name for the json manifest', :default => 'manifest.json', :type => :string
+  opt :bucket, 'destination bucket', :type => :string
+  opt :prefix, 'destination key prefix', :type => :string
+  opt :id, 'aws access key id', :type => :string
+  opt :key, 'aws secret access key', :type => :string
+  opt :threads, 'number of upload threads', :default => 5, :type => :int
+  opt :force, 'force upload'
+end
+
+s3up = S3up.new( options )
+
+if options[ :'generate-manifest' ]
+  s3up.generate_manifest
+else
+  s3up.upload
+end

data/lib/s3up.rb

@@ -0,0 +1,78 @@
+require 'rubygems'
+require 'aws'
+require 'work_queue'
+require 'mime/types'
+require 'json'
+require 'yaml'
+
+class S3up
+
+  def initialize( options = {} )
+    setup_options( options )
+    @work_queue = WorkQueue.new( @options[ :threads ] )
+    glob = "#{ @options[ :folder ] }/*.{#{ @options[ :types ] }}"
+    @files = Dir.glob( glob, File::FNM_CASEFOLD )
+  end
+
+  def overwrite_options( options )
+    options.each { | key, value | @options[ key.to_sym ] = value if value }
+  end
+
+  def load_options( file )
+    if File.exists?( File.expand_path( file ) )
+      options = YAML.load_file( File.expand_path( file ) )
+      overwrite_options( options )
+    end
+  end
+
+  def setup_options( options )
+    @options = {}
+    load_options( '~/.s3uprc' )
+    load_options( './.s3uprc' )
+    overwrite_options( options )
+    puts @options if @options[ :verbose ]
+  end
+
+  def upload_file( bucket_name, filename, key_name = filename )
+    s3 = Aws::S3.new( @options[ :id ], @options[ :key ], :connection_mode  => :per_thread )
+    bucket = s3.bucket( bucket_name )
+    key = Aws::S3::Key.create( bucket ,key_name )
+    key.put( File.open( filename ).read, 'public-read' )
+    puts "uploaded: #{ key.full_name }" if @options[ :verbose ]
+  rescue Exception => e
+    puts e
+  end
+
+  def create_bucket( bucket_name )
+    s3 = Aws::S3.new( @options[ :id ], @options[ :key ] )
+    bucket = s3.bucket( bucket_name, true, 'public-read' )
+  rescue Exception => e
+    puts e
+  end
+  
+  def upload
+    create_bucket( @options[ :bucket ] )
+    @files.each do | file |
+      @work_queue.enqueue_b {
+        dest_key = "#{ @options[ :prefix ] }#{ file }"
+        upload_file( @options[ :bucket ], file, dest_key )
+      }
+    end
+    @work_queue.join
+  end
+
+  def generate_manifest
+    items = @files.map do | file |
+      stat = File.stat( file )
+      {
+        :uri => "http://#{ @options[ :bucket ] }.s3.amazonaws.com/#{ @options[ :prefix ] }#{ file }",
+        :size => stat.size,
+        :type => MIME::Types.of( file ).first
+      }
+    end
+    File.open( @options[ :manifest ], 'w' ) do | file |
+      file.puts( JSON.pretty_generate( { :objects => items } ) )
+    end
+  end
+
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: s3up
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- James Prudente
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-01-11 00:00:00.000000000 -08:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: &2153277080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153277080
+- !ruby/object:Gem::Dependency
+  name: work_queue
+  requirement: &2153276520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153276520
+- !ruby/object:Gem::Dependency
+  name: aws
+  requirement: &2153275920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153275920
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: &2153275360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153275360
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2153274720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153274720
+description: Directory recursive multithreaded s3 uploader
+email: opensource@dashwire.com
+executables:
+- s3up
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/s3up.rb
+- bin/s3up
+has_rdoc: true
+homepage: http://rubygems.org/gems/s3up
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: S3 uploader
+test_files: []