s3-object-processor 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+
+gem "cli", "~> 1.3"
+gem "right_aws", "~> 3.0"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.8.0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 1.8.7"
+end

data/Gemfile.lock

@@ -0,0 +1,69 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    builder (3.2.2)
+    cli (1.3.1)
+    diff-lcs (1.1.3)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    git (1.2.6)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.20)
+    httpauth (0.2.0)
+    jeweler (1.8.8)
+      builder
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    multi_json (1.8.2)
+    multi_xml (0.5.5)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.1.0)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    right_aws (3.1.0)
+      right_http_connection (>= 1.2.5)
+    right_http_connection (1.4.0)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  cli (~> 1.3)
+  jeweler (~> 1.8.7)
+  rdoc (~> 3.12)
+  right_aws (~> 3.0)
+  rspec (~> 2.8.0)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Jakub Pastuszek
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# s3-object-processor
+
+This library tries to help in development of CLI programs that can process all objects (or from list file) stored in given S3 bucket.
+
+It is using multi-threaded worker model to allow for processing parallelism.
+
+# Example usage
+
+This example program can be used to call one of two HTTP endpoints for each object that matches a regexp and some other criteria.
+In addition total handled objects size is counted and time spent on getting S3 object data and spend on API call is measured and reported at the end of the run.
+
+```ruby
+require 's3-object-processor/cli'
+require 'httpclient'
+
+S3ObjectProcessor::CLI.new do
+	cli do
+		option :endpoint,
+			description: 'API endpoint URI for JPEG uploads',
+			default: '/iss/v2/pictures'
+		option :endpoint_as_is,
+			description: 'API endpoint URI for as-is uploads',
+			default: '/iss/v2/images'
+		switch :as_is,
+			description: 'upload images without conversion to JPEG'
+		option :httpimagestore,
+			description: 'URL to HTTP Image Store',
+			default: 'http://localhost:3000'
+	end
+
+	cli_process do |settings|
+	end
+
+	report :input_object_size, 0 do
+		report "total input object size [KiB]", "%d" do |value|
+			(value.to_f / 1024).round
+		end
+	end
+	report :s3_body_get_time, 0.0 do
+		report "total S3 get body time [s]", "%.3f"
+	end
+	report :httpimagestore_time, 0.0 do
+		report "total ISS request time [s]", "%.3f"
+	end
+
+	processor do |bucket, key, settings, log, reporter|
+		unless key.to_s =~ %r{(^|.*?/)([0-f]{16})(|/.*)\.(.{3,4})$}
+			log.warn "skipping bad format: #{key}"
+			reporter.report :skipped_key, key
+			next
+		end
+
+		dir = $1
+		hash = $2
+		name = $3
+		extension = $4
+
+		if name =~ /-(search|original|search_thumb|brochure|brochure_thumb|admin|admin_thumb|treatment_thumb|staff_member_thumb|consultation|clinic_google_map_thumb)$/
+			log.debug "skipping not original: #{key}"
+			reporter.report :skipped_key, key
+			next
+		end
+
+		log.debug "processing original dir: '#{dir}' hash: '#{hash}' name: '#{name}' extension: '#{extension}'"
+
+		data = nil
+		reporter.time :s3_body_get_time do
+			data = key.data
+		end
+		fail "no data for key; key not found?!" unless data
+		reporter.report :input_object_size, data.length
+
+		if settings.noop
+			reporter.report :noop_key, key
+			next
+		end
+
+		reporter.time :httpimagestore_time do
+			if settings.as_is
+				response = HTTPClient.put(settings.httpimagestore + settings.endpoint_as_is + "/#{hash}.#{extension}", data)
+			else
+				response = HTTPClient.put(settings.httpimagestore + settings.endpoint + "/#{hash}.jpg", data)
+			end
+			fail "bad HTTP Image Store response: #{response.status}: #{response.body}" if response.status != 200
+		end
+		reporter.report :handled_key, key
+	end
+end
+```
+
+## Contributing to s3-object-processor
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2013 Jakub Pastuszek. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "s3-object-processor"
+  gem.homepage = "http://github.com/jpastuszek/s3-object-processor"
+  gem.license = "MIT"
+  gem.summary = "S3 key-by-kye processor builder"
+  gem.description = "DSL tools for building programs that can process S3 object key-by-key using threaded worker pool"
+  gem.email = "jpastuszek@gmail.com"
+  gem.authors = ["Jakub Pastuszek"]
+  # dependencies defined in Gemfile
+
+	gem.files.select{|f| f =~ /^.idea/}.each do |file|
+		gem.files.exclude file
+	end
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "s3-object-processor #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/s3-object-processor.rb

@@ -0,0 +1,360 @@
+require 'thread'
+Thread.abort_on_exception = true
+
+class Runnable
+	def on_finish(&callback)
+		(@on_finish ||= []) << callback
+		self
+	end
+
+	def run
+		@thread = Thread.new do
+			begin
+				yield
+			rescue Interrupt
+				# ignored
+			ensure
+				begin
+					@on_finish.each{|on_finish| on_finish.call} if @on_finish
+				rescue
+					# ignored
+				end
+			end
+		end
+		self
+	end
+
+	def join
+		@thread.join if @thread
+	end
+end
+
+class Lister < Runnable
+	def initialize(bucket, key_queue, fetch_size, max_keys = nil)
+		@bucket = bucket
+		@key_queue = key_queue
+		@fetch_size = fetch_size
+		@max_keys = max_keys
+	end
+
+	def on_keys_chunk(&callback)
+		@on_keys_chunk = callback
+		self
+	end
+
+	def run(prefix = nil)
+		super() do
+			catch :done do
+				marker = ''
+				total_keys = 0
+				loop do
+					keys_chunk = @bucket.keys(prefix: prefix, 'max-keys' => @fetch_size, marker: marker)
+					break if keys_chunk.empty?
+					@on_keys_chunk.call(keys_chunk) if @on_keys_chunk
+					keys_chunk.each do |key|
+						throw :done if @max_keys and total_keys >= @max_keys
+						@key_queue << key
+						total_keys += 1
+					end
+					marker = keys_chunk.last.name
+				end
+			end
+		end
+	end
+end
+
+class ListLister < Runnable
+	def initialize(bucket, key_queue, max_keys = nil)
+		@bucket = bucket
+		@key_queue = key_queue
+		@max_keys = max_keys
+	end
+
+	def on_keys_chunk(&callback)
+		@on_keys_chunk = callback
+		self
+	end
+
+	def run(list)
+		super() do
+			total_keys = 0
+			@on_keys_chunk.call(list) if @on_keys_chunk
+			list.each do |key|
+				@key_queue << @bucket.key(key)
+				break if @max_keys and total_keys >= @max_keys
+				total_keys += 1
+			end
+		end
+	end
+end
+
+class Worker < Runnable
+	def initialize(no, key_queue, &process_key)
+		@no = no
+		@key_queue = key_queue
+		@process_key = process_key
+	end
+
+	def on_error(&callback)
+		@on_error = callback
+		self
+	end
+
+	def run
+		super do
+			until (key = @key_queue.pop) == :end
+				begin
+					@process_key.call(key)
+				rescue => error
+					@on_error.call(key, error) if @on_error
+				end
+			end
+		end
+	end
+end
+
+class Reporter < Runnable
+	class Report
+		class DSL
+			attr_reader :update_callback, :description, :value_pattern, :value_processor
+
+			def initialize(&setup)
+				@update_callback = ->(t,v){t + v}
+				@value_processor = ->(v){v}
+				instance_eval &setup
+			end
+
+			def update(&callback)
+				@update_callback = callback
+			end
+
+			def report(description, value_pattern, &value_processor)
+				@description = description
+				@value_pattern = value_pattern
+				@value_processor = value_processor if value_processor
+			end
+		end
+
+		def initialize(init_value, &setup)
+			@dsl = DSL.new(&setup)
+			@value = init_value
+		end
+
+		def update(value)
+			@value = @dsl.update_callback.call(@value, value)
+		end
+
+		def value
+			@dsl.value_pattern % [@dsl.value_processor.call(@value)]
+		end
+
+		def description
+			@dsl.description
+		end
+
+		def to_s
+			"#{description}: #{value.rjust(6)}"
+		end
+
+		def final
+			"#{description}: ".ljust(40) + value
+		end
+	end
+
+	def initialize(queue_size, &callback)
+		@report_queue = SizedQueue.new(queue_size)
+		@processor = callback
+
+		on_finish do
+			# flush thread waiting on queue
+			@report_queue.max = 999999
+		end
+	end
+
+	def run
+		super do
+			@processor.call(self) if @processor
+			until (report = @report_queue.pop) == :end
+				@sink.call(*report) if @sink
+			end
+		end
+	end
+
+	def each(&callback)
+		@sink = callback
+	end
+
+	def report(key, value)
+		@report_queue << [key, value]
+	end
+
+	def time(key)
+		time = Time.now.to_f
+		yield
+		report key, Time.now.to_f - time
+	end
+
+	def join
+		@report_queue << :end
+		super
+	end
+end
+
+class BucketProcessor
+	def initialize(key_id, key_secret, bucket, options = {}, &callback)
+		protocol = options[:no_https] ? 'http' : 'https'
+		port = options[:no_https] ? 80 : 443
+		@log = options[:log] || Logger.new(STDERR)
+		workers = options[:workers] || 10
+		lister_fetch_size = options[:lister_fetch_size] || 200
+		lister_backlog = options[:lister_backlog] || 1000
+		max_keys = options[:max_keys]
+	 	reporter_backlog = options[:reporter_backlog] || 1000
+		reporter_summary_interval = options[:reporter_summary_interval] || 100
+		reporter_average_contribution = options[:reporter_average_contribution] || 0.10
+		custom_reports = options[:reports] || []
+		@key_list = options[:key_list]
+
+		s3 = RightAws::S3.new(key_id, key_secret, multi_thread: true, logger: @log, protocol: protocol, port: port)
+		bucket = s3.bucket(bucket)
+
+		@key_queue = SizedQueue.new(lister_backlog)
+
+		@reporter = Reporter.new(reporter_backlog) do |reports|
+			total_listed_keys = 0
+			total_processed_keys = 0
+			total_succeeded_keys = 0
+			total_failed_keys = 0
+			total_handled_keys = 0
+			total_skipped_keys = 0
+			total_nooped_keys = 0
+
+			processed_avg = 0.0
+			last_time = nil
+			last_total = 0
+
+			reports.each do |key, value|
+				case key
+				when :new_keys_count
+					total_listed_keys += value
+				when :processed_key
+					total_processed_keys += 1
+					if total_processed_keys % reporter_summary_interval == 0
+						if last_time
+							contribution = reporter_average_contribution
+							new = (total_processed_keys - last_total).to_f / (Time.now.to_f - last_time)
+							processed_avg = processed_avg * (1.0 - contribution) + new * contribution
+						end
+						last_time = Time.now.to_f
+						last_total = total_processed_keys
+
+						log_line = "-[%s]- processed %6d: failed: %6d (%6.2f %%) handled: %6d skipped: %6d (%6.2f %%)" % [
+							value.to_s[0...2].ljust(2),
+							total_processed_keys,
+							total_failed_keys,
+							total_failed_keys.to_f / total_processed_keys * 100,
+							total_handled_keys,
+							total_skipped_keys,
+							total_skipped_keys.to_f / total_processed_keys * 100
+						]
+						log_line << custom_reports.each_value.map{|v| ' ' + v.to_s}.join
+						log_line << " [backlog: %4d] @ %.1f op/s" % [
+							@key_queue.size,
+							processed_avg
+						]
+
+						@log.info log_line
+					end
+				when :succeeded_key
+					total_succeeded_keys += 1
+				when :failed_key
+					key, error = *value
+					@log.error "Key processing failed: #{key}: #{error.class.name}, #{error.message}"
+					total_failed_keys += 1
+				when :handled_key
+					total_handled_keys += 1
+				when :skipped_key
+					total_skipped_keys += 1
+				when :noop_key
+					total_nooped_keys += 1
+				else
+					#@log.debug "custom report event: #{key}: #{value}"
+					custom_reports[key].update(value)
+				end
+				#@log.debug("Report: #{key}: #{value}")
+			end
+
+			reports.on_finish do
+				@log.info("total listed keys:                      #{total_listed_keys}")
+				@log.info("total processed keys:                   #{total_processed_keys}")
+				@log.info("total succeeded keys:                   #{total_succeeded_keys}")
+				@log.info("total failed keys:                      #{total_failed_keys}")
+				@log.info("total handled keys:                     #{total_handled_keys}")
+				@log.info("total skipped keys:                     #{total_skipped_keys}")
+				@log.info("total nooped keys:                      #{total_nooped_keys}")
+				custom_reports.each_value do |report|
+					@log.info report.final
+				end
+			end
+		end
+
+		# create lister
+		@lister = if @key_list
+			@log.info "processing #{@key_list.length} keys from list file"
+			@lister = ListLister.new(bucket, @key_queue, max_keys)
+		else
+			@lister = Lister.new(bucket, @key_queue, lister_fetch_size, max_keys)
+		end
+		.on_keys_chunk do |keys_chunk|
+			@log.debug "Got #{keys_chunk.length} new keys"
+			@reporter.report(:new_keys_count, keys_chunk.length)
+		end
+		.on_finish do
+			@log.debug "Done listing keys"
+			# notify all workers that no more messages will be posted
+			workers.times{ @key_queue << :end }
+		end
+
+		# create workers
+		@log.info "Launching #{workers} workers"
+		@workers = (1..workers).to_a.map do |worker_no|
+			Worker.new(worker_no, @key_queue) do |key|
+				@log.debug "Worker[#{worker_no}]: Processing key #{key}"
+				yield bucket, key, @reporter
+				@reporter.report :processed_key, key
+				@reporter.report :succeeded_key, key
+			end
+			.on_error do |key, error|
+				@reporter.report :processed_key, key
+				@reporter.report :failed_key, [key, error]
+			end
+			.on_finish do
+				@log.debug "Worker #{worker_no} done"
+			end
+		end
+	end
+
+	def run(prefix = nil)
+		begin
+			@reporter.run
+			if @key_list
+				@lister.run(@key_list)
+			else
+				@lister.run(prefix)
+			end
+			@workers.each(&:run)
+
+			# wait for all to finish
+			@workers.each(&:join)
+			@log.info "All workers done"
+
+			@lister.join
+			@reporter.join
+		rescue Interrupt => error
+			@log.warn 'Interrupted'
+			# flush thread waiting on queues
+			@key_queue.max = 999999
+			@reporter.join
+		end
+	end
+end

data/lib/s3-object-processor/cli.rb

@@ -0,0 +1,137 @@
+require 's3-object-processor'
+require 'cli'
+require 'logger'
+require 'right_aws'
+require 'time'
+
+module S3ObjectProcessor
+	class CLI
+		def initialize(&config)
+			@reports = {}
+			instance_eval &config
+
+			cli_setup = @cli_setup
+			cli_process_setup = @cli_process_setup
+
+			settings = ::CLI.new do
+				description 'Set header of S3 object'
+
+				option :key_id,
+					short: :i,
+					description: 'AWS access key ID',
+					default_label: 'AWS_SECRET_KEY_ID environment variable',
+					default: ENV['AWS_ACCESS_KEY_ID'],
+					required: true
+				option :key_secret,
+					short: :s,
+					description: 'AWS access key secret',
+					default_label: 'AWS_SECRET_ACCESS_KEY environment variable',
+					default: ENV['AWS_SECRET_ACCESS_KEY'],
+					required: true
+				switch :no_https,
+					description: 'use plain HTTP S3 connections'
+
+				option :bucket,
+					short: :b,
+					description: 'bucket to process',
+					required: true
+				option :prefix,
+					short: :p,
+					description: 'process only objects of key starting with given prefix'
+
+				option :lister_fetch_size,
+					description: 'fetch no more that that number of keys per request',
+					cast: Integer,
+					default: 200
+				option :lister_backlog,
+					description: 'maximum length of to be processed key queue',
+					cast: Integer,
+					default: 1000
+				option :key_list,
+					description: 'file with keys to process (one per line)',
+					default_label: 'process all keys in S3 bucket',
+					cast: Pathname
+
+				option :reporter_backlog,
+					description: 'maximum length of to be processed report queue',
+					cast: Integer,
+					default: 1000
+				option :reporter_summary_interval,
+					description: 'pring summary every some number of processed objects',
+					cast: Integer,
+					default: 100
+				option :reporter_average_contribution,
+					description: 'how much does last average calculation contribute in the printed value - less => more stable',
+					cast: Float,
+					default: 0.10
+
+				option :workers,
+					short: :t,
+					description: 'number of processing threads to start',
+					cast: Integer,
+					default: 10
+
+				switch :noop,
+					short: :n,
+					description: 'do not change any object; just say what would be done'
+
+				switch :debug,
+					short: :d,
+					description: 'log at DEBUG level'
+
+				option :max_keys,
+					description: 'stop after processing this amout of keys',
+					cast: Integer
+
+				instance_eval &cli_setup if cli_setup
+			end.parse! do |settings|
+				instance_eval &cli_process_setup if cli_process_setup
+			end
+
+			log = Logger.new(STDERR)
+			log.level = settings.debug ? Logger::DEBUG : Logger::INFO
+
+			log.debug(settings.inspect)
+
+			trap 'QUIT' do
+				Thread.list.each do |thread|
+					STDERR.puts "Thread-#{thread.object_id.to_s(36)}"
+					STDERR.puts thread.backtrace.join("\n    \\_ ")
+				end
+			end
+
+			BucketProcessor.new(settings.key_id, settings.key_secret, settings.bucket,
+				no_https: settings.no_https,
+				log: log,
+				workers: settings.workers,
+				max_keys: settings.max_keys,
+				lister_fetch_size: settings.lister_fetch_size,
+				lister_backlog: settings.lister_backlog,
+				key_list: settings.key_list && settings.key_list.readlines.map(&:strip),
+				reporter_backlog: settings.reporter_backlog,
+				reporter_summary_interval: settings.reporter_summary_interval,
+				reporter_average_contribution: settings.reporter_average_contribution,
+				reports: @reports
+			) do |bucket, key, reporter|
+				@processor.call(bucket, key, settings, log, reporter)
+			end
+			.run(settings.prefix)
+		end
+
+		def cli(&setup)
+			@cli_setup = setup
+		end
+
+		def cli_process(&setup)
+			@cli_process_setup = setup
+		end
+
+		def report(name, init_value, &setup)
+			@reports[name] = Reporter::Report.new(init_value, &setup)
+		end
+
+		def processor(&callback)
+			@processor = callback
+		end
+	end
+end

data/s3-object-processor.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "s3-object-processor"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jakub Pastuszek"]
+  s.date = "2013-10-29"
+  s.description = "DSL tools for building programs that can process S3 object key-by-key using threaded worker pool"
+  s.email = "jpastuszek@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "lib/s3-object-processor.rb",
+    "lib/s3-object-processor/cli.rb",
+    "s3-object-processor.gemspec",
+    "spec/s3-object-processor_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/jpastuszek/s3-object-processor"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.25"
+  s.summary = "S3 key-by-kye processor builder"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<cli>, ["~> 1.3"])
+      s.add_runtime_dependency(%q<right_aws>, ["~> 3.0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.7"])
+    else
+      s.add_dependency(%q<cli>, ["~> 1.3"])
+      s.add_dependency(%q<right_aws>, ["~> 3.0"])
+      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.7"])
+    end
+  else
+    s.add_dependency(%q<cli>, ["~> 1.3"])
+    s.add_dependency(%q<right_aws>, ["~> 3.0"])
+    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.7"])
+  end
+end
+

data/spec/s3-object-processor_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "S3ObjectProcessor" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 's3_object_processor'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: s3-object-processor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Jakub Pastuszek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cli
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: right_aws
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.7
+description: DSL tools for building programs that can process S3 object key-by-key
+  using threaded worker pool
+email: jpastuszek@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- .document
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- lib/s3-object-processor.rb
+- lib/s3-object-processor/cli.rb
+- s3-object-processor.gemspec
+- spec/s3-object-processor_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/jpastuszek/s3-object-processor
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3595802382691813695
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: S3 key-by-kye processor builder
+test_files: []