aws_pipes 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aws_pipes.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Joe Nelson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+Send messages between Amazon EC2 instances through Unix pipes.
+
+This gem is built on top of the Amazon [Simple Queue
+Service](http://aws.amazon.com/sqs/) (SQS) which lets you
+
+- Move data between distributed components of your application without
+  losing messages or requiring each component to be always available.
+- Get started with no extra installed software or special firewall
+  configurations.
+- Connect machines on different networks, developed with different
+  technologies, and running at different times.
+- Save messages in the queue for up to 14 days.
+
+Text is the universal interface, and any application that can read and
+write text can use this gem – no knowledge of the Amazon API is
+required.
+
+## Usage
+
+    # write data to an SQS queue named "foo"
+    your_program | aws_queue write foo
+    
+    # read data from an SQS queue named "foo"
+    aws_queue read foo | your_program
+
+To use this program you will need to [create a
+queue](https://console.aws.amazon.com/sqs/) in the Amazon Web Console.
+
+## Installation
+
+1. Sign up for an [AWS account](http://aws.amazon.com/).
+1. Find your secret key and key id in *My Account* > *Security Credentials*.
+1. (optionally) Set your environment variables AWS_ACCESS_KEY_ID, and
+   AWS_ACCESS_KEY accordingly.
+1. Run `gem install aws_pipes` from the command line.
+
+This will install the `aws_queue` command to your path. If you haven't
+stored your Amazon credentials in environment variables, you can pass
+them in as command line options. For more info, run
+
+    aws_queue --help
+
+## Examples
+
+### Downloading a massive list of urls in parallel.
+
+One computer can feed a list of urls to workers which download them.
+Suppose the urls are stored in `urls.txt`. Just redirect the file into a
+queue:
+
+    aws_queue write to_be_downloaded < urls.txt
+
+Then have each worker pull from the `to_be_downloaded` queue and
+repeatedly run a command to download each url. The queue supports many
+simultaneous readers and prevents duplicate work.
+
+    aws_queue read to_be_downloaded | xargs -L1 wget

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/aws_pipes.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aws_pipes/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "aws_pipes"
+  gem.version       = AwsPipes::VERSION
+  gem.authors       = ["Joe Nelson"]
+  gem.email         = ["cred+github@begriffs.com"]
+  gem.description   = %q{Send messages between Amazon EC2 instances through Unix pipes.}
+  gem.summary       = %q{AWS queues à la Unix}
+  gem.homepage      = "https://github.com/begriffs/aws_pipes"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "aws-sdk"
+  gem.add_dependency "trollop"
+end

data/bin/aws_queue

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'aws-sdk'
+require_relative '../lib/aws_pipes/version'
+
+## USAGE ###########################################################################
+
+opts = Trollop::options do
+  version "aws_queue #{AwsPipes::VERSION} (c) 2013 Joe Nelson"
+  banner <<-EOS
+Read and write Amazon SQS through unix pipes.
+
+You may provide Amazon authentication through these environment variables:
+AWS_ACCESS_KEY_ID - AWS access key id
+AWS_ACCESS_KEY    - AWS secret access key
+
+Usage:
+       aws_queue [options] write <queue-name>
+       Send STDIN to an Amazon SQS queue, one message per line.
+
+       aws_queue [options] read <queue-name>
+       Print messages from Amazon SQS queue to STDOUT, one message per line.
+
+where [options] are:
+EOS
+
+  opt :keyid, "AWS access key id", :type => :string
+  opt :key, "AWS secret access key", :type => :string
+end
+
+## SETUP ###########################################################################
+
+access_key_id     = ENV['AWS_ACCESS_KEY_ID'] || opts[:keyid] || Trollop::die("Missing access key id")
+secret_access_key = ENV['AWS_ACCESS_KEY']    || opts[:key]   || Trollop::die("Missing secret access key")
+sqs = AWS::SQS.new(
+  :access_key_id     => access_key_id,
+  :secret_access_key => secret_access_key
+)
+
+operation = ARGV.shift
+unless %w(read write).include? operation
+  Trollop::die "Unknown operation \"#{operation}\" -- expecting \"read\" or \"write\""
+end
+
+q_name = ARGV.shift
+Trollop::die "Missing queue name" unless q_name
+
+## OPERATION #######################################################################
+
+begin
+  q = sqs.queues.named q_name
+  case operation
+  when 'read'
+    $stdout.sync = true
+    q.poll do |msg|
+      puts msg.body
+    end
+  when 'write'
+    $stdin.sync = true
+    while msg = gets
+      q.send_message msg
+    end
+  end
+rescue AWS::SQS::Errors::NonExistentQueue => e
+  warn e
+  exit 1
+end

data/lib/aws_pipes/version.rb

@@ -0,0 +1,3 @@
+module AwsPipes
+  VERSION = "0.0.1"
+end

data/lib/aws_pipes.rb

@@ -0,0 +1,5 @@
+require "aws_pipes/version"
+
+module AwsPipes
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: aws_pipes
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Joe Nelson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Send messages between Amazon EC2 instances through Unix pipes.
+email:
+- cred+github@begriffs.com
+executables:
+- aws_queue
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- aws_pipes.gemspec
+- bin/aws_queue
+- lib/aws_pipes.rb
+- lib/aws_pipes/version.rb
+homepage: https://github.com/begriffs/aws_pipes
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: AWS queues à la Unix
+test_files: []