hydro 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d7df05d40327ea1fa200dd9deecaf9f741f4b320
+  data.tar.gz: 9dd5896a15f7c8d78ea7e72b693439957c6cc37d
+SHA512:
+  metadata.gz: 3469bee36b9ebb2fc23714955f6acaca5b1e6913c9a98098237683e34e688a8e6edc540d8754dc758274efbb731332e3d600dc83cd029a8f8aa171724cb42de5
+  data.tar.gz: 2e692f636a1eaa6fb3570df868a82a4bd16c0b654c03c931ec4f563568e66780c2d9698ed87e999c43bd68e486f4f22965f0f79dda0ff1a7afaa17276758e8f5

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+*.pid

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hydro.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 James Dabbs
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Hydro
+
+Packages a few simple utilities for cycling data up and down from the cloud
+
+## Installation and Configuration
+
+Install with `gem install hydro`
+
+Expects a configuration file in `/etc/hydro/config.yml`
+
+```yml
+---
+s3: {
+  id:     ...,
+  key:    ...,
+  bucket: ...
+},
+log: ...,
+rules: [
+  { to: ..., ext: ... }
+]
+```
+
+## Usage
+
+`evaporate [file]` - uploads file to bucket
+`precipitate` - downloads and sorts files from bucket
+`precipitated [start|stop|...]` - daemon version of `precipitate`
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/hydro/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/evaporate

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'hydro'
+
+path = ARGV.shift || raise("Provide a file to upload")
+
+precipitator = Hydro::Precipitator.new
+precipitator.upload_file path

data/bin/precipitate

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'hydro'
+
+precipitator = Hydro::Precipitator.new
+precipitator.poll 60

data/bin/precipitated

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+Daemons.run File.expand_path '../precipitate', __FILE__

data/hydro.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hydro/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hydro"
+  spec.version       = Hydro::VERSION
+  spec.authors       = ["James Dabbs"]
+  spec.email         = ["jamesdabbs@gmail.com"]
+  spec.summary       = %q{Simple utilities for cycling files through the cloud.}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/jamesdabbs/hydro"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_dependency "aws-sdk", "~> 2"
+  spec.add_dependency "daemons"
+end

data/lib/hydro/precipitator.rb

@@ -0,0 +1,58 @@
+module Hydro
+  class Precipitator
+    def initialize
+      path = "/etc/hydro/config.yml"
+      raise "Could not find config file at #{path}" unless File.exists? path
+      conf = YAML.load_file path
+
+      @logger = Logger.new conf.fetch "log"
+
+      s3 = conf.fetch "s3"
+      @client = Aws::S3::Client.new(
+        region: 'us-east-1',
+        credentials: Aws::Credentials.new(s3.fetch("id"), s3.fetch("key"))
+      )
+      @bucket = Aws::S3::Bucket.new client: @client, name: s3.fetch("bucket")
+
+      @rules = conf.fetch("rules").map do |rule|
+        Rule.new to: rule.fetch("to"), ext: rule.fetch("ext")
+      end
+    end
+
+    def upload_file path
+      raise "Can't upload `#{path}`: file not found" unless File.exists? path
+      @logger.info "Uploading #{path} => #{@bucket.name}"
+      object = @bucket.object(File.basename path)
+      object.put body: File.open(path)
+    end
+
+    def download_file object, path
+      @logger.info "Downloading #{object.key} => #{path}"
+      File.open path, "w" do |f|
+        object.get { |chunk| f.write chunk }
+      end
+      object.delete
+    end
+
+    def download_by_rules
+      @bucket.objects.each do |object|
+        if rule = @rules.find { |r| r.matches? object }
+          download_file object, rule.location_for(object)
+        end
+      end
+    end
+
+    def poll frequency
+      @logger.info "Checking for new uploads every #{frequency} seconds"
+
+      at_exit do
+        @logger.info "Halting checks"
+      end
+
+      loop do
+        download_by_rules
+        sleep frequency
+      end
+    end
+  end
+end

data/lib/hydro/rule.rb

@@ -0,0 +1,22 @@
+module Hydro
+  class Rule
+    attr_reader :ext, :to
+
+    def initialize ext:, to:
+      @ext, @to = ext, to
+      FileUtils.mkdir_p @to
+    end
+
+    def matches? object
+      if ext
+        object.key.end_with? ext
+      else
+        true
+      end
+    end
+
+    def location_for object
+      "#{to}/#{object.key}"
+    end
+  end
+end

data/lib/hydro/version.rb

@@ -0,0 +1,3 @@
+module Hydro
+  VERSION = "0.0.1"
+end

data/lib/hydro.rb

@@ -0,0 +1,11 @@
+require "aws-sdk"
+require "fileutils"
+require "logger"
+require "yaml"
+
+require "hydro/precipitator"
+require "hydro/rule"
+require "hydro/version"
+
+module Hydro
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: hydro
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- James Dabbs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple utilities for cycling files through the cloud.
+email:
+- jamesdabbs@gmail.com
+executables:
+- evaporate
+- precipitate
+- precipitated
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/evaporate
+- bin/precipitate
+- bin/precipitated
+- hydro.gemspec
+- lib/hydro.rb
+- lib/hydro/precipitator.rb
+- lib/hydro/rule.rb
+- lib/hydro/version.rb
+homepage: https://github.com/jamesdabbs/hydro
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Simple utilities for cycling files through the cloud.
+test_files: []