sluice 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: c06649f92ec59027bc5d30eb5146ee66b4d2a518
-  data.tar.gz: c28481f3eecd7a60d23e50b5809a7c11d94ae5c0
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzUwNTNjNWE1ZTIyNjhiMjIyN2FjN2UzMjliMjEyNGNlNWI5N2IzYQ==
+  data.tar.gz: !binary |-
+    OGFhOGIyOTUwMmJlMzNmZWU1M2YzYWJkZjZiYTkzN2FmYjc5ZGRmMw==
 SHA512:
-  metadata.gz: 99ead2169b18814ab2f993ae559ae03210bff3d899e06a29371d6fdcc2131bc21f682f538e9b68f0de20ee80c4f7133ab7da82c98dc63a70e64839beae11e89b
-  data.tar.gz: 6f640d9e1caff23b2f0a89ecc90453dc2d7bbb5d769af434570e5fd79048c592a853a7adca50bb40ef4fae616fe87a426abf946c490e282e1231f0e78d681c44
+  metadata.gz: !binary |-
+    MDQxYTFjNjdjZjA1YzZiM2QyMTA2ZDBhMjlmYWIyNjE1OGVmMmViZjA0ZDBj
+    OGE1YzAzYWMxMzc1YThlMDM4M2NmN2JiYWZlMzVjMjg1Mzk3N2E3MGY2NGJh
+    ZDcxNGUwNzI0NDQ4YzQwYzRjYzJlZTlkZDMxN2NjMzBkMTY0M2M=
+  data.tar.gz: !binary |-
+    ZGUzNDA5OTE5MzZiNTlhMDgwMWIyMzBkMDYxMWQ1N2MxZGVjZDE3OWM2NWM3
+    ZmJiNDQyZDM1OWE5NTkyYjMxNjBmOWYyZWI4YzdlZTYzMzRhZTNhOGQ2YTI4
+    ZTA2ODBjMjllNGMwNGRmMDQ5ODRlNDcxN2Y1OWFiMDNhZWYwYWY=

data/.gitignore

@@ -15,3 +15,6 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+
+# Vagrant
+.vagrant

data/CHANGELOG

@@ -1,3 +1,11 @@
+Version 0.2.2 (2015-04-08)
+--------------------------
+Stuck Fog dependency to version 1.24 to prevent AWS Regions issue, thanks @epantera! (#33)
+Now tolerates nil passed to trail_slash (#38)
+Added License button to README (#34)
+Added Vagrant quickstart to README (#36)
+Added dedicated Vagrant setup (#35)
+
 Version 0.2.1 (2014-05-20)
 --------------------------
 Renamed lambda and alter_filename_lambda in rename_file (#27)

data/README.md

@@ -3,6 +3,7 @@
 [![Build Status](https://travis-ci.org/snowplow/sluice.png)](https://travis-ci.org/snowplow/sluice)
 [![Code Climate](https://codeclimate.com/github/snowplow/sluice.png)](https://codeclimate.com/github/snowplow/sluice)
 [![Coverage Status](https://coveralls.io/repos/snowplow/sluice/badge.png?branch=master)](https://coveralls.io/r/snowplow/sluice?branch=master)
+[![License][license-image]][license]
 
 Sluice is a Ruby gem (built with [Bundler] [bundler]) to help you build cloud-friendly ETL (extract, transform, load) processes.
 
@@ -25,34 +26,34 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
 
 Or in your Gemfile:
 
-    gem 'sluice', '~> 0.2.0'
+    gem 'sluice', '~> 0.2.2'
 
 ## Usage
 
 Rubydoc and usage examples to come.
 
-## Hacking and contributing
+## Developer quickstart
 
-To hack on Sluice locally:
+Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed:
 
-    $ gem build sluice.gemspec
-    $ sudo gem install sluice-0.2.0.gem
+```bash
+ host$ git clone https://github.com/snowplow/sluice.git
+ host$ cd sluice
+ host$ vagrant up && vagrant ssh
+guest$ cd /vagrant
+guest$ gem install bundler
+guest$ rspec
+```
 
-To contribute:
+## Publishing
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
-
-## Credits and thanks
-
-Sluice was developed by [Alex Dean] [alexanderdean] ([Snowplow Analytics] [snowplow-analytics]) and [Michael Tibben] [mtibben] ([99designs] [99designs]).
+```bash
+ host$ vagrant push
+```
 
 ## Copyright and license
 
-Sluice is copyright 2012-2014 Snowplow Analytics Ltd.
+Sluice is copyright 2012-2015 Snowplow Analytics Ltd.
 
 Licensed under the [Apache License, Version 2.0] [license] (the "License");
 you may not use this software except in compliance with the License.
@@ -65,6 +66,9 @@ limitations under the License.
 
 [bundler]: http://gembundler.com/
 
+[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
+[license]: http://www.apache.org/licenses/LICENSE-2.0
+
 [snowplow-analytics]: http://snowplowanalytics.com
 [alexanderdean]: https://github.com/alexanderdean
 [mtibben]: https://github.com/mtibben
@@ -73,4 +77,7 @@ limitations under the License.
 [emr-etl-runner]: https://github.com/snowplow/snowplow/tree/master/3-enrich/emr-etl-runner
 [storage-loader]: https://github.com/snowplow/snowplow/tree/master/4-storage/storage-loader
 
+[vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html
+[virtualbox-install]: https://www.virtualbox.org/wiki/Downloads
+
 [license]: http://www.apache.org/licenses/LICENSE-2.0

data/Vagrantfile

@@ -0,0 +1,23 @@
+Vagrant.configure("2") do |config|
+
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.hostname = "sluice"
+  config.ssh.forward_agent = true
+
+  config.vm.provider :virtualbox do |vb|
+    vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
+    vb.memory = 2048
+  end
+
+  config.vm.provision :shell do |sh|
+    sh.path = "vagrant/up.bash"
+  end
+
+  # Requires Vagrant 1.7.0+
+  config.push.define "gem", strategy: "local-exec" do |push|
+    push.script = "vagrant/push.bash"
+  end
+
+end

data/lib/sluice/storage/storage.rb

@@ -38,13 +38,15 @@ module Sluice
     end
     module_function :files_between
 
-    # Add a trailing slash to a path if missing
+    # Add a trailing slash to a path if missing.
+    # Tolerates a nil path.
     #
     # Parameters:
     # +path+:: path to add a trailing slash to
     def trail_slash(path)
-
-      path[-1].chr != '/' ? path << '/' : path
+      unless path.nil?
+        path[-1].chr != '/' ? path << '/' : path
+      end
     end
     module_function :trail_slash
 

data/lib/sluice/version.rb

@@ -15,5 +15,5 @@
 
 module Sluice
   NAME    = "sluice"
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/sluice.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |gem|
 
   # Dependencies
   gem.add_dependency 'contracts', '~> 0.4'
-  gem.add_dependency 'fog', '~> 1.22'
+  gem.add_dependency 'fog', '1.24'
 
   gem.add_development_dependency "rspec", "~> 2.14", ">= 2.14.1"
   gem.add_development_dependency "rspec-nc"

data/vagrant/.gitignore

@@ -0,0 +1,3 @@
+oss-playbooks
+ansible
+.peru

data/vagrant/ansible.hosts

@@ -0,0 +1,2 @@
+[vagrant]
+127.0.0.1:2222

data/vagrant/peru.yaml

@@ -0,0 +1,14 @@
+imports:
+    ansible: ansible
+    ansible_playbooks: oss-playbooks
+
+curl module ansible:
+    # Equivalent of git cloning tags/v1.6.6 but much, much faster
+    url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+    unpack: zip
+    export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+
+git module ansible_playbooks:
+    url: https://github.com/snowplow/ansible-playbooks.git
+    # Comment out to fetch a specific rev instead of master:
+    # rev: xxx

data/vagrant/push.bash

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+
+# Constants
+rubygems_user=snowplow
+rubygems_gem=sluice
+
+# Check if our Vagrant box is running. Expects `vagrant status` to look like:
+#
+# > Current machine states:
+# >
+# > default                   poweroff (virtualbox)
+# >
+# > The VM is powered off. To restart the VM, simply run `vagrant up`
+function running {
+    set +e
+    vagrant status | sed -n 3p | grep -q "^default\s*running (virtualbox)$"
+    local is_running=$?
+    set -e
+    echo $is_running
+}
+
+# Reads the version out of the version.rb
+function parse_version {
+    cat "lib/${rubygems_gem}/version.rb" | awk '/  VERSION =/ {v=$3; gsub(/\047/, "", v)} END {print v}'
+}
+
+# Installs RubyGems.org credentials in our guest
+#
+# Parameters:
+# 1. rubygems_password
+function install_creds {
+    vagrant ssh -c "curl -u ${rubygems_user}:${1} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials"
+    vagrant ssh -c "chmod 0600 ~/.gem/credentials"
+}
+
+# Builds our gem
+function build_gem {
+    vagrant ssh -c "cd /vagrant && gem build ${rubygems_gem}.gemspec"
+}
+
+# Installs our gem
+#
+# Parameters:
+# 1. rubygems_version
+function install_gem {
+    vagrant ssh -c "cd /vagrant && sudo gem install ./${rubygems_gem}-${1}.gem"
+}
+
+# Pushes our gem to RubyGems.org
+#
+# Parameters:
+# 1. rubygems_version
+function push_gem {
+    vagrant ssh -c "cd /vagrant && gem push ./${rubygems_gem}-${1}.gem"
+}
+
+
+# Move to the parent directory of this script
+source="${BASH_SOURCE[0]}"
+while [ -h "${source}" ] ; do source="$(readlink "${source}")"; done
+dir="$( cd -P "$( dirname "${source}" )/.." && pwd )"
+cd ${dir}
+
+# Precondition for running
+if [ $(running) != "0" ]; then
+    echo "Vagrant guest must be running to push"
+    exit 1
+fi
+
+# Can't pass args thru vagrant push so have to prompt
+read -e -p "Please enter password for RubyGems.org user ${rubygems_user}: " rubygems_password
+
+# Build, install & push
+build_gem
+rubygems_version=$(parse_version)
+install_gem ${rubygems_version}
+install_creds ${rubygems_password}
+push_gem ${rubygems_version}

data/vagrant/up.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+vagrant_dir=/vagrant/vagrant
+bashrc=/home/vagrant/.bashrc
+
+echo "========================================"
+echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES"
+echo "----------------------------------------"
+apt-get update
+apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2
+
+echo "==============="
+echo "INSTALLING PERU"
+echo "---------------"
+sudo pip3 install peru
+
+echo "======================================="
+echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU"
+echo "---------------------------------------"
+cd ${vagrant_dir} && peru sync -v
+echo "... done"
+
+env_setup=${vagrant_dir}/ansible/hacking/env-setup
+hosts=${vagrant_dir}/ansible.hosts
+
+echo "==================="
+echo "CONFIGURING ANSIBLE"
+echo "-------------------"
+touch ${bashrc}
+echo "source ${env_setup}" >> ${bashrc}
+echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc}
+echo "... done"
+
+echo "=========================================="
+echo "RUNNING PLAYBOOKS WITH ANSIBLE*"
+echo "* no output while each playbook is running"
+echo "------------------------------------------"
+while read pb; do
+    su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant
+done <${vagrant_dir}/up.playbooks
+
+guidance=${vagrant_dir}/up.guidance
+
+if [ -f ${guidance} ]; then
+    echo "==========="
+    echo "PLEASE READ"
+    echo "-----------"
+    cat $guidance
+fi

data/vagrant/up.guidance

@@ -0,0 +1,5 @@
+To get started:
+vagrant ssh
+cd /vagrant
+bundle install
+rspec

data/vagrant/up.playbooks

@@ -0,0 +1 @@
+oss-playbooks/ruby-rvm.yml

metadata

@@ -1,121 +1,122 @@
 --- !ruby/object:Gem::Specification
 name: sluice
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Alex Dean
 - Michael Tibben
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-20 00:00:00.000000000 Z
+date: 2015-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: contracts
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.4'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.4'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: fog
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.22'
-  prerelease: false
+        version: '1.24'
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.24'
 - !ruby/object:Gem::Dependency
   name: rspec
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.14'
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.14.1
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.14'
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.14.1
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rspec-nc
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: guard
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: guard-rspec
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: coveralls
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses Fog
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses
+  Fog
 email:
 - support@snowplowanalytics.com
 executables: []
@@ -130,6 +131,7 @@ files:
 - Guardfile
 - LICENSE-2.0.txt
 - README.md
+- Vagrantfile
 - lib/sluice.rb
 - lib/sluice/errors.rb
 - lib/sluice/storage/s3/contracts.rb
@@ -142,28 +144,35 @@ files:
 - spec/spec_helper.rb
 - spec/storage/s3/location_spec.rb
 - spec/storage/s3/s3_spec.rb
+- vagrant/.gitignore
+- vagrant/ansible.hosts
+- vagrant/peru.yaml
+- vagrant/push.bash
+- vagrant/up.bash
+- vagrant/up.guidance
+- vagrant/up.playbooks
 homepage: http://snowplowanalytics.com
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.2.2
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
 specification_version: 4
 summary: Ruby toolkit for cloud-friendly ETL
 test_files: