sluice-jason 0.0.1

data/lib/sluice/storage/storage.rb

@@ -0,0 +1,111 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+module Sluice
+  module Storage
+
+    # To handle negative file matching
+    NegativeRegex = Struct.new(:regex)  
+
+    # Find files within the given date range
+    # (inclusive).
+    #
+    # Parameters:
+    # +start_date+:: start date
+    # +end_date+:: end date
+    # +date_format:: format of date in filenames
+    # +file_ext:: extension on files (if any)   
+    def files_between(start_date, end_date, date_format, file_ext=nil)
+
+      dates = []
+      Date.parse(start_date).upto(Date.parse(end_date)) do |day|
+        dates << day.strftime(date_format)
+      end
+
+      '(' + dates.join('|') + ')[^/]+%s$' % regexify(file_ext)
+    end
+    module_function :files_between
+
+    # Add a trailing slash to a path if missing.
+    # Tolerates a nil path.
+    #
+    # Parameters:
+    # +path+:: path to add a trailing slash to
+    def trail_slash(path)
+      unless path.nil?
+        path[-1].chr != '/' ? path << '/' : path
+      end
+    end
+    module_function :trail_slash
+
+    # Find files up to (and including) the given date.
+    #
+    # Returns a regex in a NegativeRegex so that the
+    # matcher can negate the match.
+    #
+    # Parameters:
+    # +end_date+:: end date
+    # +date_format:: format of date in filenames
+    # +file_ext:: extension on files (if any)  
+    def files_up_to(end_date, date_format, file_ext=nil)
+
+      # Let's create a black list from the day
+      # after the end_date up to today
+      day_after = Date.parse(end_date) + 1
+      today = Date.today
+
+      dates = []
+      day_after.upto(today) do |day|
+        dates << day.strftime(date_format) # Black list
+      end
+
+      NegativeRegex.new('(' + dates.join('|') + ')[^/]+%s$' % regexify(file_ext))
+    end
+    module_function :files_up_to
+
+    # Find files starting from the given date.
+    #
+    # Parameters:
+    # +start_date+:: start date
+    # +date_format:: format of date in filenames
+    # +file_ext:: extension on files (if any); include period    
+    def files_from(start_date, date_format, file_ext=nil)
+
+      # Let's create a white list from the start_date to today
+      today = Date.today
+
+      dates = []
+      Date.parse(start_date).upto(today) do |day|
+        dates << day.strftime(date_format)
+      end
+
+      '(' + dates.join('|') + ')[^/]+%s$' % regexify(file_ext)
+    end
+    module_function :files_from
+
+    private
+
+    # Make a file extension regular expression friendly,
+    # adding a starting period (.) if missing 
+    #
+    # Parameters:
+    # +file_ext:: the file extension to make regexp friendly
+    def regexify(file_ext)
+      file_ext.nil? ? nil : file_ext[0].chr != '.' ? '\\.' << file_ext : '\\' << file_ext
+    end
+    module_function :regexify
+
+  end
+end

data/lib/sluice/version.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+module Sluice
+  NAME    = "sluice"
+  VERSION = "0.0.1"
+end

data/lib/sluice.rb

@@ -0,0 +1,21 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'sluice/errors'
+require 'sluice/storage/storage'
+require 'sluice/storage/s3/contracts'
+require 'sluice/storage/s3/location'
+require 'sluice/storage/s3/manifest'
+require 'sluice/storage/s3/s3'

data/sluice.gemspec

@@ -0,0 +1,46 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sluice/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Alex Dean", "Michael Tibben"]
+  gem.email         = ["support@snowplowanalytics.com"]
+  gem.summary       = %q{Ruby toolkit for cloud-friendly ETL}
+  gem.description   = %q{A Ruby gem to help you build ETL processes involving Amazon S3. Uses Fog}
+  gem.homepage      = "http://snowplowanalytics.com"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "sluice-jason"
+  gem.version       = Sluice::VERSION
+  gem.license       = "Apache-2.0"
+  gem.platform      = Gem::Platform::RUBY
+  gem.require_paths = ["lib"]
+
+  # Dependencies
+  gem.add_dependency 'contracts', '~> 0.4'
+  gem.add_dependency 'fog', '1.24'
+
+  gem.add_development_dependency "rspec", "~> 2.14", ">= 2.14.1"
+  gem.add_development_dependency "rspec-nc"
+  gem.add_development_dependency "guard"
+  gem.add_development_dependency "guard-rspec"
+  gem.add_development_dependency "coveralls"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'coveralls'
+Coveralls.wear!
+
+require 'sluice'

data/spec/storage/s3/location_spec.rb

@@ -0,0 +1,47 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'spec_helper'
+
+Location = Sluice::Storage::S3::Location
+
+describe Location do
+
+  it 'should successfully initialize with a valid S3 (s3://) bucket' do
+    loc = Location.new('s3://my-s3-bucket/')
+    loc.bucket.should eql 'my-s3-bucket'
+    loc.dir.should eql ''
+    loc.dir_as_path.should eql ''
+    loc.to_s.should eql 's3://my-s3-bucket/'
+  end
+
+  it 'should successfully initialize with a valid S3 (s3n://) bucket' do
+    loc = Location.new('s3n://my-s3n-bucket/')
+    loc.bucket.should eql 'my-s3n-bucket'
+    loc.dir.should eql ''
+    loc.dir_as_path.should eql ''
+    loc.to_s.should eql 's3n://my-s3n-bucket/'
+  end
+
+  it 'should support object equality tests' do
+    loc1 = Location.new('s3n://my-s3n-bucket/hello/blah/')
+    loc2 = Location.new('s3n://my-s3n-bucket/hello/blah/')
+    loc1.should eql loc2
+    loc2.should eql loc1
+    loc1.should == loc2
+    loc2.should == loc1
+  end
+
+end

data/spec/storage/s3/s3_spec.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'spec_helper'
+
+S3 = Sluice::Storage::S3
+
+describe S3 do
+
+  it 'should allow filenames to be renamed' do
+
+    concat_subdir = lambda { |basename, filepath|
+      if m = filepath.match('([^/]+)/[^/]+$')
+        return m[1] + '-' + basename
+      else
+        return basename
+      end
+    }
+
+    foobar = lambda {
+      'foobar'
+    }
+
+    S3.rename_file('/dir/subdir/file', 'file', lambda=false).should eql 'file'
+    S3.rename_file('/dir/subdir/file', nil, foobar).should eql 'foobar'
+    S3.rename_file('resources/environments/logs/publish/e-bgp9nsynv7/i-f2b831bd/_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', '_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', concat_subdir).should eql 'i-f2b831bd-_var_log_tomcat7_localhost_access_log.txt-1391958061.gz'
+
+  end
+
+end

data/vagrant/.gitignore

@@ -0,0 +1,3 @@
+oss-playbooks
+ansible
+.peru

data/vagrant/ansible.hosts

@@ -0,0 +1,2 @@
+[vagrant]
+127.0.0.1:2222

data/vagrant/peru.yaml

@@ -0,0 +1,14 @@
+imports:
+    ansible: ansible
+    ansible_playbooks: oss-playbooks
+
+curl module ansible:
+    # Equivalent of git cloning tags/v1.6.6 but much, much faster
+    url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+    unpack: zip
+    export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+
+git module ansible_playbooks:
+    url: https://github.com/snowplow/ansible-playbooks.git
+    # Comment out to fetch a specific rev instead of master:
+    # rev: xxx

data/vagrant/push.bash

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+
+# Constants
+rubygems_user=snowplow
+rubygems_gem=sluice
+
+# Check if our Vagrant box is running. Expects `vagrant status` to look like:
+#
+# > Current machine states:
+# >
+# > default                   poweroff (virtualbox)
+# >
+# > The VM is powered off. To restart the VM, simply run `vagrant up`
+function running {
+    set +e
+    vagrant status | sed -n 3p | grep -q "^default\s*running (virtualbox)$"
+    local is_running=$?
+    set -e
+    echo $is_running
+}
+
+# Reads the version out of the version.rb
+function parse_version {
+    cat "lib/${rubygems_gem}/version.rb" | awk '/  VERSION =/ {v=$3; gsub(/\047/, "", v)} END {print v}'
+}
+
+# Installs RubyGems.org credentials in our guest
+#
+# Parameters:
+# 1. rubygems_password
+function install_creds {
+    vagrant ssh -c "curl -u ${rubygems_user}:${1} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials"
+    vagrant ssh -c "chmod 0600 ~/.gem/credentials"
+}
+
+# Builds our gem
+function build_gem {
+    vagrant ssh -c "cd /vagrant && gem build ${rubygems_gem}.gemspec"
+}
+
+# Installs our gem
+#
+# Parameters:
+# 1. rubygems_version
+function install_gem {
+    vagrant ssh -c "cd /vagrant && sudo gem install ./${rubygems_gem}-${1}.gem"
+}
+
+# Pushes our gem to RubyGems.org
+#
+# Parameters:
+# 1. rubygems_version
+function push_gem {
+    vagrant ssh -c "cd /vagrant && gem push ./${rubygems_gem}-${1}.gem"
+}
+
+
+# Move to the parent directory of this script
+source="${BASH_SOURCE[0]}"
+while [ -h "${source}" ] ; do source="$(readlink "${source}")"; done
+dir="$( cd -P "$( dirname "${source}" )/.." && pwd )"
+cd ${dir}
+
+# Precondition for running
+if [ $(running) != "0" ]; then
+    echo "Vagrant guest must be running to push"
+    exit 1
+fi
+
+# Can't pass args thru vagrant push so have to prompt
+read -e -p "Please enter password for RubyGems.org user ${rubygems_user}: " rubygems_password
+
+# Build, install & push
+build_gem
+rubygems_version=$(parse_version)
+install_gem ${rubygems_version}
+install_creds ${rubygems_password}
+push_gem ${rubygems_version}

data/vagrant/up.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+vagrant_dir=/vagrant/vagrant
+bashrc=/home/vagrant/.bashrc
+
+echo "========================================"
+echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES"
+echo "----------------------------------------"
+apt-get update
+apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2
+
+echo "==============="
+echo "INSTALLING PERU"
+echo "---------------"
+sudo pip3 install peru
+
+echo "======================================="
+echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU"
+echo "---------------------------------------"
+cd ${vagrant_dir} && peru sync -v
+echo "... done"
+
+env_setup=${vagrant_dir}/ansible/hacking/env-setup
+hosts=${vagrant_dir}/ansible.hosts
+
+echo "==================="
+echo "CONFIGURING ANSIBLE"
+echo "-------------------"
+touch ${bashrc}
+echo "source ${env_setup}" >> ${bashrc}
+echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc}
+echo "... done"
+
+echo "=========================================="
+echo "RUNNING PLAYBOOKS WITH ANSIBLE*"
+echo "* no output while each playbook is running"
+echo "------------------------------------------"
+while read pb; do
+    su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant
+done <${vagrant_dir}/up.playbooks
+
+guidance=${vagrant_dir}/up.guidance
+
+if [ -f ${guidance} ]; then
+    echo "==========="
+    echo "PLEASE READ"
+    echo "-----------"
+    cat $guidance
+fi

data/vagrant/up.guidance

@@ -0,0 +1,5 @@
+To get started:
+vagrant ssh
+cd /vagrant
+bundle install
+rspec

data/vagrant/up.playbooks

@@ -0,0 +1 @@
+oss-playbooks/ruby-rvm.yml

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: sluice-jason
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alex Dean
+- Michael Tibben
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  name: contracts
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.24'
+  name: fog
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.24'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-nc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: guard
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: guard-rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: coveralls
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses Fog
+email:
+- support@snowplowanalytics.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG
+- Gemfile
+- Guardfile
+- LICENSE-2.0.txt
+- README.md
+- Vagrantfile
+- lib/sluice.rb
+- lib/sluice/errors.rb
+- lib/sluice/storage/s3/contracts.rb
+- lib/sluice/storage/s3/location.rb
+- lib/sluice/storage/s3/manifest.rb
+- lib/sluice/storage/s3/s3.rb
+- lib/sluice/storage/storage.rb
+- lib/sluice/version.rb
+- sluice.gemspec
+- spec/spec_helper.rb
+- spec/storage/s3/location_spec.rb
+- spec/storage/s3/s3_spec.rb
+- vagrant/.gitignore
+- vagrant/ansible.hosts
+- vagrant/peru.yaml
+- vagrant/push.bash
+- vagrant/up.bash
+- vagrant/up.guidance
+- vagrant/up.playbooks
+homepage: http://snowplowanalytics.com
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Ruby toolkit for cloud-friendly ETL
+test_files:
+- spec/spec_helper.rb
+- spec/storage/s3/location_spec.rb
+- spec/storage/s3/s3_spec.rb