pipely-generators 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 027a3c870e6754434a2760a0a7f5cc15f0916196
+  data.tar.gz: 44de677290bbffaa02e061849c399d8026ca7b9d
+SHA512:
+  metadata.gz: 321c1fb57252994570c39e7f3093b0f165629cb51dd08fd7ca404fe296d9d9781cd1d81e65f462a68737cf3f3662a95f39a3bae0745155c0e423ba2732302c94
+  data.tar.gz: f55423acc0a81f970b12a25970b8f7e2d19d3cf1fab20b07882b35705096769f759aa56e2a776a9a4ee6e8064b8cfccc65c5b29120fd5db179f92869eab59c85

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pipely-generators.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Matt Gillooly
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Pipely::Generators
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'pipely-generators'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pipely-generators
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,19 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'cane/rake_task'
+
+Cane::RakeTask.new(:quality) do |cane|
+  cane.canefile = '.cane'
+end
+
+task :default => [:quality]

data/bin/pipely-generate

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'active_support/core_ext/string/inflections'
+
+class PipelyGenerator < Thor::Group
+  include Thor::Actions
+
+  # Define arguments and options
+  argument :name
+  class_option :config, :default => "~/.pipely-generate/config/"
+
+  def self.source_root
+    File.join(File.dirname(__FILE__), '..', 'templates')
+  end
+
+  def create_gemfile
+    copy_file "Gemfile", "#{name}/Gemfile"
+  end
+
+  def create_rakefile
+    copy_file "Rakefile", "#{name}/Rakefile"
+  end
+
+  def create_template
+    directory "lib", "#{name}/lib"
+    directory "templates", "#{name}/templates"
+    directory "script", "#{name}/script"
+    directory "steps", "#{name}/steps"
+  end
+
+  def create_config
+    config_path = File.expand_path(options[:config])
+    config_path = "config" unless File.directory?(config_path)
+
+    directory config_path, "#{name}/config"
+  end
+
+end
+
+PipelyGenerator.start(ARGV)

data/lib/pipely/generators/version.rb

@@ -0,0 +1,5 @@
+module Pipely
+  module Generators
+    VERSION = "0.0.1"
+  end
+end

data/lib/pipely/generators.rb

@@ -0,0 +1,7 @@
+require "pipely/generators/version"
+
+module Pipely
+  module Generators
+    # Your code goes here...
+  end
+end

data/pipely-generators.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pipely/generators/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pipely-generators"
+  spec.version       = Pipely::Generators::VERSION
+  spec.authors       = ["Matt Gillooly"]
+  spec.email         = ["matt@swipely.com"]
+  spec.description   = %q{Helpers for generating new Pipely projects}
+  spec.summary       = %q{Helpers for generating new Pipely projects}
+  spec.homepage      = "http://github.com/swipely/pipely-generators"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "cane"
+  spec.add_dependency "thor"
+end

data/templates/Gemfile

@@ -0,0 +1,12 @@
+if !File.exists?('packed_flag')
+  source 'https://rubygems.org'
+end
+
+gem 'fog'
+gem "rake", '~> 10.1.0'
+gem "streamingly", '~> 0.0.4'
+gem "pipely", '~> 0.2.1'
+
+group :development, :test do
+  gem "rspec"
+end

data/templates/Rakefile

@@ -0,0 +1,51 @@
+require 'fog'
+require 'rake'
+require 'rspec/core/rake_task'
+require 'pipely/tasks'
+require_relative 'lib/template'
+
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = File.join(File.dirname(__FILE__), 'spec', '**', '*_spec.rb')
+end
+
+template = Template.new
+environment = ENV['env'] || 'staging'
+
+definition = Pipely::Build.build_definition(
+  template,
+  environment,
+  File.join(File.dirname(__FILE__), 'config', 'config.yml')
+)
+
+Pipely::Tasks::UploadSteps.new do |t|
+  t.s3_path = definition.s3_prefix
+  t.s3_bucket_name = definition.s3_path_builder.steps_bucket
+  t.local_path = File.join(File.dirname(__FILE__), 'steps')
+end
+
+Pipely::Tasks::Deploy.new do |t|
+  t.definition = definition
+end
+
+Pipely::Tasks::Graph.new do |t|
+  t.definition = definition
+end
+
+Pipely::Tasks::Definition.new do |t|
+  t.definition = definition
+end
+
+namespace :build do
+  desc "Copy pipeline definitions to dist/"
+  task :dist do
+    dist_dir = File.join(File.dirname(__FILE__), 'dist')
+    mkdir_p(dist_dir)
+
+    Rake::Task["definition"].invoke(dist_dir)
+  end
+
+  desc "Run all tests for CI"
+  task :continuous => [:spec, :dist]
+end
+
+task :default => :spec

data/templates/config/config.yml.tt

@@ -0,0 +1,27 @@
+--- !map:HashWithIndifferentAccess
+
+common: &common
+  name: <%= name.camelize %>
+  namespace: <%= name %>
+
+  s3:
+    logs: 'bucket-of-logs'
+    steps: 'bucket-of-steps'
+    assets: 'bucket-of-assets'
+
+  security_groups:
+    - 'such-secure'
+
+  database_config:
+    swipely:
+      database: 'database'
+      username: 'username'
+      password: 'password'
+      port: '3306'
+      host: 'my.db.host'
+
+staging:
+  <<: *common
+
+production:
+  <<: *common

data/templates/lib/template.rb

@@ -0,0 +1,33 @@
+require 'pipely/build'
+
+class Template < Pipely::Build::Template
+
+  DEFINITION_PATH = File.join(File.dirname(__FILE__), '..', 'templates', 'pipeline.json.erb')
+  LOCAL_STEPS_PATH = File.join(File.dirname(__FILE__), '..', 'steps')
+
+  def initialize
+    super(File.read(DEFINITION_PATH))
+  end
+
+  def s3_step_path(filename)
+    ensure_step_file_exists(filename)
+    s3_step_prefix + filename
+  end
+
+private
+
+  def s3n_step_path(path)
+    if '/' == path[0]
+      ensure_step_file_exists(path)
+      s3n_step_prefix + path
+    else
+      path
+    end
+  end
+
+  def ensure_step_file_exists(filename)
+    full_path = File.join(LOCAL_STEPS_PATH, filename)
+    raise "Missing step file: #{full_path}" unless File.exist?(full_path)
+  end
+
+end

data/templates/script/build_bootstrap

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+rm -rf tmp
+
+[ -z "$BUNDLE_INSTALL" ] && BUNDLE_INSTALL="bundle install"
+
+$BUNDLE_INSTALL 
+

data/templates/steps/bootstrap_ec2.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e
+
+# Install rbenv
+curl --silent -L https://github.com/sstephenson/rbenv/archive/master.zip -o ~/rbenv.zip
+unzip ~/rbenv.zip -d ~
+mv ~/rbenv-master ~/rbenv
+ln -s ~/rbenv ~/.rbenv
+rm -rf ~/rbenv/versions
+
+# Set up rbenv
+export PATH="~/rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+rbenv rehash
+rbenv global 1.9.3-p327
+
+# Set up rbenv for future logins
+cat >> ~/.bash_profile << 'EOF'
+if [ -d ~/rbenv ]; then
+  export PATH="~/rbenv/bin:$PATH"
+  eval "$(rbenv init -)"
+fi
+EOF
+
+# Install bundler
+gem install bundler --no-ri --no-rdoc
+rbenv rehash
+
+# Set up ssh access
+if [ ! -f ~/.ssh/id_rsa ]; then
+  mkdir -p ~/.ssh
+  ssh-keygen -P '' -f ~/.ssh/id_rsa
+  cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+  chmod 600 ~/.ssh/authorized_keys
+fi
+
+# Use ssh to bypass the sudo "require tty" setting
+ssh -o "StrictHostKeyChecking no" -t -t ec2-user@localhost <<- EOF
+  sudo su -;
+  yum -y install gcc ruby-devel libxml2 libxml2-devel libxslt libxslt-devel make mysql mysql-devel;
+  ln -s /home/ec2-user/rbenv ~/rbenv;
+  ln -s /home/ec2-user/rbenv ~/.rbenv;
+  export PATH="/home/ec2-user/rbenv/bin:\$PATH";
+  eval "\$(rbenv init -)";
+  exit;
+  exit;
+EOF
+# ^^^ Double exit above is on purpose. (1) exit su, (2) exit ssh.
+
+# Install fog.  Has to happen after libxml2-devel
+rbenv rehash
+gem install fog --no-ri --no-rdoc
+rbenv rehash

data/templates/steps/bootstrap_emr.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+sudo apt-get -y install rubygems
+sudo gem install streamingly --no-ri --no-rdoc --source http://rubygems.org
+
+# TODO: install any gems that will be needed by streaming EMR activities
+
+exit

data/templates/templates/pipeline.json.erb.tt

@@ -0,0 +1,93 @@
+{
+  "objects": [
+    {
+      "id": "Default",
+      "role": "role-aggregator",
+      "resourceRole": "role-aggregator",
+      "failureAndRerunMode": "cascade",
+      "scheduleType": "cron",
+      "onFail": { "ref": "FailureNotify" }
+    },
+
+    {
+      "id": "Nightly",
+      "type": "Schedule",
+      "startDateTime": "<%%= start_date_time %>",
+      "period": "<%%= period %>"
+    },
+
+    {
+      "id": "SuccessNotify",
+      "type": "SnsAlarm",
+      "topicArn": "<%%= arn_topic %>",
+      "subject": "[<%%= environment %> pipelines] SUCCESS: <%= name.camelize %> pipeline step #{node.name}",
+      "message": "<%= name.camelize %> pipeline step SUCCESS\n\nScheduled start: #{node.@scheduledStartTime}\nActual start: #{node.@actualStartTime}\nActual end:\n#{node.@actualEndTime}"
+    },
+
+    {
+      "id": "FailureNotify",
+      "type": "SnsAlarm",
+      "topicArn": "<%%= arn_topic %>",
+      "subject": "[<%%= environment %> pipelines] FAILURE: <%= name.camelize %> pipeline step #{node.name}",
+      "message": "<%= name.camelize %> pipeline step FAILED #{node.name}\n\nScheduled start: #{node.@scheduledStartTime}\nError message:\n#{node.errorMessage}\nError stack trace:\n#{node.errorStackTrace}"
+    },
+
+    {
+      "id": "<%= name.camelize %>EC2Resource",
+      "type": "Ec2Resource",
+      "instanceType": "m1.large",
+      "schedule": { "ref": "Nightly" },
+      "logUri": "<%%= s3_log_prefix %>/<%= name.camelize %>EC2Resource",
+      "terminateAfter": "6 hours",
+      "keyPair": "pipeline-debug",
+      "securityGroups": <%%= security_groups %>
+    },
+
+    {
+      "id": "<%= name.camelize %>EMRCluster",
+      "type": "EmrCluster",
+      "masterInstanceType": "m1.large",
+      "taskInstanceType": "m1.large",
+      "coreInstanceType": "m1.large",
+      "coreInstanceCount": "2",
+      "terminateAfter": "6 hours",
+      "schedule": { "ref": "Nightly" },
+      "enableDebugging": "true",
+      "bootstrapAction": "<%%= s3_step_path('/bootstrap_emr.sh') %>",
+      "emrLogUri": "<%%= s3_log_prefix %>/<%= name.camelize %>EMRLogs",
+      "logUri": "<%%= s3_log_prefix %>/<%= name.camelize %>EMRCluster"
+    },
+
+    {
+      "id": "BootstrapEnvironment",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "stdout": "<%%= s3_log_prefix %>/BootstrapEnvironment/stdout",
+      "stderr": "<%%= s3_log_prefix %>/BootstrapEnvironment/stderr",
+      "schedule": { "ref": "Nightly" },
+      "runsOn": { "ref": "<%= name.camelize %>EC2Resource" },
+      "scriptUri": "<%%= s3_step_path('/bootstrap_ec2.sh') %>"
+    },
+
+    {
+      "id": "TouchEMRCluster",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "schedule": { "ref": "Nightly" },
+      "runsOn": { "ref": "<%= name.camelize %>EMRCluster" },
+      "command": "true"
+    },
+
+    {
+      "id": "Complete",
+      "type": "ShellCommandActivity",
+      "onFail": { "ref": "FailureNotify" },
+      "schedule": { "ref": "Nightly" },
+      "onSuccess": { "ref": "SuccessNotify" },
+      "runsOn": { "ref": "<%= name.camelize %>EC2Resource" },
+      "dependsOn": { "ref": "TouchEMRCluster" },
+      "command": "true"
+    }
+
+  ]
+}

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: pipely-generators
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Matt Gillooly
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cane
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Helpers for generating new Pipely projects
+email:
+- matt@swipely.com
+executables:
+- pipely-generate
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/pipely-generate
+- lib/pipely/generators.rb
+- lib/pipely/generators/version.rb
+- pipely-generators.gemspec
+- templates/Gemfile
+- templates/Rakefile
+- templates/config/config.yml.tt
+- templates/lib/template.rb
+- templates/script/build_bootstrap
+- templates/steps/bootstrap_ec2.sh
+- templates/steps/bootstrap_emr.sh
+- templates/templates/pipeline.json.erb.tt
+homepage: http://github.com/swipely/pipely-generators
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Helpers for generating new Pipely projects
+test_files: []