hodor 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 084614a22bcc209a5af059347fd77a82c16a9c09
+  data.tar.gz: ff9dc14dfe18285bb32c3c58bf72dfe4537117f8
+SHA512:
+  metadata.gz: 3402deea656434bf1099015af5e0c8e3d5a0d0a94b2c164c0cd18e7bbc077db86c8527f93d9af5a47e3be21379779e812f9d4ad5bea9a0fb8e809dec37ed795a
+  data.tar.gz: a6dc9c33adaf9079a72b5fd30c9419aed906194304935580da0f70b7ba6a7b700b0c8ddd206539ab4d899f5e551b25e60db3ae4c553a72b12057245c293e6ff2

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.idea/
+**/.tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "spec/test_repo"]
+	path = spec/test_repo
+	url = https://github.com/data-wranglers/hadoop-refrepo.git

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+hodor

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2-head

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "2.2.1"
+script: bundle exec rspec spec
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hodor.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'rspec', cmd: "bundle exec rspec" do
+  # watch /lib/ files
+  watch(%r{^lib/(.+).rb$}) do |m|
+    "spec/#{m[1]}_spec.rb"
+  end
+
+  # watch /spec/ files
+  watch(%r{^spec/(.+).rb$}) do |m|
+    "spec/#{m[1]}.rb"
+  end
+end

data/README.md

@@ -0,0 +1,105 @@
+
+Introduction
+===============
+Hodor is a ruby-based framework, API and Command Line Interface that automates and simplifies the way you specify, deploy, debug and administer Hadoop and Oozie solutions. Why did we write Hodor? Because we recognized that the Hadoop ecosystem was missing something essential. While the creators of the Hadoop ecosystem have provided a cutting-edge data ingestion, storage, and transformation engine that performs well in production when provided a specific, fully debugged workflow, getting there is hard. Hadoop lacks a mature toolchain to manage a codebase with modern software development discipline. Enter Hodor: a combination of tools and conventions to address these shortcomings, by enabling in the Hadoop environment many of the software development practices, and deployment facilities we take for granted in normal software development.
+
+Background
+===============
+The Hodor framework and CLI grew out of developer necessity: we needed a developer-oriented commandline alternative to the analyst-oriented Hue web interface tool. After working with Apache's Hue web interface, we quickly realized this tool was not designed to facilitate construction of the kind of production data systems we are building. We found no good alternative to Hue compatible with modern software development practice. Dumping Hue forced us to answer several important process questions. Without Hue, how would we manage production deployments? How should we build solutions that could be commited and reviewed in a version control system? How should we specify and structure Oozie workflows conforming to Oozie's data dependency architecture? How should we submit and monitor job status? As we began to answer these questions for ourselves, Hodor began to take shape.
+
+Getting Help
+============
+Hodor includes a rich help system that can be separated into overview, detail and topic help pages. For example, here are a few help and topic commands that provide a sprectrum of depth, from overview to conceptual tutorials.
+
+```bash
+$ hodor                                 # Prints generation information about Hodor
+$ hodor -T                              # Prints a more detail overview of Hodor's namespaces and commands
+$ hodor help oozie                      # Prints an overview of the Oozie namespace, its purpose and commands
+$ hodor help oozie:display_job          # Prints detailed information about the display_job command, arguments and options
+$ hodor oozie:topic drivers.yml         # Provides an overview of a key concept (drivers.yml) utilized by the namespace
+$ hodor master:topic clusters.yml       # Provides an overview of the clusters.yml file and its central role in Hodor
+```
+
+Hodor Setup and Aliasing
+===========================
+
+This section demonstrates how one might use the hodor commands to during your Hadoop development effort. Before using `hodor`, you'll need to build and install the `hodor` gem locally:
+
+```bash
+gem build hodor.gemspec
+gem install hodor-0.0.8.gem
+```
+
+To specify the Hadoop cluster a Hodor command should target, you have two options:
+
+```bash
+$ HADOOP_ENV=target_env hodor oozie:display_job /
+```
+
+Note, the name 'target_env' indicates which section in your config/clusters.yml file to use when defining your Hadoop cluster. Alternatively, you can avoid having to type the HADOOP_ENV prefix for every command, by adding the following to your `~/.bashrc':
+
+```bash
+export HADOOP_ENV=target_env  # name of target hadoop cluster defined in config/clusters.yml
+```
+
+You may also wish to consider adding a few aliases for the more frequently used Hodor commands. For example, the following
+aliases can (optionally) be defined in your '~/.bashrc':
+
+```bash
+alias cj='hodor oozie:change_job'  # avoid trailing space with aliases. Can cause parse problems with zsh
+alias dj='hodor oozie:display_job'  # avoid trailing space with aliases. Can cause parse problems with zsh
+alias rj='hodor oozie:run_job'
+```
+
+Hodor Core Concepts & Assumptions:
+==================================
+Hodor's commands can be studied individually, but there are some larger principles that will shed light on
+the assumptions those commands are making, and the concepts they are implementing. These topics are broken
+down by Hodor Namespace as follows:
+
+Master Topics
+-------------
+ * [The "clusters.yml" file](topics/master/clusters.yml.txt)
+
+Oozie Topics
+------------
+ * [Inspecting Oozie Jobs](topics/oozie/inspecting_jobs.txt)
+ * [Composing Job Properties](topics/oozie/composing_job_properties.txt)
+ * [Workers And Drivers](topics/oozie/workers_and_drivers.txt)
+ * [Driver Run Scenarios](topics/oozie/driver_scenarios.txt)
+ * [Blocking Coordinators](topics/oozie/blocking_coordinators.txt)
+ * [Deploying & Running Jobs](topics/oozie/jobs.yml.txt)
+
+Hdfs Topics
+-----------
+ * [Corresponding Hdfs Paths](topics/hdfs/corresponding_paths.txt)
+
+## Future / Enhancements
+
+Pull requests will be very happily considered.
+
+__Maintained by Dean Hallman__
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Dean Hallman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,105 @@
+
+$:.push File.expand_path("../lib", __FILE__)
+require "hodor/version"
+
+task :default => :run_specs
+
+##################
+# Build & Release Tasks
+#
+
+require 'bundler/gem_tasks'
+
+##################
+# RuboCop Tasks
+#
+namespace :cop do
+
+  task :depends do
+    require 'rubocop/rake_task'
+  end
+
+  desc "Run rubocop on Hadoop checks"
+  task :cli => :depends do
+    cli = RuboCop::RakeTask.new do |task|
+      task.formatters = ["s"]
+      task.patterns = ["lib/", "spec/"]
+    end
+    Rake::Task["rubocop"].invoke
+  end
+
+  task :all => [:cli]
+end
+
+desc "Run lint on all sections"
+task :cop => 'cop:all'
+
+##################
+# Ruby-Lint Tasks
+#
+namespace :lint do
+
+  task :depends do
+    require 'ruby-lint/rake_task'
+  end
+
+  desc "Run lint on cli checks"
+  task :cli => :depends do
+    RubyLint::RakeTask.new do |task|
+      task.name  = 'lint'
+      task.files = ['lib/']
+    end
+    Rake::Task["lint"].invoke
+  end
+
+  task :all => [:cli]
+end
+
+desc "Run lint on all sections"
+task :lint => 'lint:all'
+
+##################
+# Rspec Tasks
+#
+namespace :spec do
+
+  desc "Ensure dependencies load once"
+  task :depends do
+    require 'rspec/core/rake_task'
+  end
+
+  desc "Run unit tests"
+  task :unit => :depends do
+    RSpec::Core::RakeTask.new("spec:unit")  do |t|
+      t.pattern = 'spec/unit/**/*_spec.rb'
+      t.fail_on_error = true
+      t.rspec_opts = %w(--color --require spec_helper --format progress)
+    end
+  end
+
+  desc "Run unit tests"
+  task :integration => :depends do
+    RSpec::Core::RakeTask.new("spec:integration")  do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+      t.fail_on_error = true
+      t.rspec_opts = %w(--color --require spec_helper --format progress)
+    end
+  end
+
+  desc "Run unit tests"
+  task :target => :depends do
+    RSpec::Core::RakeTask.new("spec:target")  do |t|
+      t.pattern = 'spec/integration/api/oozie/action_spec.rb'
+      t.fail_on_error = true
+      t.rspec_opts = %w(--color --require spec_helper --format progress)
+    end
+  end
+
+  desc "Run all specs"
+  task :all => [:integration, :unit]
+
+  task :default => :all
+end
+
+desc "Run rspec on all sections"
+task :run_specs => ['spec:all']

data/bin/hodor

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require_relative '../lib/hodor/cli'
+
+$thor_runner = true
+$hodor_runner = true
+
+begin
+  Hodor::Cli::Runner.start(ARGV)
+rescue Hodor::Cli::CommandNotFound => ex
+  namespace = ARGV.shift
+  begin
+    Kernel.exec("hodor-#{namespace} #{ARGV.join(' ')}")
+  rescue => ex
+    puts "HODOR! '#{namespace}' is unknown.\nNOTE: If this is a Hodor plugin, then perhaps you need to install the gem 'hodor-#{namespace}'..."
+  end
+end

data/hodor.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+$:.push File.expand_path("../lib", __FILE__)
+require 'hodor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hodor"
+  spec.version       = Hodor::VERSION
+  spec.authors       = ["Dean Hallman"]
+  spec.email         = ["rdhallman@gmail.com"]
+  spec.summary       = %q{Manages Hadoop and Oozie data pipelines, through development, testing, deployment and monitoring}
+  spec.description   = %q{Hodor is a ruby-based framework, API and Command Line Interface that automates and simplifies the way you specify, deploy, debug and administer Hadoop and Oozie solutions. Hadoop lacks a mature toolchain to manage a codebase with modern software development discipline. To address this need, Hodor comprises a combination of tools and conventions that enable in the Hadoop environment many of the modern software development practices, and deployment facilities we take for granted in normal software development.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "thor", ">= 0.19.1"
+  spec.add_runtime_dependency "log4r", "~> 1.1"
+  spec.add_runtime_dependency "open4"
+  spec.add_runtime_dependency "terminal-table"
+  spec.add_runtime_dependency "awesome_print"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "byebug"
+
+  # Gems needed by Hodor's Hadoop/Oozie API
+  spec.add_runtime_dependency "rest-client"
+  spec.add_runtime_dependency "chronic"
+  spec.add_runtime_dependency "ox"
+
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "wrong"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "ruby-lint"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rspec-nc"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-remote"
+  spec.add_development_dependency "pry-nav"
+end
+

data/lib/config/log4r_config.xml

@@ -0,0 +1,35 @@
+<!--
+Log Configuration
+-->
+
+<log4r_config>
+
+  <!-- Logging Levels -->
+  <pre_config>
+    <custom_levels>DEBUG, INFO, SSHCMD, STDOUT, STDERR, WARN, ERROR, FATAL</custom_levels>
+    <global level="DEBUG"/>
+  </pre_config>
+
+  <!-- Outputters -->
+  <outputter name="logconsole" type="StdoutOutputter" level="DEBUG" >
+  </outputter>
+
+  <outputter name="console" type="StdoutOutputter" level="DEBUG" >
+    <formatter type="Log4r::PatternFormatter">
+      <pattern>%5l|%M</pattern>
+    </formatter>
+  </outputter>
+
+  <!-- Loggers -->
+  <logger name="MainLogger"
+    level="INFO" additive="false" trace="true">
+    <outputter>console</outputter>
+  </logger>
+
+  <!-- Rspec Loggers -->
+  <logger name="RspecLogger"
+    level="WARN" additive="false" trace="true">
+    <outputter>console</outputter>
+  </logger>
+
+</log4r_config>

data/lib/hodor.rb

@@ -0,0 +1,83 @@
+require "hodor/version"
+
+module Hodor
+
+  # Hodor Exception Classes
+  class NestedError < StandardError
+    attr_reader :cause
+
+    def initialize(cause, kvp = {})
+      @cause = cause
+      @kvp = kvp
+    end
+
+    alias :orig_to_s :to_s
+    def to_s
+      msg = @kvp[:msg] || orig_to_s
+      if @kvp.size > 1 || (@kvp.size == 1 && !@kvp.has_key?(:msg))
+        msg << " Exception Context:\n"
+        @kvp.each_pair { |k,v|
+          next if k == :msg
+          if k.nil?
+            msg << "   nil => "
+          elsif k.is_a?(Symbol)
+            msg << "   :#{k.to_s} => "
+          else
+            msg <<  "   #{k} => "
+          end
+          if v.nil?
+            msg << "nil"
+          elsif v.is_a?(Symbol)
+            msg << ":#{v.to_s}"
+          else
+            msg << v
+          end
+          msg << "\n"
+        }
+      end
+      msg << "Root cause: #{@cause}"
+      msg << "\nBacktrace:\n         "
+      msg << "#{@cause.backtrace[0..5].join("\n         ")}"
+    end
+  end
+
+  class AbnormalExitStatus  < StandardError
+    attr_reader :exit_status
+    def initialize(exit_status, error_lines)
+      @exit_status = exit_status
+      super error_lines
+    end
+  end
+
+end
+
+class Hash
+  def symbolize_keys
+    inject({}) { |memo,(k,v)| 
+      memo[k] = v.is_a?(Hash) ? v.symbolize_keys : v;
+      memo[k.to_sym] = v.is_a?(Hash) ? v.symbolize_keys : v;
+      memo
+    }
+  end
+
+  def match strings
+    select { |key,val|
+      is_match = false
+      strings.each { |findstr|
+        is_match ||= key.downcase.include?(findstr) || val.downcase.include?(findstr)
+      }
+      is_match
+    }
+  end
+end
+
+class String
+  def unindent(count)
+    #gsub(/^#{scan(/^\s*/).min_by{|l|l.length}}/, "")
+    gsub(/^[ \t]{1,#{count}}/, "")
+  end
+end
+
+require "hodor/environment"
+
+