hodor 1.0.2

data/lib/tasks/sandbox.thor

@@ -0,0 +1,87 @@
+
+module Hodor
+  module Cli
+    class Sandbox < ::Hodor::Command
+      #
+      # Manual change required:
+      #
+      # edit /etc/hadoop/conf/core-site.xml
+      #
+      # Change the following sections to agree with:
+      #
+      # <property
+      #    <name>hadoop.proxyuser.oozie.hosts</name>
+      #    <value>*</value>
+      # </property>
+      #
+      # <property>
+      #      <name>hadoop.proxyuser.oozie.groups</name>
+      #      <value>*</value>
+      # </property>
+      #
+
+      no_tasks do
+        def ssh_user_addr(user_key)
+          va = "#{env[user_key]}@#{env[:ssh_host]}"
+          va << " -p #{env[:ssh_port] || 22}"
+        end
+
+        def deploy_ssh_key(user_key)
+          logger.info "Preventing future password prompts for '#{env[user_key]}' sandbox user."
+          logger.info "Note: this may require you to enter the password for '#{env[user_key]}'."
+          remote_cmd = %q['umask 077; mkdir -p ~/.ssh; touch ~/.ssh/authorized_keys; cat >> ~/.ssh/authorized_keys']
+          env.run_local %Q[cat ~/.ssh/id_rsa.pub | ssh #{ssh_user_addr(user_key)} #{remote_cmd}], echo: true, echo_cmd: true
+        end
+
+        def self.help(shell, subcommand = false)
+          overview = %Q[Hodor's Sandbox namespace functions as a local proxy for Hortonworks "HDP Sandbox" that you may have
+          running in a virtual machine. The commands in this namespace are responsible for performing one-time
+          initialization tasks on the sandbox virtual cluster, so that expected user accounts are created etc.
+          To be clear, Hodor generally does not require that you run Hortonwork's Sandbox. Only this particular
+          namespace expects that. So, if you aren't running one, just avoid use of this namespace.
+
+          Note: this namespace has not be used in well over a year and is probably broken right now. It needs
+          to be reviewed and updated or overhauled. Meanwhile, use at your own risk.
+
+          ].unindent(10)
+          shell.say overview
+          result = super
+
+          more_help = %Q[Getting More Help:
+          ------------------
+          To get detailed help on specific Sandbox commands (i.e. setup_ssh), run:
+
+             $ hodor help sandbox:setup_ssh
+             $ hodor sandbox:help setup_ssh    # alternate, works the same
+
+          ].unindent(10)
+          shell.say more_help
+          result
+        end
+      end
+
+      # Set up a hortonworks sandbox. Currently, all this does is copy your SSH key
+      # to avoid password prompting. In the future, we may want to install components
+      # we expect to be available, etc.
+      desc "setup_ssh", "Set up a new sandbox to include required components and SSH keys"
+      def setup_ssh
+        deploy_ssh_key(:ssh_user)
+      end
+
+      desc "setup_users", "Set up a new sandbox to include required components and SSH keys"
+      def setup_users
+        deploy_ssh_key(:oozie_user)
+      end
+
+      desc "setup_hdfs", "Set up a new sandbox to include hdfs directories with required group settings"
+      def setup_hdfs
+        oozie_root = env[:oozie_root] || 'pipeline'
+        invoke "hdfs:fs", %w[-u hdfs -mkdir /shared]
+        invoke "hdfs:fs", %W[-u hdfs -mkdir /#{oozie_root}]
+
+        invoke "hdfs:fs", %w[-u hdfs -chgrp hadoop /shared]
+        invoke "hdfs:fs", %W[-u hdfs -chgrp hadoop /#{oozie_root}]
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/action_spec.rb

@@ -0,0 +1,69 @@
+module Hodor::Oozie
+  describe Action do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Action }
+
+      # Public fields
+      it { should respond_to? :id }
+      it { should respond_to? :json }
+      it { should respond_to? :status }
+      it { should respond_to? :parent_id }
+      it { should respond_to? :error_message }
+      it { should respond_to? :data }
+      it { should respond_to? :transition }
+      it { should respond_to? :external_status }
+      it { should respond_to? :cred }
+      it { should respond_to? :type }
+      it { should respond_to? :end_time }
+      it { should respond_to? :external_id }
+      it { should respond_to? :start_time }
+      it { should respond_to? :external_child_ids }
+      it { should respond_to? :name }
+      it { should respond_to? :error_code }
+      it { should respond_to? :tracker_url }
+      it { should respond_to? :retries }
+      it { should respond_to? :to_string }
+      it { should respond_to? :console_url }
+
+      # Public methods
+      it { should respond_to? :children }
+    end
+
+    context "Request action by job id" do
+      include_context "hodor api" do
+        let(:playback) { :sample_action }
+      end
+
+      let(:request_details) {
+        /v1\/job\/0025060-151002103648730-oozie-oozi-W@run_worker/
+      }
+
+      let(:request_children) {
+        /v1\/job\/0025062-151002103648730-oozie-oozi-W/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(request_details).once.and_mimic_original(memo)
+        expect(session).to receive(:rest_call).with(request_children).once.and_mimic_original(memo)
+        @action = oozie.job_by_id "0025060-151002103648730-oozie-oozi-W@run_worker"
+        @children = @action.children
+      end
+
+      it "should have the correct type" do
+        expect(@action.class).to eql(Hodor::Oozie::Action)
+      end
+
+      it "should have correct count" do
+        expect(@action.type).to match(/sub-workflow/)
+      end
+
+      it "should have 1 child" do
+        expect(@children.size).to eql(1)
+      end
+
+      it "should have example_business_W as only child" do
+        expect(@children[0].app_name).to match(/example_business_W/)
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/bundle_spec.rb

@@ -0,0 +1,33 @@
+module Hodor::Oozie
+  describe Bundle do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Bundle.instance_methods }
+
+      # Public fields
+      it { should include :status }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "Request bundle by job id that does not exist" do
+      include_context "hodor api" do
+        let(:playback) { :sample_bundle }
+      end
+
+      let(:request_details) {
+        /v1\/job\/0023753-151002103648730-oozie-oozi-B/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(request_details).once.and_mimic_original(memo)
+      end
+
+      it "should should throw exception when searching for a bundle that does not exist" do
+        expect {
+          oozie.job_by_id "0023753-151002103648730-oozie-oozi-B"
+        }.to raise_error JSON::ParserError
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/coordinator_spec.rb

@@ -0,0 +1,66 @@
+module Hodor::Oozie
+  describe Coordinator do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Coordinator.instance_methods }
+
+      # Public fields
+      it { should include :status }
+      it { should include :id }
+      it { should include :json }
+      it { should include :name }
+      it { should include :path }
+      it { should include :timezone }
+      it { should include :frequency }
+      it { should include :conf }
+      it { should include :end_time }
+      it { should include :execution_policy }
+      it { should include :start_time }
+      it { should include :time_unit }
+      it { should include :concurrency }
+      it { should include :last_action }
+      it { should include :acl }
+      it { should include :mat_throttling }
+      it { should include :timeout }
+      it { should include :next_materialized_time }
+      it { should include :parent_id }
+      it { should include :external_id }
+      it { should include :group }
+      it { should include :user }
+      it { should include :console_url }
+      it { should include :actions }
+      it { should include :acl }
+      it { should include :materializations }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "Request coordinator by job id" do
+      include_context "hodor api" do
+        let(:playback) { :sample_coordinator }
+      end
+
+      let(:request_details) {
+        /v1\/job\/0023753-151002103648730-oozie-oozi-C/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(request_details).once.and_mimic_original(memo)
+        @coord = oozie.job_by_id "0023753-151002103648730-oozie-oozi-C"
+        @children = @coord.children
+      end
+
+      it "should have the correct type" do
+        expect(@coord.class).to eql(Hodor::Oozie::Coordinator)
+      end
+
+      it "should have 6 children" do
+        expect(@children.size).to eql(6)
+      end
+
+      it "should show success status for child 2" do
+        expect(@children[2].status).to eql("SUCCEEDED")
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/hadoop_job_spec.rb

@@ -0,0 +1,29 @@
+module Hodor::Oozie
+  describe HadoopJob do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::HadoopJob.instance_methods }
+
+      # Public fields
+      it { should include :id }
+      it { should include :parent_id }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "Request action by job id" do
+      include_context "hodor api" do
+        let(:playback) { :sample_hadoop_job }
+      end
+
+      before(:each) do
+        expect(session).not_to receive(:rest_call)
+        @job = oozie.job_by_id "job_1443733596356_96843"
+      end
+
+      it "should have the correct type" do
+        expect(@job.class).to eql(Hodor::Oozie::HadoopJob)
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/job_spec.rb

@@ -0,0 +1,15 @@
+module Hodor::Oozie
+  describe Job do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Job.instance_methods }
+
+      # Public fields
+      it { should include :id }
+
+      # Public methods
+      it { should include :children }
+      it { should include :display_properties }
+      it { should include :display_children }
+    end
+  end
+end

data/spec/integration/api/oozie/materialization_spec.rb

@@ -0,0 +1,66 @@
+module Hodor::Oozie
+  describe Materialization do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Materialization.instance_methods }
+
+      # Public fields
+      it { should include :id }
+      it { should include :json }
+      it { should include :error_message }
+      it { should include :last_modified_time }
+      it { should include :created_at }
+      it { should include :status }
+      it { should include :push_missing_dependencies }
+      it { should include :external_status }
+      it { should include :type }
+      it { should include :nominal_time }
+      it { should include :external_id }
+      it { should include :created_conf }
+      it { should include :missing_dependencies }
+      it { should include :run_conf }
+      it { should include :action_number }
+      it { should include :error_code }
+      it { should include :tracker_uri }
+      it { should include :to_string }
+      it { should include :parent_id }
+      it { should include :coord_job_id }
+      it { should include :console_url }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "Request materialization by job id" do
+      include_context "hodor api" do
+        let(:playback) { :sample_materialization }
+      end
+
+      let(:request_details) {
+        /v1\/job\/0023753-151002103648730-oozie-oozi-C/
+      }
+
+      let(:request_children) {
+        /v1\/job\/0025060-151002103648730-oozie-oozi-W/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(request_details).once.and_mimic_original(memo)
+        expect(session).to receive(:rest_call).with(request_children).once.and_mimic_original(memo)
+        @materialization = oozie.job_by_id "0023753-151002103648730-oozie-oozi-C@3"
+        @children = @materialization.children
+      end
+
+      it "should have the correct type" do
+        expect(@materialization.class).to eql(Hodor::Oozie::Materialization)
+      end
+
+      it "should have 1 child" do
+        expect(@children.size).to eql(1)
+      end
+
+      it "should show success status for child 0" do
+        expect(@children[0].status).to eql("SUCCEEDED")
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/query_spec.rb

@@ -0,0 +1,43 @@
+module Hodor::Oozie
+  describe Query do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Query.instance_methods }
+
+      # Public fields
+      it { should include :id }
+      it { should include :request }
+      it { should include :json }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "List all running coordinators" do
+      include_context "hodor api" do
+        let(:playback) { :running_coordinators }
+      end
+
+      let(:parent_request) {
+        /v2\/jobs\?jobtype=coord&filter=status%3DRUNNING/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(parent_request).once.and_mimic_original(memo)
+        @query = Hodor::Oozie::Query.new status: [:running]
+        @matches = @query.children
+      end
+
+      it "should have correct count" do
+        expect(@matches.size).to eql(4)
+      end
+
+      it "should include worker_data_source\/business_C coordinator" do
+        expect(@matches[1].name).to match(/driver_example_workflows_master_workflow.xml_C/)
+      end
+
+      it "should include hourly_master incremental coordinator" do
+        expect(@matches[0].name).to match(/example_workflows\/hourly_master_hourly_incremental-C/)
+      end
+    end
+  end
+end

data/spec/integration/api/oozie/session_spec.rb

@@ -0,0 +1,18 @@
+module Hodor::Oozie
+  describe Session do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Session }
+
+      # Public methods
+      it { should respond_to? :pwj }
+      it { should respond_to? :make_current }
+      it { should respond_to? :current_id }
+      it { should respond_to? :current_id }
+      it { should respond_to? :get_job_state }
+      it { should respond_to? :search_jobs }
+      it { should respond_to? :len }
+      it { should respond_to? :offset }
+    end
+  end
+end
+

data/spec/integration/api/oozie/workflow_spec.rb

@@ -0,0 +1,65 @@
+module Hodor::Oozie
+  describe Workflow do
+    describe "Required Public Interface" do
+      subject { Hodor::Oozie::Workflow.instance_methods }
+
+      # Public fields
+      it { should include :id }
+      it { should include :json }
+      it { should include :app_path }
+      it { should include :acl }
+      it { should include :status }
+      it { should include :created_at }
+      it { should include :conf }
+      it { should include :last_mod_time }
+      it { should include :run }
+      it { should include :end_time }
+      it { should include :external_id }
+      it { should include :app_name }
+      it { should include :start_time }
+      it { should include :materialization_id }
+      it { should include :parent_id }
+      it { should include :materialization }
+      it { should include :to_string }
+      it { should include :group }
+      it { should include :console_url }
+      it { should include :user }
+
+      # Public methods
+      it { should include :children }
+    end
+
+    context "List all running coordinators" do
+      include_context "hodor api" do
+        let(:playback) { :sample_workflow }
+      end
+
+      let(:request) {
+        /v1\/job\/0025062-151002103648730-oozie-oozi-W/
+      }
+
+      before(:each) do
+        expect(session).to receive(:rest_call).with(request).once.and_mimic_original(memo)
+        @workflow = oozie.job_by_id "0025062-151002103648730-oozie-oozi-W"
+        @children = @workflow.children
+      end
+
+      it "should have the correct type" do
+        expect(@workflow.class).to eql(Hodor::Oozie::Workflow)
+      end
+
+
+      it "should have correct count" do
+        expect(@workflow.app_name).to match(/example_business_W/)
+      end
+
+      it "should have 3 children" do
+        expect(@children.size).to eql(3)
+      end
+
+      it "should have 3 children" do
+        expect(@children[1].name).to match(/data_workflow/)
+      end
+    end
+  end
+end