hodor 1.0.2

data/lib/hodor/api/hdfs.rb

@@ -0,0 +1,222 @@
+require 'hodor'
+require 'singleton'
+
+# curl -i "http://sample_domain.com:50070/webhdfs/v1/pipeline?op=LISTSTATUS"
+module Hodor
+
+  # HDFS Api wrapper
+  class Hdfs
+    include Singleton
+
+    def env
+      Hodor::Environment.instance
+    end
+
+    def logger
+      env.logger
+    end
+
+    def hdfs_root
+      env.settings[:hdfs_root]
+    end
+
+    def pwd
+      "#{hdfs_root}#{env.pwd}"
+    end
+
+    def path_on_hdfs(file)
+      git_path = env.path_on_github(file)
+      "#{hdfs_root}/#{git_path}".sub(/\/\/\//, '/').sub(/\/\//, '/').sub(/\/\.\//, '/').sub(/\/\.$/, '')
+    end
+
+    def user
+      env.settings[:hdfs_user]
+    end
+
+    def target
+      env.settings[:target]
+    end
+
+    class FailedToRemovePath < Hodor::NestedError; end
+
+    def rm(path)
+      dest_path = path_on_hdfs(path||".")
+      rm_path_script = %Q[HADOOP_USER_NAME=#{user} hadoop fs -rm -skipTrash #{dest_path}]
+      env.ssh rm_path_script
+    rescue StandardError => ex
+      raise FailedToRemovePath.new ex,
+        msg: "Unable to remove HDFS path.",
+        ssh_user: env.ssh_user,
+        path_to_remove: dest_path
+    end
+
+    def rm_f(path)
+      dest_path = path_on_hdfs(path||".")
+      rm_path_script = %Q[HADOOP_USER_NAME=#{user} hadoop fs -rm -f -skipTrash #{dest_path}]
+      env.ssh rm_path_script
+    rescue StandardError => ex
+      raise FailedToRemovePath.new ex,
+        msg: "Unable to remove HDFS path.",
+        ssh_user: env.ssh_user,
+        path_to_remove: dest_path
+    end
+
+    def rm_rf(path)
+      hdfs_path = path_on_hdfs(path||".")
+      rm_path_script = %Q[HADOOP_USER_NAME=#{user} hadoop fs -rm -f -R -skipTrash #{hdfs_path}]
+      env.ssh rm_path_script
+    rescue StandardError => ex
+      raise FailedToRemovePath.new ex,
+        msg: "Unable to remove HDFS path.",
+        ssh_user: env.ssh_user,
+        path_to_remove: dest_path
+    end
+
+    def ls
+      dest_path = path_on_hdfs(".")
+      ls_script = %Q[HADOOP_USER_NAME=#{user} hadoop fs -ls #{dest_path}]
+      env.ssh ls_script, echo:true
+    rescue StandardError => ex
+      raise FailedToRemovePath.new ex,
+        msg: "Unable to list HDFS path.",
+        ssh_user: env.ssh_user,
+        path_to_list: dest_path
+    end
+
+    class FailedToPutFile < Hodor::NestedError; end
+
+    # put_file
+    #  Puts a local file on HDFS, preserving path and replacing if necessary. Files
+    #  with .erb extensions are ERB expanded before deployment.
+    def put_file(file, options = {})
+
+      disc_path = env.path_on_disc(file)
+      hdfs_path = path_on_hdfs(file)
+      git_path = env.path_on_github(file)
+
+      raise "File '#{disc_path}' not found." if !File.exists?(disc_path)
+
+      logger.info "\tdeploying '#{git_path}'"
+
+      src_file = file
+      if disc_path.end_with?('.erb')
+        erb_expanded = env.erb_load(disc_path)
+        src_file = "/tmp/#{File.basename(disc_path.sub(/\.erb$/,''))}"
+        hdfs_path.sub!(/\.erb$/, '')
+        puts "ends with erb srcfile = #{src_file}"
+        File.open(src_file, 'w') { |f| f.write(erb_expanded) }
+      end
+
+      raise "File '#{src_file}' not found." if !File.exists?(src_file)
+
+      put_script = "HADOOP_USER_NAME=#{user} hadoop fs -put - #{hdfs_path}"
+      unless options[:already_cleaned]
+        rm_script = "HADOOP_USER_NAME=#{user} hadoop fs -rm -f #{hdfs_path}; "
+        put_script = rm_script + put_script
+      end
+
+      env.run_local %Q[cat #{src_file} | ssh #{env.ssh_addr} "#{put_script}"],
+        echo: true, echo_cmd: true
+    rescue StandardError => ex
+      raise FailedToPutFile.new ex,
+        msg: "Unable to write file to HDFS.",
+        ssh_user: env.ssh_user,
+        path_on_disc: disc_path,
+        path_on_github: git_path,
+        path_on_hdfs: hdfs_path,
+        src_file: src_file
+    end
+
+    class FailedToPutDir < Hodor::NestedError; end
+
+    def put_dir(path, options)
+      if env.dryrun? and env.verbose?
+        logger.info ""
+        logger.info "        ********************* Dry Run *********************"
+        logger.info ""
+      end
+
+      disc_path = env.path_on_disc(path)
+      git_path = env.path_on_github(path)
+      hdfs_path = path_on_hdfs(path)
+
+      sync_file = "#{disc_path}/.hdfs-#{target}.sync"
+
+      logger.info "Deploying: #{git_path}" unless env.silent?
+
+      fail "Path '#{disc_path}' not found." unless File.exists?(disc_path)
+      fail "Path '#{disc_path}' exists but is not a directory." unless File.directory?(disc_path)
+
+      if env.clean?
+        logger.info "  cleaning: #{git_path}"
+        FileUtils.rm_f sync_file unless env.dryrun?
+        rm_rf(git_path)
+        clean_done = true
+      end
+
+      fargs = if sync_file && File.exists?(sync_file) && !env.clean?
+                "-newer '#{sync_file}'"
+              else
+                ""
+              end
+      fargs << " -maxdepth #{options[:maxdepth]}" unless options[:maxdepth].nil?
+      mod_files = env.run_local %Q[find #{disc_path} #{fargs} -type f]
+      mod_files.split("\n").each { |file|
+        basename = File.basename(file)
+        next if basename.start_with?('job.properties') ||
+          basename.eql?("run.properties") ||
+          basename.eql?(".DS_Store") ||
+          basename.eql?(".bak") ||
+          basename.eql?(".tmp") ||
+          basename.eql?(".hdfs") ||
+          basename.eql?("Rakefile") ||
+          basename.end_with?(".sync") ||
+          file.include?("migrations/") ||
+          file.include?(".bak/") ||
+          file.include?(".tmp/")
+        put_file(file, already_cleaned: clean_done)
+      }
+    rescue StandardError => ex
+      raise FailedToPutDir.new ex,
+        msg: "Unable to write directory to HDFS.",
+        ssh_user: env.ssh_user,
+        path_on_disc: disc_path,
+        path_on_github: git_path,
+        path_on_hdfs: hdfs_path,
+        sync_file: sync_file,
+        max_depth: options[:maxdepth],
+        clean: env.clean? ? "true" : "false"
+    else
+      env.run_local %Q[touch '#{sync_file}'] unless env.dryrun?
+    end
+
+    class FailedToGetFile < Hodor::NestedError; end
+
+    # get
+    #  Gets a file from HDFS and copies it to a local file
+    def get_file(file, options = {})
+      disc_path = env.path_on_disc(file)
+      hdfs_path = path_on_hdfs(file)
+      git_path = env.path_on_github(file)
+      dest_path = "#{file}.hdfs_copy"
+
+      logger.info "\tgetting '#{git_path}'. Writing to '#{dest_path}'."
+
+      get_script = %Q["rm -f #{dest_path}; HADOOP_USER_NAME=#{user} hadoop fs -get #{hdfs_path} #{dest_path}"]
+      env.ssh get_script, echo: true, echo_cmd: true
+      if options[:clobber]
+        FileUtils.rm_f dest_path
+      end
+      env.run_local %Q[scp #{env.ssh_user}@#{env[:ssh_host]}:#{dest_path} .],
+        echo: true, echo_cmd: true
+    rescue StandardError => ex
+      raise FailedToGetFile.new ex,
+        msg: "Unable to get file from HDFS.",
+        ssh_user: env.ssh_user,
+        path_on_disc: disc_path,
+        path_on_github: git_path,
+        path_on_hdfs: hdfs_path,
+        dest_file: dest_file
+    end
+  end
+end

data/lib/hodor/api/oozie.rb

@@ -0,0 +1,215 @@
+require_relative "oozie/job"
+require_relative "oozie/query"
+require_relative "oozie/session"
+require_relative "oozie/bundle"
+require_relative "oozie/coordinator"
+require_relative "oozie/materialization"
+require_relative "oozie/workflow"
+require_relative "oozie/action"
+require_relative "oozie/hadoop_job"
+require_relative "hdfs"
+
+module Hodor::Oozie
+
+    class << self
+
+      def env
+        Hodor::Environment.instance
+      end
+
+      def session
+        Hodor::Oozie::Session.instance
+      end
+
+      def hdfs
+        Hodor::Hdfs.instance
+      end
+
+      def logger
+        env.logger
+      end
+
+      def build_rest_param(filter)
+        params = filter.map { |match|
+          case match
+          when :killed; "status%3DKILLED"
+          when :succeeded; "status%3DSUCCEEDED"
+          end
+        }
+
+        if params.size > 0
+          filter_exp = "filter=#{params.join(';')}"
+        else
+          filter_exp = ""
+        end
+      end
+
+      def job_by_id(job_id, filter = nil)
+        if (job_id.nil?)
+          result = Hodor::Oozie::Query.new status: [:running_first]
+        else
+          if job_id =~ /job_\d+/
+            result = HadoopJob.new(session.current_id, job_id)
+          else
+            if filter
+              response = session.get_job_state(job_id, build_rest_param(filter))
+            else
+              response = session.get_job_state(job_id)
+            end
+            json = JSON.parse(response)
+            job_type = json["toString"]
+            case job_type.split(" ").first.downcase.to_sym
+            when :bundle;
+              result = Bundle.new(json)
+            when :coordinator;
+              result = Coordinator.new(json)
+            when :workflow;
+              result = Workflow.new(json)
+            when :action;
+              result = Action.new(json)
+            when :coordinatoraction;
+              result = Materialization.new(json)
+            else
+            end
+          end
+        end
+      end
+
+      def job_by_path(job_path, make_current = false, filter = nil)
+        if job_path.nil? || job_path.eql?(".")
+          movement = :none
+        elsif job_path.eql?("/")
+          movement = :root
+        elsif job_path.eql?("b") || job_path.eql?("back") || 
+          job_path.eql?("u") || job_path.eql?("up") || job_path.eql?("..")
+          movement = :up
+        elsif job_path.eql?("d") || job_path.eql?("down") || 
+          job_path.eql?("f") || job_path.eql?("forward") || job_path.length < 5
+          movement = :down
+        else
+          movement = :jump
+        end
+
+        job_id = session.job_relative(movement, job_path)
+        job = job_by_id(job_id, filter)
+        session.make_current(job) if make_current
+        job
+      end
+
+      def change_job(job_path, filter = nil)
+        job_by_path(job_path, true, filter)
+      end
+
+      def select_job(job)
+        # load jobs.yml file
+        pwd = Dir.pwd
+        if File.exists?("jobs.yml")
+          jobs = env.yml_load(File.expand_path('jobs.yml', pwd))
+          marked_jobs = jobs.keys.select { |key| key.start_with?('^') }
+          marked_jobs.each { |mark|
+            jobs[mark[1..-1]] = jobs[mark]
+          }
+          if job.nil?
+            # No job explicitly specified, so look for a
+            # marked job (i.e. job starting with ^)
+            jobs.each_pair { |key, val|
+              if key.to_s.strip.start_with?('^')
+                job = key.to_s
+              end
+            }
+            fail "You must specify which job from jobs.yml to run" if !job
+          end
+          jobs = jobs.symbolize_keys
+          if !jobs.has_key?(job.to_sym)
+            caret = "^#{job.to_s}"
+            fail "Job '#{job}' was not defined in jobs.yml" if !jobs.has_key?(caret.to_sym)
+          end
+          selected_job = jobs[job.to_sym]
+          env.select_job(selected_job)
+        else
+          fail "No jobs.yml file exists in the current directory. You must specify a jobs.yml file"
+        end
+      end
+
+      # collect all job.properties.erb files up to root of repo
+      # and compose them together in top down order (i.e. deeper
+      # directories override properties in higher directories.)
+      # If direct job properties file is provided, properties will
+      # be interpolated using values in that file.
+      def compose_job_file(direct_job = nil, prefix = '')
+        if direct_job.nil?
+          pwd = Dir.pwd
+          paths = env.paths_from_root(pwd)
+          composite_jobfile = paths.inject('') { |result, path|
+            jobfile = File.expand_path('job.properties.erb', path)
+            if File.exists?(jobfile)
+              result << "\nFrom Job File '#{jobfile}':\n"
+              result << File.read(jobfile)
+            end
+            result
+          }
+          FileUtils.mkdir './.tmp' unless Dir.exists?('./.tmp')
+          composite_properties_file = File.expand_path(".tmp/runjob.properties.erb", pwd)
+          File.open(composite_properties_file, "w") do |f|
+            f.puts composite_jobfile
+          end
+          out_file = composite_properties_file.sub(/\.erb$/,'')
+          dest_file = generate_and_write_job_file(out_file, composite_properties_file, prefix)
+        else
+          raise "Job file '#{direct_job}' not found" unless File.exists?(direct_job)
+          if direct_job.end_with?('.erb')
+            out_file = "/tmp/#{File.basename(direct_job.sub(/.erb$/,''))}"
+            dest_file = generate_and_write_job_file(out_file, direct_job, prefix)
+          else
+            direct_job
+            out_file = append_prefix_to_filename(direct_job, prefix)
+            unless prefix.blank?
+              FileUtils.cp(direct_job, out_file)
+              dest_file = out_file
+            end
+          end
+        end
+        dest_file
+      end
+
+      def generate_and_write_job_file(file_name, in_file, prefix = '')
+        out_file = append_prefix_to_filename(file_name, prefix)
+        File.open(out_file, 'w') { |f| f.write(env.erb_load(in_file)) }
+        out_file
+      end
+
+      def append_prefix_to_filename(file_name, prefix = '')
+        insert_index =  file_name.rindex(File::SEPARATOR)
+        String.new(file_name).insert((insert_index.nil? ? 0 : insert_index+1) , prefix)
+      end
+
+
+      def deploy_job(job, clean_deploy)
+        select_job(job)
+        fail "No deploy section for job '#{job}'." if !env.job.has_key?("deploy")
+        if env.job[:deploy].nil?
+          fail "Nothing to deploy. Check the deploy section of your jobs.yml file"
+        else
+          env.job[:deploy].split.each { |path|
+            hdfs.put_dir File.expand_path(path, env.root), { clean: clean_deploy }
+          }
+        end
+      end
+
+      # If job references a job.properties or job.properties.erb file, that file will be
+      # used directly to interpolate job property values.
+      def run_job(job = nil, dry_run = false, file_prefix = '')
+        if job && (job =~ /job.properties.erb$/ || job =~ /job.properties/)
+          jobfile = compose_job_file(job, file_prefix)
+        else
+          select_job(job)
+          jobfile = compose_job_file(nil, file_prefix)
+        end
+        unless dry_run
+          runfile = env.deploy_tmp_file(jobfile)
+          env.ssh "oozie job -oozie :oozie_url -config #{runfile} -run", echo: true, echo_cmd: true
+        end
+        jobfile
+      end
+    end
+end

data/lib/hodor/api/oozie/action.rb

@@ -0,0 +1,52 @@
+module Hodor::Oozie
+
+  class Action < Job
+
+    attr_reader :parent_id, :json, :status, :error_message, :data, :transition, :external_status, :cred,
+          :type, :end_time, :external_id, :start_time, :external_child_ids, :name, :error_code,
+          :tracker_url, :retries, :to_string, :console_url
+
+    class << self
+      def default_columns
+        [:index, :id, :name, :status, :created_at, :nominal_time]
+      end
+    end
+
+    def initialize(json)
+      super()
+      @json = json
+
+      @error_message = json["errorMessage"]
+      @status = json["status"]
+      @stats = json["stats"]
+      @data = json["data"]
+      @transition = json["transition"]
+      @external_status = json["externalStatus"]
+      @cred = json["cred"]
+      @conf = json["conf"]
+      @type = json["type"]
+      @end_time = parse_time json["endTime"]
+      @external_id = json["externalId"]
+      @id = json["id"]
+      @start_time = parse_time json["startTime"]
+      @external_child_ids = json["externalChildIDs"]
+      @name = json["name"]
+      @error_code = json["errorCode"]
+      @tracker_url = json["trackerUri"]
+      @retries = json["retries"]
+      @to_string = json["toString"]
+      @console_url = json["consoleUrl"]
+      @parent_id = @id[0..@id.index('@')-1]
+    end
+
+    def expand
+      if @external_id && !@external_id.eql?('-')
+        [ oozie.job_by_id(@external_id) ]
+      else
+        nil
+      end
+    end
+
+  end
+
+end