spark_toolkit 0.1.1-java → 0.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56d3aadfc0f28d592c7051ee0fe9427df5f73383
-  data.tar.gz: 0fa58599de96d34d0c1fcf613e288073867ce6b8
+  metadata.gz: 28e6117542c2bfe075aa07f26288d6b5d0393940
+  data.tar.gz: c46cbeb68f311feedf38da887e1e0793abac395a
 SHA512:
-  metadata.gz: 20f30636b201ede56541e852fce3828211e4ee3aad31f83155afd8b58168a143d6c8c8bac913101be49790aa4acc601c55f2457039c722da9d0f2e666db72b4b
-  data.tar.gz: 053cf2ee78cd8bf2707aaa0c255ce21cb0866bf56b43b84056a262b8d8b1b2f9f570f6869dbad6874f6997064388a48bc81e2d7932163b4dd3801077572901e4
+  metadata.gz: 2552ec0b597334361de3f60d450c769ebcf223426ad5e7a85f0b2f2cb7d63a73be3dc9f1873217d4d48f1bdf4a53e3265d7606eef10476987903613fee00cf13
+  data.tar.gz: c70b25a32b9c793ffca931e822ed9b726554d12191016551f3f10cc61dff963e645280c40fff2d5dbe66d5834a5d70cfa292f792de554c301d543d4bc7e943af

data/docs/YARN.md

@@ -38,10 +38,10 @@ app_report_detail = app_report.get_detail
 ### Get Log of YARN Application
 
 ```ruby
-log = yarn.get_application_logs(app_id)
-formatted_log_stdout = SparkToolkit::YARN::SimpleFormatter.format(app_id, :stdout)
-formatted_log_stderr = SparkToolkit::YARN::SimpleFormatter.format(app_id, :stderr)
-formatted_log_all = SparkToolkit::YARN::SimpleFormatter.format(app_id, :all)
+log_stdout = yarn.get_application_logs(app_id, :stdout)
+log_stderr = yarn.get_application_logs(app_id, :stderr)
+log_all = yarn.get_application_logs(app_id, :all)
+formatted_log = SparkToolkit::YARN::SimpleLogFormatter.format(log_all)
 ```
 
 ### Get Report of YARN Attempt

data/lib/spark_toolkit/hadoop/hdfs/file_system.rb

@@ -41,6 +41,16 @@ module SparkToolkit
         @hdfs.copy_to_local_file(false, Path.new(hdfs_src), Path.new(local_dst), true)
       end
 
+      def get_file_status(entry)
+        @hdfs.get_file_status(Path.new(entry))
+      end
+      alias_method :status, :get_file_status
+
+      def rename(src, dst)
+        @hdfs.rename(Path.new(src), Path.new(dst))
+      end
+      alias_method :mv, :rename
+
       def exists?(path)
         @hdfs.exists(Path.new(path))
       end

data/lib/spark_toolkit/hadoop/yarn/application.rb

@@ -17,7 +17,8 @@ class Java::OrgApacheHadoopYarnApiRecordsImplPb::ApplicationReportPBImpl
       tracking_url: get_tracking_url,
       start_time: get_start_time,
       finish_time: get_finish_time,
-      state: get_yarn_application_state
+      app_state: get_yarn_application_state,
+      final_status: final_status
     }
   end
 
@@ -26,4 +27,8 @@ class Java::OrgApacheHadoopYarnApiRecordsImplPb::ApplicationReportPBImpl
   def get_yarn_application_state
     getYarnApplicationState.to_s.to_sym
   end
+
+  def final_status
+    final_application_status.to_s.to_sym
+  end
 end

data/lib/spark_toolkit/hadoop/yarn/client.rb

@@ -1,7 +1,9 @@
 module SparkToolkit
   module YARN
     Client = Java::OrgApacheHadoopYarnClientApiImpl::YarnClientImpl
+
     class Client
+      attr_reader :conf
       alias_method :initalise, :initialize
       def initialize(conf=nil)
         initalise
@@ -28,6 +30,22 @@ module SparkToolkit
         getNodeReports.to_a
       end
 
+      def get_cluster_report
+        sum = get_node_reports.reduce([0,0,0,0]) do |sum, report|
+          sum[0] += report.get_total_memory
+          sum[1] += report.get_used_memory
+          sum[2] += report.get_total_vcores
+          sum[3] += report.get_used_vcores
+          sum
+        end
+        {
+          total_memory: sum[0],
+          used_memory: sum[1],
+          total_vcores: sum[2],
+          used_vcores: sum[3],
+        }
+      end
+
       # Available devs are:
       # - :all
       # - :stdout

data/lib/spark_toolkit/hadoop/yarn/node.rb

@@ -1,4 +1,4 @@
-class Java::OrgApacheHadoopYarnApiRecordsImplPb::ResourcePBImpl
+class Java::OrgApacheHadoopYarnApiRecordsImplPb::NodeReportPBImpl
   # get_node_id
 
   # get_num_containers

data/lib/spark_toolkit/version.rb

@@ -1,3 +1,3 @@
 module SparkToolkit
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/spark_toolkit.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency 'hbase-jruby'
+
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: spark_toolkit
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: java
 authors:
 - Yuli Mo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-08 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: hbase-jruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement