spark_toolkit 0.1.0-java → 0.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 724f7d68120af23d1127588d5cce846ae9982cc0
-  data.tar.gz: 2aec33d6d4aa449a721953e6f4232ae53a75621e
+  metadata.gz: 56d3aadfc0f28d592c7051ee0fe9427df5f73383
+  data.tar.gz: 0fa58599de96d34d0c1fcf613e288073867ce6b8
 SHA512:
-  metadata.gz: 4b7dbfa7811f3b45d5d4c2ab1d02e21aaf354e6e8bcf5b2812ec025b9debf307aa4b1e076a4fd5854249ca3f83e9ef02949bf971ae4a81f903b332dc84856bfa
-  data.tar.gz: 25dc646ad44d6967fdc5b2efd957356c26446b883160d13527c07c02457c70fde1ac89996fde64c4bd34155d09b068c9f4adf9fdf0ba815084acc1da7ac0ec4c
+  metadata.gz: 20f30636b201ede56541e852fce3828211e4ee3aad31f83155afd8b58168a143d6c8c8bac913101be49790aa4acc601c55f2457039c722da9d0f2e666db72b4b
+  data.tar.gz: 053cf2ee78cd8bf2707aaa0c255ce21cb0866bf56b43b84056a262b8d8b1b2f9f570f6869dbad6874f6997064388a48bc81e2d7932163b4dd3801077572901e4

data/README.md

@@ -35,7 +35,7 @@ For more details, view the doc under `docs` directory.
 
 ## TODO
 
-- Support Spark 1.x
+- ~~Support Spark 1.x~~
 - Support Spark cluster mode
 - ~~Add YARN application log analyzer~~
 

data/docs/Spark.md

@@ -22,6 +22,7 @@ args = ["--class", "org.apache.spark.deploy.PythonRunner",
         "--arg", 2]
 spark_conf.yarn_deploy_mode(:cluster) # or :client
 spark_client.is_python_job(true)
+spark_client.avtive_kerberos # If you want to submit job to secure cluster
 # Submit your job to YARN and get its app_id for query
 yarn_app_id = spark_client.yarn_submit(args)
 # Or run as client, print all output into console

data/lib/spark_toolkit/hadoop/conf/configuration.rb

@@ -3,6 +3,7 @@ module SparkToolkit
     Configuration = Java::OrgApacheHadoopConf::Configuration
     class Configuration
       java_import org.apache.hadoop.fs.Path
+      java_import org.apache.hadoop.security.UserGroupInformation
 
       alias_method :initialise, :initialize
       def initialize(opts={})
@@ -21,6 +22,12 @@ module SparkToolkit
         add_resource_java(Path.new(f))
       end
 
+      def krb_login(principle, keytab)
+        set('hadoop.security.authentication', 'kerberos')
+        UserGroupInformation.set_configuration(self)
+        UserGroupInformation.login_user_from_keytab(principle, keytab)
+      end
+
       def []=(k, v)
         set(k, v)
       end

data/lib/spark_toolkit/spark/client.rb

@@ -69,9 +69,18 @@ module SparkToolkit
         @sconf.set("spark.hadoop.hadoop.security.authentication", "kerberos")
         @sconf.set("spark.hadoop.hadoop.security.authorization", "true")
 
+        UserGroupInformation.get_login_user.check_tgt_and_relogin_from_keytab
         UserGroupInformation.set_configuration(SparkHadoopUtil.get.newConfiguration(@sconf))
-        credentials = UserGroupInformation.getLoginUser.getCredentials
-        SparkHadoopUtil.get.addCurrentUserCredentials(credentials)
+        credentials = UserGroupInformation.get_login_user.get_credentials
+        SparkHadoopUtil.get.add_current_user_credentials(credentials)
+      end
+
+      def executor_cores n
+        @sconf.set_property('spark.executor.cores', n.to_s)
+      end
+
+      def num_executors n
+        @sconf.set_property('spark.executor.instances', n.to_s)
       end
 
       private
@@ -83,7 +92,7 @@ module SparkToolkit
           @sconf.set('spark.submit.deployMode', 'cluster')
         end
 
-        java.lang.System.setProperty("SPARK_YARN_MODE", "true")
+        java.lang.System.set_property("SPARK_YARN_MODE", "true")
       end
     end
   end

data/lib/spark_toolkit/version.rb

@@ -1,3 +1,3 @@
 module SparkToolkit
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spark_toolkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: java
 authors:
 - Yuli Mo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-04 00:00:00.000000000 Z
+date: 2017-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler