ruby-spark 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd863f728212557da03e76f6e98eeed05695ea5d
-  data.tar.gz: 214b2022187727a50badcd1910313550e59aefdf
+  metadata.gz: c7435669c50b38e45f97113e7a67aa991edfaa46
+  data.tar.gz: 95bb22186a07f47f40915edcded518d081f5f8f7
 SHA512:
-  metadata.gz: 23c0c7b6ab63a2f9c191cddc4836c73cde61722b9e6f3c7e25b090afed7cda2eaff0d8718074ae3337ff5c4bd57e1223dab76f6cf7772b4c7dda3e7ed69d98c6
-  data.tar.gz: 234897b1851614ae1371b3a33417c8d036b00a4551185829b99ef398a110a614ffe1eaeac556c00859a45598bed4219e59eb98ddbffbe0fe2c25c024408b8628
+  metadata.gz: 27fd1ff26ed3478595f6b5ef2d48cb74da73c3a4f68df29ad4745f47621df8b617b21cc006ba8b2d5169d680e10a59498956a3a06be92437949e2faac4ccd1f7
+  data.tar.gz: d0d465c5e8f86ab3c8987ef6732446f10886354b8f3e7db281cb3a18eb64db362de89164680f288a015f28e242553fab3362fa35a1462eed2caa8b0322244158

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## Unreleased
+
+## 1.3.0
+
+  - new method on RDD (lookup)
+  - fix sbt url
+  - Spark 1.5.0
+
 ## 1.2.0 (15.06.2015)
 
   - target folder is now located at HOME

data/TODO.md

@@ -6,3 +6,4 @@
 - add_rb, add_inline_rb to Spark::{Context, RDD}
 - fix broadcast for cluster
 - dump to disk if there is memory limit
+- Add Partitioner to RDD

data/ext/spark/build.sbt

@@ -4,7 +4,7 @@ assemblySettings
 
 // Default values
 val defaultScalaVersion     = "2.10.4"
-val defaultSparkVersion     = "1.3.0"
+val defaultSparkVersion     = "1.5.0"
 val defaultSparkCoreVersion = "2.10"
 val defaultTargetDir        = "target"
 val defaultHadoopVersion    = "1.0.4"

data/ext/spark/sbt/sbt

@@ -3,9 +3,9 @@
 # This script launches sbt for this project. If present it uses the system
 # version of sbt. If there is no system version of sbt it attempts to download
 # sbt locally.
-SBT_VERSION=0.13.7
-URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
-URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+SBT_VERSION=0.13.9
+URL1=http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+URL2=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
 JAR=sbt/sbt-launch-${SBT_VERSION}.jar
 
 # Download sbt launch jar if it hasn't been downloaded yet
@@ -13,10 +13,10 @@ if [ ! -f ${JAR} ]; then
   # Download
   printf "Attempting to fetch sbt\n"
   JAR_DL=${JAR}.part
-  if hash curl 2>/dev/null; then
-    (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
-  elif hash wget 2>/dev/null; then
+  if hash wget 2>/dev/null; then
     (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+  elif hash curl 2>/dev/null; then
+    (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
   else
     printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
     exit -1
@@ -31,4 +31,4 @@ printf "Launching sbt from ${JAR}\n"
 java \
   -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
   -jar ${JAR} \
-  "$@"
+  "$@"

data/ext/spark/src/main/scala/RubyMLLibAPI.scala

@@ -22,10 +22,10 @@ class RubyMLLibAPI extends MLLibAPI {
   // trainLogisticRegressionModelWithLBFGS
   // trainSVMModelWithSGD
   // trainKMeansModel
-  // trainGaussianMixture
+  // trainGaussianMixtureModel
 
   // Rjb have a problem with theta: Array[Array[Double]]
-  override def trainNaiveBayes(data: JavaRDD[LabeledPoint], lambda: Double) = {
+  override def trainNaiveBayesModel(data: JavaRDD[LabeledPoint], lambda: Double) = {
     val model = NaiveBayes.train(data.rdd, lambda)
 
     List(

data/lib/spark/build.rb

@@ -3,7 +3,7 @@ module Spark
 
     DEFAULT_SCALA_VERSION  = '2.10.4'
     DEFAULT_CORE_VERSION   = '2.10'
-    DEFAULT_SPARK_VERSION  = '1.4.0'
+    DEFAULT_SPARK_VERSION  = '1.5.0'
     DEFAULT_HADOOP_VERSION = '1.0.4'
 
     SBT       = 'sbt/sbt'

data/lib/spark/mllib/classification/naive_bayes.rb

@@ -86,7 +86,7 @@ module Spark
           raise Spark::MllibError, "RDD should contains LabeledPoint, got #{first.class}"
         end
 
-        labels, pi, theta = Spark.jb.call(RubyMLLibAPI.new, 'trainNaiveBayes', rdd, lambda)
+        labels, pi, theta = Spark.jb.call(RubyMLLibAPI.new, 'trainNaiveBayesModel', rdd, lambda)
         theta = Spark::Mllib::Matrices.dense(theta.size, theta.first.size, theta)
 
         NaiveBayesModel.new(labels, pi, theta)

data/lib/spark/mllib/clustering/gaussian_mixture.rb

@@ -64,7 +64,7 @@ module Spark
     class GaussianMixture
 
       def self.train(rdd, k, convergence_tol: 0.001, max_iterations: 100, seed: nil)
-        weights, means, sigmas = Spark.jb.call(RubyMLLibAPI.new, 'trainGaussianMixture', rdd,
+        weights, means, sigmas = Spark.jb.call(RubyMLLibAPI.new, 'trainGaussianMixtureModel', rdd,
                                                k, convergence_tol, max_iterations, Spark.jb.to_long(seed))
 
         means.map! {|mu|    Spark.jb.java_to_ruby(mu)}

data/lib/spark/rdd.rb

@@ -1241,6 +1241,28 @@ module Spark
       self.map('lambda{|(_, value)| value}')
     end
 
+    # Return the list of values in the RDD for key `key`.
+    # TODO: add Partitioner for efficiently searching
+    #
+    # == Example:
+    #   rdd = $sc.parallelize(0..10)
+    #   rdd = rdd.group_by(lambda {|x| x%3})
+    #   rdd.lookup(2)
+    #   # => [[2, 5, 8]]
+    #
+    #   rdd = $sc.parallelize(0..10)
+    #   rdd = rdd.key_by(lambda{|x| x.even?})
+    #   rdd.lookup(true)
+    #   # => [0, 2, 4, 6, 8, 10]
+    #
+    def lookup(key)
+      lookup_key = "lookup_key_#{object_id}"
+
+      self.filter("lambda{|(key, _)| key == #{lookup_key}}")
+          .bind(lookup_key => key)
+          .values
+          .collect
+    end
 
     # Aliases
     alias_method :partitionsSize, :partitions_size

data/lib/spark/version.rb

@@ -1,3 +1,3 @@
 module Spark
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 end

data/spec/lib/key_spec.rb

@@ -39,4 +39,21 @@ RSpec.describe 'Spark::RDD' do
     # it_behaves_like 'a keying by', rand(2..10)
   end
 
+  it 'lookup' do
+    numbers = Generator.numbers
+    rdd_numbers = $sc.parallelize(numbers, 2)
+
+    rdd = rdd_numbers.group_by(lambda {|x| x%3})
+    rdd.lookup(2)
+
+    expect(rdd.lookup(2).first).to eq(
+      numbers.group_by{|x| x%3}[2]
+    )
+
+    rdd = rdd_numbers.key_by(lambda{|x| x.even?})
+    expect(rdd.lookup(true)).to eq(
+      numbers.select(&:even?)
+    )
+  end
+
 end

data/spec/lib/mllib/regression_spec.rb

@@ -99,7 +99,7 @@ RSpec.describe 'Spark::Mllib regression' do
 
       expect(lrm.weights[0]).to be_between(1.9, 2.1)
       expect(lrm.weights[1]).to be_between(-1.60, -1.40)
-      expect(lrm.weights[2]).to be_between(-1.0e-3, 1.0e-3)
+      expect(lrm.weights[2]).to be_between(-1.0e-2, 1.0e-2)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-spark
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Ondřej Moravčík
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-15 00:00:00.000000000 Z
+date: 2015-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rjb
@@ -346,7 +346,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - java, scala
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Ruby wrapper for Apache Spark
@@ -402,3 +402,4 @@ test_files:
 - spec/lib/statistic_spec.rb
 - spec/lib/whole_text_files_spec.rb
 - spec/spec_helper.rb
+has_rdoc: