ruby-spark 1.0.0

data/ext/ruby_c/murmur.h

@@ -0,0 +1,9 @@
+#ifndef MURMUR_INCLUDED
+#define MURMUR_INCLUDED
+
+#include "ruby.h"
+
+VALUE method_portable_hash(int argc, VALUE *argv, VALUE klass);
+VALUE method_murmur2_digest(int argc, VALUE *argv, VALUE klass);
+
+#endif

data/ext/ruby_c/ruby-spark.c

@@ -0,0 +1,18 @@
+#include "ruby.h"
+#include "murmur.h"
+
+
+VALUE SparkModule;
+VALUE SparkDigestModule;
+VALUE SparkDigestMurmur2Class;
+
+
+void Init_ruby_spark_ext()
+{
+  SparkModule             = rb_define_module("Spark");
+  SparkDigestModule       = rb_define_module_under(SparkModule, "Digest");
+  SparkDigestMurmur2Class = rb_define_class_under(SparkDigestModule, "Murmur2", rb_cObject);
+
+  rb_define_singleton_method(SparkDigestModule, "portable_hash", method_portable_hash, -1);
+  rb_define_singleton_method(SparkDigestMurmur2Class, "digest", method_murmur2_digest, -1);
+}

data/ext/ruby_java/Digest.java

@@ -0,0 +1,36 @@
+import org.jruby.Ruby;
+import org.jruby.RubyModule;
+import org.jruby.RubyObject;
+import org.jruby.RubyClass;
+import org.jruby.RubyString;
+import org.jruby.RubyFixnum;
+import org.jruby.anno.JRubyModule;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+@JRubyModule(name="Spark::Digest")
+public class Digest extends RubyObject{
+
+  // Have to be the same as in C extension
+  final static long PORTABLE_HASH_SEED = 16154832;
+
+  public Digest(final Ruby ruby, RubyClass rubyClass) {
+    super(ruby, rubyClass);
+  }
+
+  @JRubyMethod(module=true)
+  public static IRubyObject portable_hash(ThreadContext context, IRubyObject self, IRubyObject arg) {
+    Ruby ruby = self.getRuntime();
+
+    RubyString keyString = (RubyString)arg;
+
+    long hash = Murmur2.hash64(keyString.getBytes(), (int)keyString.length().getLongValue(), PORTABLE_HASH_SEED);
+
+    RubyFixnum result = new RubyFixnum(ruby, hash);
+
+    return result;
+  }
+
+}
+

data/ext/ruby_java/Murmur2.java

@@ -0,0 +1,98 @@
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyObject;
+import org.jruby.RubyString;
+import org.jruby.RubyFixnum;
+import org.jruby.anno.JRubyClass;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+/** Murmur hash 2.0.
+ * 
+ * The murmur hash is a relative fast hash function from
+ * http://murmurhash.googlepages.com/ for platforms with efficient
+ * multiplication.
+ *
+ * http://d3s.mff.cuni.cz/~holub/sw/javamurmurhash/
+ *
+ */
+
+@JRubyClass(name="Spark::Digest::Murmur2")
+public class Murmur2 extends RubyObject {
+
+  public Murmur2(final Ruby ruby, RubyClass rubyClass) {
+    super(ruby, rubyClass);
+  }
+
+  @JRubyMethod(required=1, optional=1, module=true)
+  public static IRubyObject digest(ThreadContext context, IRubyObject self, IRubyObject[] args) {
+    Ruby ruby = context.getRuntime();
+
+    RubyString keyString = (RubyString)args[0];
+    long seed;
+
+    if(args.length > 1){
+      RubyFixnum rb_seed = (RubyFixnum)args[1];
+      seed = rb_seed.getLongValue();
+    }
+    else{
+      seed = 0;
+    }
+
+    long hash = hash64(keyString.getBytes(), (int)keyString.length().getLongValue(), seed);
+
+    RubyFixnum result = new RubyFixnum(ruby, hash);
+    return result;
+  }
+
+
+  /** Generates 64 bit hash from byte array of the given length and seed.
+   * 
+   * @param data byte array to hash
+   * @param length length of the array to hash
+   * @param seed initial seed value
+   * @return 64 bit hash of the given array
+   */
+  public static long hash64(final byte[] data, int length, long seed) {
+    final long m = 0xc6a4a7935bd1e995L;
+    final int r = 47;
+
+    long h = (seed&0xffffffffl)^(length*m);
+
+    int length8 = length/8;
+
+    for (int i=0; i<length8; i++) {
+      final int i8 = i*8;
+      long k =  ((long)data[i8+0]&0xff)      +(((long)data[i8+1]&0xff)<<8)
+          +(((long)data[i8+2]&0xff)<<16) +(((long)data[i8+3]&0xff)<<24)
+          +(((long)data[i8+4]&0xff)<<32) +(((long)data[i8+5]&0xff)<<40)
+          +(((long)data[i8+6]&0xff)<<48) +(((long)data[i8+7]&0xff)<<56);
+      
+      k *= m;
+      k ^= k >>> r;
+      k *= m;
+      
+      h ^= k;
+      h *= m; 
+    }
+    
+    switch (length%8) {
+    case 7: h ^= (long)(data[(length&~7)+6]&0xff) << 48;
+    case 6: h ^= (long)(data[(length&~7)+5]&0xff) << 40;
+    case 5: h ^= (long)(data[(length&~7)+4]&0xff) << 32;
+    case 4: h ^= (long)(data[(length&~7)+3]&0xff) << 24;
+    case 3: h ^= (long)(data[(length&~7)+2]&0xff) << 16;
+    case 2: h ^= (long)(data[(length&~7)+1]&0xff) << 8;
+    case 1: h ^= (long)(data[length&~7]&0xff);
+            h *= m;
+    };
+   
+    h ^= h >>> r;
+    h *= m;
+    h ^= h >>> r;
+
+    return h;
+  }
+
+}

data/ext/ruby_java/RubySparkExtService.java

@@ -0,0 +1,28 @@
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyModule;
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.runtime.load.BasicLibraryService;
+
+public class RubySparkExtService implements BasicLibraryService
+{
+  public boolean basicLoad(final Ruby ruby) throws java.io.IOException {
+
+    RubyModule sparkModule = ruby.defineModule("Spark");
+    RubyModule sparkDigestModule = sparkModule.defineModuleUnder("Digest");
+    RubyClass  sparkDigestMurmur2Class = sparkDigestModule.defineClassUnder("Murmur2", ruby.getObject(), sparkDigestMurmur2Allocator);
+
+    sparkDigestModule.defineAnnotatedMethods(Digest.class);
+    sparkDigestMurmur2Class.defineAnnotatedMethods(Murmur2.class);
+
+    return true;
+  }
+
+  public static ObjectAllocator sparkDigestMurmur2Allocator = new ObjectAllocator() {
+    public IRubyObject allocate(Ruby ruby, RubyClass rubyClass) {
+      return new Murmur2(ruby, rubyClass);
+    }
+  };
+
+}

data/ext/ruby_java/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("ruby_spark_ext")

data/ext/spark/build.sbt

@@ -0,0 +1,73 @@
+import AssemblyKeys._
+
+assemblySettings
+
+// Default values
+val defaultScalaVersion     = "2.10.4"
+val defaultSparkVersion     = "1.3.0"
+val defaultSparkCoreVersion = "2.10"
+val defaultSparkHome        = "target"
+val defaultHadoopVersion    = "1.0.4"
+
+// Values
+val _scalaVersion     = scala.util.Properties.envOrElse("SCALA_VERSION", defaultScalaVersion)
+val _sparkVersion     = scala.util.Properties.envOrElse("SPARK_VERSION", defaultSparkVersion)
+val _sparkCoreVersion = scala.util.Properties.envOrElse("SPARK_CORE_VERSION", defaultSparkCoreVersion)
+val _sparkHome        = scala.util.Properties.envOrElse("SPARK_HOME", defaultSparkHome)
+val _hadoopVersion    = scala.util.Properties.envOrElse("HADOOP_VERSION", defaultHadoopVersion)
+
+// Project settings
+name := "ruby-spark"
+
+version := "1.0.0"
+
+scalaVersion := _scalaVersion
+
+javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
+
+// Jar target folder
+artifactPath in Compile in packageBin := file(s"${_sparkHome}/ruby-spark.jar")
+outputPath in packageDependency := file(s"${_sparkHome}/ruby-spark-deps.jar")
+
+// Protocol buffer support
+seq(sbtprotobuf.ProtobufPlugin.protobufSettings: _*)
+
+// Additional libraries
+libraryDependencies ++= Seq(
+  "org.apache.spark"  %% "spark-core"    % _sparkVersion excludeAll(ExclusionRule(organization = "org.apache.hadoop")),
+  "org.apache.spark"  %% "spark-graphx"  % _sparkVersion,
+  "org.apache.spark"  %% "spark-mllib"   % _sparkVersion,
+  "org.apache.hadoop" %  "hadoop-client" % _hadoopVersion,
+  "com.github.fommil.netlib" % "all" % "1.1.2",
+  "org.scalatest" % "scalatest_2.10" % "2.2.1" % "test"
+)
+
+// Repositories
+resolvers ++= Seq(
+  "JBoss Repository"     at "http://repository.jboss.org/nexus/content/repositories/releases/",
+  "Spray Repository"     at "http://repo.spray.cc/",
+  "Cloudera Repository"  at "https://repository.cloudera.com/artifactory/cloudera-repos/",
+  "Akka Repository"      at "http://repo.akka.io/releases/",
+  "Twitter4J Repository" at "http://twitter4j.org/maven2/",
+  "Apache HBase"         at "https://repository.apache.org/content/repositories/releases",
+  "Twitter Maven Repo"   at "http://maven.twttr.com/",
+  "scala-tools"          at "https://oss.sonatype.org/content/groups/scala-tools",
+  "Typesafe repository"  at "http://repo.typesafe.com/typesafe/releases/",
+  "Second Typesafe repo" at "http://repo.typesafe.com/typesafe/maven-releases/",
+  "Mesosphere Public Repository" at "http://downloads.mesosphere.io/maven",
+  Resolver.sonatypeRepo("public")
+)
+
+// Merge strategy
+mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
+  {
+    case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
+    case m if m.startsWith("META-INF") => MergeStrategy.discard
+    case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
+    case PathList("org", "apache", xs @ _*) => MergeStrategy.first
+    case PathList("org", "jboss", xs @ _*) => MergeStrategy.first
+    case "about.html"  => MergeStrategy.rename
+    case "reference.conf" => MergeStrategy.concat
+    case _ => MergeStrategy.first
+  }
+}

data/ext/spark/project/plugins.sbt

@@ -0,0 +1,9 @@
+resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+
+resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/"
+
+resolvers += "Spray Repository" at "http://repo.spray.cc/"
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.10.2")
+
+addSbtPlugin("com.github.gseitz" % "sbt-protobuf" % "0.3.3")

data/ext/spark/sbt/sbt

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# This script launches sbt for this project. If present it uses the system
+# version of sbt. If there is no system version of sbt it attempts to download
+# sbt locally.
+SBT_VERSION=0.13.7
+URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+JAR=sbt/sbt-launch-${SBT_VERSION}.jar
+
+# Download sbt launch jar if it hasn't been downloaded yet
+if [ ! -f ${JAR} ]; then
+  # Download
+  printf "Attempting to fetch sbt\n"
+  JAR_DL=${JAR}.part
+  if hash curl 2>/dev/null; then
+    (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+  elif hash wget 2>/dev/null; then
+    (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+  else
+    printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
+    exit -1
+  fi
+fi
+if [ ! -f ${JAR} ]; then
+  # We failed to download
+  printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
+  exit -1
+fi
+printf "Launching sbt from ${JAR}\n"
+java \
+  -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
+  -jar ${JAR} \
+  "$@"

data/ext/spark/src/main/scala/Exec.scala

@@ -0,0 +1,91 @@
+package org.apache.spark.api.ruby
+
+import java.io.{File, FileOutputStream, InputStreamReader, BufferedReader}
+
+import scala.collection.JavaConversions._
+
+import org.apache.spark.{SparkEnv, Logging}
+import org.apache.spark.util._
+
+
+/* =================================================================================================
+ * class FileCommand
+ * =================================================================================================
+ *
+ * Save command to file and than execute him because from Scala you cannot simply run
+ * something like "bash --norc -i -c 'source .zshrc; ruby master.rb'"
+ */
+
+class FileCommand(command: String) extends Logging {
+
+  var pb: ProcessBuilder = null
+  var file: File = null
+
+  // Command is complete.
+  def this(command: String, env: SparkEnv) = {
+    this(command)
+    create(env)
+  }
+
+  // Template must contains %s which will be replaced for command
+  def this(template: String, command: String, env: SparkEnv, envVars: Map[String, String]) = {
+    this(template.format(command), env)
+    setEnvVars(envVars)
+  }
+
+  private def create(env: SparkEnv) {
+    val dir = new File(env.sparkFilesDir)
+    val ext = if(Utils.isWindows) ".cmd" else ".sh"
+    val shell = if(Utils.isWindows) "cmd" else "bash"
+
+    file = File.createTempFile("command", ext, dir)
+
+    val out = new FileOutputStream(file)
+    out.write(command.getBytes)
+    out.close
+
+    logInfo(s"New FileCommand at ${file.getAbsolutePath}")
+
+    pb = new ProcessBuilder(shell, file.getAbsolutePath)
+  }
+
+  def setEnvVars(vars: Map[String, String]) {
+    pb.environment().putAll(vars)
+  }
+
+  def run = {
+    new ExecutedFileCommand(pb.start)
+  }
+}
+
+
+/* =================================================================================================
+ * class ExecutedFileCommand
+ * =================================================================================================
+ *
+ * Represent process executed from file.
+ */
+
+class ExecutedFileCommand(process: Process) {
+
+  var reader: BufferedReader = null
+
+  def readLine = {
+    openInput
+    reader.readLine.toString.trim
+  }
+
+  def openInput {
+    if(reader != null){
+      return
+    }
+
+    val input = new InputStreamReader(process.getInputStream)
+    reader = new BufferedReader(input)
+  }
+
+  // Delegation
+  def destroy = process.destroy
+  def getInputStream = process.getInputStream
+  def getErrorStream = process.getErrorStream
+}

data/ext/spark/src/main/scala/MLLibAPI.scala

@@ -0,0 +1,4 @@
+package org.apache.spark.mllib.api.python
+
+// PythonMLLibAPI is private for python
+class MLLibAPI extends PythonMLLibAPI {}

data/ext/spark/src/main/scala/Marshal.scala

@@ -0,0 +1,52 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+
+
+/* =================================================================================================
+ * object Marshal
+ * =================================================================================================
+ */
+object Marshal {
+  def load(bytes: Array[Byte]) = {
+    val is = new DataInputStream(new ByteArrayInputStream(bytes))
+
+    val majorVersion = is.readUnsignedByte // 4
+    val minorVersion = is.readUnsignedByte // 8
+
+    (new MarshalLoad(is)).load
+  }
+
+  def dump(data: Any) = {
+    val aos = new ByteArrayOutputStream
+    val os = new DataOutputStream(aos)
+
+    os.writeByte(4)
+    os.writeByte(8)
+
+    (new MarshalDump(os)).dump(data)
+    aos.toByteArray
+  }
+}
+
+
+/* =================================================================================================
+ * class IterableMarshaller
+ * =================================================================================================
+ */
+class IterableMarshaller(iter: Iterator[Any]) extends Iterator[Array[Byte]] {
+  private val buffer = new ArrayBuffer[Any]
+
+  override def hasNext: Boolean = iter.hasNext
+
+  override def next(): Array[Byte] = {
+    while (iter.hasNext) {
+      buffer += iter.next()
+    }
+
+    Marshal.dump(buffer)
+  }
+}