cassandra_model_spark 0.0.1.5-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 15373fcc369058da2966c722c6b35ac0c164b073
+  data.tar.gz: d840cc0cd750dea5d198f37ced59608340cced97
+SHA512:
+  metadata.gz: c356202b4c8fc59c8936d4af43ad844a0f70895ee09a5453741354cb957c179486fc7491ec28ad8ef5445f742063a9a86897031e0feaae890b9e886d4a0422cf
+  data.tar.gz: a0045c5c883ba0148482c97fbe293b303c30e81289be36c83eca45f3fdebe34429f23d1305f6377b250902186837bf5bfd7d8f2e58d4c5a37edb3dfc220c55ac

data/README.md

@@ -0,0 +1,17 @@
+# cassandra_model_spark
+
+## Copyright
+
+Copyright 2015 Thomas Rogers.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+
+
+
+
+

data/bin/cmodel-spark-build

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'cassandra_model_spark/build'
+

data/bin/cmodel-spark-env.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'bundler/setup'
+require 'cassandra_model_spark/spark'
+
+print "export SPARK_HOME=#{CassandraModel::Spark.home}"
+
+

data/bin/cmodel-spark-master

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'bundler/setup'
+require 'cassandra_model_spark'
+require 'cassandra_model_spark/launcher'
+
+command = ARGV.shift.downcase.to_sym
+launcher = CassandraModel::Spark::Launcher.new
+case command
+  when :start
+    launcher.start_master
+  when :run
+    launcher.run_master
+  when :stop
+    launcher.stop_master
+  else
+    puts '=> only supports start, run, or stop'
+end
+

data/bin/cmodel-spark-run-master

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+bin_path = File.expand_path('../', __FILE__)
+`#{bin_path}/cmodel-spark-run-master.sh`

data/bin/cmodel-spark-run-master.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "${CMODEL_SPARK_HOME}" ]; then
+  export CMODEL_SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+`"${CMODEL_SPARK_HOME}/bin/cmodel-spark-env.rb"`
+"${CMODEL_SPARK_HOME}/bin/cmodel-spark-master" run

data/bin/cmodel-spark-run-slave

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+bin_path = File.expand_path('../', __FILE__)
+`#{bin_path}/cmodel-spark-run-slave.sh`

data/bin/cmodel-spark-run-slave.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "${CMODEL_SPARK_HOME}" ]; then
+  export CMODEL_SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+`"${CMODEL_SPARK_HOME}/bin/cmodel-spark-env.rb"`
+"${CMODEL_SPARK_HOME}/bin/cmodel-spark-slaves" run

data/bin/cmodel-spark-slaves

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'bundler/setup'
+require 'cassandra_model_spark'
+require 'cassandra_model_spark/launcher'
+
+command = ARGV.shift.downcase.to_sym
+launcher = CassandraModel::Spark::Launcher.new
+case command
+  when :start
+    launcher.start_slaves
+  when :run
+    launcher.run_slave
+  when :stop
+    launcher.stop_slaves
+  else
+    puts '=> only supports start, run, or stop'
+end
+

data/ext/scala_helper/bin/load-spark-env.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script loads spark-env.sh if it exists, and ensures it is only loaded once.
+# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within the current directory's
+# conf/ subdirectory.
+
+# Figure out where Spark is installed
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -z "$SPARK_ENV_LOADED" ]; then
+  export SPARK_ENV_LOADED=1
+
+  # Returns the parent of the directory this script lives in.
+  parent_dir="${SPARK_HOME}"
+
+  user_conf_dir="${SPARK_CONF_DIR:-"$parent_dir"/conf}"
+
+  if [ -f "${user_conf_dir}/spark-env.sh" ]; then
+    # Promote all variable declarations to environment (exported) variables
+    set -a
+    . "${user_conf_dir}/spark-env.sh"
+    set +a
+  fi
+fi
+
+# Setting SPARK_SCALA_VERSION if not already set.
+
+if [ -z "$SPARK_SCALA_VERSION" ]; then
+
+  ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
+  ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10"
+
+  if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
+    echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
+    echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
+    exit 1
+  fi
+
+  if [ -d "$ASSEMBLY_DIR2" ]; then
+    export SPARK_SCALA_VERSION="2.11"
+  else
+    export SPARK_SCALA_VERSION="2.10"
+  fi
+fi

data/ext/scala_helper/bin/spark-class

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "${SPARK_HOME}" ]; then
+  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${SPARK_HOME}"/bin/load-spark-env.sh
+
+# Find the java binary
+if [ -n "${JAVA_HOME}" ]; then
+  RUNNER="${JAVA_HOME}/bin/java"
+else
+  if [ `command -v java` ]; then
+    RUNNER="java"
+  else
+    echo "JAVA_HOME is not set" >&2
+    exit 1
+  fi
+fi
+
+# Find assembly jar
+SPARK_ASSEMBLY_JAR=
+if [ -f "${SPARK_HOME}/RELEASE" ]; then
+  ASSEMBLY_DIR="${SPARK_HOME}/lib"
+else
+  ASSEMBLY_DIR="${SPARK_HOME}/assembly/target/scala-$SPARK_SCALA_VERSION"
+fi
+
+GREP_OPTIONS=
+num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^spark-assembly.*hadoop.*\.jar$" | wc -l)"
+if [ "$num_jars" -eq "0" -a -z "$SPARK_ASSEMBLY_JAR" -a "$SPARK_PREPEND_CLASSES" != "1" ]; then
+  echo "Failed to find Spark assembly in $ASSEMBLY_DIR." 1>&2
+  echo "You need to build Spark before running this program." 1>&2
+  exit 1
+fi
+if [ -d "$ASSEMBLY_DIR" ]; then
+  ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^spark-assembly.*hadoop.*\.jar$" || true)"
+  if [ "$num_jars" -gt "1" ]; then
+    echo "Found multiple Spark assembly jars in $ASSEMBLY_DIR:" 1>&2
+    echo "$ASSEMBLY_JARS" 1>&2
+    echo "Please remove all but one jar." 1>&2
+    exit 1
+  fi
+fi
+
+SPARK_ASSEMBLY_JAR="${ASSEMBLY_DIR}/${ASSEMBLY_JARS}"
+
+LAUNCH_CLASSPATH="$SPARK_ASSEMBLY_JAR"
+
+# Add the launcher build dir to the classpath if requested.
+if [ -n "$SPARK_PREPEND_CLASSES" ]; then
+  LAUNCH_CLASSPATH="${SPARK_HOME}/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH"
+fi
+
+export _SPARK_ASSEMBLY="$SPARK_ASSEMBLY_JAR"
+
+# For tests
+if [[ -n "$SPARK_TESTING" ]]; then
+  unset YARN_CONF_DIR
+  unset HADOOP_CONF_DIR
+fi
+
+# The launcher library will print arguments separated by a NULL character, to allow arguments with
+# characters that would be otherwise interpreted by the shell. Read that in a while loop, populating
+# an array that will be used to exec the final command.
+CMD=()
+while IFS= read -d '' -r ARG; do
+  CMD+=("$ARG")
+done < <("$RUNNER" -cp "$LAUNCH_CLASSPATH" org.apache.spark.launcher.Main "$@")
+exec "${CMD[@]}"

data/ext/scala_helper/build.sbt

@@ -0,0 +1,62 @@
+import AssemblyKeys._
+
+assemblySettings
+
+name := "cmodel_scala_helper"
+version := "0.0.1"
+scalaVersion := "2.10.4"
+
+val sparkVersion = "1.5.2"
+val hadoopClientVersion = "1.0.4"
+val cassandraConnectionVersion = "1.5.0-M3"
+
+val _targetDir = scala.util.Properties.envOrElse("TARGET_DIR", "target")
+
+javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
+
+artifactPath in Compile in packageBin := file(s"${_targetDir}/cmodel_scala_helper.jar")
+outputPath in packageDependency := file(s"${_targetDir}/spark-assembly-${sparkVersion}-cassandra_model-hadoop${hadoopClientVersion}.jar")
+
+libraryDependencies ++= Seq(
+  "org.apache.spark" %% "spark-core" % sparkVersion excludeAll(ExclusionRule(organization = "org.apache.hadoop")),
+  "org.apache.spark" %% "spark-sql" % sparkVersion,
+  "org.apache.spark" %% "spark-hive" % sparkVersion,
+  "org.apache.spark" %% "spark-streaming" % sparkVersion,
+  "org.apache.spark" %% "spark-streaming-kafka" % sparkVersion,
+  "org.apache.spark" %% "spark-streaming-flume" % sparkVersion,
+  "org.apache.spark" %% "spark-graphx" % sparkVersion,
+  "org.apache.spark" %% "spark-mllib" % sparkVersion,
+  "org.apache.hadoop" %  "hadoop-client" % hadoopClientVersion,
+  "com.datastax.spark" %% "spark-cassandra-connector" % cassandraConnectionVersion,
+  "com.datastax.spark" %% "spark-cassandra-connector-java" % cassandraConnectionVersion,
+  "com.github.fommil.netlib" % "all" % "1.1.2",
+  "org.scalatest" % "scalatest_2.10" % "2.2.1" % "test"
+)
+
+resolvers ++= Seq(
+  "JBoss Repository"     at "http://repository.jboss.org/nexus/content/repositories/releases/",
+  "Spray Repository"     at "http://repo.spray.cc/",
+  "Cloudera Repository"  at "https://repository.cloudera.com/artifactory/cloudera-repos/",
+  "Akka Repository"      at "http://repo.akka.io/releases/",
+  "Twitter4J Repository" at "http://twitter4j.org/maven2/",
+  "Apache HBase"         at "https://repository.apache.org/content/repositories/releases",
+  "Twitter Maven Repo"   at "http://maven.twttr.com/",
+  "scala-tools"          at "https://oss.sonatype.org/content/groups/scala-tools",
+  "Typesafe repository"  at "http://repo.typesafe.com/typesafe/releases/",
+  "Second Typesafe repo" at "http://repo.typesafe.com/typesafe/maven-releases/",
+  "Mesosphere Public Repository" at "http://downloads.mesosphere.io/maven",
+  Resolver.sonatypeRepo("public")
+)
+
+mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
+  {
+    case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
+    case m if m.startsWith("META-INF") => MergeStrategy.discard
+    case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
+    case PathList("org", "apache", xs @ _*) => MergeStrategy.first
+    case PathList("org", "jboss", xs @ _*) => MergeStrategy.first
+    case "about.html"  => MergeStrategy.rename
+    case "reference.conf" => MergeStrategy.concat
+    case _ => MergeStrategy.first
+  }
+}

data/ext/scala_helper/cassandra_helper.scala

@@ -0,0 +1,23 @@
+package org.apache.spark.api.cassandra_model
+
+import org.apache.spark._
+import com.datastax.spark.connector._
+import com.datastax.spark.connector.rdd._
+import java.util._
+import scala.collection.JavaConversions.mapAsScalaMap
+
+object CassandraHelper {
+  def cassandraTable(sc: SparkContext, keyspace: String, table: String) = {
+    sc.cassandraTable(keyspace, table)
+  }
+  def filterRDD(rdd: CassandraRDD[CassandraRow], restriction: HashMap[String, Any]) = {
+    var result = rdd
+    for ((k,v) <- restriction) {
+      result = v match {
+        case (a: Array[Any]) => result.where(k, a : _*)
+        case _ => result.where(k, v)
+      }
+    }
+    result
+  }
+}

data/ext/scala_helper/data_type_helper.scala

@@ -0,0 +1,27 @@
+package org.apache.spark.api.cassandra_model
+
+import org.apache.spark.sql.types._
+
+object DataTypeHelper {
+  def getArrayType(key_type: DataType) = ArrayType(key_type)
+  def getBinaryType = BinaryType
+  def getBooleanType = BooleanType
+  def getByteType = ByteType
+  def getDataType = DataType
+  def getDateType = DateType
+  def getDecimal = Decimal
+  def getDecimalType = DecimalType
+  def getDoubleType = DoubleType
+  def getFloatType = FloatType
+  def getIntegerType = IntegerType
+  def getLongType = LongType
+  def getMapType(key_type: DataType, value_type: DataType) = MapType(key_type, value_type)
+  def getMetadata = Metadata
+  def getNullType = NullType
+  def getPrecisionInfo = PrecisionInfo
+  def getShortType = ShortType
+  def getStringType = StringType
+  def getStructField = StructField
+  def getStructType = StructType
+  def getTimestampType = TimestampType
+}

data/ext/scala_helper/marshal_loader.scala

@@ -0,0 +1,204 @@
+package org.apache.spark.api.cassandra_model
+
+import scala.collection.mutable._
+
+class MarshalLoader (dump: Array[Byte]) {
+  private val bytes: Array[Byte] = dump
+  private var parse_index: Int = 0
+  private var symbol_table: List[String] = List()
+  private var object_table: List[AnyRef] = List()
+
+  private def getBytes() = {
+    bytes
+  }
+
+  private def recordObject(obj: AnyRef) = {
+    object_table :+= obj
+  }
+
+  private def nextBytes(amount: Int): Array[Byte] = {
+    val result = bytes.slice(parse_index, parse_index + amount)
+    parse_index = parse_index + amount
+    result
+  }
+
+  private def nextByte(): Byte = {
+    val result = bytes(parse_index)
+
+    parse_index = parse_index + 1
+
+    result
+  }
+
+  private def scanIntBits(num_bytes: Int) = {
+    var bit: Int = 0
+    var value: Int = 0
+
+    for (bit <- 0 to num_bytes-1) {
+      val next_value = 0xff & nextByte()
+      value += (next_value << (bit * 8))
+    }
+    value
+  }
+
+  private def decodeInt(): java.lang.Integer = {
+    val first_byte: Int = nextByte()
+    var value: Int = 0
+
+    if (first_byte == 0)
+      return 0
+    if (first_byte >= 6)
+      return first_byte - 5
+    if (first_byte <= -6)
+      return 5 + first_byte
+
+    var num_bytes = first_byte
+    if (num_bytes > 0) {
+      value = scanIntBits(num_bytes)
+    } else {
+      num_bytes = -num_bytes
+      value = scanIntBits(num_bytes)
+      val long_bytes = num_bytes
+      val long_value: Long = value - (1L << (long_bytes * 8L))
+      value = long_value.toInt
+    }
+
+    value
+  }
+
+  private def decodeDouble(): java.lang.Double = {
+    val length = decodeInt()
+    val str_value = new String(nextBytes(length))
+
+    val result: java.lang.Double = str_value.toDouble
+    recordObject(result)
+    result
+  }
+
+  private def decodeASCIIString(): String = {
+    val length = decodeInt()
+
+    val result = new String(nextBytes(length))
+    recordObject(result)
+    result
+  }
+
+  private def decodeSymbol(): String = {
+    val length = decodeInt()
+    val string_bytes = nextBytes(length)
+    val result = new String(string_bytes)
+
+    symbol_table :+= result
+
+    result
+  }
+
+  private def decodeSymLink(): String = {
+    val index = decodeInt()
+
+    symbol_table(index)
+  }
+
+  private def decodeString(): String = {
+    val string_code = nextByte()
+    val length = decodeInt()
+    val str_bytes = nextBytes(length)
+    val var_count = decodeInt()
+    val encoding = decodeAny()
+
+    if (encoding == "E") {
+      val is_utf8 = decodeAny()
+    }
+
+    val result = new String(str_bytes)
+    recordObject(result)
+    result
+  }
+
+  private def decodeMagic(): String = {
+    val magic = new String(nextBytes(4))
+
+    if (magic != "MRSH") {
+      throw new IllegalArgumentException("Invalid format header: '" + magic + "'")
+    }
+
+    magic
+  }
+
+  private def decodeVersion(): Array[Byte] = {
+    val version = nextBytes(2)
+
+    if (version(0) != 0x04 || version(1) != 0x08) {
+      throw new IllegalArgumentException("Invalid Marshal version: [" + version(0) + "], [" + version(1) + "]")
+    }
+
+    version
+  }
+
+  private def decodeHashItem(): String = {
+    val ivar_code = nextByte()
+    decodeString()
+  }
+
+  private def decodeHash(): HashMap[AnyRef, AnyRef] = {
+    var result = new HashMap[AnyRef, AnyRef]
+    val length = decodeInt()
+
+    var item = 0
+    for (item <- 0 to length-1) {
+      val key = decodeAny()
+      val value = decodeAny()
+      result(key) = value
+    }
+
+    recordObject(result)
+    result
+  }
+
+  private def decodeArray(): Array[AnyRef] = {
+    var list_result: List[AnyRef] = List()
+    val length = decodeInt()
+
+    var item = 0
+    for (item <- 0 to length-1) {
+      val value = decodeAny()
+      list_result :+= value
+    }
+
+    val result = list_result.toArray
+    recordObject(result)
+    result
+  }
+
+  private def decodeObjectReference(): AnyRef = {
+      val index = decodeInt()-1
+
+      object_table(index)
+  }
+
+  private def decodeAny(): AnyRef = {
+    val code = nextByte()
+
+    code match {
+      case 0x30 => null
+      case 0x54 => true: java.lang.Boolean
+      case 0x46 => false: java.lang.Boolean
+      case 0x69 => decodeInt()
+      case 0x66 => decodeDouble()
+      case 0x3a => decodeSymbol()
+      case 0x3b => decodeSymLink()
+      case 0x7b => decodeHash()
+      case 0x5b => decodeArray()
+      case 0x22 => decodeASCIIString()
+      case 0x49 => decodeString()
+      case 0x40 => decodeObjectReference()
+      case _ => throw new IllegalArgumentException("Unsupported code type: " + code)
+    }
+  }
+
+  private val magic = decodeMagic()
+  private val version = decodeVersion()
+  private val value = decodeAny()
+
+  def getValue(): AnyRef = value
+}