cassandra_model_spark 0.0.1.5 → 0.0.4

data/lib/cassandra_model_spark/java_bridge.rb

@@ -1,20 +1,38 @@
+module CassandraModel
+  module Spark
+    module Lib
+
+    end
+  end
+end
+
 if RUBY_ENGINE == 'jruby'
   class Hash
     def to_java
-      JavaHashMap.new(self)
+      CassandraModel::Spark::Lib::JavaHashMap.new(self)
     end
   end
 
   class Array
+    def self.from_java_array_list(array_list)
+      array_list.to_a
+    end
+
     def to_java_argv
       to_java(:string)
     end
   end
 
+  class String
+    def self.from_java_string(string)
+      string
+    end
+  end
+
 else
   class Hash
     def to_java
-      JavaHashMap.new.tap do |map|
+      CassandraModel::Spark::Lib::JavaHashMap.new.tap do |map|
         each do |key, value|
           map.put(key, value)
         end
@@ -23,6 +41,10 @@ else
   end
 
   class Array
+    def self.from_java_array_list(array_list)
+      array_list.toArray
+    end
+
     def to_java
       self
     end
@@ -31,6 +53,12 @@ else
       self
     end
   end
+
+  class String
+    def self.from_java_string(string)
+      string.toString
+    end
+  end
 end
 
 module JavaBridge
@@ -38,7 +66,7 @@ module JavaBridge
     def import_java_object(path, options = {})
       name = options.fetch(:as) { path.split('.').last }.to_sym
       klass = "Java::#{path}"
-      Object.const_set(name, eval(klass))
+      set_import_const(name, eval(klass))
     end
 
     def initialize_java_engine
@@ -47,7 +75,7 @@ module JavaBridge
   else
     def import_java_object(path, options = {})
       name = options.fetch(:as) { path.split('.').last }.to_sym
-      Object.const_set(name, load_java_class(path))
+      set_import_const(name, load_java_class(path))
     end
 
     def require(path)
@@ -79,6 +107,14 @@ module JavaBridge
     end
   end
 
+  private
+
+  def set_import_const(name, value)
+    CassandraModel::Spark::Lib.const_set(name, value)
+  end
+
+  public
+
   def import_quiet
     prev_verbox = $VERBOSE
     $VERBOSE = nil

data/lib/cassandra_model_spark/java_classes.rb

@@ -1,36 +1,51 @@
 import_java_object 'java.util.ArrayList'
 import_java_object 'org.apache.spark.SparkConf'
 import_java_object 'org.apache.spark.api.java.JavaSparkContext'
+import_java_object 'org.apache.spark.streaming.api.java.JavaStreamingContext', as: 'JavaSparkStreamingContext'
+import_java_object 'org.apache.spark.streaming.Duration', as: 'SparkDuration'
 import_java_object 'org.apache.spark.sql.cassandra.CassandraSQLContext'
 import_java_object 'java.util.HashMap', as: 'JavaHashMap'
 import_java_object 'org.apache.spark.sql.SQLContext', as: 'SparkSQLContext'
 import_java_object 'org.apache.spark.sql.RowFactory', as: 'SparkRowFactory'
+import_java_object 'org.apache.spark.sql.catalyst.expressions.GenericRow', as: 'SqlGenericRow'
 import_java_object 'org.apache.log4j.Logger', as: 'JLogger'
 import_java_object 'org.apache.log4j.Level', as: 'JLevel'
 import_java_object 'org.apache.log4j.Priority', as: 'JPriority'
 import_java_object 'org.apache.spark.util.Utils', as: 'SparkUtils'
 import_java_object 'org.apache.spark.storage.StorageLevel', as: 'JStorageLevel'
+
+import_java_object 'org.apache.spark.api.cassandra_model.ColumnDeserializer', as: 'SparkColumnDeserializer'
+import_java_object 'org.apache.spark.api.cassandra_model.RowConversions', as: 'SqlRowConversions'
 import_java_object 'org.apache.spark.api.cassandra_model.CassandraHelper', as: 'SparkCassandraHelper'
-import_java_object 'org.apache.spark.api.cassandra_model.SchemaBuilder', as: 'SparkSchemaBuilder'
 import_java_object 'org.apache.spark.api.cassandra_model.DataTypeHelper', as: 'SparkSqlDataTypeHelper'
 import_java_object 'org.apache.spark.api.cassandra_model.MarshalLoader', as: 'ScalaMarshalLoader'
 import_java_object 'org.apache.spark.api.cassandra_model.MapStringStringRowMapping', as: 'SparkMapStringStringRowMapping'
 import_java_object 'org.apache.spark.api.cassandra_model.SparkRowRowMapping', as: 'SparkSparkRowRowMapping'
+import_java_object 'org.apache.spark.api.cassandra_model.LuaRDD'
+
 import_java_object 'org.apache.spark.deploy.master.Master', as: 'SparkMaster'
 import_java_object 'org.apache.spark.deploy.worker.RubyWorkerStarter', as: 'SparkWorkerStarter'
 
+if CassandraModel.const_defined?('TESTING_SCALA')
+  import_java_object 'com.datastax.spark.connector.CassandraRow', as: 'SparkCassandraRow'
+  import_java_object 'org.apache.spark.api.cassandra_model.LuaRowValue'
+  import_java_object 'org.apache.spark.api.cassandra_model.LuaRowLib'
+end
+
 %w(BinaryType BooleanType ByteType DataType
    DateType Decimal DecimalType DoubleType FloatType IntegerType
    LongType Metadata NullType PrecisionInfo ShortType
+   ArrayType MapType
    StringType StructField StructType TimestampType).each do |sql_type|
-  Object.const_set(:"Sql#{sql_type}", import_quiet { SparkSqlDataTypeHelper.public_send(:"get#{sql_type}") })
+  type = import_quiet { CassandraModel::Spark::Lib::SparkSqlDataTypeHelper.public_send(:"get#{sql_type}") }
+  CassandraModel::Spark::Lib.const_set(:"Sql#{sql_type}", type)
 end
 
 #noinspection RubyConstantNamingConvention
-SqlStringArrayType = SparkSqlDataTypeHelper.getArrayType(SqlStringType)
+SqlStringArrayType = CassandraModel::Spark::Lib::SparkSqlDataTypeHelper.getArrayType(CassandraModel::Spark::Lib::SqlStringType)
 
 #noinspection RubyConstantNamingConvention
-SqlIntegerArrayType = SparkSqlDataTypeHelper.getArrayType(SqlIntegerType)
+SqlIntegerArrayType = CassandraModel::Spark::Lib::SparkSqlDataTypeHelper.getArrayType(CassandraModel::Spark::Lib::SqlIntegerType)
 
 #noinspection RubyConstantNamingConvention
-SqlStringStringMapType = SparkSqlDataTypeHelper.getMapType(SqlStringType, SqlStringType)
+SqlStringStringMapType = CassandraModel::Spark::Lib::SparkSqlDataTypeHelper.getMapType(CassandraModel::Spark::Lib::SqlStringType, CassandraModel::Spark::Lib::SqlStringType)

data/lib/cassandra_model_spark/query_builder.rb

@@ -16,7 +16,7 @@ module CassandraModel
 
     def data_frame_from_frame(options)
       query_frame = @record_klass.query(@params, @options)
-      Spark::DataFrame.new(@record_klass.record_klass, nil, options.merge(spark_data_frame: query_frame))
+      Spark::DataFrame.new(options.delete(:class) || @record_klass.record_klass, nil, options.merge(spark_data_frame: query_frame))
     end
 
     def data_frame_from_model(options)
@@ -30,8 +30,8 @@ module CassandraModel
                       end
         memo.merge!(updated_key => value)
       end.stringify_keys.to_java
-      rdd = SparkCassandraHelper.filterRDD(@record_klass.rdd, updated_restriction)
-      Spark::DataFrame.new(@record_klass, rdd, options)
+      rdd = Spark::Lib::SparkCassandraHelper.filterRDD(@record_klass.rdd, updated_restriction)
+      Spark::DataFrame.new(options.delete(:class) || @record_klass, rdd, options)
     end
   end
 end

data/lib/cassandra_model_spark/raw_connection.rb

@@ -2,7 +2,7 @@ module CassandraModel
   class RawConnection
     def java_spark_context
       @spark_context ||= begin
-        JavaSparkContext.new(spark_conf).tap do |java_spark_context|
+        Spark::Lib::JavaSparkContext.new(spark_conf).tap do |java_spark_context|
           java_spark_context.sc.addJar("#{Spark.classpath}/cmodel_scala_helper.jar")
         end
       end
@@ -12,10 +12,19 @@ module CassandraModel
       java_spark_context.sc
     end
 
+    def has_spark_context?
+      !!@spark_context
+    end
+
+    #noinspection RubyInstanceMethodNamingConvention
+    def create_java_spark_streaming_context
+      Spark::Lib::JavaSparkStreamingContext.new(java_spark_context, Spark::Lib::SparkDuration.new(2000))
+    end
+
     private
 
     def spark_conf
-      @spark_conf ||= SparkConf.new(true).tap do |conf|
+      @spark_conf ||= Spark::Lib::SparkConf.new(true).tap do |conf|
         conf.set('spark.app.name', 'cassandra_model_spark')
         conf.set('spark.master', 'local[*]')
         conf.set('spark.cassandra.connection.host', config[:hosts].first)
@@ -44,4 +53,4 @@ module CassandraModel
       config.slice(:spark)
     end
   end
-end
+end

data/lib/cassandra_model_spark/record.rb

@@ -1,7 +1,7 @@
 module CassandraModel
   class Record
     def self.rdd
-      @spark_rdd ||= SparkCassandraHelper.cassandra_table(
+      @spark_rdd ||= Spark::Lib::SparkCassandraHelper.cassandra_table(
           table.connection.spark_context,
           table.connection.config[:keyspace],
           table_name)
@@ -14,5 +14,9 @@ module CassandraModel
     def self.count
       rdd.count
     end
+
+    def self.sql_schema
+      Spark::SqlSchema.new(cassandra_columns)
+    end
   end
 end

data/lib/cassandra_model_spark/schema.rb

@@ -0,0 +1,47 @@
+module CassandraModel
+  module Spark
+    class Schema
+      attr_reader :schema
+
+      def initialize(sql_schema)
+        @schema = sql_schema.fields.inject({}) do |memo, field|
+          column = field.name
+          type = field.dataType
+          memo.merge!(column.to_sym => sql_type(type))
+        end
+      end
+
+      def ==(rhs)
+        rhs.is_a?(Schema) && rhs.schema == schema
+      end
+
+      private
+
+      def sql_type(type)
+        case sql_type_name(type)
+          when 'ArrayType'
+            [:list, sql_type(type.elementType)]
+          when 'MapType'
+            [:map, sql_type(type.keyType), sql_type(type.valueType) ]
+          when 'IntegerType'
+            :int
+          when 'BooleanType'
+            :boolean
+          when 'DoubleType'
+            :double
+          when 'BinaryType'
+            :blob
+          when 'TimestampType'
+            :timestamp
+          else
+            :text
+        end
+      end
+
+      def sql_type_name(data_type)
+        data_type.getClass.getSimpleName
+      end
+
+    end
+  end
+end

data/lib/cassandra_model_spark/sql_schema.rb

@@ -0,0 +1,49 @@
+module CassandraModel
+  module Spark
+    class SqlSchema
+      attr_reader :schema
+
+      def initialize(cassandra_schema)
+        fields = cassandra_schema.map do |column, type|
+          Lib::SqlStructField.apply(column.to_s, sql_type(type), true, Lib::SqlMetadata.empty)
+        end
+        if RUBY_ENGINE == 'jruby'
+          fields = fields.to_java('org.apache.spark.sql.types.StructField')
+        end
+        @schema = Lib::SqlStructType.apply(fields)
+      end
+
+      def ==(rhs)
+        rhs.is_a?(SqlSchema) && rhs.schema == schema
+      end
+
+      private
+
+      def sql_type(type)
+        case type
+          when Array
+            base_type, first_type, second_type = type
+            case base_type
+              when :map
+                Lib::SqlMapType.apply(sql_type(first_type), sql_type(second_type), true)
+              else
+                Lib::SqlArrayType.apply(sql_type(first_type))
+            end
+          when :int
+            Lib::SqlIntegerType
+          when :boolean
+            Lib::SqlBooleanType
+          when :double
+            Lib::SqlDoubleType
+          when :blob
+            Lib::SqlBinaryType
+          when :timestamp
+            Lib::SqlTimestampType
+          else
+            Lib::SqlStringType
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,43 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: cassandra_model_spark
 version: !ruby/object:Gem::Version
-  version: 0.0.1.5
+  version: 0.0.4
 platform: ruby
 authors:
 - Thomas RM Rogers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-29 00:00:00.000000000 Z
+date: 2016-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cassandra_model
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - <=
       - !ruby/object:Gem::Version
-        version: 0.9.16
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - <=
       - !ruby/object:Gem::Version
-        version: 0.9.16
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: thomas_utils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.1.16
+    - - <
+      - !ruby/object:Gem::Version
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.1.16
+    - - <
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: rjb
   requirement: !ruby/object:Gem::Requirement
@@ -80,13 +92,16 @@ files:
 - ext/scala_helper/bin/spark-class
 - ext/scala_helper/build.sbt
 - ext/scala_helper/cassandra_helper.scala
+- ext/scala_helper/column_deserializer.scala
 - ext/scala_helper/data_type_helper.scala
+- ext/scala_helper/lua_rdd.scala
+- ext/scala_helper/lua_row_lib.scala
 - ext/scala_helper/marshal_loader.scala
 - ext/scala_helper/marshal_row_mapping.scala
 - ext/scala_helper/project/plugins.sbt
+- ext/scala_helper/row_conversions.scala
 - ext/scala_helper/sbin/spark-config.sh
 - ext/scala_helper/sbin/spark-daemon.sh
-- ext/scala_helper/schema_builder.scala
 - ext/scala_helper/worker.scala
 - lib/cassandra_model_spark.rb
 - lib/cassandra_model_spark/build.rb
@@ -99,7 +114,9 @@ files:
 - lib/cassandra_model_spark/query_builder.rb
 - lib/cassandra_model_spark/raw_connection.rb
 - lib/cassandra_model_spark/record.rb
+- lib/cassandra_model_spark/schema.rb
 - lib/cassandra_model_spark/spark.rb
+- lib/cassandra_model_spark/sql_schema.rb
 homepage: https://www.github.com/thomasrogers03/cassandra_model_spark
 licenses:
 - Apache License 2.0

data/ext/scala_helper/schema_builder.scala

@@ -1,35 +0,0 @@
-package org.apache.spark.api.cassandra_model
-
-import org.apache.spark.rdd._
-import com.datastax.spark.connector._
-import com.datastax.spark.connector.rdd._
-import org.apache.spark.sql._
-import org.apache.spark.sql.types._
-
-class SchemaBuilder {
-  var fields = Array[StructField]()
-
-  def build = StructType(fields)
-
-  def cassandraRDDToRDD(rdd: RDD[CassandraRow]) = {
-    rdd.map(
-      p => Row.fromSeq(
-        p.columnValues.map{
-          p => p match {
-              case (d: java.util.Date) => new java.sql.Timestamp(d.getTime())
-              case _ => p
-            }
-          }
-        )
-    )
-  }
-
-  def createDataFrame(sqlc: SQLContext, rdd: RDD[CassandraRow]) = {
-    val new_rdd = cassandraRDDToRDD(rdd)
-    sqlc.createDataFrame(new_rdd, build)
-  }
-
-  def addColumn(name: String, sql_type: DataType) = {
-    fields :+= StructField(name, sql_type, true)
-  }
-}