cassandra_model_spark 0.0.1.5-java

data/lib/cassandra_model_spark/java_classes.rb

@@ -0,0 +1,36 @@
+import_java_object 'java.util.ArrayList'
+import_java_object 'org.apache.spark.SparkConf'
+import_java_object 'org.apache.spark.api.java.JavaSparkContext'
+import_java_object 'org.apache.spark.sql.cassandra.CassandraSQLContext'
+import_java_object 'java.util.HashMap', as: 'JavaHashMap'
+import_java_object 'org.apache.spark.sql.SQLContext', as: 'SparkSQLContext'
+import_java_object 'org.apache.spark.sql.RowFactory', as: 'SparkRowFactory'
+import_java_object 'org.apache.log4j.Logger', as: 'JLogger'
+import_java_object 'org.apache.log4j.Level', as: 'JLevel'
+import_java_object 'org.apache.log4j.Priority', as: 'JPriority'
+import_java_object 'org.apache.spark.util.Utils', as: 'SparkUtils'
+import_java_object 'org.apache.spark.storage.StorageLevel', as: 'JStorageLevel'
+import_java_object 'org.apache.spark.api.cassandra_model.CassandraHelper', as: 'SparkCassandraHelper'
+import_java_object 'org.apache.spark.api.cassandra_model.SchemaBuilder', as: 'SparkSchemaBuilder'
+import_java_object 'org.apache.spark.api.cassandra_model.DataTypeHelper', as: 'SparkSqlDataTypeHelper'
+import_java_object 'org.apache.spark.api.cassandra_model.MarshalLoader', as: 'ScalaMarshalLoader'
+import_java_object 'org.apache.spark.api.cassandra_model.MapStringStringRowMapping', as: 'SparkMapStringStringRowMapping'
+import_java_object 'org.apache.spark.api.cassandra_model.SparkRowRowMapping', as: 'SparkSparkRowRowMapping'
+import_java_object 'org.apache.spark.deploy.master.Master', as: 'SparkMaster'
+import_java_object 'org.apache.spark.deploy.worker.RubyWorkerStarter', as: 'SparkWorkerStarter'
+
+%w(BinaryType BooleanType ByteType DataType
+   DateType Decimal DecimalType DoubleType FloatType IntegerType
+   LongType Metadata NullType PrecisionInfo ShortType
+   StringType StructField StructType TimestampType).each do |sql_type|
+  Object.const_set(:"Sql#{sql_type}", import_quiet { SparkSqlDataTypeHelper.public_send(:"get#{sql_type}") })
+end
+
+#noinspection RubyConstantNamingConvention
+SqlStringArrayType = SparkSqlDataTypeHelper.getArrayType(SqlStringType)
+
+#noinspection RubyConstantNamingConvention
+SqlIntegerArrayType = SparkSqlDataTypeHelper.getArrayType(SqlIntegerType)
+
+#noinspection RubyConstantNamingConvention
+SqlStringStringMapType = SparkSqlDataTypeHelper.getMapType(SqlStringType, SqlStringType)

data/lib/cassandra_model_spark/launcher.rb

@@ -0,0 +1,150 @@
+require 'socket'
+
+module CassandraModel
+  module Spark
+    class Launcher
+      def start_master
+        system(env, "#{spark_daemon} start #{start_master_args}")
+        add_master_jars
+      end
+
+      def run_master
+        validate_env!
+
+        result = SparkMaster.startRpcEnvAndEndpoint(master_config[:host], master_config[:master_port], master_config[:ui_port], spark_conf)._1
+        wait_for_shutdown do
+          result.shutdown
+          result.awaitTermination
+        end
+      end
+
+      def start_slaves
+        workers.map do |worker|
+          system(env, "#{spark_daemon} start #{start_slave_args(worker)}")
+        end
+      end
+
+      def run_slave
+        validate_env!
+
+        result = SparkWorkerStarter.startWorker(master_url, slave_config[:host], master_config[:master_port], master_config[:ui_port], spark_conf)
+        wait_for_shutdown do
+          result.shutdown
+          result.awaitTermination
+        end
+      end
+
+      def stop_master
+        system(env, "#{spark_daemon} stop #{master_args}")
+      end
+
+      def stop_slaves
+        workers.map do |worker|
+          system(env, "#{spark_daemon} stop #{slave_args(worker)}")
+        end
+      end
+
+      private
+
+      def spark_conf
+        @spark_conf ||= ConnectionCache[nil].send(:spark_conf)
+      end
+
+      def wait_for_shutdown
+        begin
+          loop { sleep 0.2 }
+        rescue Interrupt
+          yield
+        end
+      end
+
+      def to_argv(args)
+        args.split.to_java_argv
+      end
+
+      def validate_env!
+        unless ENV['SPARK_HOME'] && File.expand_path(ENV['SPARK_HOME']) == Spark.home
+          raise 'Spark enviroment not set correctly'
+        end
+      end
+
+      def add_master_jars
+        ConnectionCache[nil].tap do |connection|
+          connection.config = {spark: {master: master_url}}
+          connection.spark_context.addJar("#{Spark.classpath}/cmodel_scala_helper.jar")
+        end
+        ConnectionCache.clear
+      end
+
+      def workers
+        slave_config[:worker_count].to_i.times.map { |index| index + 1 }
+      end
+
+      def start_master_args
+        "#{master_args} #{run_master_args}"
+      end
+
+      def run_master_args
+        "--ip #{Socket.gethostname} --port #{master_config[:master_port]} --webui-port #{master_config[:ui_port]} -h #{master_config[:host]}"
+      end
+
+      def start_slave_args(id)
+        "#{slave_args(id)} #{run_slave_args}"
+      end
+
+      def run_slave_args
+        "--webui-port #{slave_config[:ui_port]} #{master_url}"
+      end
+
+      def master_args
+        "org.apache.spark.deploy.master.Master #{master_config[:id]}"
+      end
+
+      def slave_args(id)
+        "org.apache.spark.deploy.worker.Worker #{id}"
+      end
+
+      def spark_daemon
+        "#{Spark.home}/sbin/spark-daemon.sh"
+      end
+
+      def master_url
+        "spark://#{master_config[:host]}:#{master_config[:master_port]}"
+      end
+
+      def master_config
+        config.merge(config.fetch(:master) { {} })
+      end
+
+      def slave_config
+        config.merge(config.fetch(:slave) { {} })
+      end
+
+      def config
+        @config ||= begin
+          override_config = ConnectionCache[nil].config.fetch(:spark_daemon) { {} }
+          {
+              id: 1,
+              ui_port: 8180,
+              master_port: 7077,
+              worker_count: 1,
+              host: Socket.gethostname,
+          }.merge(override_config)
+        end
+      end
+
+      def env
+        @env ||= spark_env.merge(ENV.to_hash)
+      end
+
+      def spark_env
+        @spark_env ||= {
+            'SPARK_HOME' => Spark.home,
+            'SPARK_CLASSPATH' => Spark.classpath,
+            'SPARK_JARS' => Dir["#{Spark.classpath}/*.jar"] * ',',
+        }
+      end
+
+    end
+  end
+end

data/lib/cassandra_model_spark/query_builder.rb

@@ -0,0 +1,37 @@
+module CassandraModel
+  class QueryBuilder
+    def group(*columns)
+      append_option(columns, :group)
+    end
+
+    def as_data_frame(options = {})
+      if @record_klass.is_a?(Spark::DataFrame)
+        data_frame_from_frame(options)
+      else
+        data_frame_from_model(options)
+      end
+    end
+
+    private
+
+    def data_frame_from_frame(options)
+      query_frame = @record_klass.query(@params, @options)
+      Spark::DataFrame.new(@record_klass.record_klass, nil, options.merge(spark_data_frame: query_frame))
+    end
+
+    def data_frame_from_model(options)
+      updated_restriction = @record_klass.restriction_attributes(@params).inject({}) do |memo, (key, value)|
+        updated_key = if value.is_a?(Array)
+                        value = value.to_java
+                        updated_key = key.is_a?(ThomasUtils::KeyComparer) ? key.to_s : "#{key} IN"
+                        "#{updated_key} (#{(%w(?)*value.count)*','})"
+                      else
+                        key.is_a?(ThomasUtils::KeyComparer) ? "#{key} ?" : "#{key} = ?"
+                      end
+        memo.merge!(updated_key => value)
+      end.stringify_keys.to_java
+      rdd = SparkCassandraHelper.filterRDD(@record_klass.rdd, updated_restriction)
+      Spark::DataFrame.new(@record_klass, rdd, options)
+    end
+  end
+end

data/lib/cassandra_model_spark/raw_connection.rb

@@ -0,0 +1,47 @@
+module CassandraModel
+  class RawConnection
+    def java_spark_context
+      @spark_context ||= begin
+        JavaSparkContext.new(spark_conf).tap do |java_spark_context|
+          java_spark_context.sc.addJar("#{Spark.classpath}/cmodel_scala_helper.jar")
+        end
+      end
+    end
+
+    def spark_context
+      java_spark_context.sc
+    end
+
+    private
+
+    def spark_conf
+      @spark_conf ||= SparkConf.new(true).tap do |conf|
+        conf.set('spark.app.name', 'cassandra_model_spark')
+        conf.set('spark.master', 'local[*]')
+        conf.set('spark.cassandra.connection.host', config[:hosts].first)
+        flat_spark_config.each { |key, value| conf.set(key, value) }
+      end
+    end
+
+    def flat_spark_config(config = spark_config)
+      config.inject({}) do |memo, (key, value)|
+        if value.is_a?(Hash)
+          memo.merge!(child_spark_conf(key, value))
+        else
+          memo.merge!(key.to_s => value)
+        end
+      end
+    end
+
+    def child_spark_conf(key, value)
+      child_conf = flat_spark_config(value)
+      child_conf.inject({}) do |child_memo, (child_key, child_value)|
+        child_memo.merge!("#{key}.#{child_key}" => child_value)
+      end
+    end
+
+    def spark_config
+      config.slice(:spark)
+    end
+  end
+end

data/lib/cassandra_model_spark/record.rb

@@ -0,0 +1,18 @@
+module CassandraModel
+  class Record
+    def self.rdd
+      @spark_rdd ||= SparkCassandraHelper.cassandra_table(
+          table.connection.spark_context,
+          table.connection.config[:keyspace],
+          table_name)
+    end
+
+    def self.rdd_row_mapping
+      nil
+    end
+
+    def self.count
+      rdd.count
+    end
+  end
+end

data/lib/cassandra_model_spark/spark.rb

@@ -0,0 +1,33 @@
+require 'fileutils'
+
+module CassandraModel
+  module Spark
+    class << self
+      def root
+        @gem_root ||= File.expand_path('../../..', __FILE__)
+      end
+
+      def home
+        @home ||= (ENV['SPARK_HOME'] || default_home)
+      end
+
+      def classpath
+        @classpath ||= (ENV['SPARK_CLASSPATH'] || default_classpath)
+      end
+
+      private
+
+      def default_classpath
+        File.expand_path('./lib/', home).tap do |path|
+          FileUtils.mkdir_p(path)
+        end
+      end
+
+      def default_home
+        File.expand_path('~/.cassandra_model_spark').tap do |path|
+          FileUtils.mkdir_p(path)
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: cassandra_model_spark
+version: !ruby/object:Gem::Version
+  version: 0.0.1.5
+platform: java
+authors:
+- Thomas RM Rogers
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cassandra_model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.16
+- !ruby/object:Gem::Dependency
+  name: thomas_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.16
+description: |-
+  Spark integration for cassandra_model.
+  Get high-performance data analytics with the ease of cassandra_model.
+  Inspired by the ruby-spark gem.
+email: thomasrogers03@gmail.com
+executables:
+- cmodel-spark-build
+- cmodel-spark-env.rb
+- cmodel-spark-master
+- cmodel-spark-slaves
+- cmodel-spark-run-master
+- cmodel-spark-run-slave
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/cmodel-spark-build
+- bin/cmodel-spark-env.rb
+- bin/cmodel-spark-master
+- bin/cmodel-spark-run-master
+- bin/cmodel-spark-run-master.sh
+- bin/cmodel-spark-run-slave
+- bin/cmodel-spark-run-slave.sh
+- bin/cmodel-spark-slaves
+- ext/scala_helper/bin/load-spark-env.sh
+- ext/scala_helper/bin/spark-class
+- ext/scala_helper/build.sbt
+- ext/scala_helper/cassandra_helper.scala
+- ext/scala_helper/data_type_helper.scala
+- ext/scala_helper/marshal_loader.scala
+- ext/scala_helper/marshal_row_mapping.scala
+- ext/scala_helper/project/plugins.sbt
+- ext/scala_helper/sbin/spark-config.sh
+- ext/scala_helper/sbin/spark-daemon.sh
+- ext/scala_helper/schema_builder.scala
+- ext/scala_helper/worker.scala
+- lib/cassandra_model_spark.rb
+- lib/cassandra_model_spark/build.rb
+- lib/cassandra_model_spark/column_cast.rb
+- lib/cassandra_model_spark/connection_cache.rb
+- lib/cassandra_model_spark/data_frame.rb
+- lib/cassandra_model_spark/java_bridge.rb
+- lib/cassandra_model_spark/java_classes.rb
+- lib/cassandra_model_spark/launcher.rb
+- lib/cassandra_model_spark/query_builder.rb
+- lib/cassandra_model_spark/raw_connection.rb
+- lib/cassandra_model_spark/record.rb
+- lib/cassandra_model_spark/spark.rb
+homepage: https://www.github.com/thomasrogers03/cassandra_model_spark
+licenses:
+- Apache License 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Spark integration for cassandra_model
+test_files: []