cassandra_model_spark 0.0.1.5

data/lib/cassandra_model_spark/raw_connection.rb

@@ -0,0 +1,47 @@
+module CassandraModel
+  class RawConnection
+    def java_spark_context
+      @spark_context ||= begin
+        JavaSparkContext.new(spark_conf).tap do |java_spark_context|
+          java_spark_context.sc.addJar("#{Spark.classpath}/cmodel_scala_helper.jar")
+        end
+      end
+    end
+
+    def spark_context
+      java_spark_context.sc
+    end
+
+    private
+
+    def spark_conf
+      @spark_conf ||= SparkConf.new(true).tap do |conf|
+        conf.set('spark.app.name', 'cassandra_model_spark')
+        conf.set('spark.master', 'local[*]')
+        conf.set('spark.cassandra.connection.host', config[:hosts].first)
+        flat_spark_config.each { |key, value| conf.set(key, value) }
+      end
+    end
+
+    def flat_spark_config(config = spark_config)
+      config.inject({}) do |memo, (key, value)|
+        if value.is_a?(Hash)
+          memo.merge!(child_spark_conf(key, value))
+        else
+          memo.merge!(key.to_s => value)
+        end
+      end
+    end
+
+    def child_spark_conf(key, value)
+      child_conf = flat_spark_config(value)
+      child_conf.inject({}) do |child_memo, (child_key, child_value)|
+        child_memo.merge!("#{key}.#{child_key}" => child_value)
+      end
+    end
+
+    def spark_config
+      config.slice(:spark)
+    end
+  end
+end

data/lib/cassandra_model_spark/record.rb

@@ -0,0 +1,18 @@
+module CassandraModel
+  class Record
+    def self.rdd
+      @spark_rdd ||= SparkCassandraHelper.cassandra_table(
+          table.connection.spark_context,
+          table.connection.config[:keyspace],
+          table_name)
+    end
+
+    def self.rdd_row_mapping
+      nil
+    end
+
+    def self.count
+      rdd.count
+    end
+  end
+end

data/lib/cassandra_model_spark/spark.rb

@@ -0,0 +1,33 @@
+require 'fileutils'
+
+module CassandraModel
+  module Spark
+    class << self
+      def root
+        @gem_root ||= File.expand_path('../../..', __FILE__)
+      end
+
+      def home
+        @home ||= (ENV['SPARK_HOME'] || default_home)
+      end
+
+      def classpath
+        @classpath ||= (ENV['SPARK_CLASSPATH'] || default_classpath)
+      end
+
+      private
+
+      def default_classpath
+        File.expand_path('./lib/', home).tap do |path|
+          FileUtils.mkdir_p(path)
+        end
+      end
+
+      def default_home
+        File.expand_path('~/.cassandra_model_spark').tap do |path|
+          FileUtils.mkdir_p(path)
+        end
+      end
+    end
+  end
+end

data/lib/cassandra_model_spark.rb

@@ -0,0 +1,42 @@
+#--
+# Copyright 2015 Thomas RM Rogers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+
+require 'yaml'
+require 'logger'
+
+require 'concurrent'
+require 'cassandra'
+require 'active_support/all'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'thomas_utils'
+require 'batch_reactor'
+require 'cassandra_model'
+require 'rjb' unless RUBY_ENGINE == 'jruby' || CassandraModel.const_defined?('NO_BRIDGE')
+require 'cassandra_model_spark/spark'
+
+unless CassandraModel.const_defined?('NO_BRIDGE')
+  require 'cassandra_model_spark/java_bridge'
+  Dir["#{CassandraModel::Spark.classpath}/*.jar"].each { |file| require file }
+  initialize_java_engine
+  require 'cassandra_model_spark/java_classes'
+end
+
+require 'cassandra_model_spark/raw_connection'
+require 'cassandra_model_spark/connection_cache'
+require 'cassandra_model_spark/record'
+require 'cassandra_model_spark/query_builder'
+require 'cassandra_model_spark/data_frame'
+require 'cassandra_model_spark/column_cast'

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: cassandra_model_spark
+version: !ruby/object:Gem::Version
+  version: 0.0.1.5
+platform: ruby
+authors:
+- Thomas RM Rogers
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cassandra_model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.16
+- !ruby/object:Gem::Dependency
+  name: thomas_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.16
+- !ruby/object:Gem::Dependency
+  name: rjb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+description: |-
+  Spark integration for cassandra_model.
+  Get high-performance data analytics with the ease of cassandra_model.
+  Inspired by the ruby-spark gem.
+email: thomasrogers03@gmail.com
+executables:
+- cmodel-spark-build
+- cmodel-spark-env.rb
+- cmodel-spark-master
+- cmodel-spark-slaves
+- cmodel-spark-run-master
+- cmodel-spark-run-slave
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/cmodel-spark-build
+- bin/cmodel-spark-env.rb
+- bin/cmodel-spark-master
+- bin/cmodel-spark-run-master
+- bin/cmodel-spark-run-master.sh
+- bin/cmodel-spark-run-slave
+- bin/cmodel-spark-run-slave.sh
+- bin/cmodel-spark-slaves
+- ext/scala_helper/bin/load-spark-env.sh
+- ext/scala_helper/bin/spark-class
+- ext/scala_helper/build.sbt
+- ext/scala_helper/cassandra_helper.scala
+- ext/scala_helper/data_type_helper.scala
+- ext/scala_helper/marshal_loader.scala
+- ext/scala_helper/marshal_row_mapping.scala
+- ext/scala_helper/project/plugins.sbt
+- ext/scala_helper/sbin/spark-config.sh
+- ext/scala_helper/sbin/spark-daemon.sh
+- ext/scala_helper/schema_builder.scala
+- ext/scala_helper/worker.scala
+- lib/cassandra_model_spark.rb
+- lib/cassandra_model_spark/build.rb
+- lib/cassandra_model_spark/column_cast.rb
+- lib/cassandra_model_spark/connection_cache.rb
+- lib/cassandra_model_spark/data_frame.rb
+- lib/cassandra_model_spark/java_bridge.rb
+- lib/cassandra_model_spark/java_classes.rb
+- lib/cassandra_model_spark/launcher.rb
+- lib/cassandra_model_spark/query_builder.rb
+- lib/cassandra_model_spark/raw_connection.rb
+- lib/cassandra_model_spark/record.rb
+- lib/cassandra_model_spark/spark.rb
+homepage: https://www.github.com/thomasrogers03/cassandra_model_spark
+licenses:
+- Apache License 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Spark integration for cassandra_model
+test_files: []