ruby-spark 1.1.0.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0c7cdadb3ef29b9ff9c4a4d24c545be97937e6d0
+  data.tar.gz: fd949d2b46717f81e3fefe6ae00f3cc5ca741c32
+SHA512:
+  metadata.gz: 80595e1ff9ae32831f2bf501c78150796e037460d70b7476cc7beab9c73035324fcabc6e4ce143eca919af9f71ec6850f46b399a11c8453fc411b0ca249a7a79
+  data.tar.gz: 112f079c1b024df1b2b35e008945109afb35553373998ce49496f7fa9361d0bc89fcd78bed10d10734fb4842dc9f35024a8c86ee4c41c33fcbe626b0c93bb1dc

data/.gitignore

@@ -0,0 +1,37 @@
+/.gemtags
+/.tags
+/java/spark.jar
+.jbundler
+target/*
+*.class
+*.jar
+pom.xml
+vendor/*
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+ext/spark/target/*
+ext/spark/project/target/*
+ext/spark/project/project/target/*
+wiki
+/benchmark/performance/spark/*
+/benchmark/performance/rspark/*

data/Gemfile

@@ -0,0 +1,47 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'sourcify', '0.6.0.rc4'
+gem 'method_source'
+gem 'commander'
+gem 'pry'
+gem 'nio4r'
+gem 'distribution'
+
+platform :mri do
+  gem 'rjb'
+  gem 'msgpack'
+  gem 'oj'
+  gem 'narray'
+end
+
+platform :jruby do
+  gem 'msgpack-jruby', require: 'msgpack'
+
+  # NameError: no constructorfor arguments (org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.joda.time.chrono.GJChronology) on Java::OrgJodaTime::DateTime
+  # gem 'mdarray'
+end
+
+group :stats do
+  # gem 'nmatrix'
+  # gem 'statsample'
+  # gem 'statsample-glm'
+  # gem 'statsample-timeseries'
+  # gem 'statistics2'
+  # gem 'statsample-optimization' # libgsl0-dev
+  # gem 'narray'
+  # gem 'gsl-nmatrix'
+end
+
+group :development do
+  gem 'benchmark-ips'
+  gem 'rspec'
+  gem 'rake-compiler'
+  gem 'guard'
+  gem 'guard-rspec'
+end
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ondřej Moravčík
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,252 @@
+# Ruby-Spark
+
+Apache Spark™ is a fast and general engine for large-scale data processing.
+
+This Gem allows you use Spark functionality on Ruby.
+
+> Word count in Spark's Ruby API
+
+```ruby
+file = spark.text_file("hdfs://...")
+
+file.flat_map(:split)
+    .map(lambda{|word| [word, 1]})
+    .reduce_by_key(lambda{|a, b| a+b})
+```
+
+- [Apache Spark](http://spark.apache.org)
+- [Wiki](https://github.com/ondra-m/ruby-spark/wiki)
+- [Ruby-doc](http://www.rubydoc.info/github/ondra-m/ruby-spark)
+
+## Installation
+
+### Requirments
+
+- Java 7+
+- Ruby 2+
+- MRI or JRuby
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ruby-spark'
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+Or install it yourself as:
+
+```
+$ gem install ruby-spark
+```
+
+Run `rake compile` if you are using gem from local filesystem.
+
+### Build Apache Spark
+
+This command will download Spark and build extensions for this gem ([SBT](ext/spark/build.sbt) is used for compiling). For more informations check [wiki](https://github.com/ondra-m/ruby-spark/wiki/Installation). Everything is stored by default at [GEM_ROOT]/target,
+
+```
+$ ruby-spark build
+```
+
+## Usage
+
+You can use Ruby Spark via interactive shell (Pry is used)
+
+```
+$ ruby-spark shell
+```
+
+Or on existing project
+
+```ruby
+require 'ruby-spark'
+
+# Create a SparkContext
+Spark.start
+
+# Context reference
+Spark.sc
+```
+
+If you want configure Spark first. See [configurations](https://github.com/ondra-m/ruby-spark/wiki/Configuration) for more details.
+
+```ruby
+require 'ruby-spark'
+
+# Use if you have custom SPARK_HOME
+Spark.load_lib(spark_home)
+
+# Configuration
+Spark.config do
+   set_app_name "RubySpark"
+   set 'spark.ruby.serializer', 'oj'
+   set 'spark.ruby.serializer.batch_size', 100
+end
+
+# Start Apache Spark
+Spark.start
+```
+
+Finally, to stop the cluster. On the shell is Spark stopped automatically when you exist.
+
+```ruby
+Spark.stop
+```
+
+
+
+## Creating RDD (upload data)
+
+Single text file:
+
+```ruby
+rdd = sc.text_file(FILE, workers_num, serializer=nil)
+```
+
+All files on directory:
+
+```ruby
+rdd = sc.whole_text_files(DIRECTORY, workers_num, serializer=nil)
+```
+
+Direct uploading structures from ruby (choosen serializer must be able to serialize it):
+
+```ruby
+rdd = sc.parallelize([1,2,3,4,5], workers_num, serializer=nil)
+rdd = sc.parallelize(1..5, workers_num, serializer=nil)
+```
+
+### Options
+
+<dl>
+  <dt>workers_num</dt>
+  <dd>
+    Min count of works computing this task.<br>
+    <i>(This value can be overwriten by spark)</i>
+  </dd>
+
+  <dt>serializer</dt>
+  <dd>
+    Custom serializer.<br>
+    <i>(default: by <b>spark.ruby.serializer</b> options)</i>
+  </dd>
+</dl>
+
+## Operations
+
+All operations can be divided into 2 groups:
+
+- **Transformations:** append new operation to current RDD and return new
+- **Actions:** add operation and start calculations
+
+See [wiki page](https://github.com/ondra-m/ruby-spark/wiki/RDD) or [Ruby-doc](http://www.rubydoc.info/github/ondra-m/ruby-spark/master/Spark/RDD) for more details.
+
+#### Transformations
+
+```ruby
+rdd.map(lambda{|item| ...})
+rdd.flat_map(lambda{|item| ...})
+rdd.filter(lambda{|item| ...})
+rdd.union(rdd)
+rdd.map_paritions(lambda{|iterator| ...})
+# ...
+```
+
+#### Actions
+
+```ruby
+rdd.count
+rdd.take(n)
+rdd.collect
+# ...
+```
+
+
+## Examples
+
+Sum of numbers
+
+```ruby
+sc.parallelize(0..10).sum
+# => 55
+```
+
+Words count using methods
+
+```ruby
+# Content:
+# "first line"
+# "second line"
+rdd = sc.text_file(PATH)
+
+# ["first", "line", "second", "line"]
+rdd = rdd.flat_map(lambda{|line| line.split})
+
+# [["first", 1], ["line", 1], ["second", 1], ["line", 1]]
+rdd = rdd.map(lambda{|word| [word, 1]})
+
+# [["first", 1], ["line", 2], ["second", 1]]
+rdd = rdd.reduce_by_key(lambda{|a, b| a+b})
+
+# {"first"=>1, "line"=>2, "second"=>1}
+rdd.collect_as_hash
+```
+
+Estimating PI with a custom serializer
+
+```ruby
+slices = 3
+n = 100000 * slices
+
+def map(_)
+  x = rand * 2 - 1
+  y = rand * 2 - 1
+
+  if x**2 + y**2 < 1
+    return 1
+  else
+    return 0
+  end
+end
+
+rdd = Spark.context.parallelize(1..n, slices, serializer: 'oj')
+rdd = rdd.map(method(:map))
+
+puts 'Pi is roughly %f' % (4.0 * rdd.sum / n)
+```
+
+Estimating PI
+
+```ruby
+rdd = sc.parallelize([10_000], 1)
+rdd = rdd.add_library('bigdecimal/math')
+rdd = rdd.map(lambda{|x| BigMath.PI(x)})
+rdd.collect # => #<BigDecimal, '0.31415926...'>
+```
+
+Linear regression
+
+```ruby
+# Import Mllib classes into Object
+# Otherwise are accessible via Spark::Mllib::LinearRegressionWithSGD
+Spark::Mllib.import(Object)
+
+# Training data
+data = [
+  LabeledPoint.new(0.0, [0.0]),
+  LabeledPoint.new(1.0, [1.0]),
+  LabeledPoint.new(3.0, [2.0]),
+  LabeledPoint.new(2.0, [3.0])
+]
+
+# Train a model
+lrm = LinearRegressionWithSGD.train(sc.parallelize(data), initial_weights: [1.0])
+
+lrm.predict([0.0])
+```

data/Rakefile

@@ -0,0 +1,35 @@
+#-*- mode: ruby -*-
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task default: :spec
+task test:    :spec
+
+def java?
+  RUBY_PLATFORM =~ /java/
+end
+
+if java?
+  require "rake/javaextensiontask"
+  Rake::JavaExtensionTask.new("ruby_java") do |ext|
+    ext.name = "ruby_spark_ext"
+  end
+else
+  require "rake/extensiontask"
+  Rake::ExtensionTask.new("ruby_c") do |ext|
+    ext.name = "ruby_spark_ext"
+  end
+end
+
+
+task :clean do
+  Dir['lib/*.{jar,o,so}'].each do |path|
+    puts "Deleting #{path} ..."
+    File.delete(path)
+  end
+  FileUtils.rm_rf('./pkg')
+  FileUtils.rm_rf('./tmp')
+end

data/TODO.md

@@ -0,0 +1,6 @@
+- refactor JavaBridge: to_java, from_java
+- add Streaming
+- add SQL
+- worker informations (time, memory, ...)
+- killing zombie workers
+- global config to ~/.ruby-spark.conf (e.g. target folder for spark)

data/benchmark/aggregate.rb

@@ -0,0 +1,33 @@
+require 'benchmark'
+require 'benchmark/ips'
+
+data = 0..1_000_000
+zero_value = rand(100_000)
+function = Proc.new{|sum, n| sum+n}
+
+Benchmark.ips do |r|  
+  r.report('each') do
+    sum = zero_value
+    data.each do |n|
+      sum += n
+    end
+  end
+
+  r.report('reduce') do
+    data.reduce(zero_value){|sum, n| sum+n}
+  end
+
+  r.report('each with function') do
+    sum = zero_value
+    data.each do |n|
+      sum = function.call(sum, n)
+    end
+  end
+
+  r.report('reduce with function') do
+    data.reduce(zero_value, &function)
+  end
+
+  r.compare!
+end
+