ruby-spark 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 13c074c360edf1875745bf728b274f5bfb6e0d0a
+  data.tar.gz: a59ba09fac91e2e4c0a58aab99dd60ffb2ec5e3f
+SHA512:
+  metadata.gz: 2e667dabd55b05100831cf3d0e58044941ce965965de1d6dce9a4e8fa5be843bfac1d57d5b6674ea056b9e9395650b84e3e77a86f0a256a1b1c4bfd8ca257340
+  data.tar.gz: 690b88857fa4f841c8c0a5940af75d926285f42a81ff6753337ccc926473a4a3ce510d1c7e9b07ce2ebfd2559c05dd77432cf1dba132f15629669fbf8dfc51b6

data/.gitignore

@@ -0,0 +1,37 @@
+/.gemtags
+/.tags
+/java/spark.jar
+.jbundler
+target/*
+*.class
+*.jar
+pom.xml
+vendor/*
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+ext/spark/target/*
+ext/spark/project/target/*
+ext/spark/project/project/target/*
+wiki/*
+/benchmark/performance/spark/*
+/benchmark/performance/rspark/*

data/Gemfile

@@ -0,0 +1,47 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'sourcify', '0.6.0.rc4'
+gem 'method_source'
+gem 'commander'
+gem 'pry'
+gem 'nio4r'
+gem 'distribution'
+
+platform :mri do
+  gem 'rjb'
+  gem 'msgpack'
+  gem 'oj'
+  gem 'narray'
+end
+
+platform :jruby do
+  gem 'msgpack-jruby', require: 'msgpack'
+
+  # NameError: no constructorfor arguments (org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.jruby.RubyFixnum,org.joda.time.chrono.GJChronology) on Java::OrgJodaTime::DateTime
+  # gem 'mdarray'
+end
+
+group :stats do
+  # gem 'nmatrix'
+  # gem 'statsample'
+  # gem 'statsample-glm'
+  # gem 'statsample-timeseries'
+  # gem 'statistics2'
+  # gem 'statsample-optimization' # libgsl0-dev
+  # gem 'narray'
+  # gem 'gsl-nmatrix'
+end
+
+group :development do
+  gem 'benchmark-ips'
+  gem 'rspec'
+  gem 'rake-compiler'
+  gem 'guard'
+  gem 'guard-rspec'
+end
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ondřej Moravčík
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,185 @@
+# Ruby-Spark
+
+Apache Spark™ is a fast and general engine for large-scale data processing.
+
+This Gem allows you use Spark functionality on Ruby.
+
+> Word count in Spark's Ruby API
+
+```ruby
+file = spark.text_file("hdfs://...")
+
+file.flat_map(:split)
+    .map(lambda{|word| [word, 1]})
+    .reduce_by_key(lambda{|a, b| a+b})
+```
+
+- [Apache Spark](http://spark.apache.org)
+- [Wiki](https://github.com/ondra-m/ruby-spark/wiki)
+- [Ruby-doc](http://www.rubydoc.info/github/ondra-m/ruby-spark)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ruby-spark'
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+Or install it yourself as:
+
+```
+$ gem install ruby-spark
+```
+
+### Install Apache Spark
+
+To install latest supported Spark. First compile native extension:
+
+```
+$ rake compile
+```
+Then build Spark, [SBT](ext/spark/build.sbt) is used for compiling.
+```
+$ ruby-spark build
+```
+
+## Usage
+
+You can use Ruby Spark via interactive shell
+
+```
+$ ruby-spark pry
+```
+
+Or on existing project
+
+```ruby
+require 'ruby-spark'
+Spark.start
+
+Spark.sc # => context
+```
+
+If you want configure Spark first. See [configurations](https://github.com/ondra-m/ruby-spark/wiki/Configuration) for more details.
+
+```ruby
+require 'ruby-spark'
+
+Spark.load_lib(spark_home)
+Spark.config do
+   set_app_name "RubySpark"
+   set 'spark.ruby.batch_size', 100
+   set 'spark.ruby.serializer', 'oj'
+end
+Spark.start
+
+Spark.sc # => context
+```
+
+## Uploading a data
+
+Single file
+
+```ruby
+$sc.text_file(FILE, workers_num, custom_options)
+```
+
+All files on directory
+
+```ruby
+$sc.whole_text_files(DIRECTORY, workers_num, custom_options)
+```
+
+Direct
+
+```ruby
+$sc.parallelize([1,2,3,4,5], workers_num, custom_options)
+$sc.parallelize(1..5, workers_num, custom_options)
+```
+
+### Options
+
+<dl>
+  <dt>workers_num</dt>
+  <dd>
+    Min count of works computing this task.<br>
+    <i>(This value can be overwriten by spark)</i>
+  </dd>
+
+  <dt>custom_options</dt>
+  <dd>
+    <b>serializer</b>: name of serializator used for this RDD<br>
+    <b>batch_size</b>: see configuration<br>
+    <br>
+    <i>(Available only for parallelize)</i><br>
+    <b>use</b>: <i>direct (upload direct to java)</i>, <i>file (upload throught a file)</i>
+  </dd>
+</dl>
+
+
+## Examples
+
+Sum of numbers
+
+```ruby
+$sc.parallelize(0..10).sum
+# => 55
+```
+
+Words count using methods
+
+```ruby
+rdd = $sc.text_file(PATH)
+
+rdd = rdd.flat_map(lambda{|line| line.split})
+         .map(lambda{|word| [word, 1]})
+         .reduce_by_key(lambda{|a, b| a+b})
+
+rdd.collect_as_hash
+```
+
+Estimating pi with a custom serializer
+
+```ruby
+slices = 3
+n = 100000 * slices
+
+def map(_)
+  x = rand * 2 - 1
+  y = rand * 2 - 1
+
+  if x**2 + y**2 < 1
+    return 1
+  else
+    return 0
+  end
+end
+
+rdd = Spark.context.parallelize(1..n, slices, serializer: 'oj')
+rdd = rdd.map(method(:map))
+
+puts 'Pi is roughly %f' % (4.0 * rdd.sum / n)
+```
+
+Linear regression
+
+```ruby
+Spark::Mllib.import
+
+data = [
+  LabeledPoint.new(0.0, [0.0]),
+  LabeledPoint.new(1.0, [1.0]),
+  LabeledPoint.new(3.0, [2.0]),
+  LabeledPoint.new(2.0, [3.0])
+]
+lrm = LinearRegressionWithSGD.train($sc.parallelize(data), initial_weights: [1.0])
+
+lrm.predict([0.0])
+```

data/Rakefile

@@ -0,0 +1,35 @@
+#-*- mode: ruby -*-
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task default: :spec
+task test:    :spec
+
+def java?
+  RUBY_PLATFORM =~ /java/
+end
+
+if java?
+  require "rake/javaextensiontask"
+  Rake::JavaExtensionTask.new("ruby_java") do |ext|
+    ext.name = "ruby_spark_ext"
+  end
+else
+  require "rake/extensiontask"
+  Rake::ExtensionTask.new("ruby_c") do |ext|
+    ext.name = "ruby_spark_ext"
+  end
+end
+
+
+task :clean do
+  Dir['lib/*.{jar,o,so}'].each do |path|
+    puts "Deleting #{path} ..."
+    File.delete(path)
+  end
+  FileUtils.rm_rf('./pkg')
+  FileUtils.rm_rf('./tmp')
+end

data/TODO.md

@@ -0,0 +1,7 @@
+- add compress
+- refactor JavaBridge: to_java, from_java
+- add Streaming
+- add SQL
+- autobatch serializer
+- worker informations (time, memory, ...)
+- encoding parameter to context.text_file

data/benchmark/aggregate.rb

@@ -0,0 +1,33 @@
+require 'benchmark'
+require 'benchmark/ips'
+
+data = 0..1_000_000
+zero_value = rand(100_000)
+function = Proc.new{|sum, n| sum+n}
+
+Benchmark.ips do |r|  
+  r.report('each') do
+    sum = zero_value
+    data.each do |n|
+      sum += n
+    end
+  end
+
+  r.report('reduce') do
+    data.reduce(zero_value){|sum, n| sum+n}
+  end
+
+  r.report('each with function') do
+    sum = zero_value
+    data.each do |n|
+      sum = function.call(sum, n)
+    end
+  end
+
+  r.report('reduce with function') do
+    data.reduce(zero_value, &function)
+  end
+
+  r.compare!
+end
+

data/benchmark/bisect.rb

@@ -0,0 +1,88 @@
+require "benchmark"
+
+def bisect_left1(a, x, opts={})
+  return nil if a.nil?
+  return 0 if a.empty?
+
+  lo = (opts[:lo] || opts[:low]).to_i
+  hi = opts[:hi] || opts[:high] || a.length
+
+  while lo < hi
+    mid = (lo + hi) / 2
+    v = a[mid]
+    if v < x
+      lo = mid + 1
+    else
+      hi = mid
+    end
+  end
+  return lo
+end
+
+def bisect_left2(list, item)
+  count = 0
+  list.each{|i|
+    return count if i >= item
+    count += 1
+  }
+  nil
+end
+
+def bisect_left3(list, item, lo = 0, hi = list.size)
+  while lo < hi
+    i = (lo + hi - 1) >> 1
+
+    if 0 <= (list[i] <=> item)
+      hi = i
+    else
+      lo = i + 1
+    end
+  end
+  return hi
+end
+
+array = Array.new(1000000) { rand(0..1000000) };
+to_find = Array.new(500) { rand(0..10000) };
+
+Benchmark.bm(20) do |x|
+  x.report("bisect_left1") do
+    to_find.each do |item|
+      bisect_left1(array, item)
+    end
+  end
+
+  x.report("bisect_left2") do
+    to_find.each do |item|
+      bisect_left2(array, item)
+    end
+  end
+
+  x.report("bisect_left3") do
+    to_find.each do |item|
+      bisect_left3(array, item)
+    end
+  end
+end
+
+array = Array.new(100000) { Array.new(rand(1..10)){(97+rand(26)).chr}.join };
+to_find = Array.new(500) { (97+rand(26)).chr };
+
+Benchmark.bm(20) do |x|
+  x.report("bisect_left1") do
+    to_find.each do |item|
+      bisect_left1(array, item)
+    end
+  end
+
+  x.report("bisect_left2") do
+    to_find.each do |item|
+      bisect_left2(array, item)
+    end
+  end
+
+  x.report("bisect_left3") do
+    to_find.each do |item|
+      bisect_left3(array, item)
+    end
+  end
+end