ruby-spark 1.1.0.1-java

data/spec/lib/whole_text_files_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+
+RSpec::shared_examples "a whole_text_files" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map(get_numbers)
+    result = files.size
+
+    expect(rdd2.collect.size).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.flat_map(get_numbers)
+
+    result = 0
+    files.each{|f| result += File.read(f).split.map(&:to_i).reduce(:+)}
+
+    expect(rdd3.sum).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::Context" do
+  let(:get_numbers) { lambda{|file, content| content.split.map(&:to_i)} }
+
+  let(:dir)   { File.join("spec", "inputs", "numbers") }
+  let(:files) { Dir.glob(File.join(dir, "*")) }
+
+  def rdd(workers)
+    $sc.whole_text_files(dir, workers)
+  end
+
+  it_behaves_like "a whole_text_files", nil
+  it_behaves_like "a whole_text_files", 1
+  it_behaves_like "a whole_text_files", rand(2..10)
+end

data/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+# require 'simplecov'
+# SimpleCov.start
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'ruby-spark'
+require 'generator'
+
+# Loading
+Spark.load_lib
+Spark.jb.load_test
+Spark::Mllib.import
+
+# Keep it on method because its called from config test
+def spark_start
+  Spark.logger.disable
+  Spark.config do
+    set 'spark.ruby.serializer.batch_size', 100
+  end
+  Spark.start
+  $sc = Spark.context
+end
+
+def windows?
+  RbConfig::CONFIG['host_os'] =~ /mswin|mingw/
+end
+
+RSpec.configure do |config|
+  config.default_formatter = 'doc'
+  config.color = true
+  config.tty   = true
+
+  config.before(:suite) do
+    spark_start
+  end
+  config.after(:suite) do
+    Spark.stop
+  end
+end

metadata

@@ -0,0 +1,389 @@
+--- !ruby/object:Gem::Specification
+name: ruby-spark
+version: !ruby/object:Gem::Version
+  version: 1.1.0.1
+platform: java
+authors:
+- Ondřej Moravčík
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sourcify
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.0.rc4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.0.rc4
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: method_source
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: commander
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: pry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: distribution
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: ''
+email:
+- moravcik.ondrej@gmail.com
+executables:
+- ruby-spark
+extensions:
+- ext/ruby_java/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO.md
+- benchmark/aggregate.rb
+- benchmark/bisect.rb
+- benchmark/comparison/prepare.sh
+- benchmark/comparison/python.py
+- benchmark/comparison/r.r
+- benchmark/comparison/ruby.rb
+- benchmark/comparison/run-all.sh
+- benchmark/comparison/scala.scala
+- benchmark/custom_marshal.rb
+- benchmark/digest.rb
+- benchmark/enumerator.rb
+- benchmark/serializer.rb
+- benchmark/sort.rb
+- benchmark/sort2.rb
+- benchmark/take.rb
+- bin/ruby-spark
+- example/pi.rb
+- example/website_search.rb
+- ext/ruby_c/extconf.rb
+- ext/ruby_c/murmur.c
+- ext/ruby_c/murmur.h
+- ext/ruby_c/ruby-spark.c
+- ext/ruby_java/Digest.java
+- ext/ruby_java/Murmur2.java
+- ext/ruby_java/RubySparkExtService.java
+- ext/ruby_java/extconf.rb
+- ext/spark/build.sbt
+- ext/spark/project/plugins.sbt
+- ext/spark/sbt/sbt
+- ext/spark/src/main/scala/Exec.scala
+- ext/spark/src/main/scala/MLLibAPI.scala
+- ext/spark/src/main/scala/Marshal.scala
+- ext/spark/src/main/scala/MarshalDump.scala
+- ext/spark/src/main/scala/MarshalLoad.scala
+- ext/spark/src/main/scala/RubyAccumulatorParam.scala
+- ext/spark/src/main/scala/RubyBroadcast.scala
+- ext/spark/src/main/scala/RubyConstant.scala
+- ext/spark/src/main/scala/RubyMLLibAPI.scala
+- ext/spark/src/main/scala/RubyMLLibUtilAPI.scala
+- ext/spark/src/main/scala/RubyPage.scala
+- ext/spark/src/main/scala/RubyRDD.scala
+- ext/spark/src/main/scala/RubySerializer.scala
+- ext/spark/src/main/scala/RubyTab.scala
+- ext/spark/src/main/scala/RubyUtils.scala
+- ext/spark/src/main/scala/RubyWorker.scala
+- ext/spark/src/test/scala/MarshalSpec.scala
+- lib/ruby-spark.rb
+- lib/spark.rb
+- lib/spark/accumulator.rb
+- lib/spark/broadcast.rb
+- lib/spark/build.rb
+- lib/spark/cli.rb
+- lib/spark/command.rb
+- lib/spark/command/base.rb
+- lib/spark/command/basic.rb
+- lib/spark/command/pair.rb
+- lib/spark/command/sort.rb
+- lib/spark/command/statistic.rb
+- lib/spark/command_builder.rb
+- lib/spark/command_validator.rb
+- lib/spark/config.rb
+- lib/spark/constant.rb
+- lib/spark/context.rb
+- lib/spark/error.rb
+- lib/spark/ext/hash.rb
+- lib/spark/ext/integer.rb
+- lib/spark/ext/io.rb
+- lib/spark/ext/ip_socket.rb
+- lib/spark/ext/module.rb
+- lib/spark/ext/object.rb
+- lib/spark/ext/string.rb
+- lib/spark/helper.rb
+- lib/spark/helper/logger.rb
+- lib/spark/helper/parser.rb
+- lib/spark/helper/serialize.rb
+- lib/spark/helper/statistic.rb
+- lib/spark/helper/system.rb
+- lib/spark/java_bridge.rb
+- lib/spark/java_bridge/base.rb
+- lib/spark/java_bridge/jruby.rb
+- lib/spark/java_bridge/rjb.rb
+- lib/spark/logger.rb
+- lib/spark/mllib.rb
+- lib/spark/mllib/classification/common.rb
+- lib/spark/mllib/classification/logistic_regression.rb
+- lib/spark/mllib/classification/naive_bayes.rb
+- lib/spark/mllib/classification/svm.rb
+- lib/spark/mllib/clustering/gaussian_mixture.rb
+- lib/spark/mllib/clustering/kmeans.rb
+- lib/spark/mllib/matrix.rb
+- lib/spark/mllib/regression/common.rb
+- lib/spark/mllib/regression/labeled_point.rb
+- lib/spark/mllib/regression/lasso.rb
+- lib/spark/mllib/regression/linear.rb
+- lib/spark/mllib/regression/ridge.rb
+- lib/spark/mllib/ruby_matrix/matrix_adapter.rb
+- lib/spark/mllib/ruby_matrix/vector_adapter.rb
+- lib/spark/mllib/stat/distribution.rb
+- lib/spark/mllib/vector.rb
+- lib/spark/rdd.rb
+- lib/spark/sampler.rb
+- lib/spark/serializer.rb
+- lib/spark/serializer/auto_batched.rb
+- lib/spark/serializer/base.rb
+- lib/spark/serializer/batched.rb
+- lib/spark/serializer/cartesian.rb
+- lib/spark/serializer/compressed.rb
+- lib/spark/serializer/marshal.rb
+- lib/spark/serializer/message_pack.rb
+- lib/spark/serializer/oj.rb
+- lib/spark/serializer/pair.rb
+- lib/spark/serializer/text.rb
+- lib/spark/sort.rb
+- lib/spark/stat_counter.rb
+- lib/spark/storage_level.rb
+- lib/spark/version.rb
+- lib/spark/worker/master.rb
+- lib/spark/worker/spark_files.rb
+- lib/spark/worker/worker.rb
+- ruby-spark.gemspec
+- spec/generator.rb
+- spec/inputs/lorem_300.txt
+- spec/inputs/numbers/1.txt
+- spec/inputs/numbers/10.txt
+- spec/inputs/numbers/11.txt
+- spec/inputs/numbers/12.txt
+- spec/inputs/numbers/13.txt
+- spec/inputs/numbers/14.txt
+- spec/inputs/numbers/15.txt
+- spec/inputs/numbers/16.txt
+- spec/inputs/numbers/17.txt
+- spec/inputs/numbers/18.txt
+- spec/inputs/numbers/19.txt
+- spec/inputs/numbers/2.txt
+- spec/inputs/numbers/20.txt
+- spec/inputs/numbers/3.txt
+- spec/inputs/numbers/4.txt
+- spec/inputs/numbers/5.txt
+- spec/inputs/numbers/6.txt
+- spec/inputs/numbers/7.txt
+- spec/inputs/numbers/8.txt
+- spec/inputs/numbers/9.txt
+- spec/inputs/numbers_0_100.txt
+- spec/inputs/numbers_1_100.txt
+- spec/lib/collect_spec.rb
+- spec/lib/command_spec.rb
+- spec/lib/config_spec.rb
+- spec/lib/context_spec.rb
+- spec/lib/ext_spec.rb
+- spec/lib/external_apps_spec.rb
+- spec/lib/filter_spec.rb
+- spec/lib/flat_map_spec.rb
+- spec/lib/group_spec.rb
+- spec/lib/helper_spec.rb
+- spec/lib/key_spec.rb
+- spec/lib/manipulation_spec.rb
+- spec/lib/map_partitions_spec.rb
+- spec/lib/map_spec.rb
+- spec/lib/mllib/classification_spec.rb
+- spec/lib/mllib/clustering_spec.rb
+- spec/lib/mllib/matrix_spec.rb
+- spec/lib/mllib/regression_spec.rb
+- spec/lib/mllib/vector_spec.rb
+- spec/lib/reduce_by_key_spec.rb
+- spec/lib/reduce_spec.rb
+- spec/lib/sample_spec.rb
+- spec/lib/serializer_spec.rb
+- spec/lib/sort_spec.rb
+- spec/lib/statistic_spec.rb
+- spec/lib/whole_text_files_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- java, scala
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: Ruby wrapper for Apache Spark
+test_files:
+- spec/generator.rb
+- spec/inputs/lorem_300.txt
+- spec/inputs/numbers/1.txt
+- spec/inputs/numbers/10.txt
+- spec/inputs/numbers/11.txt
+- spec/inputs/numbers/12.txt
+- spec/inputs/numbers/13.txt
+- spec/inputs/numbers/14.txt
+- spec/inputs/numbers/15.txt
+- spec/inputs/numbers/16.txt
+- spec/inputs/numbers/17.txt
+- spec/inputs/numbers/18.txt
+- spec/inputs/numbers/19.txt
+- spec/inputs/numbers/2.txt
+- spec/inputs/numbers/20.txt
+- spec/inputs/numbers/3.txt
+- spec/inputs/numbers/4.txt
+- spec/inputs/numbers/5.txt
+- spec/inputs/numbers/6.txt
+- spec/inputs/numbers/7.txt
+- spec/inputs/numbers/8.txt
+- spec/inputs/numbers/9.txt
+- spec/inputs/numbers_0_100.txt
+- spec/inputs/numbers_1_100.txt
+- spec/lib/collect_spec.rb
+- spec/lib/command_spec.rb
+- spec/lib/config_spec.rb
+- spec/lib/context_spec.rb
+- spec/lib/ext_spec.rb
+- spec/lib/external_apps_spec.rb
+- spec/lib/filter_spec.rb
+- spec/lib/flat_map_spec.rb
+- spec/lib/group_spec.rb
+- spec/lib/helper_spec.rb
+- spec/lib/key_spec.rb
+- spec/lib/manipulation_spec.rb
+- spec/lib/map_partitions_spec.rb
+- spec/lib/map_spec.rb
+- spec/lib/mllib/classification_spec.rb
+- spec/lib/mllib/clustering_spec.rb
+- spec/lib/mllib/matrix_spec.rb
+- spec/lib/mllib/regression_spec.rb
+- spec/lib/mllib/vector_spec.rb
+- spec/lib/reduce_by_key_spec.rb
+- spec/lib/reduce_spec.rb
+- spec/lib/sample_spec.rb
+- spec/lib/serializer_spec.rb
+- spec/lib/sort_spec.rb
+- spec/lib/statistic_spec.rb
+- spec/lib/whole_text_files_spec.rb
+- spec/spec_helper.rb
+has_rdoc: