ruby-spark 1.1.0.1-java

data/example/website_search.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+# Parse sitemap and search word on every page
+
+require 'optparse'
+require 'open-uri'
+require 'nokogiri'
+require 'ruby-spark'
+
+options = {
+  sitemap: 'http://fit.cvut.cz/sitemap.xml',
+  query: 'cvut',
+  workers: 2
+}
+
+opt_parser = OptionParser.new do |opts|
+  opts.banner = 'Usage: website_search.rb [options]'
+
+  opts.separator ''
+  opts.separator 'Specific options:'
+
+  opts.on('-s', '--sitemap SITEMAP', 'Sitemap URL') do |sitemap|
+    options[:sitemap] = sitemap
+  end
+
+  opts.on('-q', '--query QUERY', 'Query to search') do |query|
+    options[:query] = query
+  end
+
+  opts.on('-w', '--workers WORKERS_NUM', Integer, 'Number of workers') do |workers|
+    options[:workers] = workers
+  end
+
+  opts.on('--quite', 'Run quitely') do |v|
+    Spark.logger.disabled
+  end
+
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end
+
+opt_parser.parse!
+
+@links = []
+
+def parse_sitemap(doc)
+  doc.xpath('//sitemapindex/sitemap/loc').each do |loc|
+    next_doc = Nokogiri::HTML(open(loc.text))
+    parse_sitemap(next_doc)
+  end
+
+  doc.xpath('//url/loc').each do |loc|
+    @links << loc.text
+  end
+end
+
+doc = Nokogiri::HTML(open(options[:sitemap]))
+parse_sitemap(doc)
+
+# Map function
+func = Proc.new do |url|
+  begin
+    open(url) {|f|
+      [url, f.read.scan(query).count]
+    }
+  rescue
+    [url, 0]
+  end
+end
+
+Spark.start
+
+rdd = Spark.sc.parallelize(@links, options[:workers])
+              .add_library('open-uri')
+              .bind(query: options[:query])
+              .map(func)
+              .sort_by(lambda{|(_, value)| value}, false)
+
+rdd.collect.each do |(url, count)|
+  puts "#{url} => #{count}"
+end

data/ext/ruby_c/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("ruby_spark_ext")

data/ext/ruby_c/murmur.c

@@ -0,0 +1,158 @@
+#include "murmur.h"
+
+#if defined(_MSC_VER)
+#define BIG_CONSTANT(x) (x)
+#else
+#define BIG_CONSTANT(x) (x##LLU)
+#endif
+
+/*-----------------------------------------------------------------------------
+// MurmurHash2, 64-bit versions, by Austin Appleby
+//
+// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment 
+// and endian-ness issues if used across multiple platforms.
+//
+// 64-bit hash for 64-bit platforms
+*/
+
+uint64_t MurmurHash64A(const void * key, int len, uint64_t seed)
+{
+  const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
+  const int r = 47;
+
+  uint64_t h = seed ^ (len * m);
+
+  const uint64_t * data = (const uint64_t *)key;
+  const uint64_t * end = data + (len/8);
+
+  while(data != end)
+  {
+    uint64_t k = *data++;
+
+    k *= m; 
+    k ^= k >> r; 
+    k *= m; 
+    
+    h ^= k;
+    h *= m; 
+  }
+
+  const unsigned char * data2 = (const unsigned char*)data;
+
+  switch(len & 7)
+  {
+  case 7: h ^= ((uint64_t) data2[6]) << 48;
+  case 6: h ^= ((uint64_t) data2[5]) << 40;
+  case 5: h ^= ((uint64_t) data2[4]) << 32;
+  case 4: h ^= ((uint64_t) data2[3]) << 24;
+  case 3: h ^= ((uint64_t) data2[2]) << 16;
+  case 2: h ^= ((uint64_t) data2[1]) << 8;
+  case 1: h ^= ((uint64_t) data2[0]);
+          h *= m;
+  };
+ 
+  h ^= h >> r;
+  h *= m;
+  h ^= h >> r;
+
+  return h;
+} 
+
+/* 64-bit hash for 32-bit platforms */
+
+uint64_t MurmurHash64B(const void * key, int len, uint64_t seed)
+{
+  const uint32_t m = 0x5bd1e995;
+  const int r = 24;
+
+  uint32_t h1 = ((uint32_t) seed) ^ len;
+  uint32_t h2 = ((uint32_t) (seed >> 32));
+
+  const uint32_t * data = (const uint32_t *)key;
+
+  while(len >= 8)
+  {
+    uint32_t k1 = *data++;
+    k1 *= m; k1 ^= k1 >> r; k1 *= m;
+    h1 *= m; h1 ^= k1;
+    len -= 4;
+
+    uint32_t k2 = *data++;
+    k2 *= m; k2 ^= k2 >> r; k2 *= m;
+    h2 *= m; h2 ^= k2;
+    len -= 4;
+  }
+
+  if(len >= 4)
+  {
+    uint32_t k1 = *data++;
+    k1 *= m; k1 ^= k1 >> r; k1 *= m;
+    h1 *= m; h1 ^= k1;
+    len -= 4;
+  }
+
+  switch(len)
+  {
+  case 3: h2 ^= ((unsigned char*)data)[2] << 16;
+  case 2: h2 ^= ((unsigned char*)data)[1] << 8;
+  case 1: h2 ^= ((unsigned char*)data)[0];
+      h2 *= m;
+  };
+
+  h1 ^= h2 >> 18; h1 *= m;
+  h2 ^= h1 >> 22; h2 *= m;
+  h1 ^= h2 >> 17; h1 *= m;
+  h2 ^= h1 >> 19; h2 *= m;
+
+  uint64_t h = h1;
+
+  h = (h << 32) | h2;
+
+  return h;
+}
+
+
+
+// ================================================================================================
+// Ruby methods
+
+#define PORTABLE_HASH_SEED 16154832
+
+
+VALUE murmur2_digest(VALUE rb_str, uint64_t seed)
+{
+  StringValue(rb_str);
+
+  void * key = RSTRING_PTR(rb_str);
+  long   len = RSTRING_LEN(rb_str);
+
+  uint64_t result = MurmurHash64A(key, len, seed);
+
+  return LONG2FIX(result);
+}
+
+// ------------------------------------------------------------------------------------------------
+// Spark::Digest::Murmur2.digest
+
+VALUE method_murmur2_digest(int argc, VALUE *argv, VALUE klass)
+{
+  if(argc == 0 || argc > 2){
+    rb_raise(rb_eArgError, "wrong number of arguments (%d for 1..2)", argc);
+  }
+
+  uint64_t seed = (argc == 1 ? 0 : NUM2UINT(argv[1]));
+
+  return murmur2_digest(argv[0], seed);
+}
+
+// ------------------------------------------------------------------------------------------------
+// Spark::Digest.portable_hash
+
+VALUE method_portable_hash(int argc, VALUE *argv, VALUE klass)
+{
+  if(argc != 1){
+    rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc);
+  }
+
+  return murmur2_digest(argv[0], PORTABLE_HASH_SEED);
+}

data/ext/ruby_c/murmur.h

@@ -0,0 +1,9 @@
+#ifndef MURMUR_INCLUDED
+#define MURMUR_INCLUDED
+
+#include "ruby.h"
+
+VALUE method_portable_hash(int argc, VALUE *argv, VALUE klass);
+VALUE method_murmur2_digest(int argc, VALUE *argv, VALUE klass);
+
+#endif

data/ext/ruby_c/ruby-spark.c

@@ -0,0 +1,18 @@
+#include "ruby.h"
+#include "murmur.h"
+
+
+VALUE SparkModule;
+VALUE SparkDigestModule;
+VALUE SparkDigestMurmur2Class;
+
+
+void Init_ruby_spark_ext()
+{
+  SparkModule             = rb_define_module("Spark");
+  SparkDigestModule       = rb_define_module_under(SparkModule, "Digest");
+  SparkDigestMurmur2Class = rb_define_class_under(SparkDigestModule, "Murmur2", rb_cObject);
+
+  rb_define_singleton_method(SparkDigestModule, "portable_hash", method_portable_hash, -1);
+  rb_define_singleton_method(SparkDigestMurmur2Class, "digest", method_murmur2_digest, -1);
+}

data/ext/ruby_java/Digest.java

@@ -0,0 +1,36 @@
+import org.jruby.Ruby;
+import org.jruby.RubyModule;
+import org.jruby.RubyObject;
+import org.jruby.RubyClass;
+import org.jruby.RubyString;
+import org.jruby.RubyFixnum;
+import org.jruby.anno.JRubyModule;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+@JRubyModule(name="Spark::Digest")
+public class Digest extends RubyObject{
+
+  // Have to be the same as in C extension
+  final static long PORTABLE_HASH_SEED = 16154832;
+
+  public Digest(final Ruby ruby, RubyClass rubyClass) {
+    super(ruby, rubyClass);
+  }
+
+  @JRubyMethod(module=true)
+  public static IRubyObject portable_hash(ThreadContext context, IRubyObject self, IRubyObject arg) {
+    Ruby ruby = self.getRuntime();
+
+    RubyString keyString = (RubyString)arg;
+
+    long hash = Murmur2.hash64(keyString.getBytes(), (int)keyString.length().getLongValue(), PORTABLE_HASH_SEED);
+
+    RubyFixnum result = new RubyFixnum(ruby, hash);
+
+    return result;
+  }
+
+}
+

data/ext/ruby_java/Murmur2.java

@@ -0,0 +1,98 @@
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyObject;
+import org.jruby.RubyString;
+import org.jruby.RubyFixnum;
+import org.jruby.anno.JRubyClass;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+/** Murmur hash 2.0.
+ * 
+ * The murmur hash is a relative fast hash function from
+ * http://murmurhash.googlepages.com/ for platforms with efficient
+ * multiplication.
+ *
+ * http://d3s.mff.cuni.cz/~holub/sw/javamurmurhash/
+ *
+ */
+
+@JRubyClass(name="Spark::Digest::Murmur2")
+public class Murmur2 extends RubyObject {
+
+  public Murmur2(final Ruby ruby, RubyClass rubyClass) {
+    super(ruby, rubyClass);
+  }
+
+  @JRubyMethod(required=1, optional=1, module=true)
+  public static IRubyObject digest(ThreadContext context, IRubyObject self, IRubyObject[] args) {
+    Ruby ruby = context.getRuntime();
+
+    RubyString keyString = (RubyString)args[0];
+    long seed;
+
+    if(args.length > 1){
+      RubyFixnum rb_seed = (RubyFixnum)args[1];
+      seed = rb_seed.getLongValue();
+    }
+    else{
+      seed = 0;
+    }
+
+    long hash = hash64(keyString.getBytes(), (int)keyString.length().getLongValue(), seed);
+
+    RubyFixnum result = new RubyFixnum(ruby, hash);
+    return result;
+  }
+
+
+  /** Generates 64 bit hash from byte array of the given length and seed.
+   * 
+   * @param data byte array to hash
+   * @param length length of the array to hash
+   * @param seed initial seed value
+   * @return 64 bit hash of the given array
+   */
+  public static long hash64(final byte[] data, int length, long seed) {
+    final long m = 0xc6a4a7935bd1e995L;
+    final int r = 47;
+
+    long h = (seed&0xffffffffl)^(length*m);
+
+    int length8 = length/8;
+
+    for (int i=0; i<length8; i++) {
+      final int i8 = i*8;
+      long k =  ((long)data[i8+0]&0xff)      +(((long)data[i8+1]&0xff)<<8)
+          +(((long)data[i8+2]&0xff)<<16) +(((long)data[i8+3]&0xff)<<24)
+          +(((long)data[i8+4]&0xff)<<32) +(((long)data[i8+5]&0xff)<<40)
+          +(((long)data[i8+6]&0xff)<<48) +(((long)data[i8+7]&0xff)<<56);
+      
+      k *= m;
+      k ^= k >>> r;
+      k *= m;
+      
+      h ^= k;
+      h *= m; 
+    }
+    
+    switch (length%8) {
+    case 7: h ^= (long)(data[(length&~7)+6]&0xff) << 48;
+    case 6: h ^= (long)(data[(length&~7)+5]&0xff) << 40;
+    case 5: h ^= (long)(data[(length&~7)+4]&0xff) << 32;
+    case 4: h ^= (long)(data[(length&~7)+3]&0xff) << 24;
+    case 3: h ^= (long)(data[(length&~7)+2]&0xff) << 16;
+    case 2: h ^= (long)(data[(length&~7)+1]&0xff) << 8;
+    case 1: h ^= (long)(data[length&~7]&0xff);
+            h *= m;
+    };
+   
+    h ^= h >>> r;
+    h *= m;
+    h ^= h >>> r;
+
+    return h;
+  }
+
+}

data/ext/ruby_java/RubySparkExtService.java

@@ -0,0 +1,28 @@
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyModule;
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.runtime.load.BasicLibraryService;
+
+public class RubySparkExtService implements BasicLibraryService
+{
+  public boolean basicLoad(final Ruby ruby) throws java.io.IOException {
+
+    RubyModule sparkModule = ruby.defineModule("Spark");
+    RubyModule sparkDigestModule = sparkModule.defineModuleUnder("Digest");
+    RubyClass  sparkDigestMurmur2Class = sparkDigestModule.defineClassUnder("Murmur2", ruby.getObject(), sparkDigestMurmur2Allocator);
+
+    sparkDigestModule.defineAnnotatedMethods(Digest.class);
+    sparkDigestMurmur2Class.defineAnnotatedMethods(Murmur2.class);
+
+    return true;
+  }
+
+  public static ObjectAllocator sparkDigestMurmur2Allocator = new ObjectAllocator() {
+    public IRubyObject allocate(Ruby ruby, RubyClass rubyClass) {
+      return new Murmur2(ruby, rubyClass);
+    }
+  };
+
+}

data/ext/ruby_java/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("ruby_spark_ext")