fairy 0.6.0

data/etc/fairy.conf.tmpl

@@ -0,0 +1,118 @@
+#						-*- ruby -*-
+# encoding: UTF-8
+
+module Fairy
+
+## Double sharp comment is a system default value.
+
+##CONF.RUBY_BIN = ENV["FAIRY_RUBY"] || "ruby" 
+
+##CONF.MASTER_HOST = #NO SYSTEM DEFAULT VALUE
+CONF.MASTER_HOST = NO SYSTEM DEFAULT VALUE
+
+##CONF.MASTER_PORT = "19999"
+
+##CONF.HOME = ENV["FAIRY_HOME"]
+##CONF.BIN = CONF.HOME+"/bin" 
+##CONF.LIB = CONF.HOME+"/lib"
+##CONF.SUBCMD = CONF.BIN+"/subcmd"
+##CONF.CONTROLLER_BIN = CONF.SUBCMD+"/controller"
+##CONF.PROCESSOR_BIN = CONF.SUBCMD+"/processor"
+
+##CONF.DEFAULT_EXTERNAL = nil
+##CONF.DEFAULT_INTERNAL = nil
+
+##CONF.IGNORE_EXCEPTION_ON_FILTER = false
+
+##CONF.MASTER_MAX_ACTIVE_PROCESSORS = 4
+##CONF.CONTROLLER_INPUT_PROCESSOR_N = 10
+
+## 生成されるプロセッサの上限. nilなら無制限. 
+##CONF.CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR = 4
+
+##CONF.CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR = 1
+
+##  CONF.POSTMAPPING_POLICY = nil
+
+##CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
+##CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
+
+##CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
+##CONF.POOLQUEUE_POOL_THRESHOLD = 10000
+
+##CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
+##CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
+
+##CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
+##CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
+##CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
+
+##CONF.SORTEDQUEUE_POOL_THRESHOLD = CONF.POOLQUEUE_POOL_THRESHOLD
+##CONF.SORTEDQUEUE_THRESHOLD = 10000/2
+##CONF.SORTEDQUEUE_SORTBY = %{|v| v}
+
+##CONF.INPUT_LOCAL_FILE_BUFFER_SIZE = 1024*1024
+##CONF.HERE_POOL_THRESHOLD = 32000
+
+##CONF.GROUP_BY_NO_SEGMENT = 4
+
+##CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
+##CONF.GROUP_BY_GROUPING_OPTIMIZE = false
+
+##CONF.GROUP_BY_BUFFERING_POLICY = {:buffering_class => :DirectMergeSortBuffer}
+##CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
+##CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
+
+##CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
+##CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
+##CONF.SORT_SAMPLING_MIN = 100
+##CONF.SORT_SAMPLING_MAX = 10000
+##CONF.SORT_SAMPLING_RATIO_1_TO = 100
+##CONF.SORT_NO_SEGMENT = CONF.GROUP_BY_NO_SEGMENT
+##CONF.SORT_CMP_OPTIMIZE = false
+
+##CONF.IOTA_SPLIT_NO = 4
+
+##CONF.TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE = false
+
+##CONF.VF_ROOT = CONF.HOME+"/Repos"
+##CONF.VF_PREFIX = `hostname`.chomp 
+#- CONF.VF_PREFIX use client setting.
+#  (ja_JP.utf-8: CONF.VF_PREFIXはクライアントのものが使われる)
+
+##CONF.VF_SPLIT_SIZE = 64*1024*1024
+
+##CONF.TMP_DIR = "/tmp/fairy/tmpbuf"
+
+##CONF.LOG_FILE = "/tmp/fairy/log"
+##CONF.LOG_FLUSH_INTERVAL = 1
+##CONF.LOG_MARK_INTERVAL = 300
+##CONF.LOG_LEVEL = :INFO
+##CONF.LOG_IMPORT_NTIMES_POP = 100000
+##CONF.LOG_LOCAL_OUTPUT_DEV = :$stderr
+##CONF.LOG_ROTATE_INTERVAL = 60*60*24
+##CONF.LOG_ROTATE_N = 7
+
+##CONF.SUBCMD_EXEC_TIMEOUT = 60
+
+##  CONF.PROCESSOR_MON_ON = false
+##  CONF.PROCESSOR_MON_INTERVAL = 300
+##  CONF.PROCESSOR_MON_PSFORMAT = "stat,vsz,rss,sz,pmem,pcpu,nlwp,time,wchan"
+##  CONF.PROCESSOR_MON_OBJECTSPACE_INSPECT_ON = false
+
+##CONF.SOCK_DO_NOT_REVERSE_LOOKUP = true
+##CONF.USE_RESOLV_REPLACE = false
+
+##CONF.BLOCK_USE_STDOUT = true
+
+##CONF.DEBUG_PORT_WAIT = false
+##CONF.DEBUG_FULL_BACKTRACE = false
+##CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION = false
+##CONF.DEBUG_MONITOR_ON = false
+##CONF.DEBUG_PROCESSOR_TRACE_ON = false
+##CONF.DEBUG_BUG49 = false
+
+##CONF.BUG234 = false
+
+##CONF.PROCESS_LIFE_MANAGE_INTERVAL = nil
+end

data/ext/simple_hash/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+
+create_makefile("simple_hash")
+

data/ext/simple_hash/simple_hash.c

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2007-2010 Rakuten, Inc.
+ */
+
+#include <ruby.h>
+
+#define MULTIPLIER  137
+
+static VALUE simple_hash(VALUE self, VALUE vstr);
+
+
+static VALUE mFairy;
+static VALUE mSimpleHash;
+
+
+static VALUE simple_hash(VALUE self, VALUE vstr) {
+    VALUE vh;
+    char *str;
+    int len;
+    char *p;
+    unsigned int h = 0;
+
+    str = StringValuePtr(vstr);
+    len = RSTRING_LEN(vstr);
+
+    for (p = str; p - str < len; p++) {
+        h = h * MULTIPLIER + *p;
+    }
+
+    /* vh = UINT2NUM(h); */
+    vh = INT2FIX(h);
+    return vh;
+}
+
+void Init_simple_hash(void) {
+    mFairy = rb_define_module("Fairy");
+    mSimpleHash = rb_define_module_under(mFairy, "SimpleHash");
+    
+    rb_define_module_function(mSimpleHash, "hash", simple_hash, 1);
+}
+
+

data/fairy.gemspec

@@ -0,0 +1,60 @@
+
+require "rubygems"
+
+v = `ruby -Ilib -e 'require "fairy/version"; print Fairy::Version'`
+v, p = v.scan(/^([0-9]+\.[0-9]+\.[0-9]+)-([0-9]+)/).first
+if p.to_i > 1
+  v += "."+p
+end
+
+Gem::Specification.new do |s|
+  s.name = "fairy"
+  s.authors = "Rakuten, Inc."
+  s.email = "hajime.masuda@mail.rakuten.co.jp"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "fairy is a framework for distributed processing in Ruby, originally designed at Rakuten Institute of Technology with Yukihiro Matsumoto, the founder of Ruby."
+  s.rubyforge_project = s.name
+  s.homepage = "http://code.google.com/p/fairy-prj/"
+  s.version = v
+  s.require_path = "lib"
+  s.test_file = "spec/run_all.sh"
+  s.executables = ["fairy", "fairy-cat", "fairy-cp", "fairy-rm"]
+  s.default_executable = "fairy"
+
+  s.files = ["Makefile", "README", "LICENSE", "fairy.gemspec", "lib/fairy.rb" ]
+  s.files.concat Dir.glob("lib/fairy/**/*.rb")
+  s.files.concat Dir.glob("lib/fairy/template/*.tmpl")
+  s.files.concat ["etc/fairy.conf.tmpl"]
+  s.files.concat Dir.glob("bin/{#{s.executables.grep(/.*[a-z]$/).join(",")}}")
+  s.files.concat Dir.glob("bin/subcmd/*[A-Za-z]")
+  s.files.concat Dir.glob("ext/**/{Makefile,*.rb,*.c}")
+  s.files.concat Dir.glob("doc/*.{rd,html}")
+  s.files.concat Dir.glob("spec/{README,*.rb,run_all.sh}")
+  s.files.concat Dir.glob("sample/*.rb")
+  s.files.concat ["test/testc.rb"]
+  s.files.concat Dir.glob("tools/**/*[a-z]")
+
+  s.add_dependency("DeepConnect", ">= 0.4.06")
+  s.add_dependency("fiber-mon", ">= 0.1.0")
+  
+  s.description = <<EOF
+fairy is a framework for distributed processing in Ruby, originally
+designed at Rakuten Institute of Technology with Yukihiro Matsumoto,
+the founder of Ruby.
+
+Although fairy was inspired by MapReduce model, a well-known
+programming model for distributed processing, it's more flexible and
+suitable for wider use. That's due to fairy's programming model,
+called filter IF, and various built-in filters.
+
+fairy is implemented in Ruby and inherits its high productivity and
+simplicity. fairy's API is quite similar to Ruby. Therefore most
+programmers who know Ruby can easily understand and use it.
+EOF
+end
+
+# Editor settings
+# - Emacs -
+# local variables:
+# mode: Ruby
+# end:

data/lib/fairy/client/addins.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+job_dir = File.dirname(__FILE__)
+subdir = File.basename(File.dirname(job_dir))
+job_name = File.basename(job_dir)
+for job in Dir.glob("#{job_dir}/*.rb")
+  base = File.basename(job)
+  case base
+  when /18.rb$/
+    next if RUBY_VERSION >= "1.9.0"
+  when /19.rb$/
+    next unless RUBY_VERSION >= "1.9.0"
+  end
+  require [subdir, job_name, base].join("/")
+end
+
+Fairy::post_initialize

data/lib/fairy/client/barrier.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+require "fairy/share/block-source"
+
+module Fairy
+    module Interface
+      def barrier(opts = nil)
+	if opts[:cond].kind_of?(String)
+	  opts[:cond] = BlockSource.new(opts[:cond])
+	end
+	barrier = Barrier.new(@fairy, opts)
+	barrier.input = self
+	barrier
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+
+  class Barrier<IOFilter
+    def backend_class_name
+      "CBarrier"
+    end
+  end
+end

data/lib/fairy/client/basic-group-by.rb

@@ -0,0 +1,52 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class BasicGroupBy<IOFilter
+
+    module Interface
+      def basic_group_by(hash_block, opts = nil)
+	hash_block = BlockSource.new(hash_block) 
+	group_by = BasicGroupBy.new(@fairy, opts, hash_block)
+	group_by.input = self
+	group_by
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CBasicGroupBy"
+    end
+  end
+
+  class BasicMGroupBy<IOFilter
+
+    module Interface
+      def basic_mgroup_by(hash_block, opts = nil)
+	hash_block = BlockSource.new(hash_block) 
+	mgroup_by = BasicMGroupBy.new(@fairy, opts, hash_block)
+	mgroup_by.input = self
+	mgroup_by
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CBasicMGroupBy"
+    end
+  end
+end

data/lib/fairy/client/cat.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+  class Cat<IOFilter
+
+    module Interface
+      # jpb.cat(opts,...,filter,...,opts,...)
+      def cat(*others)
+	others, opts = others.partition{|e| e.kind_of?(Filter)}
+	if opts.last.kind_of?(Hash)
+	  h = opts.pop
+	else
+	  h = {}
+	end
+	opts.each{|e| h[e] = true}
+
+	cat = Cat.new(@fairy, h, others)
+	cat.input = self
+	cat
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, others)
+      super(fairy, opts, others.collect{|o| o.backend})
+      @others = others
+      @block_source
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CCat"
+    end
+  end
+end
+

data/lib/fairy/client/direct-product.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+
+module Fairy
+  class DirectProduct<IOFilter
+
+    module Interface
+      # jpb.direct_product(opts,...,filter,...,block_source, opts,...)
+      def direct_product(*others)
+	block_source = nil
+	if others.last.kind_of?(String)
+	  block_source = others.pop
+	end
+	others, opts = others.partition{|e| e.kind_of?(Filter)}
+	if opts.last.kind_of?(Hash)
+	  h = opts.pop
+	else
+	  h = {}
+	end
+	opts.each{|e| h[e] = true}
+
+	block_source = BlockSource.new(block_source) 
+	dp = DirectProduct.new(@fairy, h, others, block_source)
+	dp.input = self
+	dp
+      end
+      alias product direct_product
+
+      def *(other)
+	direct_product(other, %{|e| e})
+      end
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, others, block_source)
+      super(fairy, opts, others.collect{|o| o.backend}, block_source)
+      @others = others
+      @block_source
+      @opts = opts
+    end
+
+    def backend_class_name
+      "CDirectProduct"
+    end
+  end
+end
+

data/lib/fairy/client/equijoin.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/basic-group-by"
+require "fairy/client/seg-join"
+
+require "fairy/client/group-by"
+
+Fairy.def_filter(:equijoin_on_memory) do |fairy, input, other, *no|
+  puts no1 = no2 = no[0]
+  puts no2 = no[1] if no[1]
+
+#  mod = Fairy::CONF.GROUP_BY_HASH_MODULE
+#  require mod
+#  seed = Fairy::HValueGenerator.create_seed
+#  fairy.def_pool_variable(:HASH_SEED, seed)
+
+  main = input.basic_group_by(%{|e| @hgen.value(e[#{no1}]) % CONF.GROUP_BY_NO_SEGMENT},
+			:BEGIN=>%{
+                           mod = CONF.GROUP_BY_HASH_MODULE
+                           require mod
+                           @hgen = Fairy::HValueGenerator.new(@Pool[:HASH_SEED])
+                        }).barrier(:mode=>:NODE_CREATION, :cond=>:NODE_ARRIVED, :buffer=>:MEMORY)
+  other2 = other.basic_group_by(%{|e| @hgen.value(e[#{no2}]) % CONF.GROUP_BY_NO_SEGMENT},
+			:BEGIN=>%{
+                           mod = CONF.GROUP_BY_HASH_MODULE
+                           require mod
+                           @hgen = Fairy::HValueGenerator.new(@Pool[:HASH_SEED])
+                        }).barrier(:mode=>:NODE_CREATION, :cond=>:NODE_ARRIVED, :buffer=>:MEMORY)
+
+
+  main.seg_join(other2, %{|in0, in1, out_block|
+
+    next unless in0 && in1    
+
+    ary_m = in0.to_a.group_by{|e| e[#{no1}]}
+    ary_o = in1.to_a.group_by{|e| e[#{no2}]}
+
+    ary_m.each do |key, values|
+      o_values = ary_o[key]
+      next unless o_values
+      values.each do |value|
+        o_values.each do |o_value|
+          out_block.call([value, o_value])
+        end
+      end
+    end
+  }, :by => :key)
+end
+
+Fairy.def_filter(:equijoin) do |fairy, input, other, *no|
+  no1 = no2 = no[0]
+  no2 = no[1] if no[1]
+
+  main = input.map(%{|e| [e[#{no1}], 0, e]})
+  other = other.map(%{|e| [e[#{no2}], 1, e]})
+  
+  main.cat(other).group_by(%{|e| e[0]}).mapf(%{|values|
+      parted = values.group_by{|value| value[1]}
+      if parted[0] && parted[1]
+         parted[0].collect{|e| e[2]}.product(parted[1].collect{|e| e[2]})       
+      else
+         []
+      end
+  })
+
+#   main.cat(other).mod_group_by(%{|e| e[0]}).emap(%{|key, values|
+#      puts "XXXX: \#{key.inspect}"
+#      puts "XXXS: \#{values.inspect}"
+
+#      parted = values.group_by{|value| value[1]}
+#      parted[0].product(parted[1])
+#   })
+end
+
+			       
+