fairy 0.6.0

data/lib/fairy/share/debug.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  module Debug
+    def njob_status_monitor_on(fairy)
+#      require "backend/bjob"
+      
+      bjob = fairy.name2backend_class("CFilter")
+      bjob.watch_status = true
+    end
+
+    for method in self.instance_methods
+      module_function method
+    end
+
+  end
+end
+

data/lib/fairy/share/encoding.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  begin
+    if CONF.DEFAULT_EXTERNAL
+      Encoding.default_external = CONF.DEFAULT_EXTERNAL
+    end
+    if CONF.DEFAULT_INTERNAL
+      Encoding.default_internal = CONF.DEFAULT_INTERNAL
+    end
+  rescue NameError
+    ERR.Raise ERR::NoSupportRubyEncoding, RUBY_VERSION
+  end
+end

data/lib/fairy/share/fast-tempfile.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "thread"
+require "forwardable"
+
+require "fairy/share/conf.rb"
+
+module Fairy
+  class FastTempfile
+    extend Forwardable
+
+    Entry = Struct.new(:path, :io)
+
+    def self.reset
+      @DEFAULT_TMPDIR = CONF.TMP_DIR
+      @PID = $$
+      @HEAD = Time.now.strftime("%Y%m%d")+format("-%05d-", @PID)
+      @Seq = "00000"
+    end
+    reset
+    @Mutex = Mutex.new
+
+    def self.open(prefix, tmpdir = @DEFAULT_TMPDIR)
+      new(prefix, tmpdir)
+    end
+
+    def self.gen_tmpname(prefix, tmpdir)
+      @Mutex.synchronize do
+	# forkしたときのため
+	reset if @PID != $$
+	name = tmpdir+"/"+prefix+@HEAD+@Seq
+	@Seq = @Seq.succ
+	name
+      end
+    end
+
+    def initialize(prefix, tmpdir)
+      @entry = Entry.new
+      ObjectSpace.define_finalizer(self, FastTempfile.terminate_proc(@entry))
+
+      @entry.path = FastTempfile.gen_tmpname(prefix, tmpdir)
+
+      begin
+	@entry.io = File.open(path, File::RDWR|File::CREAT|File::EXCL)
+      rescue Errno::ENOENT
+	unless File.directory?(tmpdir)
+	  ERR::Fail ERR::NoTmpDir, tmpdir
+	end
+	raise
+      end
+    end
+
+    def_delegator :@entry, :path
+    def_delegator :@entry, :io
+
+    def open
+      @entry.io.close if @entry.io
+      @entry.io = File.open(path)
+    end
+
+    def close
+      @entry.io.close
+      @entry.io = nil
+    end
+
+    def close!
+      @entry.io.close if @entry.io
+      if File.exist?(@entry.path)
+	File.unlink @entry.path
+      end
+      ObjectSpace.undefine_finalizer(self)
+    end
+
+    def self.terminate_proc(entry)
+      pid = @PID
+      Proc.new {
+	if pid == $$
+	  entry.io.close if entry.io
+
+	  # keep this order for thread safeness
+	  if entry.path
+	    File.unlink(entry.path) if File.exist?(entry.path)
+	  end
+	end
+      }
+    end
+  end
+end
+
+

data/lib/fairy/share/file-place.rb

@@ -0,0 +1,176 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "thread"
+require "uri"
+
+module Fairy
+  URI_REGEXP = /:\/\//
+
+  class CFilePlace
+    def initialize(vfile)
+      @vfile = vfile
+
+      # for next_filter
+      @no = 0
+      @nfileplaces = @vfile.real_file_names
+      @nfileplaces_mutex = Mutex.new
+    end
+
+    def each_assigned_filter(&block)
+      loop do
+	fp = nil
+	@nfileplaces_mutex.synchronize do
+	  file = @nfileplaces.shift
+	  return unless file
+	  fp = PFilePlace.new(file, @no)
+	  @no += 1
+	end
+	block.call fp
+      end
+    end
+
+  end
+
+  class PFilePlace
+    def initialize(url, no)
+      @url = url
+      @no = no
+
+      @host = "localhost"
+      @path = @url
+      if URI_REGEXP =~ @url
+	uri = URI(@url)
+	@host = uri.host
+	if /^\[([0-9a-f.:]*)\]$/ =~ @host
+	  @host = $1
+	end
+	@path = uri.path
+      end
+    end
+
+    attr_reader :url
+    attr_reader :no
+    attr_reader :host
+    attr_reader :path
+  end
+
+  class CLocalIOPlace
+    def initialize(job)
+      @job = job
+      @no = 0
+    end
+
+    def each_assigned_filter(&block)
+      @job.each_assigned_filter do |io|
+	block.call PLocalIOPlace.new(io, @no)
+	@no += 1
+      end
+    end
+  end
+
+  class PLocalIOPlace
+    def initialize(io, no)
+      @io = io
+      @no = no
+    end
+
+    attr_reader :no
+    attr_reader :io
+  end
+
+  class CVarrayPlace
+    def initialize(varray)
+      @varray = varray
+      @no = 0
+    end
+
+    def each_assigned_filter(&block)
+      no = 0
+      @varray.arrays_each do |ary|
+	vp = PVarrayPlace.new(ary, no)
+	no += 1
+	block.call vp
+      end
+    end
+  end
+
+  class PVarrayPlace
+    def initialize(ary, no)
+      @ary = ary
+      @no = no
+
+      @host = "localhost"
+      @path = @url
+      if URI_REGEXP =~ @url
+	uri = URI(@url)
+	@host = uri.host
+	if /^\[([0-9a-f.:]*)\]$/ =~ @host
+	  @host = $1
+	end
+	@path = uri.path
+      end
+    end
+
+    attr_reader :ary
+    attr_reader :no
+  end
+
+  class CIotaPlace
+    def initialize(last, offset, split_no)
+      @last = last
+      @offset = offset
+      @split_no = split_no
+    end
+
+    def each_assigned_filter(&block)
+      first = @offset
+      no = -1
+
+      @split_no.times do
+	no += 1
+	Log::debug self, "NO: #{no}"
+	last = [first + @last.div(@split_no), @last].min
+	block.call PIotaPlace.new(no, first, last)
+	first = last + 1
+	break if first > @last
+      end
+#      sleep 0.1
+    end
+  end
+
+  class PIotaPlace
+    def initialize(no, first, last)
+      @no = no
+      @first = first
+      @last = last
+    end
+
+    attr_reader :no
+    attr_reader :first
+    attr_reader :last
+  end
+
+
+  class CTherePlace
+    def initialize(enum)
+      @enumerable = enum
+    end
+
+    def each_assigned_filter(&block)
+      block.call PTherePlace.new(0, @enumerable)
+    end
+  end
+
+  class PTherePlace
+    def initialize(no, enum)
+      @no = no
+      @enumerable = enum
+    end
+
+    attr_reader :no
+    attr_reader :enumerable
+  end
+end

data/lib/fairy/share/hash-1.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  module HValueGenerator
+    module Hash1
+      def Hash1.value(key)
+	key.ord
+      end
+    end
+
+    def self.create_seed;end
+    def self.new(seed)
+      Hash1
+    end
+  end
+end
+

data/lib/fairy/share/hash-md5.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "digest/md5"
+
+module Fairy
+  module HValueGenerator
+    module MD5
+      def MD5.value(key)
+	case key
+	when String
+	  Digest::MD5.digest(key).unpack("@12N").first
+	else
+	  ERR::Raise ERR::NoImpliment, "non-string key(#{key.inspect})"
+	end
+      end
+#      module_function :hash
+    end
+
+    def self.create_seed;end
+    def self.new(seed)
+      HValueGenerator::MD5
+    end
+  end
+end
+

data/lib/fairy/share/hash-murmur.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+
+  module HValueGenerator
+
+    # mhash is modiftied from MurmerHash(http://murmurhash.googlepages.com/).
+    class MurMur
+      FIX_MASK = 0x3fff_ffff
+      MASK24 = 0xffff_ffff_ffff
+      MAGIC = 0x7fd652ad & FIX_MASK
+      R = 18
+
+      def initialize(h)
+	@seed = h + 0xdeadbeef & FIX_MASK
+#	@postfix = [@seed].pack("N")[1..3]
+	@postfix = "000"
+      end
+
+      def value(data)
+	case data
+	when String
+	  str_hash(data)
+	else
+	  ERR::Raise ERR::NoImpliment, "non-string key(#{data.inspect})"
+	end
+      end
+
+      def str_hash(data)
+	h = @seed
+
+	for k in (data+@postfix).unpack("N*")
+	  k *= MAGIC
+	  k ^= k >> R
+	  k &= FIX_MASK
+	  k *= MAGIC
+	  k &= FIX_MASK
+
+	  h = @seed * MAGIC
+	  k &= FIX_MASK
+	  h ^= k
+	end
+
+	h ^= h >> 13
+	h *= MAGIC
+	h &= FIX_MASK
+	h ^= h >> 15
+	h
+      end
+
+      def MurMur.create_seed
+	rand(FIX_MASK)
+      end
+    end
+
+
+    def self.create_seed
+      MurMur.create_seed
+    end
+
+    def self.new(seed)
+      MurMur.new(seed)
+    end
+
+  end
+end

data/lib/fairy/share/hash-rb18.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+module Fairy
+  module HValueGenerator
+    module RB18
+      def RB18.value(key)
+	key.hash
+      end
+    end
+
+    def self.create_seed;end
+    def self.new(seed)
+      RB18
+    end
+  end
+end
+