fairy 0.6.0

data/lib/fairy/client/input-file.rb

@@ -0,0 +1,53 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/vfile"
+
+module Fairy
+  class InputFile<Filter
+
+    def InputFile.open(fairy, opts, ffile_descripter)
+      ffile = new(fairy, opts)
+      ffile.open(ffile_descripter)
+      ffile
+    end
+
+    def InputFile.input(fairy, opts, ffile_descripter)
+      InputFile.open(fairy, opts, ffile_descripter)
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+    end
+
+    def backend_class_name
+      "CInputFile"
+    end
+
+    def open(ffile_descripter)
+      @descripter = ffile_descripter
+      case ffile_descripter
+      when Array
+	vf = VFile.real_files(ffile_descripter)
+      when VFile
+	vf = ffile_descripter
+      when String
+	if VFile.vfile?(ffile_descripter)
+	  vf = VFile.vfile(ffile_descripter)
+	else
+	  vf = VFile.real_files([ffile_descripter])
+	end
+      else
+	ERR::Raise ERR::IllegalVFile
+      end
+      backend.open(vf)
+      self
+    end
+
+  end
+
+#  class BFile;end
+end

data/lib/fairy/client/input-iota.rb

@@ -0,0 +1,49 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+
+module Fairy
+
+  SPLIT_NO = CONF.IOTA_SPLIT_NO
+
+  class InputIota < Filter
+    module Interface
+
+      # Usage:
+      # fairy.iota(no)....
+      #
+      def iota(times, opts={})
+	InputIota.input(self, opts, times)
+      end
+      alias times iota
+      
+    end
+    Fairy::def_fairy_interface Interface
+
+    def self.input(fairy, opts, n)
+      unless opts[:SPLIT_NO]
+	opts[:SPLIT_NO] = SPLIT_NO
+      end
+      iota = new(fairy, opts, n)
+      iota.start
+      iota
+    end
+    
+    
+    def backend_class_name
+      "CInputIota"
+    end
+
+    def start
+      backend.start
+    end
+  end
+end
+
+
+    
+      
+      

data/lib/fairy/client/input-local-file.rb

@@ -0,0 +1,188 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+require "fairy/share/vfile"
+
+module Fairy
+  class InputLocalFile < Filter
+
+    def self.input(fairy, opts, filename)
+      self.start(fairy, opts, filename)
+    end
+
+    def self.start(fairy, opts, filename)
+      lfile = new(fairy, opts)
+      lfile.start(filename)
+      lfile
+    end
+
+    def initialize(fairy, opts=nil)
+      super
+      @io = nil
+    end
+
+    attr_reader :io
+
+    def backend_class_name
+      "CInputLocalFile"
+    end
+
+    def start(filename)
+      @filename = filename
+      backend.start(self)
+    end
+    
+    def each_assigned_filter(&block)
+      if @opts[:split_no]
+	each_assigned_filter_split_no(&block)
+      elsif !@opts[:split_size]
+	each_assigned_filter1(&block)
+      else
+	each_assigned_filter_split(&block)
+      end
+    end
+
+    def each_assigned_filter1(&block)
+      io = File.open(@filename)
+      yield io
+    end
+
+    def each_assigned_filter_split(&block)
+      split_size = @opts[:split_size]
+      begin
+	seek = 0
+	size = File.stat(@filename).size
+	while seek < size
+	  io = SplittedFile.open(@filename, seek, seek + split_size - 1)
+	  seek = io.seek_end + 1
+	  yield io
+	end
+      rescue
+	Log::warn_exception(self)
+	raise
+      end
+      nil
+    end
+
+    def each_assigned_filter_split_no(&block)
+      split_no = @opts[:split_no]
+      size = File.stat(@filename).size
+      split_size = Rational(size, split_no)
+      begin
+	rest_split_no = split_no
+	seek = 0
+	while seek < size
+	  io = SplittedFile.open(@filename, seek, seek + (size - seek + 1)/rest_split_no - 1)
+	  seek = io.seek_end + 1
+	  rest_split_no -= 1
+	  yield io
+	end
+	if rest_split_no > 0
+	  Log::warn(self, "Split #{split_no - rest_split_no} files. Can't split specified split_no: #{split_no}")
+	end
+      rescue
+	Log::warn_exception(self)
+	raise
+      end
+      nil
+    end
+
+#     def open
+#       if block_given?
+# 	io = File.open(@filename)
+# 	begin
+# 	  yield io
+# 	ensure
+# 	  io.close
+# 	end
+#       else
+# 	File.open(@filename)
+#       end
+#     end
+
+#     def split_opens(split_size, &block)
+#       begin
+# 	seek = 0
+# 	size = File.stat(@filename).size
+# 	while seek < size
+# 	  io = SplittedFile.open(@filename, seek, seek + split_size)
+# 	  seek = io.seek_end + 1
+# 	  yield io
+# 	end
+#       rescue
+# 	Log::warn_exception(self)
+# 	raise
+#       end
+#     end
+
+    class SplittedFile
+      include Enumerable
+
+      def self.open(fd, seek_start, seek_end, &block)
+	sf = new(fd, seek_start, seek_end)
+	if block_given?
+	  begin
+	    yield sf
+	  ensure
+	    sf.close
+	  end
+	else
+	  sf
+	end
+      end
+
+      def initialize(fd, seek_start, seek_end)
+	@io = File.open(fd)
+	@seek_start = seek_start
+
+	@io.seek(seek_end)
+	c = @io.read(1)
+	case c
+	when nil, "\n"
+	  @seek_end = seek_end
+	else
+	  @io.gets
+	  @seek_end = @io.pos - 1
+	end
+	@io.seek(seek_start)
+      end
+
+      attr_reader :seek_start
+      attr_reader :seek_end
+
+      def external_encoding
+	@io.external_encoding
+      end
+
+      def close
+	@io.close
+	@io = nil
+      end
+
+      def each(&block)
+	begin
+	  while @io.pos < @seek_end && l = @io.gets
+	    yield l
+	  end
+	rescue
+	  Log::warn_exception(self)
+	  raise
+	end
+      end
+
+      def read(length)
+	if @seek_end - @io.pos + 1 < length
+	  length = @seek_end - @io.pos + 1
+	end
+	if length == 0
+	  nil
+	else
+	  @io.read(length)
+	end
+      end
+    end
+  end
+end

data/lib/fairy/client/input-varray.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+require "fairy/client/filter"
+
+module Fairy
+
+  class InputVArray < Filter
+
+    def self.input(fairy, opts, varray)
+      input_va = new(fairy, opts, varray)
+      input_va.start
+      input_va
+    end
+    
+    def backend_class_name
+      "CInputVArray"
+    end
+
+    def start
+      backend.start
+    end
+  end
+end
+
+
+    
+      
+      

data/lib/fairy/client/input.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/share/varray"
+
+module Fairy
+  module InputInterface
+    def input(desc, *opts)
+      if opts.last.kind_of?(Hash)
+	opts_h = opts.pop
+      else
+	opts_h = {}
+      end
+
+      case desc
+#      when Enumerable
+#	There.input(self, opts_h, desc, *opts)
+      when VArray
+	  InputVArray.input(self, opts_h, desc)
+      when DeepConnect::Reference
+	if desc.peer_class.name == "Fairy::VArray"
+	  InputVArray.input(self, opts_h, desc)
+	else
+	  ERR::Raise ERR::NoImpliment "#{desc}, #{desc.peer_class}"
+	end
+      when Class
+	desc.input(self, opts_h, *opts)
+      else
+	if !desc.kind_of?(String) || VFile.vfile?(desc)
+	  InputFile.input(self, opts_h, desc)
+	else
+	  InputLocalFile.input(self, opts_h, desc)
+	end
+      end
+    end
+  end
+  def_fairy_interface InputInterface
+end
+
+

data/lib/fairy/client/io-filter.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/filter"
+
+module Fairy
+  class IOFilter < Filter
+
+    def initialize(fairy, *rests)
+      super
+    end
+
+    def backend_class_name
+      ERR::Raise ERR::INTERNAL::UndefinedBackendClass
+    end
+
+    def input=(job)
+      @input=job
+#      atom = Atom.new(backend, :input=, job.backend)
+#      @fairy.send_atom(atom)
+      backend.input=job.backend
+    end
+  end
+end

data/lib/fairy/client/junction.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+
+module Fairy
+
+  class Junction<IOFilter
+    module Interface
+      def junction(opts = nil, &block)
+	subfairy = Fairy.create_subfairy(@fairy)
+	junction_ps = Junction.new(subfairy, opts)
+	junction_ps.input = self
+
+	last_filter = yield subfairy, junction_ps
+	
+	junction_sp = Junction.new(@fairy, opts)
+	junction_sp.input = last_filter
+	junction_sp
+      end
+      alias sub junction
+    end
+    Fairy::def_filter_interface Interface
+
+    def backend_class_name
+      "CJunction"
+    end
+  end
+end

data/lib/fairy/client/map.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/client/io-filter"
+require "fairy/share/block-source"
+
+module Fairy
+
+  class Map<IOFilter
+    module Interface
+      def map(block_source, opts = nil)
+	ERR::Raise ERR::CantAcceptBlock if block_given?
+	block_source = BlockSource.new(block_source) 
+	mapper = Map.new(@fairy, opts, block_source)
+	mapper.input = self
+	mapper
+      end
+      alias collect map
+     
+    end
+    Fairy::def_filter_interface Interface
+
+    def initialize(fairy, opts, block_source)
+      super
+      @block_source = block_source
+    end
+
+    def backend_class_name
+      "CMap"
+    end
+  end
+end