fairy 0.6.0

data/lib/fairy/node/p-here.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PHere<PIOFilter
+    Processor.def_export self
+
+    include PSingleExportable
+
+    def basic_each(&block)
+      @input.each do |e|
+	block.call e
+      end
+    end
+  end
+end

data/lib/fairy/node/p-identity.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PIdentity<PSingleExportFilter
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts=nil)
+      super
+    end
+
+    def basic_each(&block)
+      @input.each do |e|
+	block.call e
+      end
+    end
+  end
+
+end

data/lib/fairy/node/p-inject.rb

@@ -0,0 +1,127 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+
+  class PInject<PIOFilter
+    def initialize(id, processor, bjob, opts, block_source)
+      super
+
+      @init_value = :__FAIRY_NO_VALUE__
+      if @opts.key?(:init_value)
+	@init_value = @opts[:init_value].dc_deep_copy
+      end
+      @block_source = block_source
+#      @inject_proc = @context.create_proc(@block_source)
+    end
+
+    def basic_each(&block)
+      @inject_proc = BBlock.new(@block_source, @context, self)
+      sum = @init_value
+      @input.each do |e|
+	if sum == :__FAIRY_NO_VALUE__
+	  sum = e
+	else
+# Log::debug(self, "sum, e, #{sum}, #{e}")
+	  if Import::CTLTOKEN_NULLVALUE === (v = @inject_proc.yield(sum, e))
+	    next
+	  end
+	  sum = v
+	end
+      end
+      finish(sum, &block)
+    end
+
+#     def start
+#       super do
+# 	@inject_proc = BBlock.new(@block_source, @context, self)
+# 	sum = @init_value
+# 	@import.each do |e|
+# 	  if sum == :__FAIRY_NO_VALUE__
+# 	    sum = e
+# 	  else
+# 	    sum = @inject_proc.yield(sum, e)
+# 	  end
+# 	end
+# 	finish(sum)
+#       end
+  end
+
+  class PLocalInject<PInject
+    include PSingleExportable
+
+    Processor.def_export self
+    
+    def finish(sum, &block)
+#      block.call sum
+      block.call [no, sum]
+    end
+
+  end
+
+  class PWideInject<PInject
+    Processor.def_export self
+
+    def initialize(*args)
+      super
+
+      @value = :__FAIRY_NO_VALUE__
+      @value_mutex = Mutex.new
+      @value_cv = ConditionVariable.new
+    end
+
+    def input=(input)
+      super
+
+      start do
+	self.super_each{}
+      end
+    end
+
+    alias super_each each
+
+    def basic_each(&block)
+      @inject_proc = BBlock.new(@block_source, @context, self)
+      sum = @init_value
+      @input.sort_by{|n, e| n}.each do |n, e|
+#Log::debug(self, "n e: #{n}, #{e}")
+	if sum == :__FAIRY_NO_VALUE__
+	  sum = e
+	else
+	  if Import::CTLTOKEN_NULLVALUE === (v = @inject_proc.yield(sum, e))
+	    next
+	  end
+	  sum = v
+	end
+      end
+      finish(sum, &block)
+    end
+
+    def each(&block)
+      block.call value
+    end
+
+    def value
+      @value_mutex.synchronize do
+	while @value == :__FAIRY_NO_VALUE__
+	  @value_cv.wait(@value_mutex)
+	end
+	@value
+      end
+    end
+    DeepConnect.def_method_spec(self, "DVAL value")
+    
+    def finish(sum, &block)
+      @value = sum
+      @value_cv.broadcast 
+      if block
+	block.call sum
+      end
+    end
+  end
+end

data/lib/fairy/node/p-input-file.rb

@@ -0,0 +1,108 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+require "fairy/node/p-single-exportable"
+
+require "fairy/share/file-place"
+
+module Fairy
+  class PInputFile<PSingleExportInput
+    Processor.def_export self
+
+    def PInputFile.open(processor, bjob, opts, fn)
+      nfile = PInputFile.new(processor, bjob, opts)
+      nfile.open(fn)
+    end
+
+    def initialize(id, ntask, bjob, opts=nil)
+      super
+      @file = nil
+    end
+
+    def open(nfileplace)
+      @file_name = nfileplace.path
+      self.no = nfileplace.no
+      begin
+	@file = File.open(@file_name)
+      rescue 
+	e = $!.exception($!.message+ "(vfile entry##{nfileplace.no}: #{nfileplace.url})")
+	e.set_backtrace($!.backtrace)
+
+	Log::error_exception(e)
+	handle_exception(e)
+	raise e
+      end
+#      start
+      self
+    end
+    DeepConnect::def_method_spec(self, "REF open(VAL)")
+
+#     def start
+#       buf = ""
+#       buf_size = 1024*20
+#       rest = nil
+#       super do
+# 	while @file.read(buf_size, buf)
+# 	  lines = buf.split
+# 	  if rest
+# 	    begin
+# 	      lines[0] = rest+lines[0]
+# 	    rescue
+# 	      Log::debug(self, "AAAAAAAAAAAAAAAA")
+# 	      Log::debug(self, buf.inspect)
+# 	      Log::debug(self, lines.inspect)
+# 	      raise
+# 	    end
+# 	  end
+# 	  rest = lines.pop
+# 	  if @export.respond_to?(:push_buf)
+# 	    @export.push_buf lines
+# 	  else
+# 	    for l in lines
+# 	      @export.push l
+# 	    end
+# 	  end
+# 	end
+# 	if rest
+# 	  @export.push rest
+# 	end
+# 	@file.close
+# 	@file = nil # FileオブジェクトをGCの対象にするため
+#       end
+#     end
+
+#     def start
+#       super do
+# 	for l in @file
+# 	  @export.push l
+# 	end
+# 	@file.close
+# 	@file = nil # FileオブジェクトをGCの対象にするため
+#       end
+#     end
+
+    def basic_each(&block)
+      begin
+	@file.each &block
+      ensure
+	@file.close
+	@file = nil # FileオブジェクトをGCの対象にするため
+      end
+    end
+
+    def basic_next
+      begin
+	ret = @file.gets
+      ensure
+	unless ret
+	  @file.close
+	  @file = nil
+	  return :END_OF_STREAM 
+	end
+      end
+    end
+  end
+end

data/lib/fairy/node/p-input-iota.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PInputIota<PSingleExportInput
+    Processor.def_export self
+
+    def initialize(id, ntask, bjob, opts)
+      super
+    end
+
+    def open(niota_place)
+      self.no = niota_place.no
+
+      @first = niota_place.first
+      @last = niota_place.last
+    end
+
+    def basic_each(&block)
+      for i in @first..@last
+	block.call i
+      end
+    end
+
+#     def start
+#       super do
+# 	for i in @first..@last
+# 	  @export.push i
+# 	end
+#       end
+#     end
+
+  end
+end
+

data/lib/fairy/node/p-input-local-file.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PInputLocalFile<PSingleExportInput
+    Processor.def_export self
+
+    def self.open(id, ntask, bjob, io, opts=nil)
+      nlfileinput = self.new(id, processor, bjob, opts)
+      nlfileinput.open(job)
+    end
+
+    def initialize(id, ntask, bjob, opts=nil)
+      super
+    end
+
+    def open(nioplace)
+      @io = nioplace.io
+      self.no = nioplace.no
+      @io_external_encoding = @io.external_encoding.dc_deep_copy
+
+      @buffer_size = @opts[:buffer_size]
+      @buffer_size = CONF.INPUT_LOCAL_FILE_BUFFER_SIZE unless @buffer_size
+    end
+
+    def basic_each(&block)
+      rest = nil
+      while (buf = @io.read(@buffer_size))
+	lines = buf.scan(/.*\n?/)
+	lines.pop # scan で末尾にゴミが出るため
+	if rest
+	  begin
+	    lines[0] = rest+lines[0]
+	  rescue
+	    Log::debug(self, @io.inspect)
+	    Log::debug(self, buf.inspect)
+	    Log::debug(self, lines.inspect)
+	    Log::debug(self, rest.inspect)
+	    raise
+	  end
+	end
+	rest = lines.pop
+	lines.each{|l| 
+	  l.force_encoding(@io_external_encoding)
+	  block.call(l)}
+      end
+      if rest
+	rest.force_encoding(@io_external_encoding)
+	block.call rest
+      end
+      @io.close
+      @io = nil # FileオブジェクトをGCの対象にするため
+    end
+   
+  end
+end

data/lib/fairy/node/p-input-varray.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PInputVArray<PSingleExportInput
+    Processor.def_export self
+
+#    def initialize(processor, bjob, opts)
+#      super
+#    end
+
+    def open(nvarrayplace)
+      @array = nvarrayplace.ary
+      self.no = nvarrayplace.no
+    end
+
+    def basic_each(&block)
+      @array.each &block
+    end
+  end
+end
+

data/lib/fairy/node/p-io-filter.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-filter"
+
+module Fairy
+  class PIOFilter<PFilter
+    Processor.def_export self
+
+    ST_WAIT_IMPORT = :ST_WAIT_IMPORT
+
+    def initialize(id, ntask, bjob, opts=nil, *rests)
+      super
+      self.status = ST_WAIT_IMPORT
+    end
+
+    def input=(input)
+      @input = input
+      if input.kind_of?(Import)
+	input.njob_id = @id
+      end
+      self.no = input.no
+      self.key = input.key
+    end
+  end
+end

data/lib/fairy/node/p-map.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+#
+# Copyright (C) 2007-2010 Rakuten, Inc.
+#
+
+require "fairy/node/p-io-filter"
+require "fairy/node/p-single-exportable"
+
+module Fairy
+  class PMap<PSingleExportFilter
+    Processor.def_export self
+
+    DeepConnect.def_single_method_spec(self, "REF new(DEFAULT, REF, REF, VAL, REF)")
+    def initialize(id, ntask, bjob, opts, block_source)
+      super
+      @block_source = block_source
+#      @map_proc = eval("proc{#{@block_source}}", TOPLEVEL_BINDING)
+#      @map_proc = @context.create_proc(@block_source)
+
+
+    end
+
+#     def start
+#       super do
+# 	@map_proc = BBlock.new(@block_source, @context, self)
+# 	@import.each do |e|
+# 	  @export.push @map_proc.yield(e)
+# 	end
+#       end
+#     end
+
+    def basic_each(&block)
+      @map_proc = BBlock.new(@block_source, @context, self)
+
+      @input.each do |e|
+	block.call @map_proc.yield(e)
+      end
+    end
+  end
+end