pmux 0.1.0

data/test/test_i_mapreduce.rb

@@ -0,0 +1,169 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+require 'stringio'
+
+require 'mock_world'
+require 'mock_mros'
+require 'mock_pipeio'
+
+Log.open('-') if __FILE__ == $0 and $show_log
+
+class MockWorker < Pmux::Worker
+  def run
+    while true
+      future = @client.call_async :get_task
+      future.attach_callback {|f|
+        task = f.get
+        result = exec_task task
+        res = @client.call :put_result, result
+      }
+      Fiber.yield
+    end
+  end
+end
+
+class MockHandler < Pmux::Handler
+  def fork_worker num_cpu, options
+    world = @server.loop.world
+    stransport = @server.listeners.detect {|c| c.kind_of? MR::MockTransport}
+    ctransport = world.new_connection MR::MockTransport
+    ctransport.mock_receiver = stransport
+    stransport.mock_receiver = ctransport
+    client = MR::Client.new ctransport, 'dummy', @server.loop
+    worker = MockWorker.new options, client
+    @fiber = Fiber.new {
+      worker.run
+    }
+    @fiber.resume
+  end
+end
+
+class MockWriter < Pmux::Writer
+  attr_reader :lines
+
+  def initialize
+    @lines = []
+  end
+
+  def write path
+    @lines += File.readlines path
+  end
+end
+
+class Pmux::Cleaner
+  def run lim_time=nil
+    lim_time = Time.now - 3600
+    clean @glob_pat, lim_time
+  end
+end
+
+class TestIMapreduce < Test::Unit::TestCase
+  # gflocatorを生成し、そのサーバへのconnection(socket)を返す
+  def setup_gflocator world, host, port
+    #handler = GFL::Handler.new
+    #server = MRIO::MockServer.start world, host, port, handler
+    #c = MockTCPSocket.new host, port
+  end
+
+  def setup_world root_dir
+    MockWorld.cleanup root_dir
+    world = MockWorld.new
+    Coolio::Loop.default.world = world
+    Net::SSH::MockChannel.handler_class = MockHandler
+    world
+  end
+
+  def setup_task_dispatcher addrs, options, files, writer=nil
+    setup_world options[:root_dir]
+    FileUtils.mkdir_p options[:tmp_dir]
+    sn = options[:storage_name] || 'local'
+    adapter = Pmux::StorageAdapter.create sn, addrs
+    msession = Pmux::MRSession.new addrs, options
+    msession.connect
+    gatherer = Pmux::Gatherer.new(writer || MockWriter.new)
+    dispatcher = Pmux::TaskDispatcher.new options, adapter, msession, gatherer
+  end
+
+  def test_show_status
+  end
+
+  # mr test; num_r=0, 4 files
+  def test_run_mr0
+    root_dir = "#{$test_dir}/.t"
+
+    # 2 nodes, 4 files
+    addrs = ['192.168.0.1', '192.168.0.2']
+    options = {:root_dir=>root_dir, :tmp_dir=>(root_dir + '/localhost'),
+      :mapper=>'cat'}
+    files = Dir.glob($test_dir + '/txt/[0-3].log')
+    writer = MockWriter.new
+    dispatcher = setup_task_dispatcher addrs, options, files, writer
+    job = Pmux::Job.new options, files
+    dispatcher.run job
+    ae 420, writer.lines.size
+  end
+
+  # mr test; num_r=1, 4 files
+  def test_run_mr1
+    root_dir = "#{$test_dir}/.t"
+    world = setup_world root_dir
+
+    # 2 nodes, 4 files, 2 reducers
+    addrs = ['192.168.0.1', '192.168.0.2']
+    options = {:root_dir=>root_dir, :tmp_dir=>(root_dir + '/localhost'),
+      :mapper=>'cat', :num_r=>1}
+    files = Dir.glob($test_dir + '/txt/[0-3].log')
+    writer = MockWriter.new
+    dispatcher = setup_task_dispatcher addrs, options, files, writer
+    job = Pmux::Job.new options, files
+    job.mk_reducer_addrs addrs
+    dispatcher.run job
+    ae 420, writer.lines.size
+  end
+
+  # mr test; num_r=2, 4 files
+  def test_run_mr2
+    root_dir = "#{$test_dir}/.t"
+    world = setup_world root_dir
+
+    # 2 nodes, 4 files, 2 reducers
+    addrs = ['192.168.0.1', '192.168.0.2']
+    options = {:root_dir=>root_dir, :tmp_dir=>(root_dir + '/localhost'),
+      :mapper=>'cat', :num_r=>2, :separator=>' '}
+    files = Dir.glob($test_dir + '/txt/[0-3].log')
+    writer = MockWriter.new
+    dispatcher = setup_task_dispatcher addrs, options, files, writer
+    job = Pmux::Job.new options, files
+    job.mk_reducer_addrs addrs
+    dispatcher.run job
+    ae 420, writer.lines.size
+
+    # error check
+    files.push '/notexist'
+    writer = MockWriter.new
+    dispatcher = setup_task_dispatcher addrs, options, files, writer
+    er = nil
+    dispatcher.on_error {|r| er = r}
+    job = Pmux::Job.new options, files
+    job.mk_reducer_addrs addrs
+    dispatcher.run job
+    assert_match %r{/notexist: No such file or directory}, er['error_message']
+    ae 420, writer.lines.size
+  end
+
+  # mr test; num_r=0, 4 files, ff=2
+  def test_run_mr_f
+    root_dir = "#{$test_dir}/.t"
+
+    # 2 nodes, 9 files, ff=2
+    addrs = ['192.168.0.1', '192.168.0.2']
+    options = {:root_dir=>root_dir, :tmp_dir=>(root_dir + '/localhost'),
+      :mapper=>'cat', :ff=>2}
+    files = Dir.glob($test_dir + '/txt/*.log')
+    writer = MockWriter.new
+    dispatcher = setup_task_dispatcher addrs, options, files, writer
+    job = Pmux::Job.new options, files
+    dispatcher.run job
+    ae 945, writer.lines.size
+  end
+end

data/test/test_i_mros.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+require 'mock_world'
+require 'mock_mros'
+
+class TestIMros < Test::Unit::TestCase
+  def test_server
+    world = MockWorld.new
+
+    options = {}
+    server = MR::Server.new
+    handler = Pmux::Handler.new server, options
+    transport = MR::MockPipeTransport.new
+    server.listen transport, handler
+    server.run
+
+    # get_status handler test
+    bin = [0, 0, 'get_status', []].to_msgpack
+    transport.receive_data bin
+    msg = MessagePack.unpack transport.data
+    # [1, 0, nil, [['key', value, :type], [], ...]
+
+    ae 1, msg[0]
+    ae 0, msg[1]
+    assert_nil msg[2]
+    assert_kind_of Array, msg[3]
+  end
+end

data/test/test_i_msession.rb

@@ -0,0 +1,27 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+require 'mock_world'
+require 'mock_mros'
+
+class TestIMultiSession < Test::Unit::TestCase
+  def setup_loop
+    loop = Coolio::MockLoop.new
+    loop.world = MockWorld.new
+    loop
+  end
+
+  def test_mr_multicast_call_async
+    loop = setup_loop
+    msession = Pmux::MRSession.new ['192.168.0.1', '192.168.0.2'], {}, loop
+    msession.connect
+
+    mf = msession.multicast_call_async :get_properties
+    mf.join_all
+    mf.all.each {|f|
+      assert_match /^192\.168\.0\.\d+$/, f.addr
+      res = f.get
+      assert_kind_of Hash, res
+      ae RUBY_VERSION, res['RUBY_VERSION']
+    }
+  end
+end

data/test/test_job.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+class TestJob < Test::Unit::TestCase
+  def test_job_id
+    job = Pmux::Job.new({}, [])
+    assert_match /^\d+$/, job.id
+  end
+
+  def test_mk_tasks
+    files = ['/tmp/f1', '/tmp/f2', '/tmp/f3']
+    params = {}
+    job = Pmux::Job.new params, files
+    ae files.size, job.tasks.size
+  end
+
+  def test_mk_reducer_addrs
+    job = Pmux::Job.new({}, [])
+
+    # 1 node
+    addrs = ['192.168.0.1']
+    res = job.mk_reducer_addrs addrs, 1
+    ae ['192.168.0.1'], res
+    res = job.mk_reducer_addrs addrs, 3
+    ae ['192.168.0.1', '192.168.0.1', '192.168.0.1'], res
+
+    # 4 nodes
+    addrs = ['192.168.0.1', '192.168.0.2', '192.168.0.3', '192.168.0.4']
+    res = job.mk_reducer_addrs addrs, 1
+    ae ['192.168.0.1'], res
+    res = job.mk_reducer_addrs addrs, 2
+    ae ['192.168.0.1', '192.168.0.3'], res
+    res = job.mk_reducer_addrs addrs, 3
+    ae ['192.168.0.1', '192.168.0.2', '192.168.0.3'], res
+  end
+end

data/test/test_joblogger.rb

@@ -0,0 +1,16 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+class TestJoblogger < Test::Unit::TestCase
+  def test_j
+    jl = Pmux::Joblogger.new nil, nil
+    assert_nil jl.path
+
+    job = Pmux::Job.new({:job_name=>'unknown'}, [])
+    jl = Pmux::Joblogger.new '/var/tmp', job
+    jl.dump job.to_jlheader
+    assert_match %r{/var/tmp/\d+\.yml}, jl.path
+    jl.close
+    assert File.size?(jl.path)
+    File.unlink jl.path
+  end
+end

data/test/test_mapper.rb

@@ -0,0 +1,60 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+require 'fileutils'
+
+class TestMapper < Test::Unit::TestCase
+  def setup
+    @tmp_dir = $test_dir + '/.t'
+    FileUtils.mkdir_p @tmp_dir
+  end
+
+  def test_r0_r1
+    task = {'task_id'=>1, 'mapper'=>'cat', 'path'=>"#{$test_dir}/txt/0.log"}
+    size = File.size task['path']
+
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    ifbase = mapper.do_map_task
+    ae size, File.size(ifbase+'-0')
+    ae 0, mapper.exitstatus
+
+    task['task_id'] = 2
+    task['num_r'] = 1
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    ifbase = mapper.do_map_task
+    ae size, File.size(ifbase+'-0')
+    ae 0, mapper.exitstatus
+  end
+
+  def test_r2
+    task = {'task_id'=>1, 'mapper'=>'cat', 'path'=>"#{$test_dir}/txt/0.log"}
+    size = File.size task['path']
+
+    task['num_r'] = 2
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    ifbase = mapper.do_map_task
+    ae size, File.size(ifbase+'-0') + File.size(ifbase+'-1')
+    ae 0, mapper.exitstatus
+  end
+
+  def test_err
+    task = {'task_id'=>1, 'mapper'=>'cat', 'path'=>"notexist"}
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    assert_raise(RuntimeError) {
+      ifbase = mapper.do_map_task
+    }
+    assert(mapper.exitstatus > 0)
+
+    task['mapper'] = 'notexist'
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    assert_raise(RuntimeError) {
+      ifbase = mapper.do_map_task
+    }
+
+    task['mapper'] = 'ruby -e "undefinedmethod()"'
+    mapper = Pmux::StreamingMapper.new task, @tmp_dir
+    assert_raise(RuntimeError) {
+      ifbase = mapper.do_map_task
+    }
+    assert(mapper.exitstatus > 0)
+  end
+end

data/test/test_pipeio.rb

@@ -0,0 +1,24 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+require 'mock_pipeio'
+
+class TestPipeio < Test::Unit::TestCase
+  def test_pipeio
+    loop = Coolio::MockLoop.new
+    loop.world = MockWorld.new
+
+    cmd = "cat #{$test_dir}/txt/0.log"
+    pipeio = Pmux::PipeIO.new cmd
+    loop.attach pipeio
+    str = nil
+    pipeio.on_receive {|data|
+      str = data
+    }
+    closed = false
+    pipeio.on_close {closed = true}
+    loop.world.run_once
+    assert_match /START: \w+ /, str
+    while !closed
+      loop.world.run_once
+    end
+  end
+end

data/test/test_storage_adapter.rb

@@ -0,0 +1,63 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+class TestStorageAdapter < Test::Unit::TestCase
+  def setup
+  end
+
+  def test_create
+    addrs = ['192.168.0.1', '192.168.0.2']
+
+    adapter = Pmux::StorageAdapter.create 'local', addrs
+    assert_kind_of Pmux::LocalAdapter, adapter
+    ae addrs, adapter.addrs
+    assert_kind_of Hash, adapter['192.168.0.1']
+    assert adapter['192.168.0.1'].empty?
+
+    adapter = Pmux::StorageAdapter.create 'glusterfs', []
+    assert_kind_of Pmux::GlusterFSAdapter, adapter
+  end
+
+  def test_local
+    addrs = ['192.168.0.1', '192.168.0.2']
+    adapter = Pmux::StorageAdapter.create 'local', addrs
+
+    args = ['/tmp/0.log', '/tmp/1.log', '/tmp/2.log', '/tmp/3.log']
+    files = adapter.get_files args
+    ae args.size, files.size
+
+    args = ["#{$test_dir}/txt/*.log"]
+    files = adapter.get_files args
+    ae args.size, files.size
+
+    files = ["#{$test_dir}/txt/*.log"]
+    files = adapter.get_files args, true
+    ae 9, files.size
+  end
+
+  def _test_pmuxfs
+    addrs = ['192.168.0.1', '192.168.0.2']
+    adapter = Pmux::StorageAdapter.create 'pmuxfs', addrs
+
+    # pmuxfs find test
+    adapter.scan_unit = 5
+    adapter.set_fs_dir "#{$test_dir}/txt"
+    res = adapter.find
+    ae 5, res.size
+    until (res = adapter.find).empty?; end
+  end
+
+  def _test_pmux_node_table
+    addrs = ['192.168.0.1', '192.168.0.2', '192.168.0.3']#, '192.168.0.4']
+    nt = Pmux::PmuxNodeTable.new
+    addrs.each {|addr| nt.push_node addr}
+    nt.reposition
+
+    counts = Hash.new 0
+    for i in 1..1000
+      s = i.to_s
+      h = Digest::SHA1.digest(s)
+      node_addr, = nt.lookup h
+      counts[node_addr] += 1
+    end
+  end
+end

data/test/test_task_queue.rb

@@ -0,0 +1,87 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+class TestTaskQueue < Test::Unit::TestCase
+  def test_task_queue
+    tq = Pmux::TaskQueue.new
+
+    # push
+    v = {:id=>0}
+    tq.push(v)
+    tq.push({:id=>1})
+    tq.push({:id=>2, :job_id=>99})
+    ae 3, tq.size
+
+    ae 2, tq.at(2)[:id]
+    assert_nil tq.at(3)
+
+    # inject_tasks, shift, delete_job
+    tq.inject_tasks [{:id=>3}, {:id=>4}, {:id=>5}, {:id=>6}, {:id=>7}]
+    #tq.each {|v| p v}
+    ae 8, tq.size
+    a = tq.shift
+    ae 0, a[:id]
+    ae 7, tq.size
+    a = tq.shift
+    ae 1, a[:id] # ae 3, a[:id]
+    ae 6, tq.size
+    tq.delete_job 99
+    ae 5, tq.size
+
+    # clear
+    tq.clear
+    ae 0, tq.size
+
+    # inject, shift
+    tasks = (0..9).map {|i|{:id=>i, :job_id=>99}}
+    tq.inject_tasks tasks
+    tmpq = Pmux::TaskQueue.new
+    while item = tq.shift
+      if item[:id] % 2 == 0
+        item.delete :_f #XXX
+        tmpq.push item
+      end
+      break if item[:id] == 6
+    end
+
+    # delete
+    tq.delete_job 99
+    ae 0, tq.size
+
+    # clear, inject
+    tmpq.clear
+    tmpq.inject_tasks [{:id=>0}, {:id=>1}]
+    ae 2, tmpq.size
+  end
+
+  # inject_tasks ĎņŹň
+  def test_inject_tasks
+    tq = Pmux::TaskQueue.new
+    ae 0, tq.size
+
+    tasks = []
+    tq.inject_tasks tasks
+    ae 0, tq.size
+
+    tasks = [{:id=>0}, {:id=>1}]
+    tq.inject_tasks tasks
+    ae 2, tq.size
+    ae 0, tq.at(0)[:id]
+    ae 1, tq.at(1)[:id]
+
+    tasks = [{:id=>2, :mapper_class=>true}, {:id=>3}, {:id=>4}]
+    tq.inject_tasks tasks
+    ae 5, tq.size
+    # TaskQueue#inject_tasks
+    ae 0, tq.at(0)[:id]
+    ae 1, tq.at(1)[:id]
+    ae 2, tq.at(2)[:id]
+    ae 3, tq.at(3)[:id]
+    ae 4, tq.at(4)[:id]
+
+    tasks = (5..6).map {|i|{:id=>i}}
+    tq.inject_tasks tasks
+    ae 7, tq.size
+    tasks = (7..14).map {|i|{:id=>i}}
+    tq.inject_tasks tasks
+  end
+end

data/test/test_task_scheduler.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
+
+class TestTaskScheduler < Test::Unit::TestCase
+  def test_map_task
+    addrs = ['192.168.0.1', '192.168.0.2', '192.168.0.3',]
+    adapter = Pmux::StorageAdapter.create 'local', addrs
+    files = ['/tmp/f1', '/tmp/f2', '/tmp/f3', '/tmp/f4',]
+    job = Pmux::Job.new({}, files)
+    sch = Pmux::TaskScheduler.new adapter
+    sent_tasks = {}
+    sch.attach_flush_callback {|node_addr, task|
+      (sent_tasks[node_addr] ||= []).push task[:task_id]
+    }
+    addrs.each {|addr| sch.shipped[addr] = true}
+    sch.push_job job
+    sch.process_queue
+    ae [1, 4], sent_tasks['192.168.0.1']
+    ae [2], sent_tasks['192.168.0.2']
+    ae [3], sent_tasks['192.168.0.3']
+  end
+
+  def test_task_fusion
+    addrs = ['192.168.0.1', '192.168.0.2', '192.168.0.3',]
+    adapter = Pmux::StorageAdapter.create 'local', addrs
+    files = (0..8).map {|n| "txt/#{n}.log"}
+    job = Pmux::Job.new({:ff=>3}, files)
+    sch = Pmux::TaskScheduler.new adapter
+    sent_tasks = {}
+    sch.attach_flush_callback {|node_addr, task|
+      (sent_tasks[node_addr] ||= []).push task[:task_id]
+    }
+    addrs.each {|addr| sch.shipped[addr] = true}
+    sch.push_job job
+    sch.process_queue
+    ae [1], sent_tasks['192.168.0.1']
+    ae [2], sent_tasks['192.168.0.2']
+    ae [3], sent_tasks['192.168.0.3']
+  end
+end

data/test/txt/0.log

@@ -0,0 +1,105 @@
+2000-12-31 21:01:01 START: foobarorController (pid 7904)
+2000-12-31 21:01:01 lock_path: "/tmp/zzzfoobaror-lock" (pid 7904)
+2000-12-31 21:01:01 sock_path: "/tmp/.foobarorsock.7904"
+2000-12-31 21:01:02 #1: (7906) start
+2000-12-31 21:01:02 foobar unknown iiiiiiiiii.example.com
+2000-12-31 21:01:02 #2: (7907) start
+2000-12-31 21:01:02 foobar unknown ssssssssss.example.com
+2000-12-31 21:01:02 #3: (7908) start
+2000-12-31 21:01:02 foobar unknown pppppppppp.example.com
+2000-12-31 21:01:02 #4: (7909) start
+2000-12-31 21:01:02 foobar unknown kkkkkkkkkk.example.com
+2000-12-31 21:01:02 #5: (7910) start
+2000-12-31 21:01:02 foobar unknown qqqqqqqqqq.example.com
+2000-12-31 21:01:02 #6: (7911) start
+2000-12-31 21:01:02 foobar unknown pppppppppp.example.com
+2000-12-31 21:01:02 #7: (7912) start
+2000-12-31 21:01:02 foobar unknown hhhhhhhhhh.example.com
+2000-12-31 21:01:02 #8: (7913) start
+2000-12-31 21:01:02 foobar unknown llllllllll.example.com
+2000-12-31 21:01:03 foobar unknown kkkkkkkkkk.example.com
+2000-12-31 21:01:03 #9: (7914) start
+2000-12-31 21:01:03 foobar unknown qqqqqqqqqq.example.com
+2000-12-31 21:01:03 #10: (7915) start
+2000-12-31 21:01:03 foobar unknown iiiiiiiiii.example.com
+2000-12-31 21:01:03 foobar unknown llllllllll.example.com
+2000-12-31 21:01:03 foobar unknown router1.example.jp
+2000-12-31 21:01:04 foobar unknown router0.example.jp
+2000-12-31 21:01:04 #4: (7909) end
+2000-12-31 21:01:04 #3: (7908) end
+2000-12-31 21:01:04 #1: (7906) end
+2000-12-31 21:01:04 #10: (7915) end
+2000-12-31 21:01:04 #9: (7914) end
+2000-12-31 21:01:05 #5: (7910) end
+2000-12-31 21:01:05 #2: (7907) end
+2000-12-31 21:01:05 #7: (7912) end
+2000-12-31 21:01:05 #8: (7913) end
+2000-12-31 21:01:06 #6: (7911) end
+2000-12-31 21:01:06 waiting children, done (pid 7904)
+2000-12-31 21:01:06 foobar unknown: elapsed 1.222222 sec
+2000-12-31 21:01:06 sock_path: "/tmp/.foobarorsock.7904"
+2000-12-31 21:01:07 #1: (7916) start
+2000-12-31 21:01:07 foobar unknown ssssssssss.example.com
+2000-12-31 21:01:07 #2: (7917) start
+2000-12-31 21:01:07 foobar unknown cccccccccc.example.com
+2000-12-31 21:01:11 ssssssssss.example.com: insert xxxxxxx: (99999 records in 1.222222 sec)
+2000-12-31 21:01:11 qqqqqqqqqq.example.com: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:11 qqqqqqqqqq.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:11 tttttttttt.example.com: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:11 tttttttttt.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:11 tttttttttt.example.com: insert xxxxxxxx: 9999 records
+2000-12-31 21:01:11 rrrrrrrrrr.example.com: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:12 foobar unknown 192.168.195.12
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxx: (9999 records in 1.222222 sec)
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxx: 9 records
+2000-12-31 21:01:13 192.168.195.12: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:13 foobar unknown iiiiiiiiii.example.com
+2000-12-31 21:01:16 eeeeeeeeee.example.com: insert xxxxxxx: (99999 records in 1.222222 sec)
+2000-12-31 21:01:16 nnnnnnnnnn.example.com: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:16 ssssssssss.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:16 kkkkkkkkkk.example.com: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:16 bbbbbbbbbb.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:16 tttttttttt.example.com: insert xxxxxxxx: 9999 records
+2000-12-31 21:01:16 aaaaaaaaaa.example.com: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:17 foobar unknown iiiiiiiiii.example.com
+2000-12-31 21:01:23 gggggggggg.example.com: insert xxxxxxx: (99999 records in 1.22222 sec)
+2000-12-31 21:01:23 gggggggggg.example.com: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:23 mmmmmmmmmm.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:23 bbbbbbbbbb.example.com: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:23 ffffffffff.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:23 ffffffffff.example.com: insert xxxxxxxx: 9999 records
+2000-12-31 21:01:23 kkkkkkkkkk.example.com: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:24 foobar unknown rrrrrrrrrr.example.com
+2000-12-31 21:01:24 foobar unknown router1.example.jp
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxx: (9999 records in 1.22222 sec)
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxxxxxxx: 9 records
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxx: 9 records
+2000-12-31 21:01:25 router1.example.jp: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:25 foobar unknown router0.example.jp
+2000-12-31 21:01:28 oooooooooo.example.com: insert xxxxxxx: (99999 records in 11.222222 sec)
+2000-12-31 21:01:28 aaaaaaaaaa.example.com: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:28 oooooooooo.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:28 rrrrrrrrrr.example.com: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:28 rrrrrrrrrr.example.com: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:28 hhhhhhhhhh.example.com: insert xxxxxxxx: 9999 records
+2000-12-31 21:01:28 nnnnnnnnnn.example.com: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:29 #2: (7917) end
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxx: (9999 records in 1.222222 sec)
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxxxx: 99 records
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxxxxxxx: 99 records
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxxx: 9 records
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxx: 9 records
+2000-12-31 21:01:29 router0.example.jp: insert xxxxxxxxxxxx: 9 records
+2000-12-31 21:01:30 #1: (7916) end
+2000-12-31 21:01:30 waiting children, done (pid 7904)
+2000-12-31 21:01:30 foobar unknown: elapsed 11.222222 sec
+2000-12-31 21:01:30 unlocked: "/tmp/zzzfoobaror-lock" (pid 7904)
+2000-12-31 21:01:30 END: foobarorController (pid 7904)