ls4 0.9.0

data/test/common.rb

@@ -0,0 +1,250 @@
+here = File.expand_path(File.dirname(__FILE__))
+
+RUBY = ENV["RUBY"] || "ruby"
+TTSERVER = ENV["TTSERVER"] || "ttserver"
+
+DATA_DIR = ENV["DATA_DIR"] || "#{here}/tmp-#{Time.now.strftime("%Y-%m-%d.%H-%M-%S")}.#{Process.pid}"
+
+CMD_BASE = "#{RUBY} -I#{here}/../lib #{here}/../lib/ls4/command"
+CMD_CS  = "#{CMD_BASE}/cs.rb"
+CMD_DS  = "#{CMD_BASE}/ds.rb"
+CMD_GW  = "#{CMD_BASE}/gw.rb"
+CMD_CTL = "#{CMD_BASE}/ctl.rb"
+CMD_STANDALONE = "#{CMD_BASE}/standalone.rb"
+
+CS_PORT = (ENV["CS_PORT"] || 49700).to_i
+DS_PORT = (ENV["DS_PORT"] || 49900).to_i
+GW_PORT = (ENV["GW_PORT"] || 49800).to_i
+MDS_PORT = (ENV["MDS_PORT"] || 49600).to_i
+DS_HTTP_PORT = (ENV["GW_HTTP_PORT"] || 49400).to_i
+GW_HTTP_PORT = (ENV["GW_HTTP_PORT"] || 49500).to_i
+
+if ENV["NO_TRACE"]
+	OPT = ENV["OPT"] || "--color-log"
+	TTOPT = ENV["TTOPT"] || ""
+else
+	OPT = ENV["OPT"] || "--trace --color-log"
+	TTOPT = ENV["TTOPT"] || "-ld"
+end
+
+begin
+require 'rubygems'
+rescue LoadError
+end
+require 'chukan'
+require 'msgpack/rpc'
+require 'fileutils'
+require 'json'
+
+require 'net/http'
+Net::HTTP.version_1_2
+
+include Chukan
+include Chukan::Test
+
+class ServerProcess < Chukan::LocalProcess
+	def host
+		"127.0.0.1"
+	end
+
+	protected
+	def init_data_dir(subdir)
+		ddir = "#{DATA_DIR}/#{subdir}"
+		FileUtils.rm_rf(ddir)
+		FileUtils.mkdir_p(ddir)
+		ddir
+	end
+end
+
+class MDSProcess < ServerProcess
+	def initialize(*args)
+		@port = MDS_PORT
+
+		ddir = init_data_dir("mds")
+		super("#{TTSERVER} -port #{@port} #{TTOPT} #{args.join(' ')} #{ddir}/mds.tct")
+	end
+
+	attr_reader :port
+
+	def join_started
+		stdout_join('started')
+	end
+end
+
+class CtlProcess < Chukan::LocalProcess
+	def initialize(host, port, *args)
+		super("#{CMD_CTL} #{host}:#{port} #{args.join(' ')}")
+		set_display("ctl #{args[0]}")
+	end
+end
+
+class CSProcess < ServerProcess
+	def initialize(*args)
+		@port = CS_PORT
+
+		ddir = init_data_dir("cs")
+		super("#{CMD_CS} --mds tt:127.0.0.1:#{MDS_PORT} -p #{@port} -s #{ddir} #{args.join(' ')} #{OPT}")
+
+		set_display("cs")
+	end
+
+	attr_reader :port
+
+	def join_started
+		stdout_join('start on')
+	end
+
+	def show_stat
+		CtlProcess.new(host, port, "stat").join
+	end
+
+	def show_nodes
+		CtlProcess.new(host, port, "nodes").join
+	end
+
+	def show_weight
+		CtlProcess.new(host, port, "weight").join
+	end
+
+	def show_version
+		CtlProcess.new(host, port, "version").join
+	end
+
+	def show_items
+		CtlProcess.new(host, port, "items").join
+	end
+
+	def show_snapshot
+		CtlProcess.new(host, port, "snapshot").join
+	end
+
+	def add_snapshot(name)
+		CtlProcess.new(host, port, "add_snapshot", name).join
+	end
+end
+
+class DSProcess < ServerProcess
+	def initialize(nid, rsid, *args)
+		@nid = nid
+		@rsid = rsid
+		@port = DS_PORT+nid
+		@http_port = DS_HTTP_PORT+nid
+
+		ddir = init_data_dir("ds#{nid}")
+		super("#{CMD_DS} -c 127.0.0.1:#{CS_PORT} -s #{ddir} -i #{nid} -n node#{nid} -g #{rsid} -a 127.0.0.1:#{@port} -t #{@http_port} #{args.join(' ')} #{OPT}")
+
+		set_display("ds#{nid} rsid=#{rsid}")
+	end
+
+	attr_reader :port
+	attr_reader :http_port
+	attr_reader :nid
+	attr_reader :rsid
+
+	def join_started
+		stdout_join('start on')
+	end
+
+	def client
+		MessagePack::RPC::Client.new(host, @port)
+	end
+
+	def http_client(path, &block)
+		Net::HTTP.start("127.0.0.1", @http_port, &block)
+	end
+end
+
+class GWProcess < ServerProcess
+	def initialize(n, *args)
+		@port = GW_PORT+n
+		@http_port = GW_HTTP_PORT+n
+
+		super("#{CMD_GW} -c 127.0.0.1:#{CS_PORT} -p #{port} -t #{@http_port} #{args.join(' ')} #{OPT}")
+
+		set_display("gw#{n}")
+	end
+
+	attr_reader :port
+	attr_reader :http_port
+
+	def join_started
+		stdout_join('start on')
+	end
+
+	def client
+		MessagePack::RPC::Client.new(host, @port)
+	end
+
+	def http_client(&block)
+		Net::HTTP.start("127.0.0.1", @http_port, &block)
+	end
+end
+
+class StandaloneProcess < ServerProcess
+	def initialize(*args)
+		@port = DS_PORT
+		@http_port = DS_HTTP_PORT
+
+		ddir = init_data_dir("standalone")
+		super("#{CMD_STANDALONE} -s #{ddir} -p #{@port} -t #{@http_port} #{args.join(' ')} #{OPT}")
+
+		set_display("standalone")
+	end
+
+	attr_reader :port
+	attr_reader :http_port
+
+	def join_started
+		stdout_join('start on')
+	end
+
+	def client
+		MessagePack::RPC::Client.new(host, @port)
+	end
+
+	def http_client(path, &block)
+		Net::HTTP.start("127.0.0.1", @http_port, &block)
+	end
+end
+
+
+def start_mds(*args)
+	mds = MDSProcess.new(*args)
+	mds.join_started
+	mds
+end
+
+def start_cs(*args)
+	cs = CSProcess.new(*args)
+	cs.join_started
+	cs
+end
+
+def start_ds(nid, rsid, *args)
+	ds = DSProcess.new(nid, rsid, *args)
+	ds.join_started
+	ds
+end
+
+def start_gw(n=0, *args)
+	gw = GWProcess.new(n, *args)
+	gw.join_started
+	gw
+end
+
+def start_standalone(*args)
+	standalone = StandaloneProcess.new(*args)
+	standalone.join_started
+	standalone
+end
+
+def start_memcached(port, *args)
+	m = spawn("memcached -U 0 -p #{port} #{args.join(' ')}")
+	m
+end
+
+def term_all(*procs)
+	procs.each {|pr| pr.term rescue p($!) }
+	procs.each {|pr| pr.join rescue p($!) }
+end
+

data/test/load_test.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+begin
+	require 'rubygems'
+rescue LoadError
+end
+require 'chukan'
+require 'msgpack/rpc'
+
+include Chukan::Test
+
+host = '127.0.0.1'
+port = ARGV[0] || 18800
+
+THREAD = 1
+LOOP = 2
+SIZE = 4*1024*1024
+NUM  = 100
+DUP  = 3
+
+pid = Process.pid
+
+data = " " * SIZE
+
+threads = (1..THREAD).map do
+
+	test "remove existent keys" do
+		c = MessagePack::RPC::Client.new(host, port)
+		(NUM/DUP*DUP).times {|i|
+			c.call(:remove, "k#{i}")
+		}
+	end
+
+	Thread.new(MessagePack::RPC::Client.new(host, port)) {|c|
+
+		test "run normally" do
+			LOOP.times {
+				puts "set #{SIZE} bytes #{NUM/DUP*DUP} items with #{DUP} async"
+				t = Time.now
+				(NUM/DUP).times {|n|
+					ar = []
+					DUP.times {|d|
+						#f = c.call_async(:add, "k#{n*DUP+d}", data, {"attr"=>"at"})
+						f = c.call_async(:add_data, "k#{pid}-#{n*DUP+d}", data)
+						ar << f
+					}
+					ar.each {|f|
+						f.get
+					}
+				}
+				e = Time.now - t
+				puts "#{e} sec."
+				puts "#{NUM/DUP*DUP/e} req/sec"
+				puts "#{NUM/DUP*DUP/e*SIZE/1024/1024} MB/sec"
+
+				puts "get #{SIZE} bytes #{NUM/DUP*DUP} items with #{DUP} async"
+				t = Time.now
+				(NUM/DUP).times {|n|
+					ar = []
+					DUP.times {|d|
+						f = c.call_async(:get, "k#{pid}-#{n*DUP+d}")
+						ar << f
+					}
+					ar.each {|f|
+						f.get
+					}
+				}
+				e = Time.now - t
+				puts "#{e} sec."
+				puts "#{NUM/DUP*DUP/e} req/sec"
+				puts "#{NUM/DUP*DUP/e*SIZE/1024/1024} MB/sec"
+			}
+		end
+
+	}
+end.each {|th|
+	th.join
+}
+

data/test/load_test_offload.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+begin
+	require 'rubygems'
+rescue LoadError
+end
+require 'chukan'
+require 'msgpack/rpc'
+require 'net/http'
+Net::HTTP.version_1_2
+
+include Chukan::Test
+
+host = '127.0.0.1'
+port = ARGV[0] || 18800
+
+THREAD = 1
+LOOP = 2
+SIZE = 4*1024*1024
+NUM  = 100
+DUP  = 3
+
+pid = Process.pid
+
+data = " " * SIZE
+
+threads = (1..THREAD).map do
+
+	test "remove existent keys" do
+		c = MessagePack::RPC::Client.new(host, port)
+		(NUM/DUP*DUP).times {|i|
+			c.call(:remove, "k#{i}")
+		}
+	end
+
+	Thread.new(MessagePack::RPC::Client.new(host, port)) {|c|
+
+		test "run normally" do
+			LOOP.times {
+				puts "set #{SIZE} bytes #{NUM/DUP*DUP} items with #{DUP} async"
+				t = Time.now
+				(NUM/DUP).times {|n|
+					ar = []
+					DUP.times {|d|
+						#f = c.call_async(:add, "k#{n*DUP+d}", data, {"attr"=>"at"})
+						f = c.call_async(:add_data, "k#{pid}-#{n*DUP+d}", data)
+						ar << f
+					}
+					ar.each {|f|
+						f.get
+					}
+				}
+				e = Time.now - t
+				puts "#{e} sec."
+				puts "#{NUM/DUP*DUP/e} req/sec"
+				puts "#{NUM/DUP*DUP/e*SIZE/1024/1024} MB/sec"
+
+				puts "get #{SIZE} bytes #{NUM/DUP*DUP} items with #{DUP} async"
+				t = Time.now
+				(NUM/DUP).times {|n|
+					ar = []
+					DUP.times {|d|
+						f = c.call_async(:url, "k#{pid}-#{n*DUP+d}")
+						ar << f
+					}
+					ar.each {|f|
+						url = f.get
+						uri = URI.parse(url)
+						Net::HTTP.start(uri.host, uri.port) {|http|
+							res = http.get(uri.path)
+							res.body
+						}
+					}
+				}
+				e = Time.now - t
+				puts "#{e} sec."
+				puts "#{NUM/DUP*DUP/e} req/sec"
+				puts "#{NUM/DUP*DUP/e*SIZE/1024/1024} MB/sec"
+			}
+		end
+
+	}
+end.each {|th|
+	th.join
+}
+

metadata

@@ -0,0 +1,295 @@
+--- !ruby/object:Gem::Specification 
+name: ls4
+version: !ruby/object:Gem::Version 
+  hash: 59
+  prerelease: false
+  segments: 
+  - 0
+  - 9
+  - 0
+  version: 0.9.0
+platform: ruby
+authors: 
+- FURUHASHI Sadayuki
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-20 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: msgpack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 4
+        - 4
+        version: 0.4.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: msgpack-rpc
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 4
+        - 3
+        version: 0.4.3
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: tokyocabinet
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 53
+        segments: 
+        - 1
+        - 29
+        version: "1.29"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: tokyotyrant
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 1
+        - 13
+        version: "1.13"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: memcache-client
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 1
+        - 8
+        - 5
+        version: 1.8.5
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rack
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 2
+        - 1
+        version: 1.2.1
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id007
+description: 
+email: frsyuki@users.sourceforge.jp
+executables: 
+- ls4-cs
+- ls4-ds
+- ls4-gw
+- ls4-standalone
+- ls4cmd
+- ls4ctl
+- ls4rpc
+- ls4stat
+- ls4top
+extensions: []
+
+extra_rdoc_files: 
+- ChangeLog
+- README.rdoc
+files: 
+- AUTHORS
+- COPYING
+- ChangeLog
+- NOTICE
+- README.rdoc
+- lib/ls4/command/cmd.rb
+- lib/ls4/command/cs.rb
+- lib/ls4/command/ctl.rb
+- lib/ls4/command/ds.rb
+- lib/ls4/command/gw.rb
+- lib/ls4/command/rpc.rb
+- lib/ls4/command/standalone.rb
+- lib/ls4/command/stat.rb
+- lib/ls4/command/top.rb
+- lib/ls4/default.rb
+- lib/ls4/lib/cclog.rb
+- lib/ls4/lib/ebus.rb
+- lib/ls4/lib/vbcode.rb
+- lib/ls4/logic/fault_detector.rb
+- lib/ls4/logic/membership.rb
+- lib/ls4/logic/node.rb
+- lib/ls4/logic/okey.rb
+- lib/ls4/logic/tsv_data.rb
+- lib/ls4/logic/weight.rb
+- lib/ls4/service/balance.rb
+- lib/ls4/service/base.rb
+- lib/ls4/service/bus.rb
+- lib/ls4/service/config.rb
+- lib/ls4/service/config_cs.rb
+- lib/ls4/service/config_ds.rb
+- lib/ls4/service/config_gw.rb
+- lib/ls4/service/data_client.rb
+- lib/ls4/service/data_server.rb
+- lib/ls4/service/data_server_url.rb
+- lib/ls4/service/gateway.rb
+- lib/ls4/service/gateway_ro.rb
+- lib/ls4/service/gw_http.rb
+- lib/ls4/service/heartbeat.rb
+- lib/ls4/service/log.rb
+- lib/ls4/service/master_select.rb
+- lib/ls4/service/mds.rb
+- lib/ls4/service/mds_cache.rb
+- lib/ls4/service/mds_cache_mem.rb
+- lib/ls4/service/mds_cache_memcached.rb
+- lib/ls4/service/mds_ha.rb
+- lib/ls4/service/mds_memcache.rb
+- lib/ls4/service/mds_tc.rb
+- lib/ls4/service/mds_tt.rb
+- lib/ls4/service/membership.rb
+- lib/ls4/service/process.rb
+- lib/ls4/service/rpc.rb
+- lib/ls4/service/rpc_cs.rb
+- lib/ls4/service/rpc_ds.rb
+- lib/ls4/service/rpc_gw.rb
+- lib/ls4/service/rts.rb
+- lib/ls4/service/rts_file.rb
+- lib/ls4/service/rts_memory.rb
+- lib/ls4/service/slave.rb
+- lib/ls4/service/stat.rb
+- lib/ls4/service/stat_cs.rb
+- lib/ls4/service/stat_ds.rb
+- lib/ls4/service/stat_gw.rb
+- lib/ls4/service/storage.rb
+- lib/ls4/service/storage_dir.rb
+- lib/ls4/service/sync.rb
+- lib/ls4/service/time_check.rb
+- lib/ls4/service/ulog.rb
+- lib/ls4/service/ulog_file.rb
+- lib/ls4/service/ulog_memory.rb
+- lib/ls4/service/weight.rb
+- lib/ls4/version.rb
+- test/01_add_get_remove.rt
+- test/02_read.rt
+- test/03_getd_readd.rt
+- test/04_version_time.rt
+- test/05_version_name.rt
+- test/06_http_get_set_remove_1.rt
+- test/07_http_get_set_remove_2.rt
+- test/08_read_only_time.rt
+- test/09_read_only_name.rt
+- test/10_http_get_set_remove_3.rt
+- test/11_mds_cache_memcached.rt
+- test/12_mds_cache_local_memory.rt
+- test/13_memcache_mds.rt
+- test/14_delete.rt
+- test/15_standalone.rt
+- test/chukan.rb
+- test/common.rb
+- test/load_test.rb
+- test/load_test_offload.rb
+- bin/ls4-cs
+- bin/ls4-ds
+- bin/ls4-gw
+- bin/ls4-standalone
+- bin/ls4cmd
+- bin/ls4ctl
+- bin/ls4rpc
+- bin/ls4stat
+- bin/ls4top
+has_rdoc: true
+homepage: http://ls4.sourceforge.net/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: LS4 - large-scale simple storage system
+test_files: 
+- test/01_add_get_remove.rt
+- test/02_read.rt
+- test/03_getd_readd.rt
+- test/04_version_time.rt
+- test/05_version_name.rt
+- test/06_http_get_set_remove_1.rt
+- test/07_http_get_set_remove_2.rt
+- test/08_read_only_time.rt
+- test/09_read_only_name.rt
+- test/10_http_get_set_remove_3.rt
+- test/11_mds_cache_memcached.rt
+- test/12_mds_cache_local_memory.rt
+- test/13_memcache_mds.rt
+- test/14_delete.rt
+- test/15_standalone.rt