ls4 0.9.0

data/lib/ls4/service/stat_cs.rb

@@ -0,0 +1,25 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class CSStatService < StatService
+end
+
+
+end

data/lib/ls4/service/stat_ds.rb

@@ -0,0 +1,40 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class DSStatService < GWStatService
+	def stat_db_items
+		StatBus.db_items
+	end
+
+	def stat_cmd_read
+		StatBus.cmd_read
+	end
+
+	def stat_cmd_write
+		StatBus.cmd_write
+	end
+
+	def stat_cmd_delete
+		StatBus.cmd_delete
+	end
+end
+
+
+end

data/lib/ls4/service/stat_gw.rb

@@ -0,0 +1,25 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class GWStatService < StatService
+end
+
+
+end

data/lib/ls4/service/storage.rb

@@ -0,0 +1,116 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class StorageBus < Bus
+	call_slot :open
+	call_slot :close
+
+	call_slot :get
+
+	call_slot :set
+
+	call_slot :read
+
+	call_slot :delete
+
+	call_slot :copy
+
+	call_slot :exist
+
+	call_slot :get_items
+end
+
+
+class StorageSelector
+	IMPLS = {}
+
+	def self.register(name, klass)
+		IMPLS[name.to_sym] = klass
+		nil
+	end
+
+	def self.select_class(uri)
+		if m = /^(\w{1,8})\:(.*)/.match(uri)
+			type = m[1].to_sym
+			expr = m[2]
+		else
+			type = :dir
+			expr = uri
+		end
+
+		klass = IMPLS[type]
+		unless klass
+			raise "unknown Storage type: #{type}"
+		end
+
+		return klass, expr
+	end
+
+	def self.select!(uri)
+		klass, expr = select_class(uri)
+		klass.init
+
+		StorageBus.open(expr)
+	end
+
+	def self.open!
+		select!(ConfigBus.get_storage_path)
+	end
+end
+
+
+class StorageService < Service
+	#def open(path)
+	#end
+
+	#def close
+	#end
+
+	#def get(vtime, key)
+	#end
+
+	#def set(vtime, key, data)
+	#end
+
+	#def read(vtime, key, offset, size)
+	#end
+
+	#def delete(vtime, key)
+	#end
+
+	#def exist(vtime, key)
+	#end
+
+	#def get_items
+	#end
+
+	ebus_connect :StorageBus,
+		:get,
+		:set,
+		:read,
+		:delete,
+		:exist,
+		:get_items,
+		:open,
+		:close
+end
+
+
+end

data/lib/ls4/service/storage_dir.rb

@@ -0,0 +1,201 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+require 'fileutils'
+require 'uri'
+
+class DirectoryStorageService < StorageService
+	StorageSelector.register(:dir, self)
+
+	def open(expr)
+		@dir = File.expand_path(expr)
+	end
+
+	def close
+	end
+
+	def get(vtime, key)
+		read(vtime, key, nil, nil)
+	end
+
+	def read(vtime, key, offset, size)
+		$log.trace { "DirectoryStorage: read vtime=#{vtime} key=#{key.dump} offset=#{offset} size=#{size}" }
+
+		path = key_to_path(vtime, key)
+		begin
+			return File.read(path, size, offset)
+		rescue
+			return nil
+		end
+	end
+
+	def set(vtime, key, data)
+		$log.trace { "DirectoryStorage: set vtime=#{vtime} key=#{key.dump} data=#{data.size}byte)" }
+
+		path = key_to_path(vtime, key)
+		make_dir(path)
+
+		tmp_path = path+".tmp"
+		File.open(tmp_path, File::WRONLY|File::CREAT|File::TRUNC) {|f|
+			#f.write(data)
+			while true
+				n = f.syswrite(data)
+				if data.size <= n
+					break
+				else
+					data = data[n..-1]
+					#data.slice!(0,n-1)
+				end
+			end
+		}
+		File.rename(tmp_path, path)
+
+		true
+	end
+
+	#def write(vtime, key, offset, data)
+	#	$log.trace { "DirectoryStorage: write vtime=#{vtime} key=#{key.dump} offset=#{offset} data=#{data.size}bytes" }
+	#
+	#	path = key_to_path(vtime, key)
+	#	make_dir(path)
+	#
+	#	File.open(path, File::WRONLY|File::CREAT) {|f|
+	#		f.pos = offset
+	#		#f.write(data)
+	#		while true
+	#			n = f.syswrite(data)
+	#			if data.size <= n
+	#				break
+	#			else
+	#				data = data[n..-1]
+	#				#data.slice!(0,n-1)
+	#			end
+	#		end
+	#	}
+	#
+	#	true
+	#end
+
+	#def append(vtime, key, data)
+	#	$log.trace { "DirectoryStorage: append vtime=#{vtime} key=#{key.dump} offset=#{offset} data=#{data.size}bytes" }
+	#
+	#	path = key_to_path(vtime, key)
+	#	make_dir(path)
+	#
+	#	size = nil
+	#	File.open(path, File::WRONLY|File::CREAT|File::APPEND) {|f|
+	#		#f.write(data)
+	#		while true
+	#			n = f.syswrite(data)
+	#			if data.size <= n
+	#				break
+	#			else
+	#				data = data[n..-1]
+	#				#data.slice!(0,n-1)
+	#			end
+	#		end
+	#		size = f.stat.size
+	#	}
+	#
+	#	size
+	#end
+
+	#def resize(vtime, key, size)
+	#	$log.trace { "DirectoryStorage: resize vtime=#{vtime} key=#{key.dump} size=#{size}" }
+	#
+	#	path = key_to_path(vtime, key)
+	#	make_dir(path)
+	#
+	#	File.open(path, File::WRONLY|File::CREAT) {|f|
+	#		f.truncate(size)
+	#	}
+	#
+	#	true
+	#end
+
+	def delete(vtime, key)
+		$log.trace { "DirectoryStorage: delete vtime=#{vtime} key=#{key.dump}" }
+
+		path = key_to_path(vtime, key)
+		begin
+			File.unlink(path)
+			Dir.rmdir(File.dirname(path)) rescue nil
+			return true
+		rescue
+			return false
+		end
+	end
+
+	def exist(vtime, key)
+		$log.trace { "DirectoryStorage: exist vtime=#{vtime} key=#{key.dump}" }
+
+		path = key_to_path(vtime, key)
+		return File.exist?(path)
+	end
+
+	def get_items
+		num = 0
+		(0..0xff).each {|d|
+			box = "%03d" % d
+			dirpath = File.join(@dir, "data", box)
+			begin
+				e = Dir.entries(dirpath).size
+				e -= 2  # skip "." and ".."
+				num += e if e > 0
+			rescue
+			end
+		}
+		num
+	end
+
+	# for DataServerURLService
+	def self.encode_okey(okey)
+		subpath = key_to_subpath(okey.vtime, okey.key)
+		File.join(*subpath)
+	end
+
+	def key_to_path(vtime, key)
+		subpath = self.class.key_to_subpath(vtime, key)
+		File.join(@dir, "data", *subpath)
+	end
+
+	def self.key_to_subpath(vtime, key)
+		digest = Digest::MD5.digest(key)
+		box = "%03d" % digest.unpack('C')[0]
+		kdir = encode_path(key)
+		fname = "v#{vtime}"
+		return box, kdir, fname
+	end
+
+	private
+	def self.encode_path(s)
+		URI.encode(s, /[^-_.a-zA-Z0-9]/n)
+	end
+
+	def self.decode_path(b)
+		URI.decode(b, /[^-_.a-zA-Z0-9]/n)
+	end
+
+	def make_dir(path)
+		FileUtils.mkdir_p File.dirname(path)
+	end
+end
+
+
+end

data/lib/ls4/service/sync.rb

@@ -0,0 +1,206 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+SYNC_MEMBERSHIP     = 0
+SYNC_FAULT_LIST     = 1
+SYNC_REPLSET_WEIGHT = 2
+SYNC_MDS_URI        = 3
+SYNC_MDS_CACHE_URI  = 4
+
+
+class SyncBus < Bus
+	call_slot :get_hash
+
+	call_slot :check_hash
+	call_slot :update
+
+	call_slot :register_callback
+	call_slot :try_sync
+end
+
+
+class SyncService < Service
+	def initialize
+		@hash_array = []
+		update_hash
+	end
+
+	def get_hash
+		@hash
+	end
+
+	ebus_connect :SyncBus,
+		:get_hash
+
+	private
+	def update_hash
+		@hash = Digest::SHA1.digest(@hash_array.to_msgpack)
+	end
+end
+
+
+class SyncServerService < SyncService
+	def initialize
+		@data_array = []
+		super
+	end
+
+	def check_hash(hash)
+		@hash == hash
+	end
+
+	def update(id, data, hash)
+		@data_array[id] = data
+		@hash_array[id] = hash
+		update_hash
+		nil
+	end
+
+	def rpc_sync_config(hash_array)
+		result = []
+
+		hash_array.each_with_index {|hash,id|
+			if h = @hash_array[id]
+				if h != hash
+					result[id] = @data_array[id]
+				end
+			end
+		}
+
+		result
+	end
+
+	ebus_connect :SyncBus,
+		:update,
+		:check_hash
+
+	ebus_connect :CSRPCBus,
+		:sync_config => :rpc_sync_config
+end
+
+
+class SyncClientService < SyncService
+	def initialize
+		@callback_array = []
+		super
+	end
+
+	def register_callback(id, initial_hash, &block)
+		@callback_array[id] = block
+		@hash_array[id] = initial_hash
+		update_hash
+		nil
+	end
+
+	def try_sync
+		do_sync
+		nil
+	end
+
+	def sync_blocking!
+		do_sync.join
+	end
+
+	ebus_connect :SyncBus,
+		:register_callback,
+		:try_sync
+
+	private
+	def do_sync
+		get_cs_session.callback(:sync_config, @hash_array) do |future|
+			begin
+				data_array = future.get
+				ack_sync(data_array)
+			rescue
+				$log.error "sync error: #{$!}"
+				$log.error_backgrace $!.backtrace
+			end
+		end
+	end
+
+	def ack_sync(data_array)
+		data_array.each_with_index {|obj,id|
+			if obj
+				if callback = @callback_array[id]
+					$log.debug "sync: #{id} => #{obj.inspect}"
+					@hash_array[id] = callback.call(obj)
+				end
+			end
+		}
+		update_hash
+	end
+
+	def get_cs_session
+		ProcessBus.get_session(ConfigBus.get_cs_address)
+	end
+end
+
+
+class StandaloneSyncService < SyncServerService
+	def initialize
+		@callback_array = []
+		@client_hash_array = []
+		super
+	end
+
+	def register_callback(id, initial_hash, &block)
+		@callback_array[id] = block
+		@client_hash_array[id] = initial_hash
+		nil
+	end
+
+	def try_sync
+		do_sync
+		nil
+	end
+
+	def sync_blocking!
+		do_sync.join
+	end
+
+	ebus_connect :SyncBus,
+		:register_callback,
+		:try_sync
+
+	private
+	def do_sync
+		result = []
+
+		@client_hash_array.each_with_index {|hash,id|
+			if h = @hash_array[id]
+				if h != hash
+					result[id] = MessagePack.unpack(@data_array[id].to_msgpack)
+				end
+			end
+		}
+
+		result.each_with_index {|obj,id|
+			if obj
+				if callback = @callback_array[id]
+					$log.debug "sync: #{id} => #{obj.inspect}"
+					@client_hash_array[id] = callback.call(obj)
+				end
+			end
+		}
+	end
+end
+
+
+end