ls4 0.9.0

data/lib/ls4/service/mds_cache.rb

@@ -0,0 +1,294 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class MDSCacheBus < Bus
+	call_slot :get
+	call_slot :set
+	call_slot :invalidate
+end
+
+
+class MDSCacheConfigService < Service
+	def run
+		@uri = ConfigBus.get_initial_mds_cache_uri
+		@uri ||= "null"
+		on_change
+	end
+
+	def rpc_get_mds_cache_uri
+		@uri
+	end
+
+	def rpc_set_mds_cache_uri(uri)
+		@uri = uri
+		on_change
+		nil
+	end
+
+	def self.hash_uri(uri)
+		Digest::SHA1.digest(uri)
+	end
+
+	def on_change
+		SyncBus.update(SYNC_MDS_CACHE_URI,
+							@uri, MDSCacheConfigService.hash_uri(@uri))
+	end
+
+	ebus_connect :ProcessBus,
+		:run
+
+	ebus_connect :CSRPCBus,
+		:get_mds_cache_uri => :rpc_get_mds_cache_uri,
+		:set_mds_cache_uri => :rpc_set_mds_cache_uri
+end
+
+
+class MDSCacheService < Service
+	def initialize
+		@uri = ""
+		@cache = NullMDSCache.new
+	end
+
+	def run
+		SyncBus.register_callback(SYNC_MDS_CACHE_URI,
+							MDSCacheConfigService.hash_uri(@uri)) do |obj|
+			uri = obj
+			reopen(uri)
+			@uri = uri
+			MDSCacheConfigService.hash_uri(@uri)
+		end
+	end
+
+	def shutdown
+		if @cache
+			@cache.close rescue nil
+		end
+	end
+
+	def reopen(uri)
+		klass, expr = MDSCacheSelector.select_class(uri)
+
+		cache = klass.new
+		cache.open(expr)
+
+		old_cache = @cache
+		@cache = cache
+
+		$log.info "using MDS cache: #{@cache}"
+
+		begin
+			old_cache.close
+		rescue
+			$log.error "MDSCache close error: #{$!}"
+			$log.error_backtrace $!.backtrace
+		end
+	end
+
+	ebus_connect :ProcessBus,
+		:run,
+		:shutdown
+
+	ebus_connect :MDSCacheBus,
+		:get,
+		:set,
+		:invalidate
+
+	extend Forwardable
+
+	def_delegators :@cache,
+		:get,
+		:set,
+		:invalidate
+end
+
+
+module MDSCacheSelector
+	IMPLS = {}
+
+	def self.register(name, klass)
+		IMPLS[name.to_sym] = klass
+		nil
+	end
+
+	def self.select_class(uri)
+		if uri.empty?
+			return NullMDSCache
+		end
+
+		if m = /^(\w{1,8})\:(.*)/.match(uri)
+			type = m[1].to_sym
+			expr = m[2]
+		else
+			type = :null
+			expr = uri
+		end
+
+		klass = IMPLS[type]
+		unless klass
+			raise "unknown MDSCache type: #{type}"
+		end
+
+		return klass, expr
+	end
+end
+
+
+class MDSCache
+	#def open(expr)
+	#end
+
+	#def close
+	#end
+
+	#def get(key)
+	#end
+
+	#def set(key, val)
+	#end
+
+	#def invalidate(key)
+	#end
+end
+
+
+class NullMDSCache < MDSCache
+	MDSCacheSelector.register(:null, self)
+
+	def open(expr)
+	end
+
+	def close
+	end
+
+	def get(key)
+		nil
+	end
+
+	def set(key, val)
+	end
+
+	def invalidate(key)
+	end
+
+	def to_s
+		"no-cache"
+	end
+end
+
+
+class CachedMDSBus < Bus
+	call_slot :get_okey
+	call_slot :get_attrs
+	call_slot :get_okey_attrs
+	call_slot :add
+	call_slot :update_attrs
+	call_slot :remove
+	call_slot :delete
+	call_slot :util_locate
+end
+
+
+class CachedMDSService < Service
+	def get_okey(key, version=nil, &cb)
+		if version == nil
+			if okey = get_cache(key)
+				cb.call(okey, nil)
+				return
+			end
+		end
+		MDSBus.get_okey(key, version) {|okey,error|
+			set_cache(key, okey) if okey
+			cb.call(okey, error)
+		}
+	end
+
+	def get_attrs(key, version=nil, &cb)
+		MDSBus.get_attrs(key, version, &cb)
+	end
+
+	def get_okey_attrs(key, version=nil, &cb)
+		MDSBus.get_okey_attrs(key, version, &cb)
+	end
+
+	def add(key, attrs={}, vname=nil, &cb)
+		invalidate_cache(key)
+		MDSBus.add(key, attrs, vname, &cb)
+	end
+
+	def update_attrs(key, attrs, &cb)
+		invalidate_cache(key)
+		MDSBus.update_attrs(key, attrs, &cb)
+	end
+
+	def remove(key, &cb)
+		invalidate_cache(key)
+		MDSBus.remove(key, &cb)
+	end
+
+	def delete(key, version=nil, &cb)
+		invalidate_cache(key)
+		MDSBus.delete(key, version, &cb)
+	end
+
+	def util_locate(key, &cb)
+		MDSBus.util_locate(key, &cb)
+	end
+
+	ebus_connect :CachedMDSBus,
+		:get_okey,
+		:get_attrs,
+		:get_okey_attrs,
+		:add,
+		:update_attrs,
+		:remove,
+		:delete,
+		:util_locate
+
+	private
+	def get_cache(key)
+		if val = MDSCacheBus.get(key)
+			rsid, vtime = MessagePack.unpack(val)
+			return ObjectKey.new(key, vtime, rsid)
+		end
+		return nil
+	rescue
+		$log.warn("error when getting MDS cache: key=#{key.inspect}: #{$!.to_s}")
+		$log.debug_backtrace $!.backtrace
+		return nil
+	end
+
+	def set_cache(key, okey)
+		val = [okey.rsid, okey.vtime].to_msgpack
+		MDSCacheBus.set(key, val)
+	rescue
+		$log.warn("error when setting MDS cache: key=#{key.inspect}: #{$!.to_s}")
+		$log.debug_backtrace $!.backtrace
+	end
+
+	def invalidate_cache(key)
+		MDSCacheBus.invalidate(key)
+	rescue
+		$log.warn("error when invalidating MDS cache: key=#{key.inspect}: #{$!.to_s}")
+		$log.debug_backtrace $!.backtrace
+	end
+end
+
+
+end

data/lib/ls4/service/mds_cache_mem.rb

@@ -0,0 +1,63 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class LocalMemoryMDSCache < MDSCache
+	MDSCacheSelector.register(:local, self)
+
+	def initialize
+		require 'tokyocabinet'
+	end
+
+	def open(expr)
+		@db = TokyoCabinet::ADB.new
+		if expr.empty?
+			@size = "32m"
+		else
+			@size = expr
+		end
+		name = "+#capsiz=#{@size}"
+		unless @db.open(name)
+			raise "failed to MDS local memory cache database: #{@db.errmsg(@db.ecode)}"
+		end
+	end
+
+	def close
+		@db.close
+	end
+
+	def get(key)
+		@db[key]
+	end
+
+	def set(key, val)
+		@db[key] = val
+	end
+
+	def invalidate(key)
+		@db.delete(key)
+	end
+
+	def to_s
+		"<LocalMemoryMDSCache size=#{@size}>"
+	end
+end
+
+
+end

data/lib/ls4/service/mds_cache_memcached.rb

@@ -0,0 +1,65 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class MemcachedMDSCache < MDSCache
+	MDSCacheSelector.register(:mc, self)
+
+	def initialize
+		require 'memcache'
+	end
+
+	def open(expr)
+		@expire = 0
+		if m = /\;expire\=(\d+)/.match(expr)
+			servers_line = expr[0,m.begin(0)]
+			@expire = m[1].to_i
+		else
+			servers_line = expr
+		end
+		@servers = servers_line.split(/\s*\,\s*/)
+		if @expire == 0
+			@expire = 60*60*24
+		end
+		@mc = MemCache.new(@servers, {:urlencode => false, :compression => false, :multithread => true, :timeout => 1.0})
+	end
+
+	def close
+		@mc.reset
+	end
+
+	def get(key)
+		@mc.get(key, true)
+	end
+
+	def set(key, val)
+		@mc.set(key, val, @expire, true)
+	end
+
+	def invalidate(key)
+		@mc.delete(key)
+	end
+
+	def to_s
+		"<MemcachedMDSCache servers=#{@servers.join(',')} expire=#{@expire}>"
+	end
+end
+
+
+end

data/lib/ls4/service/mds_ha.rb

@@ -0,0 +1,176 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+module LS4
+
+
+# single node:
+#   host1:port1
+#
+# master-slave:
+#   host1:port1,host2:port2
+#
+# master-slave with read weight:
+#   host1:port1,host2:port2;0,1
+#
+# dual-master:
+#   host1:port1--host2:port2
+#
+class BasicHADB
+	DEFAULT_WEIGHT = 10
+
+	def initialize(expr)
+		@dbmap = {}    # {Address => DB}
+		@writers = []   # [Address]
+		@readers = []   # [Address]
+		@readers_rr = 0
+
+		expr.split('--').each {|line|
+			nodes, weights = line.strip.split(';',2)
+
+			addrs = nodes.strip.split(',').map {|addr|
+				parse_addr(addr)
+			}
+
+			weights = (weights||"").strip.split(',').map {|x| x.to_i }
+
+			@writers << addrs.first
+
+			addrs.each_with_index {|addr,i|
+				weight = weights[i] ||= DEFAULT_WEIGHT
+				weight.times {
+					@readers << addr
+				}
+				@dbmap[addr] = nil
+			}
+
+			$log.info "MDS -- #{addrs.join(',')};#{weights.join(',')}"
+		}
+
+		if @dbmap.empty?
+			raise "empty expression"
+		end
+
+		if @dbmap.size == 1
+			# single node
+			@readers = [@readers[0]]
+		else
+			@readers = @readers.sort_by {|addr| rand }
+		end
+
+		# open remote database
+		@dbmap.keys.each {|addr|
+			@dbmap[addr] = open_db(addr)
+		}
+
+	rescue
+		@dbmap.each_pair {|addr,db|
+			if db
+				close_db(db) rescue nil
+			end
+		}
+		$log.error $!
+		$log.error_backtrace $!.backtrace
+		raise "MDS: invlaid address expression: #{$!}"
+	end
+
+	def write(shard_key, &block)
+		if @writers.size == 1
+			n = 0
+		else
+			n = hash_key(shard_key) % @writers.size
+		end
+		ha_call(@writers, n) {|db|
+			block.call(db)
+		}
+	end
+
+	def read(shard_key, &block)
+		@readers_rr += 1
+		@readers_rr = 0 if @readers_rr >= @readers.size
+		ha_call(@readers, @readers_rr) {|db|
+			block.call(db)
+		}
+	end
+
+	def close
+		@dbmap.each_pair {|addr,db|
+			close_db(db) rescue nil
+		}
+	end
+
+	protected
+	def parse_addr(addr)
+		host, port = addr.split(':',2)
+		port ||= self.class::DEFAULT_PORT
+		port = port.to_i
+		host.strip!
+		Address.new(host, port)
+	end
+
+	def open_db(addr)
+		raise "LOGIC ERROR: not implemented!"
+	end
+
+	def ensure_db(db, addr)
+		true
+	end
+
+	def error_result?(db, result)
+		nil
+	end
+
+	def close_db(db)
+		db.close
+	end
+
+	def hash_key(key)
+		digest = Digest::MD5.digest(key)
+		digest.unpack('C')[0]
+	end
+
+	def ha_call(array, idx, &block)
+		db = nil
+		failed = []
+		sz = array.size
+		sz.times {
+			addr = array[idx % sz]
+			unless failed.include?(addr)
+				db = @dbmap[addr]
+				if ensure_db(db, addr)
+					@dbmap[addr] = db  # FIXME
+					begin
+						result = block.call(db)
+						if err = error_result?(db, result)
+							failed << [addr, err]
+						else
+							return result
+						end
+					rescue
+						failed << [addr, $!]
+					end
+				end
+			end
+			idx += 1
+		}
+		raise "MDS error: #{failed.inspect}"  # TODO error message
+	end
+end
+
+
+end