ls4 0.9.0

data/lib/ls4/service/mds_memcache.rb

@@ -0,0 +1,209 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class MemcacheMDS < MDS
+	MDSSelector.register(:mc, self)
+
+	class HADB < BasicHADB
+		DEFAULT_PORT = 11211
+
+		def open_db(addr)
+			MemCache.new(addr.to_s, {:urlencode => false, :compression => :false, :multithread => true, :timeout => 5.0})
+		end
+
+		def ensure_db(db, addr)
+			true
+		end
+
+		def error_result?(db, result)
+			nil
+		end
+	end
+
+	def initialize
+		require 'memcache'
+	end
+
+	def open(expr)
+		@hadb = HADB.new(expr)
+	end
+
+	def close
+		@hadb.close
+	end
+
+	def get_okey(key, version=nil, &cb)
+		e = get_impl(key, version)
+		if e
+			cb.call(e.to_okey(key), nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def get_attrs(key, version=nil, &cb)
+		e = get_impl(key, version)
+		if e
+			cb.call(e.attrs, nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def get_okey_attrs(key, version=nil, &cb)
+		e = get_impl(key, version)
+		if e
+			cb.call([e.to_okey(key), e.attrs], nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def add(key, attrs={}, vname=nil, &cb)
+		okey = set_impl(key, attrs, vname)
+		cb.call(okey, nil) rescue nil
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def update_attrs(key, attrs, &cb)
+		okey = update_impl(key) {|old_attrs|
+			attrs
+		}
+		cb.call(okey, nil) rescue nil
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def remove(key, &cb)
+		e = get_impl(key)
+
+		if e
+			result = nil
+			@hadb.write(key) {|mc|
+				result = mc.delete(key)
+			}
+			#if result !~ /DELETED/
+			# TODO
+			#end
+
+			cb.call(e.to_okey(key), nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def delete(key, version=nil, &cb)
+		raise "version is not supported on memcache MDS" if version
+		remove(key, &cb)
+	end
+
+	private
+	class Entry
+		def initialize(rsid=nil, vtime=nil, attrs=nil)
+			@rsid = rsid
+			@vtime = vtime
+			@attrs = attrs
+		end
+
+		attr_accessor :rsid
+		attr_accessor :vtime
+		attr_accessor :attrs
+
+		def to_okey(key)
+			ObjectKey.new(key, @vtime, @rsid)
+		end
+
+		def to_msgpack(out = '')
+			[@rsid, @vtime, @attrs].to_msgpack(out)
+		end
+
+		def from_msgpack(obj)
+			@rsid  = obj[0]
+			@vtime = obj[1]
+			@attrs = obj[2]
+			self
+		end
+	end
+
+	def get_impl(key, version=nil)
+		raise "version is not supported on memcache MDS" if version
+
+		raw = nil
+		@hadb.read(key) {|mc|
+			raw = mc.get(key, true)
+		}
+
+		if raw
+			obj = MessagePack.unpack(raw)
+			e = Entry.new.from_msgpack(obj)
+			return e
+		else
+			return nil
+		end
+	end
+
+	def set_impl(key, attrs, vname=nil)
+		raise "version is not supported on memcache MDS" if vname
+
+		okey = new_okey(key)
+		e = Entry.new(okey.rsid, okey.vtime, attrs)
+		raw = e.to_msgpack
+		result = nil
+		@hadb.write(key) {|mc|
+			# TODO add/cas?
+			result = mc.set(key, raw, 0, true)
+		}
+
+		#if result !~ /STORED/
+		# TODO
+		#end
+
+		return okey
+	end
+
+	def update_impl(key, &modifier)
+		e = get_impl(key)
+
+		if e
+			e.attrs = modifier.call(e.attrs)
+			raw = e.to_msgpack
+			@hadb.write(key) {|mc|
+				# TODO cas?
+				mc.set(key, raw, 0, true)
+			}
+			return e.to_okey(key)
+		else
+			return nil
+		end
+	end
+end
+
+
+end

data/lib/ls4/service/mds_tc.rb

@@ -0,0 +1,508 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+module LS4
+
+
+class LocalTokyoCabinetMDS < MDS
+	MDSSelector.register(:local, self)
+
+	COL_PK      = ''
+	COL_KEY     = '_key'
+	COL_VTIME   = '_time'
+	COL_RSID    = '_rsid'
+	COL_VNAME   = '_vname'
+	COL_REMOVED = '_removed'
+
+	COLS_RESERVED = [COL_PK, COL_KEY, COL_VTIME, COL_RSID, COL_VNAME, COL_REMOVED]
+	COLS_REQUIRED = [COL_PK, COL_KEY, COL_VTIME, COL_RSID]
+
+	def self.define_consts
+		unless const_defined?(:QRY)
+			require 'tokyocabinet'
+			const_set(:QRY, TokyoCabinet::TDBQRY)
+			HADB.const_set(:TDB, TokyoCabinet::TDB)
+			HADB.const_set(:FATAL_ERROR, [
+				TokyoCabinet::TDB::ETHREAD,
+				TokyoCabinet::TDB::EINVALID,
+				TokyoCabinet::TDB::ENOFILE,
+				TokyoCabinet::TDB::ENOPERM,
+				TokyoCabinet::TDB::EMETA,
+				TokyoCabinet::TDB::ERHEAD,
+				TokyoCabinet::TDB::EOPEN,
+				TokyoCabinet::TDB::ECLOSE,
+				TokyoCabinet::TDB::ETRUNC,
+				TokyoCabinet::TDB::ESYNC,
+				TokyoCabinet::TDB::ESTAT,
+				TokyoCabinet::TDB::ESEEK,
+				TokyoCabinet::TDB::EREAD,
+				TokyoCabinet::TDB::EWRITE,
+				TokyoCabinet::TDB::EMMAP,
+				TokyoCabinet::TDB::ELOCK,
+				TokyoCabinet::TDB::EUNLINK,
+				TokyoCabinet::TDB::ERENAME,
+				TokyoCabinet::TDB::EMKDIR,
+				TokyoCabinet::TDB::ERMDIR,
+				#TokyoCabinet::TDB::EKEEP,
+				#TokyoCabinet::TDB::ENOREC,
+				TokyoCabinet::TDB::EMISC
+			])
+		end
+	end
+
+	class HADB < BasicHADB
+		def parse_addr(addr)
+			addr
+		end
+
+		def open_db(addr)
+			db = TDB.new
+			db.instance_eval("@enc = 'ASCII-8BIT'")  # FIXME
+			unless db.open(addr, TDB::OCREAT|TDB::OWRITER)
+				$log.warn "failed to connect local TokyoCabinet MDS: #{addr}"
+			end
+			db.setindex(COL_KEY, TDB::ITLEXICAL)
+			db
+		end
+
+		def ensure_db(db, addr)
+			if FATAL_ERROR.include?(db.ecode)
+				db.close rescue nil
+				db.instance_eval("@ecode = ESUCCESS")  # FIXME
+				db.open(*addr)
+			end
+			return db
+		rescue
+			return nil
+		end
+
+		def error_result?(db, result)
+			if FATAL_ERROR.include?(db.ecode)
+				return db.errmsg(db.ecode)
+			end
+			return nil
+		end
+	end
+
+	if defined?(Random)
+		def initialize
+			self.class.define_consts
+			@random = Random.new
+			@pid = Process.pid
+		end
+	else
+		# Ruby 1.8
+		class PseudoRandom
+			def rand(n)
+				Kernel::rand(n)
+			end
+		end
+		def initialize
+			self.class.define_consts
+			@random = PseudoRandom.new
+			@pid = Process.pid
+		end
+	end
+
+	def open(expr)
+		@hadb = HADB.new(expr)
+	end
+
+	def close
+		@hadb.close
+	end
+
+	def get_okey(key, version=nil, &cb)
+		map = get_impl(key, version, COLS_RESERVED)
+		if map && !is_removed(map)
+			okey = to_okey(map)
+			cb.call(okey, nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def get_attrs(key, version=nil, &cb)
+		map = get_impl(key, version)
+		if map && !is_removed(map)
+			attrs = to_attrs(map)
+			cb.call(attrs, nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def get_okey_attrs(key, version=nil, &cb)
+		map = get_impl(key, version)
+		if map && !is_removed(map)
+			okey = to_okey(map)
+			attrs = to_attrs(map)
+			cb.call([okey, attrs], nil) rescue nil
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def add(key, attrs={}, vname=nil, &cb)
+		okey = add_impl(key, attrs, vname)
+		cb.call(okey, nil) rescue nil
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def update_attrs(key, attrs, &cb)
+		okey = update_impl(key) {|old_attrs|
+			attrs
+		}
+		cb.call(okey, nil) rescue nil
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	#def merge_attrs(key, attrs, &cb)
+	#	okey = update_impl(key) {|old_attrs|
+	#		old_attrs.merge(attrs)
+	#	}
+	#	cb.call(okey, nil) rescue nil
+	#rescue
+	#	cb.call(nil, $!) rescue nil
+	#end
+
+	def remove(key, &cb)
+		map = get_impl_head(key, COLS_REQUIRED)
+		if map && !is_removed(map)
+			okey = to_okey(map)
+
+			# optional: inherit rsid
+			rsid = map[COL_RSID].to_i
+
+			# get current vtime later than old vtime
+			vtime = get_current_vtime(map[COL_VTIME].to_i)
+
+			remove_okey = new_okey(key, vtime, rsid)
+
+			# insert
+			pk = new_pk(map[COL_PK])
+			map = to_map({}, remove_okey, nil, true)
+			@hadb.write(key) {|rdb| rdb.put(pk, map) }
+
+			cb.call(okey, nil) rescue nil
+
+		else
+			cb.call(nil, nil) rescue nil
+		end
+
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def delete(key, version=nil, &cb)
+		map = get_impl(key, version, COLS_RESERVED)
+		if map && !is_removed(map)
+			okey = to_okey(map)
+
+			pk = map[COL_PK]
+			@hadb.write(key) {|rdb| rdb.delete(pk) }
+
+			cb.call(okey, nil) rescue nil
+
+		else
+			cb.call(nil, nil) rescue nil
+		end
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	def util_locate(key, &cb)
+		array = @hadb.read(key) {|rdb|
+			qry = QRY.new(rdb)
+			qry.addcond(COL_KEY, QRY::QCSTREQ, key)
+			qry.search.map {|key|
+				rdb.get(key)
+				map[COL_PK] = key
+				map
+			}
+		}
+
+		array.reject! {|map|
+			!is_valid_map(map) || is_removed(map)
+		}
+
+		array.map! {|map|
+			[to_okey(map), map[COL_VNAME]]
+		}
+
+		cb.call(array, nil) rescue nil
+
+	rescue
+		cb.call(nil, $!) rescue nil
+	end
+
+	private
+	def to_okey(map)
+		key = map[COL_KEY]
+		rsid = map[COL_RSID].to_i
+		vtime = map[COL_VTIME].to_i
+		new_okey(key, vtime, rsid)
+	end
+
+	def to_attrs(map)
+		map.reject {|k,v| COLS_RESERVED.include?(k) }
+	end
+
+	def to_map(attrs, okey, vname=nil, removed=false)
+		map = attrs.dup
+		map.delete(COL_PK)
+		map[COL_KEY] = okey.key
+		map[COL_RSID] = okey.rsid.to_s
+		map[COL_VTIME] = okey.vtime.to_s
+		if vname
+			map[COL_VNAME] = vname.to_s
+		end
+		if removed
+			map[COL_REMOVED] = "1"
+		end
+		map
+	end
+
+	def get_impl(key, version=nil, cols=nil)
+		if version == nil
+			get_impl_head(key, cols)
+		elsif version.is_a?(String)
+			get_impl_vname(key, version, cols)
+		else
+			get_impl_vtime(key, version, cols)
+		end
+	end
+
+	def get_impl_head(key, cols=nil)
+		map = @hadb.read(key) {|rdb|
+			qry = QRY.new(rdb)
+			qry.addcond(COL_KEY, QRY::QCSTREQ, key)
+			qry.setorder(COL_VTIME, QRY::QONUMDESC)
+			qry.setlimit(1)
+			array = qry.search.map {|key|
+				map = rdb.get(key)
+				map[COL_PK] = key
+				map
+			}
+			array[0]
+		}
+
+		if map == nil
+			return nil
+		elsif !is_valid_map(map)
+			return nil
+		else
+			return map
+		end
+	end
+
+	def get_impl_vname(key, vname, cols=nil)
+		map = @hadb.read(key) {|rdb|
+			qry = QRY.new(rdb)
+			qry.addcond(COL_KEY, QRY::QCSTREQ, key)
+			qry.addcond(COL_VNAME, QRY::QCSTREQ, vname)
+			qry.setorder(COL_VTIME, QRY::QONUMDESC)
+			qry.setlimit(1)
+			array = qry.search.map {|key|
+				rdb.get(key)
+				map[COL_PK] = key
+				map
+			}
+			array[0]
+		}
+
+		if map == nil
+			return nil
+		elsif !is_valid_map(map)
+			return nil
+		else
+			return map
+		end
+	end
+
+	def get_impl_vtime(key, vtime, cols)
+		map = @hadb.read(key) {|rdb|
+			qry = QRY.new(rdb)
+			qry.addcond(COL_KEY, QRY::QCSTREQ, key)
+			qry.addcond(COL_VTIME, QRY::QCNUMLE, vtime.to_s)
+			qry.setorder(COL_VTIME, QRY::QONUMDESC)
+			qry.setlimit(1)
+			array = qry.search.map {|key|
+				rdb.get(key)
+				map[COL_PK] = key
+				map
+			}
+			array[0]
+		}
+
+		if map == nil
+			return nil
+		elsif !is_valid_map(map)
+			return nil
+		else
+			return map
+		end
+	end
+
+	#def select_latest_valid(array)
+	#	latest = select_latest(array)
+	#	if is_valid_map(latest)
+	#		return latest
+	#	else
+	#		return nil
+	#	end
+	#end
+
+	#def select_latest(array)
+	#	if array.size == 1
+	#		return array[0]
+	#	end
+	#
+	#	max_vtime = 0
+	#	marray = []
+	#
+	#	array.each {|map|
+	#		vtime = map[COL_VTIME].to_i
+	#		if max_vtime < vtime
+	#			marray.clear
+	#			marray << map
+	#			max_vtime = vtime
+	#		elsif max_vtime == vtime
+	#			marray << map
+	#		end
+	#	}
+	#
+	#	if marray.size == 1
+	#		return marray[0]
+	#	else
+	#		return marray.sort_by {|map| map[COL_PK] }.last
+	#	end
+	#end
+
+	def is_valid_map(map)
+		COLS_REQUIRED.all? {|col| map.has_key?(col) }
+	end
+
+	def is_removed(map)
+		map.has_key?(COL_REMOVED)
+	end
+
+	def add_impl(key, attrs, vname=nil)
+		map = get_impl_head(key, COLS_REQUIRED)
+
+		if map
+			# optional: inherit rsid
+			rsid = map[COL_RSID].to_i
+
+			# get current vtime later than old vtime
+			vtime = get_current_vtime(map[COL_VTIME].to_i)
+
+			okey = new_okey(key, vtime, rsid)
+
+			# insert
+			pk = new_pk(map[COL_PK])
+			map = to_map(attrs, okey, vname)
+			@hadb.write(key) {|rdb| rdb.put(pk, map) }
+
+		else
+			attrs ||= {}
+
+			okey = new_okey(key)
+
+			# insert
+			pk = new_pk()
+			map = to_map(attrs, okey, vname)
+			@hadb.write(key) {|rdb| rdb.put(pk, map) }
+		end
+
+		return okey
+	end
+
+	def update_impl(key, &modifier)
+		map = get_impl_head(key, nil)
+
+		if map && !is_removed(map)
+			okey = to_okey(map)
+
+			# create new attrs
+			attrs = modifier.call( to_attrs(map) )
+
+			# reject old attributes
+			map.reject! {|k,v| !COLS_RESERVED.include?(k) }
+
+			# merge new attributes
+			map = attrs.merge(map)
+
+			# update
+			pk = map.delete(COL_PK)
+			@hadb.write(key) {|rdb| rdb.put(pk, map) }
+
+			return okey
+
+		else
+			return nil
+		end
+	end
+
+	# +----------+----+------+---+
+	# |    30    | 10 |  16  | 8 |
+	# +----------+----+------+---+
+	# UNIX time
+	#            millisec
+	#                 rand
+	#                        generator-id
+	# +--------------------------+
+	#            64 bits
+	#
+	def new_pk(at_least=nil)
+		nowtime = Time.now.utc
+		sec = nowtime.sec
+		msec = nowtime.usec / 1000
+
+		if at_least && at_least.size == 8
+			time_u, time_d = at_least.unpack('NC')
+			asec = time_u>>2
+			amsec = (time_u&0x3)<<2 & time_d
+			if sec < asec || (sec == asec && msec <= amsec)
+				sec = asec + 1
+				msec = amsec
+			end
+		end
+
+		gid = @pid
+
+		r = @random.rand(2**16)
+		raw = [sec<<2|msec>>8, msec&0xff, r, gid].pack('NCnC')
+
+		# FIXME base64
+		raw = [raw].pack('m')
+		raw.gsub!(/[\n\=]+/,'')
+		raw
+	end
+end
+
+
+end
+