@ohos-ports/lmdb 2.5.3 → 2.8.5-beta.1

package/src/compression.cpp

@@ -16,7 +16,7 @@ Compression::Compression(const CallbackInfo& info) : ObjectWrap<Compression>(inf
 				throwError(info.Env(), "Dictionary must be a buffer");
 				return;
 			}
-			napi_get_typedarray_info(info.Env(), dictionaryOption, nullptr, &dictSize, (void**) &dictionary, nullptr, nullptr);
+			napi_get_buffer_info(info.Env(), dictionaryOption, (void**) &dictionary, &dictSize);
 			dictSize = (dictSize >> 3) << 3; // make sure it is word-aligned
 		}
 		auto thresholdOption = info[0].As<Object>().Get("threshold");
@@ -34,10 +34,13 @@ Compression::Compression(const CallbackInfo& info) : ObjectWrap<Compression>(inf
 }
 
 Napi::Value Compression::setBuffer(const CallbackInfo& info) {
-	napi_get_typedarray_info(info.Env(), info[0], nullptr, nullptr, (void**) &this->decompressTarget, nullptr, nullptr);
-	this->decompressSize = info[1].As<Number>();
-	napi_get_typedarray_info(info.Env(), info[2], nullptr, nullptr, (void**) &this->dictionary, nullptr, nullptr);
-	this->dictionarySize = info[3].As<Number>();
+	size_t length;
+	napi_get_arraybuffer_info(info.Env(), info[0], (void**) &this->decompressTarget, &length);
+	unsigned int offset = info[1].As<Number>();
+	this->decompressTarget += offset;
+	this->decompressSize = info[2].As<Number>();
+	this->dictionarySize = info[4].As<Number>();
+	napi_get_buffer_info(info.Env(), info[3], (void**) &this->dictionary, &length);
 	return info.Env().Undefined();
 }
 void Compression::decompress(MDB_val& data, bool &isValid, bool canAllocate) {
@@ -180,10 +183,10 @@ class CompressionWorker : public AsyncWorker {
 
 NAPI_FUNCTION(EnvWrap::compress) {
 	ARGS(3)
-    GET_INT64_ARG(0);
-    EnvWrap* ew = (EnvWrap*) i64;	
-    napi_get_value_int64(env, args[1], &i64);
-    double* compressionAddress = (double*) i64;
+   GET_INT64_ARG(0);
+   EnvWrap* ew = (EnvWrap*) i64;
+   napi_get_value_int64(env, args[1], &i64);
+   double* compressionAddress = (double*) i64;
 	CompressionWorker* worker = new CompressionWorker(ew, (double*) compressionAddress, Function(env, args[2]));
 	worker->Queue();
 	RETURN_UNDEFINED;

package/src/cursor.cpp

@@ -3,6 +3,15 @@
 
 using namespace Napi;
 
+const int INCLUDE_VALUES = 0x100;
+const int REVERSE = 0x400;
+const int VALUES_FOR_KEY = 0x800;
+const int ONLY_COUNT = 0x1000;
+const int RENEW_CURSOR = 0x2000;
+const int EXACT_MATCH = 0x4000;
+const int INCLUSIVE_END = 0x8000;
+const int EXCLUSIVE_START = 0x10000;
+
 CursorWrap::CursorWrap(const CallbackInfo& info) : Napi::ObjectWrap<CursorWrap>(info) {
 	this->keyType = LmdbKeyType::StringKey;
 	this->freeKey = nullptr;
@@ -14,10 +23,11 @@ CursorWrap::CursorWrap(const CallbackInfo& info) : Napi::ObjectWrap<CursorWrap>(
 
 	DbiWrap *dw;
 	napi_unwrap(info.Env(), info[0], (void**)&dw);
-
+	int64_t tw_address = 0;
+	napi_get_value_int64(info.Env(), info[1], &tw_address);
 	// Open the cursor
 	MDB_cursor *cursor;
-	MDB_txn *txn = dw->ew->getReadTxn();
+	MDB_txn *txn = dw->ew->getReadTxn(tw_address);
 	int rc = mdb_cursor_open(txn, dw->dbi, &cursor);
 	if (rc != 0) {
 		throwLmdbError(info.Env(), rc);
@@ -77,22 +87,30 @@ int CursorWrap::returnEntry(int lastRC, MDB_val &key, MDB_val &data) {
 	}
 	if (endKey.mv_size > 0) {
 		int comparison;
-		if (flags & 0x800)
+		if (flags & VALUES_FOR_KEY)
 			comparison = mdb_dcmp(txn, dw->dbi, &endKey, &data);
 		else
 			comparison = mdb_cmp(txn, dw->dbi, &endKey, &key);
-		if ((flags & 0x400) ? comparison >= 0 : (comparison <= 0)) {
-			return 0;
+		if ((flags & REVERSE) ? comparison >= 0 : (comparison <= 0)) {
+			if (!((flags & INCLUSIVE_END) && comparison == 0))
+				return 0;
 		}
 	}
 	char* keyBuffer = dw->ew->keyBuffer;
-	if (flags & 0x100) {
-		bool result = getVersionAndUncompress(data, dw);
-		if (result)
-			result = valToBinaryFast(data, dw);
-		*((size_t*)keyBuffer) = data.mv_size;
+	if (flags & INCLUDE_VALUES) {
+		int result = getVersionAndUncompress(data, dw);
+		bool fits = true;
+		if (result) {
+			fits = valToBinaryFast(data, dw); // it fit in the global/compression-target buffer
+		}
+		if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// if it was decompressed
+			*((uint32_t*)keyBuffer) = data.mv_size;
+			*((uint32_t*)(keyBuffer + 4)) = 0; // buffer id of 0
+		} else {
+			EnvWrap::toSharedBuffer(dw->ew->env, (uint32_t*) dw->ew->keyBuffer, data);
+		}
 	}
-	if (!(flags & 0x800)) {
+	if (!(flags & VALUES_FOR_KEY)) {
 		memcpy(keyBuffer + 32, key.mv_data, key.mv_size);
 		*(keyBuffer + 32 + key.mv_size) = 0; // make sure it is null terminated for the sake of better ordered-binary performance
 	}
@@ -105,7 +123,10 @@ int32_t CursorWrap::doPosition(uint32_t offset, uint32_t keySize, uint64_t endKe
 	//char* keyBuffer = dw->ew->keyBuffer;
 	MDB_val key, data;
 	int rc;
-	if (flags & 0x2000) { // TODO: check the txn_id to determine if we need to renew
+	if (dw->ew->env == nullptr) {
+		return MDB_BAD_TXN;
+	}
+	if (flags & RENEW_CURSOR) { // TODO: check the txn_id to determine if we need to renew
 		rc = mdb_cursor_renew(txn = dw->ew->getReadTxn(), cursor);
 		if (rc) {
 			if (rc > 0)
@@ -119,25 +140,28 @@ int32_t CursorWrap::doPosition(uint32_t offset, uint32_t keySize, uint64_t endKe
 		endKey.mv_data = (char*)(keyBuffer + 1);
 	} else
 		endKey.mv_size = 0;
-	iteratingOp = (flags & 0x400) ?
-		(flags & 0x100) ?
-			(flags & 0x800) ? MDB_PREV_DUP : MDB_PREV :
+	iteratingOp = (flags & REVERSE) ?
+		(flags & INCLUDE_VALUES) ?
+			(flags & VALUES_FOR_KEY) ? MDB_PREV_DUP : MDB_PREV :
 			MDB_PREV_NODUP :
-		(flags & 0x100) ?
-			(flags & 0x800) ? MDB_NEXT_DUP : MDB_NEXT :
+		(flags & INCLUDE_VALUES) ?
+			(flags & VALUES_FOR_KEY) ? MDB_NEXT_DUP : MDB_NEXT :
 			MDB_NEXT_NODUP;
 	key.mv_size = keySize;
 	key.mv_data = dw->ew->keyBuffer;
 	if (keySize == 0) {
-		rc = mdb_cursor_get(cursor, &key, &data, flags & 0x400 ? MDB_LAST : MDB_FIRST);  
+		rc = mdb_cursor_get(cursor, &key, &data, flags & REVERSE ? MDB_LAST : MDB_FIRST);  
 	} else {
-		if (flags & 0x800) { // only values for this key
+		if (flags & VALUES_FOR_KEY) { // only values for this key
 			// take the next part of the key buffer as a pointer to starting data
 			uint32_t* startValueBuffer = (uint32_t*)(size_t)(*(double*)(dw->ew->keyBuffer + START_ADDRESS_POSITION));
 			data.mv_size = endKeyAddress ? *((uint32_t*)startValueBuffer) : 0;
 			data.mv_data = startValueBuffer + 1;
-			if (flags & 0x400) {// reverse through values
-				MDB_val startValue = data; // save it for comparison
+			MDB_val startValue;
+			if (flags & EXCLUSIVE_START)
+				startValue = data; // save it for comparison
+			if (flags & REVERSE) {// reverse through values
+				startValue = data; // save it for comparison
 				rc = mdb_cursor_get(cursor, &key, &data, data.mv_size ? MDB_GET_BOTH_RANGE : MDB_SET_KEY);
 				if (rc) {
 					if (startValue.mv_size) {
@@ -154,33 +178,54 @@ int32_t CursorWrap::doPosition(uint32_t offset, uint32_t keySize, uint64_t endKe
 				}
 			} else // forward, just do a get by range
 				rc = mdb_cursor_get(cursor, &key, &data, data.mv_size ?
-					(flags & 0x4000) ? MDB_GET_BOTH : MDB_GET_BOTH_RANGE : MDB_SET_KEY);
+					(flags & EXACT_MATCH) ? MDB_GET_BOTH : MDB_GET_BOTH_RANGE : MDB_SET_KEY);
 
 			if (rc == MDB_NOTFOUND)
 				return 0;
-			if (flags & 0x1000 && (!endKeyAddress || (flags & 0x4000))) {
+			if (flags & ONLY_COUNT && (!endKeyAddress || (flags & EXACT_MATCH))) {
 				size_t count;
 				rc = mdb_cursor_count(cursor, &count);
 				if (rc)
 					return rc > 0 ? -rc : rc;
 				return count;
 			}
+			if (flags & EXCLUSIVE_START) {
+				while(!rc) {
+					if (mdb_dcmp(txn, dw->dbi, &startValue, &data))
+						break;
+					rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
+				}
+			}
 		} else {
-			if (flags & 0x400) {// reverse
-				MDB_val firstKey = key; // save it for comparison
+			MDB_val firstKey;
+			if (flags & EXCLUSIVE_START)
+				firstKey = key; // save it for comparison
+			if (flags & REVERSE) {// reverse
+				firstKey = key; // save it for comparison
 				rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_RANGE);
 				if (rc)
 					rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST);
 				else if (mdb_cmp(txn, dw->dbi, &firstKey, &key)) // the range found the next entry *after* the start
 					rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV);
+				else if (dw->flags & MDB_DUPSORT)
+					// we need to go to the last value of this key
+					rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP);
 			} else // forward, just do a get by range
-				rc = mdb_cursor_get(cursor, &key, &data, (flags & 0x4000) ? MDB_SET_KEY : MDB_SET_RANGE);
+				rc = mdb_cursor_get(cursor, &key, &data, (flags & EXACT_MATCH) ? MDB_SET_KEY : MDB_SET_RANGE);
+			if (flags & EXCLUSIVE_START) {
+				while(!rc) {
+					if (mdb_cmp(txn, dw->dbi, &firstKey, &key))
+						break;
+					rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
+				}
+			}
 		}
 	}
+
 	while (offset-- > 0 && !rc) {
 		rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
 	}
-	if (flags & 0x1000) {
+	if (flags & ONLY_COUNT) {
 		uint32_t count = 0;
 		bool useCursorCount = false;
 		// if we are in a dupsort database, and we are iterating over all entries, we can just count all the values for each key
@@ -198,12 +243,13 @@ int32_t CursorWrap::doPosition(uint32_t offset, uint32_t keySize, uint64_t endKe
 		while (!rc) {
 			if (endKey.mv_size > 0) {
 				int comparison;
-				if (flags & 0x800)
+				if (flags & VALUES_FOR_KEY)
 					comparison = mdb_dcmp(txn, dw->dbi, &endKey, &data);
 				else
 					comparison = mdb_cmp(txn, dw->dbi, &endKey, &key);
-				if ((flags & 0x400) ? comparison >= 0 : (comparison <=0)) {
-					return count;
+				if ((flags & REVERSE) ? comparison >= 0 : (comparison <=0)) {
+					if (!((flags & INCLUSIVE_END) && comparison == 0))
+						return count;
 				}
 			}
 			if (useCursorCount) {
@@ -251,7 +297,10 @@ NAPI_FUNCTION(iterate) {
     GET_INT64_ARG(0);
     CursorWrap* cw = (CursorWrap*) i64;
 	MDB_val key, data;
-	int rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
+	int rc;
+	if (cw->dw->ew->env == nullptr) rc = MDB_BAD_TXN;
+	else
+		rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
 	RETURN_INT32(cw->returnEntry(rc, key, data));
 }
 
@@ -260,6 +309,8 @@ int32_t iterateFFI(double cwPointer) {
 	DbiWrap* dw = cw->dw;
 	dw->getFast = true;
 	MDB_val key, data;
+	if (cw->dw->ew->env == nullptr)
+		return MDB_BAD_TXN;
 	int rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
 	return cw->returnEntry(rc, key, data);
 }

package/src/dbi.cpp

@@ -30,7 +30,7 @@ DbiWrap::DbiWrap(const Napi::CallbackInfo& info) : ObjectWrap<DbiWrap>(info) {
 		napi_unwrap(info.Env(), info[4], (void**) &compression);
 	else
 		compression = nullptr;
-	int rc = this->open(flags & ~HAS_VERSIONS, nameBytes, flags & HAS_VERSIONS,
+	int rc = this->open(flags, nameBytes, flags & HAS_VERSIONS,
 		keyType, compression);
 	//if (nameBytes)
 		//delete nameBytes;
@@ -69,8 +69,10 @@ int DbiWrap::open(int flags, char* name, bool hasVersions, LmdbKeyType keyType,
 	this->hasVersions = hasVersions;
 	this->compression = compression;
 	this->keyType = keyType;
+	#ifndef MDB_RPAGE_CACHE
+	flags &= ~0x100; // remove use versions flag for older lmdb
+	#endif
 	this->flags = flags;
-	flags &= ~HAS_VERSIONS;
 	int rc = txn ? mdb_dbi_open(txn, name, flags, &this->dbi) : EINVAL;
 	if (rc)
 		return rc;
@@ -136,46 +138,58 @@ Value DbiWrap::stat(const Napi::CallbackInfo& info) {
 	return stats;
 }
 
-
-int32_t DbiWrap::doGetByBinary(uint32_t keySize) {
+int32_t DbiWrap::doGetByBinary(uint32_t keySize, uint32_t ifNotTxnId, int64_t txnWrapAddress) {
 	char* keyBuffer = ew->keyBuffer;
-	MDB_txn* txn = ew->getReadTxn();
+	MDB_txn* txn = ew->getReadTxn(txnWrapAddress);
 	MDB_val key, data;
 	key.mv_size = keySize;
 	key.mv_data = (void*) keyBuffer;
-
+	uint32_t* currentTxnId = (uint32_t*) (keyBuffer + 32);
+	#ifdef MDB_RPAGE_CACHE
+	int result = mdb_get_with_txn(txn, dbi, &key, &data, (mdb_size_t*) currentTxnId);
+	#else
 	int result = mdb_get(txn, dbi, &key, &data);
+	#endif
 	if (result) {
-        if (result > 0)
-            return -result;
+		if (result > 0)
+			return -result;
 		return result;
 	}
+	#ifdef MDB_RPAGE_CACHE
+	if (ifNotTxnId && ifNotTxnId == *currentTxnId)
+		return -30004;
+	#endif
 	result = getVersionAndUncompress(data, this);
-	if (result)
-		valToBinaryFast(data, this);
-	if (data.mv_size < 0x80000000)
-		return data.mv_size;
-	*((uint32_t*)keyBuffer) = data.mv_size;
-	return -30000;
-	/*
-	alternately, if we want to send over the address, which can be used for direct access to the LMDB shared memory, but all benchmarking shows it is slower
-	*((size_t*) keyBuffer) = data.mv_size;
-	*((uint64_t*) (keyBuffer + 8)) = (uint64_t) data.mv_data;
-	return 0;*/
+	bool fits = true;
+	if (result) {
+		fits = valToBinaryFast(data, this); // it fits in the global/compression-target buffer
+	}
+	if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// result = 2 if it was decompressed
+		if (data.mv_size < 0x80000000)
+			return data.mv_size;
+		*((uint32_t*)keyBuffer) = data.mv_size;
+		return -30000;
+	} else {
+		return EnvWrap::toSharedBuffer(ew->env, (uint32_t*) ew->keyBuffer, data);
+	}
 }
 
 NAPI_FUNCTION(getByBinary) {
-	ARGS(2)
-    GET_INT64_ARG(0);
-    DbiWrap* dw = (DbiWrap*) i64;
+	ARGS(4)
+	GET_INT64_ARG(0);
+	DbiWrap* dw = (DbiWrap*) i64;
 	uint32_t keySize;
 	GET_UINT32_ARG(keySize, 1);
-	RETURN_INT32(dw->doGetByBinary(keySize));
+	uint32_t ifNotTxnId;
+	GET_UINT32_ARG(ifNotTxnId, 2);
+	int64_t txnAddress = 0;
+	napi_status status = napi_get_value_int64(env, args[3], &txnAddress);
+	RETURN_INT32(dw->doGetByBinary(keySize, ifNotTxnId, txnAddress));
 }
 
-uint32_t getByBinaryFFI(double dwPointer, uint32_t keySize) {
-    DbiWrap* dw = (DbiWrap*) (size_t) dwPointer;
-    return dw->doGetByBinary(keySize);
+uint32_t getByBinaryFFI(double dwPointer, uint32_t keySize, uint32_t ifNotTxnId, uint64_t txnAddress) {
+	DbiWrap* dw = (DbiWrap*) (size_t) dwPointer;
+	return dw->doGetByBinary(keySize, ifNotTxnId, txnAddress);
 }
 
 napi_finalize noopDbi = [](napi_env, void *, void *) {
@@ -183,8 +197,8 @@ napi_finalize noopDbi = [](napi_env, void *, void *) {
 };
 NAPI_FUNCTION(getSharedByBinary) {
 	ARGS(2)
-    GET_INT64_ARG(0);
-    DbiWrap* dw = (DbiWrap*) i64;
+	GET_INT64_ARG(0);
+	DbiWrap* dw = (DbiWrap*) i64;
 	uint32_t keySize;
 	GET_UINT32_ARG(keySize, 1);
 	MDB_val key;
@@ -205,16 +219,18 @@ NAPI_FUNCTION(getSharedByBinary) {
 	return returnValue;
 }
 NAPI_FUNCTION(getStringByBinary) {
-	ARGS(2)
-    GET_INT64_ARG(0);
-    DbiWrap* dw = (DbiWrap*) i64;
+	ARGS(3)
+	GET_INT64_ARG(0);
+	DbiWrap* dw = (DbiWrap*) i64;
 	uint32_t keySize;
 	GET_UINT32_ARG(keySize, 1);
+	int64_t txnAddress = 0;
+	napi_status status = napi_get_value_int64(env, args[2], &txnAddress);
 	MDB_val key;
 	MDB_val data;
 	key.mv_size = keySize;
 	key.mv_data = (void*) dw->ew->keyBuffer;
-	MDB_txn* txn = dw->ew->getReadTxn();
+	MDB_txn* txn = dw->ew->getReadTxn(txnAddress);
 	int rc = mdb_get(txn, dw->dbi, &key, &data);
 	if (rc) {
 		if (rc == MDB_NOTFOUND) {
@@ -237,12 +253,14 @@ int DbiWrap::prefetch(uint32_t* keys) {
 	MDB_val data;
 	unsigned int flags;
 	mdb_dbi_flags(txn, dbi, &flags);
-	bool dupSort = flags & MDB_DUPSORT;
+	bool findAllValues = flags & MDB_DUPSORT;
 	int effected = 0;
+	bool findDataValue = false;
 	MDB_cursor *cursor;
 	int rc = mdb_cursor_open(txn, dbi, &cursor);
 	if (rc)
 		return rc;
+
 	while((key.mv_size = *keys++) > 0) {
 		if (key.mv_size == 0xffffffff) {
 			// it is a pointer to a new buffer
@@ -251,9 +269,20 @@ int DbiWrap::prefetch(uint32_t* keys) {
 			if (key.mv_size == 0)
 				break;
 		}
-		key.mv_data = (void*) keys;
+		if (key.mv_size & 0x80000000) {
+			// indicator of using a data value combination (with dupSort), to be followed by the corresponding key
+			data.mv_size = key.mv_size & 0x7fffffff;
+			data.mv_data = (void *) keys;
+			keys += (data.mv_size + 12) >> 2;
+			findDataValue = true;
+			findAllValues = false;
+			continue;
+		}
+		// else standard key
+		key.mv_data = (void *) keys;
 		keys += (key.mv_size + 12) >> 2;
-		int rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
+		int rc = mdb_cursor_get(cursor, &key, &data, findDataValue ? MDB_GET_BOTH : MDB_SET_KEY);
+		findDataValue = false;
 		while (!rc) {
 			// access one byte from each of the pages to ensure they are in the OS cache,
 			// potentially triggering the hard page fault in this thread
@@ -262,7 +291,7 @@ int DbiWrap::prefetch(uint32_t* keys) {
 			for (int i = 0; i < pages; i++) {
 				effected += *(((uint8_t*)data.mv_data) + (i << 12));
 			}
-			if (dupSort) // in dupsort databases, access the rest of the values
+			if (findAllValues) // in dupsort databases, access the rest of the values
 				rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
 			else
 				rc = 1; // done
@@ -283,7 +312,13 @@ class PrefetchWorker : public AsyncWorker {
 	}
 
 	void OnOK() {
-		Callback().Call({Env().Null()});
+		napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
+		napi_call_function(Env(), Env().Undefined(), Callback().Value(), 0, {}, &result);
+	}
+	void OnError(const Error& e) {
+		napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
+		napi_value arg = e.Value();
+		napi_call_function(Env(), Env().Undefined(), Callback().Value(), 1, &arg, &result);
 	}
 
   private:
@@ -293,9 +328,9 @@ class PrefetchWorker : public AsyncWorker {
 
 NAPI_FUNCTION(prefetchNapi) {
 	ARGS(3)
-    GET_INT64_ARG(0);
-    DbiWrap* dw = (DbiWrap*) i64;
-    napi_get_value_int64(env, args[1], &i64);
+	GET_INT64_ARG(0);
+	DbiWrap* dw = (DbiWrap*) i64;
+	napi_get_value_int64(env, args[1], &i64);
 	uint32_t* keys = (uint32_t*) i64;
 	PrefetchWorker* worker = new PrefetchWorker(dw, keys, Function(env, args[2]));
 	worker->Queue();
@@ -319,6 +354,9 @@ void DbiWrap::setupExports(Napi::Env env, Object exports) {
 	// TODO: wrap mdb_stat too
 }
 
+
+
+
 // This file contains code from the node-lmdb project
 // Copyright (c) 2013-2017 Timur Kristóf
 // Copyright (c) 2021 Kristopher Tate