@ohos-ports/lmdb 2.5.3 → 2.8.5-beta.0

package/src/misc.cpp

@@ -1,7 +1,9 @@
 #include "lmdb-js.h"
+#include <atomic>
 #include <string.h>
 #include <stdio.h>
 #include <node_version.h>
+#include <time.h>
 
 using namespace Napi;
 
@@ -22,12 +24,20 @@ void setupExportMisc(Napi::Env env, Object exports) {
 	#endif
 
 	exports.Set("version", versionObj);
-	exports.Set("setGlobalBuffer", Function::New(env, setGlobalBuffer));
-	exports.Set("lmdbError", Function::New(env, lmdbError));
-	exports.Set("enableDirectV8", Function::New(env, enableDirectV8));
+	EXPORT_NAPI_FUNCTION("setGlobalBuffer", setGlobalBuffer)
+	EXPORT_NAPI_FUNCTION("lmdbError", lmdbError)
+	EXPORT_NAPI_FUNCTION("enableDirectV8", enableDirectV8)
 	EXPORT_NAPI_FUNCTION("createBufferForAddress", createBufferForAddress);
-	EXPORT_NAPI_FUNCTION("getAddress", getViewAddress);
+	EXPORT_NAPI_FUNCTION("getAddress", getAddress);
+	EXPORT_NAPI_FUNCTION("getBufferAddress", getBufferAddress);
 	EXPORT_NAPI_FUNCTION("detachBuffer", detachBuffer);
+	EXPORT_NAPI_FUNCTION("startRead", startRead);
+	EXPORT_NAPI_FUNCTION("setReadCallback", setReadCallback);
+	EXPORT_NAPI_FUNCTION("enableThreadSafeCalls", enableThreadSafeCalls);
+	napi_value globalBuffer;
+	napi_create_buffer(env, SHARED_BUFFER_THRESHOLD, (void**) &globalUnsafePtr, &globalBuffer);
+	globalUnsafeSize = SHARED_BUFFER_THRESHOLD;
+	exports.Set("globalBuffer", Object(env, globalBuffer));
 }
 
 void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options) {
@@ -69,6 +79,7 @@ bool valToBinaryFast(MDB_val &data, DbiWrap* dw) {
 			// already decompressed to the target, nothing more to do
 		} else {
 			if (data.mv_size > compression->decompressSize) {
+				// indicates we could not copy, won't fit
 				return false;
 			}
 			// copy into the buffer target
@@ -76,11 +87,7 @@ bool valToBinaryFast(MDB_val &data, DbiWrap* dw) {
 		}
 	} else {
 		if (data.mv_size > globalUnsafeSize) {
-			// TODO: Provide a direct reference if for really large blocks, but we do that we need to detach that in the next turn
-			/* if(data.mv_size > 64000) {
-				dw->SetUnsafeBuffer(data.mv_data, data.mv_size);
-				return Nan::New<Number>(data.mv_size);
-			}*/
+			// indicates we could not copy, won't fit
 			return false;
 		}
 		memcpy(globalUnsafePtr, data.mv_data, data.mv_size);
@@ -93,7 +100,7 @@ Value valToBinaryUnsafe(MDB_val &data, DbiWrap* dw, Env env) {
 }
 
 
-bool getVersionAndUncompress(MDB_val &data, DbiWrap* dw) {
+int getVersionAndUncompress(MDB_val &data, DbiWrap* dw) {
 	//fprintf(stdout, "uncompressing %u\n", compressionThreshold);
 	unsigned char* charData = (unsigned char*) data.mv_data;
 	if (dw->hasVersions) {
@@ -104,27 +111,29 @@ bool getVersionAndUncompress(MDB_val &data, DbiWrap* dw) {
 		data.mv_size -= 8;
 	}
 	if (data.mv_size == 0) {
-		return true;// successFunc(data);
+		return 1;// successFunc(data);
 	}
 	unsigned char statusByte = dw->compression ? charData[0] : 0;
 		//fprintf(stdout, "uncompressing status %X\n", statusByte);
 	if (statusByte >= 250) {
 		bool isValid;
 		dw->compression->decompress(data, isValid, !dw->getFast);
-		if (!isValid)
-			return false;
-			//return Nan::Null();
+		return isValid ? 2 : 0;
 	}
-	return true;
+	return 1;
 }
 
-Value lmdbError(const CallbackInfo& info) {
-	return throwLmdbError(info.Env(), info[0].As<Number>().Int32Value());
+NAPI_FUNCTION(lmdbError) {
+	ARGS(1)
+	int32_t error_code;
+	GET_INT32_ARG(error_code, 0);
+	return throwLmdbError(env, error_code);
 }
 
-Value setGlobalBuffer(const CallbackInfo& info) {
-	napi_get_typedarray_info(info.Env(), info[0], nullptr, &globalUnsafeSize, (void**) &globalUnsafePtr, nullptr, nullptr);
-	return info.Env().Undefined();
+NAPI_FUNCTION(setGlobalBuffer) {
+	ARGS(1)
+	napi_get_buffer_info(env, args[0], (void**) &globalUnsafePtr, &globalUnsafeSize);
+	RETURN_UNDEFINED
 }
 
 /*Value getBufferForAddress) {
@@ -136,18 +145,28 @@ Value setGlobalBuffer(const CallbackInfo& info) {
 }*/
 NAPI_FUNCTION(createBufferForAddress) {
 	ARGS(2)
-    GET_INT64_ARG(0);
-    void* data = (void*) i64;   
+	GET_INT64_ARG(0)
+	void* data = (void*) i64;   
 	uint32_t length;
 	GET_UINT32_ARG(length, 1);
 	napi_create_external_buffer(env, length, data, nullptr, nullptr, &returnValue);
 	return returnValue;
 }
 
-NAPI_FUNCTION(getViewAddress) {
+NAPI_FUNCTION(getAddress) {
+	ARGS(1)
+	void* data;
+	size_t length;
+	napi_get_arraybuffer_info(env, args[0], &data, &length);
+	napi_create_double(env, (double) (size_t) data, &returnValue);
+	return returnValue;
+}
+
+NAPI_FUNCTION(getBufferAddress) {
 	ARGS(1)
 	void* data;
-	napi_get_typedarray_info(env, args[0], nullptr, nullptr, &data, nullptr, nullptr);
+	size_t length;
+	napi_get_buffer_info(env, args[0], &data, &length);
 	napi_create_double(env, (double) (size_t) data, &returnValue);
 	return returnValue;
 }
@@ -160,6 +179,232 @@ NAPI_FUNCTION(detachBuffer) {
 	RETURN_UNDEFINED;
 }
 
+class ReadWorker : public AsyncWorker {
+  public:
+	ReadWorker(uint32_t* start, const Function& callback)
+	  : AsyncWorker(callback), start(start) {}
+
+	void Execute() {
+		uint32_t instruction;
+		uint32_t* gets = start;
+		while((instruction = std::atomic_exchange((std::atomic<uint32_t>*)(gets + 2), (uint32_t)0xf0000000))) {
+
+			MDB_val key;
+			key.mv_size = *(gets + 3);
+			MDB_dbi dbi = (MDB_dbi) (instruction & 0xffff);
+			MDB_val data;
+			MDB_txn* txn = (MDB_txn*) (size_t) *((double*)gets);
+			
+			unsigned int flags;
+			mdb_dbi_flags(txn, dbi, &flags);
+			bool dupSort = flags & MDB_DUPSORT;
+			int effected = 0;
+			MDB_cursor *cursor;
+			int rc = mdb_cursor_open(txn, dbi, &cursor);
+			if (rc)
+				return SetError(mdb_strerror(rc));
+
+			key.mv_data = (void*) gets;
+			rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
+			MDB_env* env = mdb_txn_env(txn);
+			*(gets + 3) = data.mv_size;
+			*((double*)gets) = (double) (size_t) data.mv_data;
+			gets += (key.mv_size + 28) >> 2;
+			while (!rc) {
+				// access one byte from each of the pages to ensure they are in the OS cache,
+				// potentially triggering the hard page fault in this thread
+				int pages = (data.mv_size + 0xfff) >> 12;
+				// TODO: Adjust this for the page headers, I believe that makes the first page slightly less 4KB.
+				for (int i = 0; i < pages; i++) {
+					effected += *(((uint8_t*)data.mv_data) + (i << 12));
+				}
+				if (dupSort) // in dupsort databases, access the rest of the values
+					rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
+				else
+					rc = 1; // done
+
+			}
+			mdb_cursor_close(cursor);
+		}
+	}
+
+	void OnOK() {
+		// TODO: For each entry, find the shared buffer
+		uint32_t* gets = start;
+		// EnvWrap::toSharedBuffer();
+		Callback().Call({Env().Null()});
+	}
+
+  private:
+	uint32_t* start;
+};
+typedef struct { // this is read results data buffer that is actively being used by a JS thread (read results are written to it)
+	int id;
+	char* data;
+	uint32_t offset;
+	uint32_t size;
+} read_results_buffer_t;
+
+static thread_local std::unordered_map<void*, read_results_buffer_t*>* buffersByWorker;
+
+typedef struct {
+	napi_ref callback;
+	napi_ref resource;
+} read_callback_t;
+static int next_buffer_id = -1;
+typedef struct {
+	uint32_t* instructionAddress;
+	uint32_t callback_id;
+	napi_async_work work;
+	//napi_deferred deferred;
+	js_buffers_t* buffers;
+} read_instruction_t;
+const uint32_t ZERO = 0;
+void do_read(napi_env nenv, void* instruction_pointer) {
+	read_instruction_t* readInstruction = (read_instruction_t*) instruction_pointer;
+	//fprintf(stderr, "lock %p\n", &readInstruction->buffers->modification_lock);
+	uint32_t* instruction = readInstruction->instructionAddress;
+	MDB_val key;
+	key.mv_size = *(instruction + 3);
+	MDB_dbi dbi = (MDB_dbi) (*(instruction + 2) & 0xffff) ;
+	MDB_val data;
+	TxnWrap* tw = (TxnWrap*) (size_t) *((double*)instruction);
+	MDB_txn* txn = tw->txn;
+	mdb_txn_renew(txn);
+	unsigned int flags;
+	mdb_dbi_flags(txn, dbi, &flags);
+	bool dupSort = flags & MDB_DUPSORT;
+	int effected = 0;
+	MDB_cursor *cursor;
+	MDB_env* env = mdb_txn_env(txn);
+	int rc = mdb_cursor_open(txn, dbi, &cursor);
+	if (rc) {
+		*instruction = rc;
+		return;
+	}
+	key.mv_data = (void*) (instruction + 4);
+	rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
+	*(instruction + 3) = data.mv_size;
+
+	//instruction += (key.mv_size + 28) >> 2;
+	while (!rc) {
+		// access one byte from each of the pages to ensure they are in the OS cache,
+		// potentially triggering the hard page fault in this thread
+		int pages = (data.mv_size + 0xfff) >> 12;
+		// TODO: Adjust this for the page headers, I believe that makes the first page slightly less than 4KB.
+		for (int i = 0; i < pages; i++) {
+			effected += *(((uint8_t*)data.mv_data) + (i << 12));
+		}
+		if (dupSort) // in dupsort databases, access the rest of the values
+			rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
+		else
+			rc = 1; // done
+
+	}
+	*instruction = rc;
+	unsigned int env_flags = 0;
+	mdb_env_get_flags(env, &env_flags);
+	if (data.mv_size > 4096 && !(env_flags & MDB_REMAP_CHUNKS)) {
+		EnvWrap::toSharedBuffer(env, instruction, data);
+		*((double*)instruction) = (double) (size_t) data.mv_data;
+	} else {
+		if (!buffersByWorker)
+			buffersByWorker = new std::unordered_map<void*, read_results_buffer_t*>;
+		read_results_buffer_t* read_buffer;
+		auto buffer_search = buffersByWorker->find(readInstruction->buffers);
+		if (buffer_search == buffersByWorker->end()) {
+			// create new one
+			buffersByWorker->emplace(readInstruction->buffers, read_buffer = new read_results_buffer_t);
+			read_buffer->size = 0;
+			read_buffer->offset = 0; // force it re-malloc
+		} else
+			read_buffer = buffer_search->second;
+		if ((int) read_buffer->size - (int) read_buffer->offset - 4 < (int) data.mv_size) {
+			size_t size = 0x40000;// 256KB
+			read_buffer->data = (char*) malloc(size);
+			read_buffer->size = size;
+			read_buffer->offset = 0;
+			buffer_info_t buffer_info;
+			buffer_info.end = read_buffer->data + size;
+			buffer_info.env = nullptr;
+			buffer_info.isSharedMap = false;
+			pthread_mutex_lock(&readInstruction->buffers->modification_lock);
+			buffer_info.id = read_buffer->id = readInstruction->buffers->nextId++;
+			readInstruction->buffers->buffers.emplace(read_buffer->data, buffer_info);
+			pthread_mutex_unlock(&readInstruction->buffers->modification_lock);
+
+		}
+		auto position = (uint32_t*) (read_buffer->data + read_buffer->offset);
+		memcpy(position, data.mv_data, data.mv_size);
+		position += (data.mv_size + 7) >> 2;
+		*(instruction + 1) = read_buffer->id;
+		*(instruction + 2) = read_buffer->offset;
+		read_buffer->offset = (char*)position - read_buffer->data;
+	}
+	mdb_cursor_close(cursor);
+	//fprintf(stderr, "unlock %p\n", &readInstruction->buffers->modification_lock);
+}
+static thread_local napi_ref* read_callback;
+void read_complete(napi_env env, napi_status status, void* data) {
+	read_instruction_t* readInstruction = (read_instruction_t*) data;
+	napi_value callback;
+	napi_get_reference_value(env, *read_callback, &callback);
+	//uint32_t count;
+	napi_value result;
+	napi_value callback_id;
+	napi_create_int32(env, readInstruction->callback_id, &callback_id);
+	status = napi_call_function(env, callback, callback, 1, &callback_id, &result);
+	napi_delete_async_work(env, readInstruction->work);
+	delete readInstruction;
+	//napi_resolve_deferred(env, readInstruction->deferred, resolution);
+}
+NAPI_FUNCTION(enableThreadSafeCalls) {
+	WriteWorker::threadSafeCallsEnabled = true;
+    napi_value returnValue;
+	RETURN_UNDEFINED;
+}
+
+NAPI_FUNCTION(setReadCallback) {
+	ARGS(1)
+	read_callback = new napi_ref;
+	napi_create_reference(env, args[0], 1, read_callback);
+	RETURN_UNDEFINED;
+}
+NAPI_FUNCTION(startRead) {
+	ARGS(4)
+	GET_INT64_ARG(0);
+	uint32_t* instructionAddress = (uint32_t*) i64;
+	read_instruction_t* readInstruction = new read_instruction_t;
+	readInstruction->instructionAddress = instructionAddress;
+	uint32_t callback_id;
+	GET_UINT32_ARG(callback_id, 1);
+	//napi_create_reference(env, args[1], 1, &readInstruction->callback);
+	readInstruction->callback_id = callback_id;
+	readInstruction->buffers = EnvWrap::sharedBuffers;
+	napi_status status;
+	status = napi_create_async_work(env, args[2], args[3], do_read, read_complete, readInstruction, &readInstruction->work);
+	status = napi_queue_async_work(env, readInstruction->work);
+	RETURN_UNDEFINED;
+}/*
+NAPI_FUNCTION(nextRead) {
+	ARGS(1)
+	uint32_t offset;
+	GET_UINT32_ARG(offset, 0);
+	uint32_t* instructionAddress = (uint32_t*) currentReadAddress + offset;
+	read_callback_t* callback = lastCallback;
+	uint32_t count;
+	napi_reference_ref(callback->env, callback->callback, &count);
+	read_instruction_t* readInstruction = new read_instruction_t;
+	readInstruction->instructionAddress = instructionAddress;
+	readInstruction->callback = callback;
+	napi_async_work work;
+	napi_create_async_work(callback->env, nullptr, readInstruction, &work);
+	napi_queue_async_work(env, work);
+	ReadWorker* worker = new ReadWorker(instructionAddress, Function(env, args[1]));
+	worker->Queue();
+	RETURN_UNDEFINED;
+}*/
+
 Value lmdbNativeFunctions(const CallbackInfo& info) {
 	// no-op, just doing this to give a label to the native functions
 	return info.Env().Undefined();
@@ -192,7 +437,8 @@ int putWithVersion(MDB_txn *   txn,
 	if (rc == 0) {
 		// if put is successful, data->mv_data will point into the database where we copy the data to
 		memcpy((char*) data->mv_data + 8, sourceData, size);
-		*((double*) data->mv_data) = version;
+		memcpy(data->mv_data, &version, 8);
+		//*((double*) data->mv_data) = version; // this doesn't work on ARM v7 because it is not (guaranteed) memory-aligned
 	}
 	data->mv_data = sourceData; // restore this so that if it points to data that needs to be freed, it points to the right place
 	return rc;
@@ -236,9 +482,8 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex)
 	return 0;
 }
 
-int pthread_cond_init(pthread_cond_t *cond, pthread_condattr_t *attr)
+int cond_init(pthread_cond_t *cond)
 {
-	(void)attr;
 	if (cond == NULL)
 		return 1;
 	InitializeConditionVariable(cond);
@@ -286,17 +531,41 @@ int pthread_cond_broadcast(pthread_cond_t *cond)
 	return 0;
 }
 
+uint64_t get_time64() {
+    return GetTickCount64();
+}
+
 #else
+int cond_init(pthread_cond_t *cond) {
+    pthread_condattr_t attr;
+    pthread_condattr_init( &attr);
+    #if defined(__linux)
+    // only tested in linux, not available on macos
+    pthread_condattr_setclock( &attr, CLOCK_MONOTONIC);
+    #endif
+    return pthread_cond_init(cond, &attr);
+}
+
 int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, uint64_t cms)
 {
 	struct timespec ts;
+    #if defined(__linux)
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	#else
+	// without being able to set the clock for condition/mutexes, need to use default realtime clock on macos
 	clock_gettime(CLOCK_REALTIME, &ts);
+	#endif
 	uint64_t ns = ts.tv_nsec + cms * 10000;
 	ts.tv_sec += ns / 1000000000;
-	ts.tv_nsec += ns % 1000000000;
+	ts.tv_nsec = ns % 1000000000;
 	return pthread_cond_timedwait(cond, mutex, &ts);
 }
 
+uint64_t get_time64() {
+    struct timespec time;
+    clock_gettime(CLOCK_MONOTONIC, &time);
+    return time.tv_sec * 1000000000ll + time.tv_nsec;
+}
 #endif
 
 // This file contains code from the node-lmdb project

package/src/ordered-binary.cpp

@@ -1,46 +1,46 @@
-#include "lmdb-js.h"
-
-#ifdef _WIN32
-#define ntohl _byteswap_ulong
-#define htonl _byteswap_ulong
-#else
-#include <arpa/inet.h>
-#endif
-
-// compare items by 32-bit comparison, a is user provided and assumed to be zero terminated/padded
-// which allows us to do the full 32-bit comparisons safely
-int compareFast(const MDB_val *a, const MDB_val *b) {
-    uint32_t* dataA = (uint32_t*) a->mv_data;
-    uint32_t* dataB = (uint32_t*) b->mv_data;
-    size_t remaining = b->mv_size;
-    uint32_t aVal, bVal;
-    while(remaining >= 4) {
-        aVal = ntohl(*dataA);
-        bVal = ntohl(*dataB);
-        if (aVal > bVal)
-            return 1;
-        if (aVal < bVal)
-            return -1;
-        /*diff = (int64_t) ntohl(*dataA) - (int64_t) ntohl(*dataB);
-        if (diff)
-            return diff;*/
-        dataA++;
-        dataB++;
-        remaining -= 4;
-    }
-    if (remaining) {
-        if (remaining == 1) {
-            aVal = *((uint8_t*) dataA);
-            bVal = *((uint8_t*) dataB);
-        } else {
-            aVal = ntohl(*dataA);
-            bVal = remaining == 2 ? (*((uint8_t*) dataB) << 24) + (*((uint8_t*) dataB + 1) << 16) :
-                ntohl(*dataB & 0x00ffffff);
-        }
-        if (aVal > bVal)
-            return 1;
-        if (aVal < bVal)
-            return -1;
-    }
-    return a->mv_size - b->mv_size;
-}
+#include "lmdb-js.h"
+
+#ifdef _WIN32
+#define ntohl _byteswap_ulong
+#define htonl _byteswap_ulong
+#else
+#include <arpa/inet.h>
+#endif
+
+// compare items by 32-bit comparison, a is user provided and assumed to be zero terminated/padded
+// which allows us to do the full 32-bit comparisons safely
+int compareFast(const MDB_val *a, const MDB_val *b) {
+    uint32_t* dataA = (uint32_t*) a->mv_data;
+    uint32_t* dataB = (uint32_t*) b->mv_data;
+    size_t remaining = b->mv_size;
+    uint32_t aVal, bVal;
+    while(remaining >= 4) {
+        aVal = ntohl(*dataA);
+        bVal = ntohl(*dataB);
+        if (aVal > bVal)
+            return 1;
+        if (aVal < bVal)
+            return -1;
+        /*diff = (int64_t) ntohl(*dataA) - (int64_t) ntohl(*dataB);
+        if (diff)
+            return diff;*/
+        dataA++;
+        dataB++;
+        remaining -= 4;
+    }
+    if (remaining) {
+        if (remaining == 1) {
+            aVal = *((uint8_t*) dataA);
+            bVal = *((uint8_t*) dataB);
+        } else {
+            aVal = ntohl(*dataA);
+            bVal = remaining == 2 ? (*((uint8_t*) dataB) << 24) + (*((uint8_t*) dataB + 1) << 16) :
+                ntohl(*dataB & 0x00ffffff);
+        }
+        if (aVal > bVal)
+            return 1;
+        if (aVal < bVal)
+            return -1;
+    }
+    return a->mv_size - b->mv_size;
+}