@daneren2005/shared-memory-objects 0.0.0 → 0.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@daneren2005/shared-memory-objects",
-  "version": "0.0.0",
+  "version": "0.0.1",
   "author": "daneren2005@gmail.com",
   "description": "Creating objects with a SharedArrayBuffer",
   "homepage": "https://github.com/daneren2005/shared-memory-objects#readme",
@@ -8,7 +8,6 @@
     "sharedarraybuffer"
   ],
   "license": "MIT",
-  
   "repository": {
     "type": "git",
     "url": "git+ssh://git@github.com/daneren2005/shared-memory-objects.git"
@@ -16,10 +15,9 @@
   "bugs": {
     "url": "https://github.com/daneren2005/shared-memory-objects/issues"
   },
-
   "type": "module",
   "files": [
-    "dist"
+    "src/"
   ],
   "main": "./src/main.ts",
   "scripts": {

package/src/__tests__/memory-heap.spec.ts

@@ -0,0 +1,92 @@
+import AllocatedMemory from '../allocated-memory';
+import MemoryHeap from '../memory-heap';
+
+describe('MemoryHeap', () => {
+	it('auto grows memory as needed', () => {
+		let memory = new MemoryHeap({ bufferSize: 200 });
+		const startedUsed = memory.currentUsed;
+
+		let allocated = memory.allocUI32(10);
+		expect(allocated.data.length).toEqual(10);
+		memory.allocUI32(10);
+		expect(memory.buffers.length).toEqual(1);
+		// Don't try to get exact size since actual used memory depends on how much memory malloc is using for internal representation
+		expect(memory.currentUsed).toBeGreaterThan(startedUsed + 20 * 4);
+		expect(memory.totalAllocated).toEqual(200);
+		const midUsed = memory.currentUsed;
+
+		memory.allocUI32(20);
+		expect(memory.buffers.length).toEqual(2);
+		expect(memory.currentUsed).toBeGreaterThan(midUsed + 20 * 4);
+		expect(memory.totalAllocated).toEqual(400);
+	});
+	it('can re-create from raw memory and continue working', () => {
+		let mainMemory = new MemoryHeap({ bufferSize: 200 });
+		let copyMemory = new MemoryHeap({
+			buffers: mainMemory.buffers.map(buffer => buffer.buf as SharedArrayBuffer)
+		});
+
+		let mainBlock = mainMemory.allocUI32(10);
+		let copyBlock = new AllocatedMemory(copyMemory, mainBlock.getSharedMemory());
+		expect(copyBlock.data.length).toEqual(12);
+
+		mainBlock.data[1] = 10;
+		expect(copyBlock.data[1]).toEqual(10);
+	});
+
+	it('grabs free memory from any used buffer', () => {
+		let memory = new MemoryHeap({ bufferSize: 200 });
+
+		let block1 = memory.allocUI32(20);
+		let block2 = memory.allocUI32(20);
+		let block3 = memory.allocUI32(20);
+		expect(memory.buffers.length).toEqual(3);
+		let maxMemory = memory.currentUsed;
+
+		block1.free();
+		expect(memory.currentUsed).toBeLessThan(maxMemory);
+		memory.allocUI32(20);
+		expect(memory.buffers.length).toEqual(3);
+		expect(memory.currentUsed).toEqual(maxMemory);
+		
+		block2.free();
+		expect(memory.currentUsed).toBeLessThan(maxMemory);
+		memory.allocUI32(20);
+		expect(memory.buffers.length).toEqual(3);
+		expect(memory.currentUsed).toEqual(maxMemory);
+		
+		block3.free();
+		expect(memory.currentUsed).toBeLessThan(maxMemory);
+		memory.allocUI32(20);
+		expect(memory.buffers.length).toEqual(3);
+		expect(memory.currentUsed).toEqual(maxMemory);
+	});
+
+	it('growing memory by doubles while freeing previous', () => {
+		let memory = new MemoryHeap({
+			bufferSize: 1_024 * 16
+		});
+		let allocSize = 40;
+
+		let oldMemory = memory.allocUI32(allocSize);
+		for(let i = 0; i < 5; i++) {
+			allocSize *= 2;
+			let newMemory = memory.allocUI32(allocSize);
+			expect(newMemory.bufferByteOffset).toBeGreaterThan(oldMemory.bufferByteOffset);
+
+			oldMemory.free();
+			oldMemory = newMemory;
+		}
+	});
+
+	it('Block creating with bufferSize greater than 1MB', () => {
+		let error: Error | null = null;
+		try {
+			new MemoryHeap({ bufferSize: Math.pow(2, 30) });
+		} catch(e) {
+			error = e as Error;
+		}
+
+		expect(error).not.toBeNull();
+	});
+});

package/src/__tests__/serialize-object.spec.ts

@@ -0,0 +1,23 @@
+import MemoryHeap from '../memory-heap';
+import { serializeObjectToMemory, createObjectFromMemory } from '../serialize-object';
+
+describe('SerializeObject', () => {
+	let heap: MemoryHeap;
+	beforeEach(() => {
+		heap = new MemoryHeap();
+	});
+
+	it('simple', () => {
+		const object = {
+			test: 12.5,
+			test2: 'string',
+			fa: 'blue'
+		};
+		let memory = serializeObjectToMemory(heap, object);
+
+		let clone = createObjectFromMemory<typeof object>(memory);
+		expect(clone.test).toEqual(12.5);
+		expect(clone.test2).toEqual('string');
+		expect(clone.fa).toEqual('blue');
+	});
+});

package/src/__tests__/shared-data.bench.ts

@@ -0,0 +1,166 @@
+import { bench } from 'vitest';
+import SharedList from '../shared-list';
+import MemoryHeap from '../memory-heap';
+import SharedMap from '../shared-map';
+import SharedVector from '../shared-vector';
+
+const INSERT_COUNT = 1_000;
+describe(`Shared Data Structures: ${INSERT_COUNT} inserts`, () => {
+	bench('shared list', () => {
+		let memory = new MemoryHeap();
+		let list = new SharedList(memory);
+
+		for(let i = 0; i < INSERT_COUNT; i++) {
+			list.insert(Math.random() * 1_000_000);
+		}
+	});
+	bench('shared map', () => {
+		let memory = new MemoryHeap();
+		let list = new SharedMap<number>(memory);
+
+		for(let i = 0; i < INSERT_COUNT; i++) {
+			list.set(Math.random() * 1_000_000, Math.random() * 1_000_000);
+		}
+	});
+	bench('shared vector', () => {
+		let memory = new MemoryHeap();
+		let list = new SharedVector(memory);
+
+		for(let i = 0; i < INSERT_COUNT; i++) {
+			list.push(Math.random() * 1_000_000);
+		}
+	});
+	bench('shared vector with correct amount initialized', () => {
+		let memory = new MemoryHeap();
+		let list = new SharedVector(memory, {
+			bufferLength: INSERT_COUNT + 2
+		});
+
+		for(let i = 0; i < INSERT_COUNT; i++) {
+			list.push(Math.random() * 1_000_000);
+		}
+	});
+	bench('native array', () => {
+		let list: Array<number> = [];
+		for(let i = 0; i < INSERT_COUNT; i++) {
+			list.push(Math.random() * 1_000_000);
+		}
+	});
+});
+
+const DELETE_COUNT = 1_000;
+describe(`Shared Data Structures: ${DELETE_COUNT} deletes end element`, () => {
+	const INSERT_COUNT = 2_000;
+
+	let sharedList: SharedList;
+	bench('shared list', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			sharedList.deleteIndex(0);
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				let memory = new MemoryHeap();
+				sharedList = new SharedList(memory);
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					sharedList.insert(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+
+	let sharedVector: SharedVector;
+	bench('shared vector', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			sharedVector.pop();
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				let memory = new MemoryHeap({
+					bufferSize: 1024 * 16
+				});
+				sharedVector = new SharedVector(memory);
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					sharedVector.push(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+
+	let nativeList: Array<number> = [];
+	bench('native array', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			nativeList.pop();
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				nativeList = [];
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					nativeList.push(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+});
+
+describe(`Shared Data Structures: ${DELETE_COUNT} deletes random element`, () => {
+	const INSERT_COUNT = 2_000;
+
+	let sharedList: SharedList;
+	bench('shared list', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			sharedList.deleteIndex(randomIndex(sharedList));
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				let memory = new MemoryHeap();
+				sharedList = new SharedList(memory);
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					sharedList.insert(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+
+	let sharedVector: SharedVector;
+	bench('shared vector', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			sharedVector.deleteIndex(randomIndex(sharedList));
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				let memory = new MemoryHeap({
+					bufferSize: 1024 * 16
+				});
+				sharedVector = new SharedVector(memory);
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					sharedVector.push(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+
+	let nativeList: Array<number> = [];
+	bench('native array', () => {
+		for(let i = 0; i < DELETE_COUNT; i++) {
+			nativeList.splice(randomIndex(nativeList), 1);
+		}
+	}, {
+		setup: (task) => {
+			task.opts.beforeEach = () => {
+				nativeList = [];
+				for(let i = 0; i < INSERT_COUNT; i++) {
+					nativeList.push(Math.random() * 1_000_000);
+				}
+			};
+		}
+	});
+});
+
+function randomIndex(array: { length: number }) {
+	return Math.floor(Math.random() * array.length);
+}

package/src/__tests__/shared-list.spec.ts

@@ -0,0 +1,247 @@
+import SharedList from '../shared-list';
+import MemoryHeap from '../memory-heap';
+
+describe('SharedList', () => {
+	let memory: MemoryHeap;
+	beforeEach(() => {
+		memory = new MemoryHeap();
+	});
+
+	it('can insert items', () => {
+		let list = new SharedList(memory);
+
+		list.insert(5);
+		expect(list.length).toEqual(1);
+
+		list.insert(10);
+		list.insert(4);
+		expect(list.length).toEqual(3);
+
+		expect(flat(list)).toEqual([5, 10, 4]);
+	});
+	it('can delete items by index', () => {
+		let list = new SharedList(memory);
+		let startMemory = memory.currentUsed;
+		list.insert(5);
+		list.insert(10);
+		list.insert(4);
+		list.insert(8);
+		list.insert(20);
+		let fullMemory = memory.currentUsed;
+
+		list.deleteIndex(2);
+		expect(flat(list)).toEqual([5, 10, 8, 20]);
+		expect(memory.currentUsed).toBeLessThan(fullMemory);
+		expect(list.length).toEqual(4);
+
+		list.deleteIndex(0);
+		expect(flat(list)).toEqual([10, 8, 20]);
+		list.deleteIndex(2);
+		expect(flat(list)).toEqual([10, 8]);
+
+		// Do nothing for bug deletes
+		list.deleteIndex(2);
+		expect(flat(list)).toEqual([10, 8]);
+		
+		// Delete everything and should still work
+		list.deleteIndex(0);
+		list.deleteIndex(0);
+		expect(flat(list)).toEqual([]);
+		expect(list.length).toEqual(0);
+		expect(memory.currentUsed).toEqual(startMemory);
+
+		list.insert(80);
+		list.insert(52);
+		expect(flat(list)).toEqual([80, 52]);
+	});
+	it('can delete items by value', () => {
+		let list = new SharedList(memory);
+		let startMemory = memory.currentUsed;
+		list.insert(5);
+		list.insert(10);
+		list.insert(4);
+		list.insert(8);
+		list.insert(20);
+		let fullMemory = memory.currentUsed;
+
+		list.deleteValue(4);
+		expect(flat(list)).toEqual([5, 10, 8, 20]);
+		expect(memory.currentUsed).toBeLessThan(fullMemory);
+		expect(list.length).toEqual(4);
+
+		list.deleteValue(5);
+		expect(flat(list)).toEqual([10, 8, 20]);
+		list.deleteValue(20);
+		expect(flat(list)).toEqual([10, 8]);
+
+		// Do nothing for bug deletes
+		list.deleteValue(15);
+		expect(flat(list)).toEqual([10, 8]);
+		
+		// Delete everything and should still work
+		list.deleteValue(10);
+		list.deleteValue(8);
+		expect(flat(list)).toEqual([]);
+		expect(list.length).toEqual(0);
+		expect(memory.currentUsed).toEqual(startMemory);
+
+		list.insert(80);
+		list.insert(52);
+		expect(flat(list)).toEqual([80, 52]);
+	});
+
+	it('can delete during iteration', () => {
+		let list = new SharedList(memory);
+		list.insert(5);
+		list.insert(10);
+		list.insert(4);
+		list.insert(8);
+		list.insert(20);
+		let fullMemory = memory.currentUsed;
+
+		for(let { data, deleteCurrent } of list) {
+			if(data[0] === 10 || data[0] === 20) {
+				deleteCurrent();
+			}
+		}
+		
+		expect(flat(list)).toEqual([5, 4, 8]);
+		expect(list.length).toEqual(3);
+		expect(memory.currentUsed).toBeLessThan(fullMemory);
+
+		// Delete concurrent indexes
+		for(let { data, deleteCurrent } of list) {
+			if(data[0] === 5 || data[0] === 4) {
+				deleteCurrent();
+			}
+		}
+		expect(flat(list)).toEqual([8]);
+		expect(list.length).toEqual(1);
+	});
+
+	it('can insert and delete over and over again without leaking memory', () => {
+		let list = new SharedList(memory);
+		list.insert(5);
+		list.insert(10);
+		let startMemory = memory.currentUsed;
+
+		for(let i = 0; i < 10; i++) {
+			list.insert(70);
+			list.deleteIndex(2);
+		}
+		for(let i = 0; i < 10; i++) {
+			list.insert(70);
+			list.deleteValue(70);
+		}
+
+		expect(memory.currentUsed).toEqual(startMemory);
+	});
+
+	it('can share memory and insert/delete items from either instance', () => {
+		let mainList = new SharedList(memory);
+		let secondList = new SharedList(memory, mainList.getSharedMemory());
+
+		mainList.insert(5);
+		mainList.insert(60);
+		secondList.insert(14);
+		mainList.insert(8);
+		secondList.deleteIndex(1);
+
+		expect(flat(mainList)).toEqual([5, 14, 8]);
+		expect(flat(secondList)).toEqual([5, 14, 8]);
+	});
+	it.skip('can delete item mid-iteration');
+
+	it('free', () => {
+		let startMemory = memory.currentUsed;
+		let list = new SharedList(memory);
+		list.insert(5);
+		list.insert(10);
+		list.insert(4);
+
+		list.free();
+		expect(memory.currentUsed).toEqual(startMemory);
+	});
+
+	it('with int32', () => {
+		let list = new SharedList(memory, {
+			type: Int32Array
+		});
+
+		list.insert(5);
+		expect(list.length).toEqual(1);
+
+		list.insert(-10);
+		list.insert(4);
+		expect(list.length).toEqual(3);
+
+		expect(flat(list)).toEqual([5, -10, 4]);
+	});
+	it('with float32', () => {
+		let list = new SharedList(memory, {
+			type: Float32Array
+		});
+
+		list.insert(5.5);
+		expect(list.length).toEqual(1);
+
+		list.insert(-10);
+		list.insert(4);
+		expect(list.length).toEqual(3);
+
+		expect(flat(list)).toEqual([5.5, -10, 4]);
+	});
+
+	it('with dataLength = 3', () => {
+		let list = new SharedList(memory, {
+			type: Int32Array,
+			dataLength: 3
+		});
+
+		list.insert(5);
+		expect(list.length).toEqual(1);
+
+		list.insert([-10, 20, 1]);
+		list.insert([4, -40]);
+		expect(list.length).toEqual(3);
+
+		expect(flat(list)).toEqual([
+			5, 0, 0,
+			-10, 20, 1,
+			4, -40, 0
+		]);
+
+		// Don't delete middle values
+		list.deleteValue(20);
+		expect(flat(list)).toEqual([
+			5, 0, 0,
+			-10, 20, 1,
+			4, -40, 0
+		]);
+
+		// Allow deleting first value only
+		list.deleteValue(-10);
+		expect(flat(list)).toEqual([
+			5, 0, 0,
+			4, -40, 0
+		]);
+		expect(list.length).toEqual(2);
+
+		// Allow deleting entire set of values
+		list.deleteValue([4, 10, 0]);
+		list.deleteValue([5, 0, 0]);
+		expect(flat(list)).toEqual([
+			4, -40, 0
+		]);
+		expect(list.length).toEqual(1);
+	});
+});
+
+function flat(list: SharedList<any>) {
+	return [...list].reduce((array, value) => {
+		// @ts-expect-error
+		array.push(...value.data);
+
+		return array;
+	}, []);
+}

package/src/__tests__/shared-map.spec.ts

@@ -0,0 +1,98 @@
+import MemoryHeap from '../memory-heap';
+import SharedMap from '../shared-map';
+
+describe('SharedMap', () => {
+	let memory: MemoryHeap;
+	beforeEach(() => {
+		memory = new MemoryHeap();
+	});
+
+	describe('SharedMap<string>', () => {
+		it('insert and remove items', () => {
+			let map = new SharedMap<string>(memory);
+
+			// NOTE: If values are changed make sure we have at least one hash collision
+			map.set('ds', 3);
+			map.set('asd', 72);
+			map.set('gfredf', 8);
+			map.set('z', 65);
+			// Map should only have one instance of each key - replace
+			map.set('ds', 1);
+			map.set('red', 10);
+			expect(map.length).toEqual(5);
+
+			expect(map.get('gfredf')).toEqual(8);
+			expect(map.get('ds')).toEqual(1);
+			expect(map.get('z')).toEqual(65);
+			expect(map.get('blue')).toBeUndefined();
+			expect(map.get('red')).toEqual(10);
+
+			map.delete('ds');
+			expect(map.length).toEqual(4);
+			expect(map.get('ds')).toEqual(undefined);
+			map.delete('ds');
+			expect(map.length).toEqual(4);
+
+			map.delete('dfsdfsd');
+			expect(map.length).toEqual(4);
+			map.delete('gfredf');
+			expect(map.length).toEqual(3);
+		});
+
+		it('free', () => {
+			let startMemory = memory.currentUsed;
+			let map = new SharedMap<string>(memory);
+			map.set('ds', 3);
+			map.set('asd', 72);
+			map.set('gfredf', 8);
+			map.set('z', 65);
+
+			map.free();
+			expect(memory.currentUsed).toEqual(startMemory);
+		});
+
+		it('can be created from shared memory', () => {
+			let startMemory = memory.currentUsed;
+			let mainMap = new SharedMap<string>(memory);
+			let cloneMap = new SharedMap<string>(memory, mainMap.getSharedMemory());
+
+			mainMap.set('ds', 3);
+			mainMap.set('asd', 72);
+			cloneMap.set('ds', 1);
+			cloneMap.set('red', 10);
+
+			expect(mainMap.length).toEqual(3);
+			expect(cloneMap.length).toEqual(3);
+			
+			expect(mainMap.get('ds')).toEqual(1);
+			expect(cloneMap.get('ds')).toEqual(1);
+
+			cloneMap.free();
+			expect(memory.currentUsed).toEqual(startMemory);
+		});
+
+		it('inserting many items will grow hash table', () => {
+			let map = new SharedMap<string>(memory);
+			const previousMaxHash = map.maxHash;
+			for(let i = 0; i < 5; i++) {
+				insertRandom(map);
+			}
+			expect(map.length).toEqual(5);
+
+			expect(map.maxHash).toEqual(previousMaxHash);
+			for(let i = 0; i < 20; i++) {
+				insertRandom(map);
+			}
+			expect(map.length).toEqual(25);
+			expect(map.maxHash).toBeGreaterThan(previousMaxHash);
+
+			for(let i = 0; i < map.length; i++) {
+				expect(map.get(`${i + 1}`)).not.toBeUndefined();
+			}
+		});
+
+		function insertRandom(map: SharedMap<string>) {
+			map.set(`${map.length + 1}`, Math.random() * 1_000_000);
+		}
+	});
+});