embedded-raptor 1.0.0 → 1.1.0

package/README.md

@@ -1,21 +1,21 @@
-# Raptor
+# Embedded Raptor
 
-[![CI](https://img.shields.io/github/actions/workflow/status/artmann/raptor/ci.yml?branch=main&label=CI&logo=github)](https://github.com/artmann/raptor/actions/workflows/ci.yml)
-[![npm version](https://img.shields.io/npm/v/raptor.svg?logo=npm)](https://www.npmjs.com/package/raptor)
-[![npm downloads](https://img.shields.io/npm/dm/raptor.svg)](https://www.npmjs.com/package/raptor)
+[![CI](https://img.shields.io/github/actions/workflow/status/artmann/embedded-raptor/ci.yml?branch=main&label=CI&logo=github)](https://github.com/artmann/embedded-raptor/actions/workflows/ci.yml)
+[![npm version](https://img.shields.io/npm/v/embedded-raptor.svg?logo=npm)](https://www.npmjs.com/package/embedded-raptor)
+[![npm downloads](https://img.shields.io/npm/dm/embedded-raptor.svg)](https://www.npmjs.com/package/embedded-raptor)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![TypeScript](https://img.shields.io/badge/TypeScript-5.0+-blue.svg?logo=typescript)](https://www.typescriptlang.org/)
 
 > A lightweight semantic search database with text embeddings for Node.js and
 > Bun
 
-Raptor lets you build semantic search into your applications with just a few
+Embedded Raptor lets you build semantic search into your applications with just a few
 lines of code. Store text, search by meaning, and find similar content—perfect
 for RAG systems, chatbots, and recommendation engines.
 
-## What is Raptor?
+## What is Embedded Raptor?
 
-Raptor is an embedding database that automatically converts text into vector
+Embedded Raptor is an embedding database that automatically converts text into vector
 embeddings and stores them in an efficient binary format. Instead of searching
 by exact keywords, you can search by semantic similarity—finding documents that
 mean the same thing, even if they use different words.
@@ -23,7 +23,7 @@ mean the same thing, even if they use different words.
 **Example:** Search for "how to reset password" and find results like "forgot my
 login credentials" or "change account password".
 
-## Why Raptor?
+## Why Embedded Raptor?
 
 - **Simple API** - No complex setup, just store and search
 - **Semantic Search** - Find content by meaning, not just keywords
@@ -44,10 +44,10 @@ login credentials" or "change account password".
 
 ```bash
 # Using npm
-npm install raptor
+npm install embedded-raptor
 
 # Using bun
-bun add raptor
+bun add embedded-raptor
 ```
 
 ## Quick Start
@@ -55,7 +55,7 @@ bun add raptor
 ### Programmatic API
 
 ```typescript
-import { EmbeddingEngine } from 'raptor'
+import { EmbeddingEngine } from 'embedded-raptor'
 
 const engine = new EmbeddingEngine({
   storePath: './my-database.raptor'
@@ -203,11 +203,11 @@ raptor store key1 "Some text" --storePath ./data/custom.raptor
 
 ## How It Works
 
-1. **Text → Embeddings**: Raptor uses the BGE-Base-EN model to convert text into
+1. **Text → Embeddings**: Embedded Raptor uses the BGE-Base-EN model to convert text into
    768-dimensional vector embeddings
 2. **Storage**: Embeddings are stored in an efficient binary format (.raptor
    files)
-3. **Search**: When you search, Raptor compares your query embedding against all
+3. **Search**: When you search, Embedded Raptor compares your query embedding against all
    stored embeddings using cosine similarity
 4. **Results**: Returns the most similar results ranked by similarity score
 

package/dist/cli.cjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const require_engine = require('./engine-KhnrAv7v.cjs');
+const require_engine = require('./engine-CTvg_66e.cjs');
 let node_fs = require("node:fs");
 node_fs = require_engine.__toESM(node_fs);
 let node_path = require("node:path");

package/dist/cli.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as EmbeddingEngine } from "./engine-DISO9uFr.mjs";
+import { t as EmbeddingEngine } from "./engine-ez6nFONK.mjs";
 import { readFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { cli, command } from "cleye";

package/dist/engine-CTvg_66e.cjs

@@ -0,0 +1,404 @@
+//#region rolldown:runtime
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+	if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
+		key = keys[i];
+		if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
+			get: ((k) => from[k]).bind(null, key),
+			enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+		});
+	}
+	return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
+	value: mod,
+	enumerable: true
+}) : target, mod));
+
+//#endregion
+let __xenova_transformers = require("@xenova/transformers");
+__xenova_transformers = __toESM(__xenova_transformers);
+let node_fs = require("node:fs");
+node_fs = __toESM(node_fs);
+let node_fs_promises = require("node:fs/promises");
+node_fs_promises = __toESM(node_fs_promises);
+let node_path = require("node:path");
+node_path = __toESM(node_path);
+let tiny_invariant = require("tiny-invariant");
+tiny_invariant = __toESM(tiny_invariant);
+
+//#region src/binary-format.ts
+const magicBytes = "EMBD";
+const currentVersion = 1;
+const headerSize = 16;
+function calculateRecordLength(keyLength, dimension) {
+	return 2 + keyLength + dimension * 4 + 4;
+}
+async function writeHeader(filePath, dimension) {
+	const buffer = new ArrayBuffer(headerSize);
+	const view = new DataView(buffer);
+	for (let i = 0; i < 4; i++) view.setUint8(i, magicBytes.charCodeAt(i));
+	view.setUint16(4, currentVersion, true);
+	view.setUint32(6, dimension, true);
+	await (0, node_fs_promises.writeFile)(filePath, new Uint8Array(buffer));
+}
+async function readHeader(filePath) {
+	const file = await (0, node_fs_promises.open)(filePath, "r");
+	try {
+		const buffer = new ArrayBuffer(headerSize);
+		const uint8View = new Uint8Array(buffer);
+		const { bytesRead } = await file.read(uint8View, 0, headerSize, 0);
+		if (bytesRead < headerSize) throw new Error(`File too small: expected at least ${headerSize} bytes, got ${bytesRead}`);
+		const view = new DataView(buffer);
+		const magic = new Uint8Array(buffer, 0, 4);
+		const magicString = String.fromCharCode(...magic);
+		if (magicString !== magicBytes) throw new Error(`Invalid file format: magic bytes expected "${magicBytes}", got "${magicString}"`);
+		const version = view.getUint16(4, true);
+		if (version !== currentVersion) throw new Error(`Unsupported version: ${version}. Current version is ${currentVersion}`);
+		return {
+			version,
+			dimension: view.getUint32(6, true)
+		};
+	} finally {
+		await file.close();
+	}
+}
+async function writeRecord(filePath, key, embedding) {
+	const keyBytes = new TextEncoder().encode(key);
+	const keyLength = keyBytes.length;
+	const dimension = embedding.length;
+	const recordLength = calculateRecordLength(keyLength, dimension);
+	const buffer = new ArrayBuffer(recordLength);
+	const view = new DataView(buffer);
+	const uint8View = new Uint8Array(buffer);
+	let offset = 0;
+	view.setUint16(offset, keyLength, true);
+	offset += 2;
+	uint8View.set(keyBytes, offset);
+	offset += keyLength;
+	for (let i = 0; i < dimension; i++) {
+		view.setFloat32(offset, embedding[i], true);
+		offset += 4;
+	}
+	view.setUint32(offset, recordLength, true);
+	await (0, node_fs_promises.appendFile)(filePath, uint8View);
+}
+async function writeRecords(filePath, records) {
+	if (records.length === 0) return;
+	let totalSize = 0;
+	const recordBuffers = [];
+	for (const record of records) {
+		const keyBytes = new TextEncoder().encode(record.key);
+		const keyLength = keyBytes.length;
+		const dimension = record.embedding.length;
+		const recordLength = calculateRecordLength(keyLength, dimension);
+		const buffer = new ArrayBuffer(recordLength);
+		const view = new DataView(buffer);
+		const uint8View = new Uint8Array(buffer);
+		let offset = 0;
+		view.setUint16(offset, keyLength, true);
+		offset += 2;
+		uint8View.set(keyBytes, offset);
+		offset += keyLength;
+		for (let i = 0; i < dimension; i++) {
+			view.setFloat32(offset, record.embedding[i], true);
+			offset += 4;
+		}
+		view.setUint32(offset, recordLength, true);
+		recordBuffers.push(uint8View);
+		totalSize += recordLength;
+	}
+	const combinedBuffer = new Uint8Array(totalSize);
+	let position = 0;
+	for (const buffer of recordBuffers) {
+		combinedBuffer.set(buffer, position);
+		position += buffer.length;
+	}
+	await (0, node_fs_promises.appendFile)(filePath, combinedBuffer);
+}
+
+//#endregion
+//#region src/binary-file-reader.ts
+const chunkSize = 64 * 1024;
+var BinaryFileReader = class {
+	constructor(storePath) {
+		this.storePath = storePath;
+	}
+	async *entries() {
+		const dimension = (await readHeader(this.storePath)).dimension;
+		const fileSize = (await (0, node_fs_promises.stat)(this.storePath)).size;
+		if (fileSize <= 16) return;
+		const seenKeys = /* @__PURE__ */ new Set();
+		const file = await (0, node_fs_promises.open)(this.storePath, "r");
+		try {
+			let currentPosition = fileSize;
+			while (currentPosition > 16) {
+				const chunkStart = Math.max(16, currentPosition - chunkSize);
+				const bytesToRead = currentPosition - chunkStart;
+				const buffer = new ArrayBuffer(bytesToRead);
+				const uint8View = new Uint8Array(buffer);
+				await file.read(uint8View, 0, bytesToRead, chunkStart);
+				let chunkPosition = bytesToRead;
+				while (chunkPosition > 0) {
+					if (chunkPosition < 4) break;
+					const recordLength = new DataView(buffer).getUint32(chunkPosition - 4, true);
+					if (recordLength > chunkPosition) {
+						const recordStart = chunkStart + chunkPosition - recordLength;
+						const recordBuffer = new ArrayBuffer(recordLength);
+						const recordUint8View = new Uint8Array(recordBuffer);
+						await file.read(recordUint8View, 0, recordLength, recordStart);
+						const record = this.parseRecord(recordBuffer, dimension);
+						if (record && !seenKeys.has(record.key)) {
+							seenKeys.add(record.key);
+							yield record;
+						}
+						chunkPosition -= recordLength;
+					} else {
+						const recordStart = chunkPosition - recordLength;
+						const recordView = new DataView(buffer, recordStart, recordLength);
+						const record = this.parseRecordFromView(recordView, dimension, recordLength);
+						if (record && !seenKeys.has(record.key)) {
+							seenKeys.add(record.key);
+							yield record;
+						}
+						chunkPosition -= recordLength;
+					}
+				}
+				currentPosition = chunkStart + chunkPosition;
+			}
+		} finally {
+			await file.close();
+		}
+	}
+	parseRecord(buffer, dimension) {
+		const view = new DataView(buffer);
+		return this.parseRecordFromView(view, dimension, buffer.byteLength);
+	}
+	parseRecordFromView(view, dimension, _recordLength) {
+		try {
+			let offset = 0;
+			const keyLength = view.getUint16(offset, true);
+			offset += 2;
+			const keyBytes = new Uint8Array(view.buffer, view.byteOffset + offset, keyLength);
+			const key = new TextDecoder().decode(keyBytes);
+			offset += keyLength;
+			const embedding = new Float32Array(dimension);
+			for (let i = 0; i < dimension; i++) {
+				embedding[i] = view.getFloat32(offset, true);
+				offset += 4;
+			}
+			return {
+				key,
+				text: "",
+				embedding: Array.from(embedding),
+				timestamp: 0
+			};
+		} catch {
+			return null;
+		}
+	}
+};
+
+//#endregion
+//#region src/candidate-set.ts
+var CandidateSet = class {
+	size;
+	entries = [];
+	constructor(size = 5) {
+		(0, tiny_invariant.default)(size > 0, "Size must be a positive integer.");
+		this.size = size;
+	}
+	add(key, value) {
+		(0, tiny_invariant.default)(key, "Key must be provided.");
+		(0, tiny_invariant.default)(value, "Value must be provided.");
+		if (this.entries.length < this.size) {
+			this.entries.push(new CandidateSetEntry(key, value));
+			return;
+		}
+		let minIndex = 0;
+		let minValue = this.entries[0].value;
+		for (let i = 1; i < this.entries.length; i++) if (this.entries[i].value < minValue) {
+			minValue = this.entries[i].value;
+			minIndex = i;
+		}
+		if (value > minValue) this.entries[minIndex] = new CandidateSetEntry(key, value);
+	}
+	count() {
+		return this.entries.length;
+	}
+	getEntries() {
+		return this.entries.slice().sort((a, b) => b.value - a.value);
+	}
+	getKeys() {
+		return this.getEntries().map((entry) => entry.key);
+	}
+};
+var CandidateSetEntry = class {
+	key;
+	value;
+	constructor(key, value) {
+		this.key = key;
+		this.value = value;
+	}
+};
+
+//#endregion
+//#region src/engine.ts
+var EmbeddingEngine = class {
+	fileReader;
+	storePath;
+	extractor;
+	constructor(options) {
+		this.storePath = options.storePath;
+		this.fileReader = new BinaryFileReader(options.storePath);
+		__xenova_transformers.env.cacheDir = "./.cache";
+	}
+	/**
+	* Gets or initializes the embedding model
+	* Caches the model instance to avoid repeated initialization overhead
+	* @returns Initialized feature extraction pipeline
+	*/
+	async getOrInitModel() {
+		this.extractor ??= await (0, __xenova_transformers.pipeline)("feature-extraction", "Xenova/bge-small-en-v1.5");
+		return this.extractor;
+	}
+	/**
+	* Generates embedding from text using Transformers.js bge-small-en-v1.5 model
+	* @param text - Text to embed
+	* @returns 384-dimensional embedding vector (normalized)
+	*/
+	async generateEmbedding(text) {
+		const output = await (await this.getOrInitModel())(text, {
+			pooling: "mean",
+			normalize: true
+		});
+		return Array.from(output.data);
+	}
+	/**
+	* Retrieves an embedding entry by key
+	* Reads the file in reverse order for efficiency (most recent first)
+	* @param key - Unique identifier for the entry
+	* @returns The embedding entry, or null if not found
+	*/
+	async get(key) {
+		(0, tiny_invariant.default)(key, "Key must be provided.");
+		if (!(0, node_fs.existsSync)(this.storePath)) return null;
+		for await (const entry of this.fileReader.entries()) if (entry.key === key) return entry;
+		return null;
+	}
+	/**
+	* Searches for similar embeddings using cosine similarity
+	* @param query - Text query to search for
+	* @param limit - Maximum number of results to return (default: 10)
+	* @param minSimilarity - Minimum similarity threshold (default: 0, range: -1 to 1)
+	* @returns Array of search results sorted by similarity (highest first)
+	*/
+	async search(query, limit = 10, minSimilarity = .5) {
+		(0, tiny_invariant.default)(query, "Query text must be provided.");
+		(0, tiny_invariant.default)(limit > 0, "Limit must be a positive integer.");
+		(0, tiny_invariant.default)(minSimilarity >= 0 && minSimilarity <= 1, "minSimilarity must be between 0 and 1.");
+		if (!(0, node_fs.existsSync)(this.storePath)) return [];
+		const queryEmbedding = await this.generateEmbedding(query);
+		const candidateSet = new CandidateSet(limit);
+		for await (const entry of this.fileReader.entries()) {
+			const similarity = this.cosineSimilarity(queryEmbedding, entry.embedding);
+			if (similarity < minSimilarity) continue;
+			candidateSet.add(entry.key, similarity);
+		}
+		return candidateSet.getEntries().map((entry) => ({
+			key: entry.key,
+			similarity: entry.value
+		}));
+	}
+	/**
+	* Stores a text embedding in the binary append-only file
+	* Creates header on first write
+	* @param key - Unique identifier for this entry
+	* @param text - Text to embed and store
+	*/
+	async store(key, text) {
+		const embedding = await this.generateEmbedding(text);
+		const embeddingFloat32 = new Float32Array(embedding);
+		await (0, node_fs_promises.mkdir)((0, node_path.dirname)(this.storePath), { recursive: true });
+		if (!(0, node_fs.existsSync)(this.storePath)) await writeHeader(this.storePath, embedding.length);
+		await writeRecord(this.storePath, key, embeddingFloat32);
+	}
+	/**
+	* Stores multiple text embeddings in batch
+	* More efficient than calling store() multiple times
+	* Generates embeddings in a single batch and writes all records at once
+	* @param items - Array of {key, text} objects to store
+	*/
+	async storeMany(items) {
+		(0, tiny_invariant.default)(items.length > 0, "Items array must not be empty.");
+		const texts = items.map((item) => item.text);
+		const output = await (await this.getOrInitModel())(texts, {
+			pooling: "mean",
+			normalize: true
+		});
+		const batchSize = output.dims[0];
+		const embeddingDim = output.dims[1];
+		const embeddingsList = [];
+		for (let i = 0; i < batchSize; i++) {
+			const start = i * embeddingDim;
+			const end = start + embeddingDim;
+			const data = Array.from(output.data);
+			embeddingsList.push(data.slice(start, end));
+		}
+		(0, tiny_invariant.default)(embeddingsList.length === items.length, "Number of embeddings must match number of items.");
+		await (0, node_fs_promises.mkdir)((0, node_path.dirname)(this.storePath), { recursive: true });
+		if (!(0, node_fs.existsSync)(this.storePath)) await writeHeader(this.storePath, embeddingsList[0].length);
+		const records = items.map((item, index) => ({
+			key: item.key,
+			embedding: new Float32Array(embeddingsList[index])
+		}));
+		await writeRecords(this.storePath, records);
+	}
+	/**
+	* Calculates cosine similarity between two embeddings
+	* @param a - First embedding vector
+	* @param b - Second embedding vector
+	* @returns Cosine similarity score between -1 and 1 (1 = identical, -1 = opposite)
+	*/
+	cosineSimilarity(a, b) {
+		if (a.length !== b.length) throw new Error("Embeddings must have the same dimensions");
+		let dotProduct = 0;
+		let magnitudeA = 0;
+		let magnitudeB = 0;
+		for (let i = 0; i < a.length; i++) {
+			dotProduct += a[i] * b[i];
+			magnitudeA += a[i] * a[i];
+			magnitudeB += b[i] * b[i];
+		}
+		magnitudeA = Math.sqrt(magnitudeA);
+		magnitudeB = Math.sqrt(magnitudeB);
+		if (magnitudeA === 0 || magnitudeB === 0) return 0;
+		return dotProduct / (magnitudeA * magnitudeB);
+	}
+	/**
+	* Disposes of the cached embedding model and releases resources
+	* Call this when you're done using the engine to free up memory
+	*/
+	dispose() {
+		this.extractor = void 0;
+	}
+};
+
+//#endregion
+Object.defineProperty(exports, 'EmbeddingEngine', {
+  enumerable: true,
+  get: function () {
+    return EmbeddingEngine;
+  }
+});
+Object.defineProperty(exports, '__toESM', {
+  enumerable: true,
+  get: function () {
+    return __toESM;
+  }
+});

package/dist/engine-ez6nFONK.mjs

@@ -0,0 +1,365 @@
+import { env, pipeline } from "@xenova/transformers";
+import { existsSync } from "node:fs";
+import { appendFile, mkdir, open, stat, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import invariant from "tiny-invariant";
+
+//#region src/binary-format.ts
+const magicBytes = "EMBD";
+const currentVersion = 1;
+const headerSize = 16;
+function calculateRecordLength(keyLength, dimension) {
+	return 2 + keyLength + dimension * 4 + 4;
+}
+async function writeHeader(filePath, dimension) {
+	const buffer = new ArrayBuffer(headerSize);
+	const view = new DataView(buffer);
+	for (let i = 0; i < 4; i++) view.setUint8(i, magicBytes.charCodeAt(i));
+	view.setUint16(4, currentVersion, true);
+	view.setUint32(6, dimension, true);
+	await writeFile(filePath, new Uint8Array(buffer));
+}
+async function readHeader(filePath) {
+	const file = await open(filePath, "r");
+	try {
+		const buffer = new ArrayBuffer(headerSize);
+		const uint8View = new Uint8Array(buffer);
+		const { bytesRead } = await file.read(uint8View, 0, headerSize, 0);
+		if (bytesRead < headerSize) throw new Error(`File too small: expected at least ${headerSize} bytes, got ${bytesRead}`);
+		const view = new DataView(buffer);
+		const magic = new Uint8Array(buffer, 0, 4);
+		const magicString = String.fromCharCode(...magic);
+		if (magicString !== magicBytes) throw new Error(`Invalid file format: magic bytes expected "${magicBytes}", got "${magicString}"`);
+		const version = view.getUint16(4, true);
+		if (version !== currentVersion) throw new Error(`Unsupported version: ${version}. Current version is ${currentVersion}`);
+		return {
+			version,
+			dimension: view.getUint32(6, true)
+		};
+	} finally {
+		await file.close();
+	}
+}
+async function writeRecord(filePath, key, embedding) {
+	const keyBytes = new TextEncoder().encode(key);
+	const keyLength = keyBytes.length;
+	const dimension = embedding.length;
+	const recordLength = calculateRecordLength(keyLength, dimension);
+	const buffer = new ArrayBuffer(recordLength);
+	const view = new DataView(buffer);
+	const uint8View = new Uint8Array(buffer);
+	let offset = 0;
+	view.setUint16(offset, keyLength, true);
+	offset += 2;
+	uint8View.set(keyBytes, offset);
+	offset += keyLength;
+	for (let i = 0; i < dimension; i++) {
+		view.setFloat32(offset, embedding[i], true);
+		offset += 4;
+	}
+	view.setUint32(offset, recordLength, true);
+	await appendFile(filePath, uint8View);
+}
+async function writeRecords(filePath, records) {
+	if (records.length === 0) return;
+	let totalSize = 0;
+	const recordBuffers = [];
+	for (const record of records) {
+		const keyBytes = new TextEncoder().encode(record.key);
+		const keyLength = keyBytes.length;
+		const dimension = record.embedding.length;
+		const recordLength = calculateRecordLength(keyLength, dimension);
+		const buffer = new ArrayBuffer(recordLength);
+		const view = new DataView(buffer);
+		const uint8View = new Uint8Array(buffer);
+		let offset = 0;
+		view.setUint16(offset, keyLength, true);
+		offset += 2;
+		uint8View.set(keyBytes, offset);
+		offset += keyLength;
+		for (let i = 0; i < dimension; i++) {
+			view.setFloat32(offset, record.embedding[i], true);
+			offset += 4;
+		}
+		view.setUint32(offset, recordLength, true);
+		recordBuffers.push(uint8View);
+		totalSize += recordLength;
+	}
+	const combinedBuffer = new Uint8Array(totalSize);
+	let position = 0;
+	for (const buffer of recordBuffers) {
+		combinedBuffer.set(buffer, position);
+		position += buffer.length;
+	}
+	await appendFile(filePath, combinedBuffer);
+}
+
+//#endregion
+//#region src/binary-file-reader.ts
+const chunkSize = 64 * 1024;
+var BinaryFileReader = class {
+	constructor(storePath) {
+		this.storePath = storePath;
+	}
+	async *entries() {
+		const dimension = (await readHeader(this.storePath)).dimension;
+		const fileSize = (await stat(this.storePath)).size;
+		if (fileSize <= 16) return;
+		const seenKeys = /* @__PURE__ */ new Set();
+		const file = await open(this.storePath, "r");
+		try {
+			let currentPosition = fileSize;
+			while (currentPosition > 16) {
+				const chunkStart = Math.max(16, currentPosition - chunkSize);
+				const bytesToRead = currentPosition - chunkStart;
+				const buffer = new ArrayBuffer(bytesToRead);
+				const uint8View = new Uint8Array(buffer);
+				await file.read(uint8View, 0, bytesToRead, chunkStart);
+				let chunkPosition = bytesToRead;
+				while (chunkPosition > 0) {
+					if (chunkPosition < 4) break;
+					const recordLength = new DataView(buffer).getUint32(chunkPosition - 4, true);
+					if (recordLength > chunkPosition) {
+						const recordStart = chunkStart + chunkPosition - recordLength;
+						const recordBuffer = new ArrayBuffer(recordLength);
+						const recordUint8View = new Uint8Array(recordBuffer);
+						await file.read(recordUint8View, 0, recordLength, recordStart);
+						const record = this.parseRecord(recordBuffer, dimension);
+						if (record && !seenKeys.has(record.key)) {
+							seenKeys.add(record.key);
+							yield record;
+						}
+						chunkPosition -= recordLength;
+					} else {
+						const recordStart = chunkPosition - recordLength;
+						const recordView = new DataView(buffer, recordStart, recordLength);
+						const record = this.parseRecordFromView(recordView, dimension, recordLength);
+						if (record && !seenKeys.has(record.key)) {
+							seenKeys.add(record.key);
+							yield record;
+						}
+						chunkPosition -= recordLength;
+					}
+				}
+				currentPosition = chunkStart + chunkPosition;
+			}
+		} finally {
+			await file.close();
+		}
+	}
+	parseRecord(buffer, dimension) {
+		const view = new DataView(buffer);
+		return this.parseRecordFromView(view, dimension, buffer.byteLength);
+	}
+	parseRecordFromView(view, dimension, _recordLength) {
+		try {
+			let offset = 0;
+			const keyLength = view.getUint16(offset, true);
+			offset += 2;
+			const keyBytes = new Uint8Array(view.buffer, view.byteOffset + offset, keyLength);
+			const key = new TextDecoder().decode(keyBytes);
+			offset += keyLength;
+			const embedding = new Float32Array(dimension);
+			for (let i = 0; i < dimension; i++) {
+				embedding[i] = view.getFloat32(offset, true);
+				offset += 4;
+			}
+			return {
+				key,
+				text: "",
+				embedding: Array.from(embedding),
+				timestamp: 0
+			};
+		} catch {
+			return null;
+		}
+	}
+};
+
+//#endregion
+//#region src/candidate-set.ts
+var CandidateSet = class {
+	size;
+	entries = [];
+	constructor(size = 5) {
+		invariant(size > 0, "Size must be a positive integer.");
+		this.size = size;
+	}
+	add(key, value) {
+		invariant(key, "Key must be provided.");
+		invariant(value, "Value must be provided.");
+		if (this.entries.length < this.size) {
+			this.entries.push(new CandidateSetEntry(key, value));
+			return;
+		}
+		let minIndex = 0;
+		let minValue = this.entries[0].value;
+		for (let i = 1; i < this.entries.length; i++) if (this.entries[i].value < minValue) {
+			minValue = this.entries[i].value;
+			minIndex = i;
+		}
+		if (value > minValue) this.entries[minIndex] = new CandidateSetEntry(key, value);
+	}
+	count() {
+		return this.entries.length;
+	}
+	getEntries() {
+		return this.entries.slice().sort((a, b) => b.value - a.value);
+	}
+	getKeys() {
+		return this.getEntries().map((entry) => entry.key);
+	}
+};
+var CandidateSetEntry = class {
+	key;
+	value;
+	constructor(key, value) {
+		this.key = key;
+		this.value = value;
+	}
+};
+
+//#endregion
+//#region src/engine.ts
+var EmbeddingEngine = class {
+	fileReader;
+	storePath;
+	extractor;
+	constructor(options) {
+		this.storePath = options.storePath;
+		this.fileReader = new BinaryFileReader(options.storePath);
+		env.cacheDir = "./.cache";
+	}
+	/**
+	* Gets or initializes the embedding model
+	* Caches the model instance to avoid repeated initialization overhead
+	* @returns Initialized feature extraction pipeline
+	*/
+	async getOrInitModel() {
+		this.extractor ??= await pipeline("feature-extraction", "Xenova/bge-small-en-v1.5");
+		return this.extractor;
+	}
+	/**
+	* Generates embedding from text using Transformers.js bge-small-en-v1.5 model
+	* @param text - Text to embed
+	* @returns 384-dimensional embedding vector (normalized)
+	*/
+	async generateEmbedding(text) {
+		const output = await (await this.getOrInitModel())(text, {
+			pooling: "mean",
+			normalize: true
+		});
+		return Array.from(output.data);
+	}
+	/**
+	* Retrieves an embedding entry by key
+	* Reads the file in reverse order for efficiency (most recent first)
+	* @param key - Unique identifier for the entry
+	* @returns The embedding entry, or null if not found
+	*/
+	async get(key) {
+		invariant(key, "Key must be provided.");
+		if (!existsSync(this.storePath)) return null;
+		for await (const entry of this.fileReader.entries()) if (entry.key === key) return entry;
+		return null;
+	}
+	/**
+	* Searches for similar embeddings using cosine similarity
+	* @param query - Text query to search for
+	* @param limit - Maximum number of results to return (default: 10)
+	* @param minSimilarity - Minimum similarity threshold (default: 0, range: -1 to 1)
+	* @returns Array of search results sorted by similarity (highest first)
+	*/
+	async search(query, limit = 10, minSimilarity = .5) {
+		invariant(query, "Query text must be provided.");
+		invariant(limit > 0, "Limit must be a positive integer.");
+		invariant(minSimilarity >= 0 && minSimilarity <= 1, "minSimilarity must be between 0 and 1.");
+		if (!existsSync(this.storePath)) return [];
+		const queryEmbedding = await this.generateEmbedding(query);
+		const candidateSet = new CandidateSet(limit);
+		for await (const entry of this.fileReader.entries()) {
+			const similarity = this.cosineSimilarity(queryEmbedding, entry.embedding);
+			if (similarity < minSimilarity) continue;
+			candidateSet.add(entry.key, similarity);
+		}
+		return candidateSet.getEntries().map((entry) => ({
+			key: entry.key,
+			similarity: entry.value
+		}));
+	}
+	/**
+	* Stores a text embedding in the binary append-only file
+	* Creates header on first write
+	* @param key - Unique identifier for this entry
+	* @param text - Text to embed and store
+	*/
+	async store(key, text) {
+		const embedding = await this.generateEmbedding(text);
+		const embeddingFloat32 = new Float32Array(embedding);
+		await mkdir(dirname(this.storePath), { recursive: true });
+		if (!existsSync(this.storePath)) await writeHeader(this.storePath, embedding.length);
+		await writeRecord(this.storePath, key, embeddingFloat32);
+	}
+	/**
+	* Stores multiple text embeddings in batch
+	* More efficient than calling store() multiple times
+	* Generates embeddings in a single batch and writes all records at once
+	* @param items - Array of {key, text} objects to store
+	*/
+	async storeMany(items) {
+		invariant(items.length > 0, "Items array must not be empty.");
+		const texts = items.map((item) => item.text);
+		const output = await (await this.getOrInitModel())(texts, {
+			pooling: "mean",
+			normalize: true
+		});
+		const batchSize = output.dims[0];
+		const embeddingDim = output.dims[1];
+		const embeddingsList = [];
+		for (let i = 0; i < batchSize; i++) {
+			const start = i * embeddingDim;
+			const end = start + embeddingDim;
+			const data = Array.from(output.data);
+			embeddingsList.push(data.slice(start, end));
+		}
+		invariant(embeddingsList.length === items.length, "Number of embeddings must match number of items.");
+		await mkdir(dirname(this.storePath), { recursive: true });
+		if (!existsSync(this.storePath)) await writeHeader(this.storePath, embeddingsList[0].length);
+		const records = items.map((item, index) => ({
+			key: item.key,
+			embedding: new Float32Array(embeddingsList[index])
+		}));
+		await writeRecords(this.storePath, records);
+	}
+	/**
+	* Calculates cosine similarity between two embeddings
+	* @param a - First embedding vector
+	* @param b - Second embedding vector
+	* @returns Cosine similarity score between -1 and 1 (1 = identical, -1 = opposite)
+	*/
+	cosineSimilarity(a, b) {
+		if (a.length !== b.length) throw new Error("Embeddings must have the same dimensions");
+		let dotProduct = 0;
+		let magnitudeA = 0;
+		let magnitudeB = 0;
+		for (let i = 0; i < a.length; i++) {
+			dotProduct += a[i] * b[i];
+			magnitudeA += a[i] * a[i];
+			magnitudeB += b[i] * b[i];
+		}
+		magnitudeA = Math.sqrt(magnitudeA);
+		magnitudeB = Math.sqrt(magnitudeB);
+		if (magnitudeA === 0 || magnitudeB === 0) return 0;
+		return dotProduct / (magnitudeA * magnitudeB);
+	}
+	/**
+	* Disposes of the cached embedding model and releases resources
+	* Call this when you're done using the engine to free up memory
+	*/
+	dispose() {
+		this.extractor = void 0;
+	}
+};
+
+//#endregion
+export { EmbeddingEngine as t };

package/dist/index.cjs

@@ -1,3 +1,3 @@
-const require_engine = require('./engine-KhnrAv7v.cjs');
+const require_engine = require('./engine-CTvg_66e.cjs');
 
 exports.EmbeddingEngine = require_engine.EmbeddingEngine;

package/dist/index.mjs

@@ -1,3 +1,3 @@
-import { t as EmbeddingEngine } from "./engine-DISO9uFr.mjs";
+import { t as EmbeddingEngine } from "./engine-ez6nFONK.mjs";
 
 export { EmbeddingEngine };

package/package.json

@@ -1,16 +1,16 @@
 {
   "name": "embedded-raptor",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Lightweight semantic search database with text embeddings for Node.js and Bun",
   "author": "Christoffer Artmann <artgaard@gmail.com>",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/artmann/raptor.git"
+    "url": "https://github.com/artmann/embedded-raptor.git"
   },
-  "homepage": "https://github.com/artmann/raptor#readme",
+  "homepage": "https://github.com/artmann/embedded-raptor#readme",
   "bugs": {
-    "url": "https://github.com/artmann/raptor/issues"
+    "url": "https://github.com/artmann/embedded-raptor/issues"
   },
   "keywords": [
     "embeddings",