@rekal/mem 0.0.0

package/dist/store-DE7S35SS.mjs

@@ -0,0 +1,137 @@
+import { t as Progress } from "./progress-B1JdNapX.mjs";
+import { t as Doc } from "./doc-DnYN4jAU.mjs";
+import { performance } from "node:perf_hooks";
+//#region src/store.ts
+var Store = class Store {
+	constructor(db, ctx) {
+		this.db = db;
+		this.ctx = ctx;
+	}
+	static async load(ctx) {
+		return new Store(await ctx.db(), ctx);
+	}
+	add(doc) {
+		const row = this.db.getDoc(doc.path);
+		if (row?.hash === doc.hash) {
+			this.db.touchDoc(row.id);
+			return row.id;
+		}
+		if (row) this.db.deleteDoc(row.id, { vec: true });
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		return this.db.addDoc({
+			body: doc.body,
+			description: doc.$description ?? "",
+			entities: doc.entities.join(","),
+			hash: doc.hash,
+			path: doc.path,
+			synced_at: now,
+			tags: doc.tags.join(","),
+			title: doc.title,
+			updated_at: now
+		});
+	}
+	async chunk(id, doc) {
+		const title = doc.title.trim();
+		const description = (doc.description ?? "").trim();
+		const body = doc.body.trim();
+		const chunks = [];
+		const embedder = await this.ctx.embedder();
+		if (description.length > 0) chunks.push(...await embedder.chunk({
+			text: description,
+			title
+		}));
+		if (body.length > 0) {
+			const offset = Math.max(1, chunks.length);
+			const bodyChunks = await embedder.chunk({
+				text: body,
+				title
+			});
+			chunks.push(...bodyChunks.map((c) => Object.assign(c, { seq: c.seq + offset })));
+		}
+		return chunks.map((chunk) => Object.assign(chunk, {
+			doc,
+			doc_id: id
+		}));
+	}
+	async index() {
+		const docs = [];
+		const nodes = /* @__PURE__ */ new Map();
+		const vfs = await this.ctx.vfs();
+		for await (const entry of vfs.find()) docs.push(Doc.load(entry));
+		const loaded = await Promise.all(docs);
+		this.db.transaction(() => {
+			for (const doc of loaded) if (doc) nodes.set(this.add(doc), doc);
+		})();
+		this.ctx.success(`Indexed ${nodes.size} docs from disk`);
+		return nodes;
+	}
+	async embed(docs) {
+		const todo = this.db.getUnembeddedDocs();
+		if (todo.length === 0) {
+			this.ctx.success("All docs are already embedded");
+			return;
+		}
+		this.ctx.info(`Sync found ${todo.length} unembedded docs`);
+		let doneBytes = 0;
+		let doneDocs = 0;
+		const queue = [];
+		const embedder = await this.ctx.embedder();
+		await embedder.backend();
+		const start = performance.now();
+		const progress = new Progress("Embedding", { max: todo.length });
+		this.ctx.events.emit("progress", progress);
+		const updateProgress = () => {
+			const secs = (performance.now() - start) / 1e3;
+			const kbPerSec = (doneBytes / secs / 1024).toFixed(0);
+			progress.set({
+				status: `${progress.max - todo.length}/${progress.max} docs embedded ${kbPerSec}kb/s...`,
+				value: doneDocs
+			});
+		};
+		const markEmbedded = (id) => {
+			doneDocs++;
+			const hash = docs.get(id)?.hash;
+			if (hash) this.db.markEmbedded(id, hash);
+		};
+		const embed = async (flush) => {
+			while (queue.length >= (flush ? 1 : embedder.opts.batchSize)) {
+				const batch = queue.splice(0, embedder.opts.batchSize);
+				const embeddings = await embedder.embed(batch);
+				doneBytes += batch.reduce((sum, c) => sum + c.prompt.length, 0);
+				batch.forEach((chunk, i) => chunk.embedding = embeddings[i]);
+				this.db.insertEmbeddings(batch);
+				updateProgress();
+				const completed = new Set(batch.map((c) => c.doc_id));
+				for (const c of queue) completed.delete(c.doc_id);
+				completed.forEach((id) => markEmbedded(id));
+			}
+		};
+		while (todo.length > 0) {
+			const { id } = todo.pop();
+			const doc = docs.get(id);
+			if (!doc) continue;
+			this.db.deleteEmbeddings(id);
+			const chunks = await this.chunk(id, doc);
+			queue.push(...chunks);
+			if (chunks.length === 0) markEmbedded(id);
+			await embed();
+		}
+		await embed(true);
+		progress.stop();
+		this.ctx.success("Sync complete");
+	}
+	async sync(opts) {
+		const syncStart = (/* @__PURE__ */ new Date()).toISOString();
+		const docs = await this.index();
+		await this.prune(syncStart);
+		if (opts?.embed) await this.embed(docs);
+	}
+	async prune(syncStart) {
+		let dels = 0;
+		const vfs = await this.ctx.vfs();
+		for (const { path } of vfs.folders) dels += this.db.deleteStaleDocs(syncStart, path);
+		if (dels > 0) this.ctx.warn(`Removed ${dels} stale docs`);
+	}
+};
+//#endregion
+export { Store };

package/dist/transformers-CJ3QA2PK.mjs

@@ -0,0 +1,55 @@
+import { n as parseModelUri } from "./models-DFQSgBNr.mjs";
+//#region src/embed/transformers.ts
+var TransformersBackend = class TransformersBackend {
+	device = "cpu";
+	maxTokens;
+	dims;
+	#pipeline;
+	#ctx;
+	normalize;
+	constructor(pipeline, ctx) {
+		this.#pipeline = pipeline;
+		this.#ctx = ctx;
+		this.maxTokens = pipeline.model.config.max_position_embeddings;
+		this.dims = pipeline.model.config.hidden_size ?? 0;
+		this.device = pipeline.model.sessions.model?.config?.device ?? "cpu";
+	}
+	static async load(ctx) {
+		const parsed = parseModelUri(ctx.opts.model.uri);
+		const { pipeline, layer_norm } = await import("@huggingface/transformers");
+		const backend = new TransformersBackend(await pipeline("feature-extraction", parsed.model, {
+			dtype: parsed.variant ?? "auto",
+			progress_callback: (event) => TransformersBackend.onProgress(ctx, event),
+			session_options: { intraOpNumThreads: ctx.opts.threads }
+		}), ctx);
+		const dims = ctx.opts.maxDims;
+		if (dims < backend.dims) backend.normalize = (output) => layer_norm(output, [output.dims[1] ?? 0]).slice(null, [0, dims]).normalize(2, -1);
+		return backend;
+	}
+	async embed(texts) {
+		const output = await this.#pipeline(texts, {
+			normalize: !this.normalize,
+			pooling: this.#ctx.opts.model.pooling
+		});
+		return (this.normalize?.(output) ?? output).tolist();
+	}
+	toks(input) {
+		return this.#pipeline.tokenizer.tokenize(input).length;
+	}
+	static onProgress(ctx, event) {
+		if (event.status === "initiate") ctx.status.child(event.name).child(event.file).status = event.status;
+		else if (event.status === "download") ctx.status.child(event.name).child(event.file).status = event.status;
+		else if (event.status === "progress") ctx.status.child(event.name).child(event.file).set({
+			max: event.total,
+			status: event.status,
+			value: event.loaded
+		});
+		else if (event.status === "done") ctx.status.child(event.name).child(event.file).set({ status: event.status }).stop();
+		else if (event.status === "ready") {
+			ctx.status.name = `model \`${ctx.opts.model.uri}\` loaded`;
+			ctx.status.child(event.task).set({ status: event.status }).stop();
+		}
+	}
+};
+//#endregion
+export { TransformersBackend };

package/dist/uri-CehXVDGB.mjs

@@ -0,0 +1,28 @@
+//#region src/uri.ts
+const URI_PREFIX = "rekal://";
+function assertUri(uri) {
+	if (!uri.startsWith("rekal://")) throw new Error(`URI must start with ${URI_PREFIX}, got: ${uri}`);
+}
+function normUri(uri, dir) {
+	if (uri === void 0) return URI_PREFIX;
+	if (typeof uri !== "string") throw new Error(`URI must be a string, got: ${JSON.stringify(uri)}`);
+	uri = uri.trim();
+	uri = uri.replace(/^rekall?:/, "");
+	uri = uri.replace(/[\\/]+/g, "/");
+	uri = uri.replace(/^\/+/, "");
+	if (uri === "") return URI_PREFIX;
+	uri = URI_PREFIX + uri;
+	if (uri.endsWith("/index.md")) return uri.replace(/\/index\.md$/, "/");
+	uri = dir ? uri.replace(/\/?$/, "/") : uri;
+	uri = dir === false ? uri.replace(/\/?$/, "") : uri;
+	return uri;
+}
+function parentUri(uri) {
+	uri = normUri(uri);
+	if (uri === "rekal://") return;
+	uri = uri.replace(/\/?$/, "");
+	uri = uri.replace(/\/[^/]+$/, "");
+	return uri === "rekal://" ? URI_PREFIX : `${uri}/`;
+}
+//#endregion
+export { parentUri as i, assertUri as n, normUri as r, URI_PREFIX as t };

package/dist/util-DNyrmcA3.mjs

@@ -0,0 +1,11 @@
+import "./runtime.node-DlQPaGrV.mjs";
+import { createHash } from "node:crypto";
+//#region src/util.ts
+function hash(content) {
+	return createHash("sha256").update(content).digest("hex");
+}
+function toError(err) {
+	return err instanceof Error ? err : new Error(String(err));
+}
+//#endregion
+export { toError as n, hash as t };

package/dist/vfs-CNQbkhsf.mjs

@@ -0,0 +1,222 @@
+import { a as sstat, i as normPath } from "./fs-DMp26Byo.mjs";
+import { r as normUri, t as URI_PREFIX } from "./uri-CehXVDGB.mjs";
+import { basename, join, relative } from "pathe";
+//#region src/vfs.ts
+const DEFAULT_EXCLUDE = [".git", "node_modules/"];
+var Vfs = class {
+	#folders = /* @__PURE__ */ new Map();
+	#root = {
+		children: /* @__PURE__ */ new Map(),
+		depth: 0,
+		name: "#root",
+		paths: [],
+		uri: URI_PREFIX
+	};
+	constructor(ctx) {
+		this.ctx = ctx;
+		for (const folder of ctx.opts.folders ?? []) this.addFolder(folder);
+	}
+	get folders() {
+		return [...this.#folders.values()].flat();
+	}
+	isFolder(path) {
+		path = normPath(path).replace(/\/?$/, "/");
+		return this.#folders.has(path);
+	}
+	getScope(uri, opts) {
+		uri = normUri(uri, true);
+		const view = this.resolve(uri, opts);
+		return {
+			...view,
+			map: (path) => {
+				path = normPath(path);
+				let best;
+				for (const p of view.paths) {
+					const rel = relative(p.path, path);
+					if (rel.startsWith("..")) continue;
+					const candidate = p.node.uri + rel;
+					if (!best || candidate.length < best.length) best = candidate;
+				}
+				return best;
+			}
+		};
+	}
+	getNode(uri, create = false) {
+		uri = normUri(uri);
+		let node = this.#root;
+		const parts = uri.slice(URI_PREFIX.length).split("/").filter(Boolean);
+		for (const part of parts) {
+			let child = node.children.get(part);
+			if (!child) {
+				child = {
+					children: /* @__PURE__ */ new Map(),
+					depth: node.depth + 1,
+					name: part,
+					parent: node,
+					paths: [],
+					uri: `${node.uri}${part}/`
+				};
+				if (create) node.children.set(part, child);
+			}
+			node = child;
+		}
+		return node;
+	}
+	resolve(node, opts) {
+		node = typeof node === "string" ? this.getNode(node) : node;
+		const nodes = [node];
+		let { parent } = node;
+		while (parent) {
+			nodes.unshift(parent);
+			parent = parent.parent;
+		}
+		const paths = [];
+		const folders = [];
+		for (const n of nodes) {
+			for (const p of paths) p.path = join(p.path, n.name);
+			for (const path of n.paths) {
+				paths.push({
+					node,
+					path
+				});
+				if (this.isFolder(path)) folders.push({
+					node: n,
+					path
+				});
+			}
+		}
+		const stack = opts?.children === false ? [] : [...node.children.values()];
+		while (stack.length > 0) {
+			const n = stack.pop();
+			for (const path of n.paths) {
+				paths.push({
+					node: n,
+					path
+				});
+				if (this.isFolder(path)) folders.push({
+					node: n,
+					path
+				});
+			}
+			stack.push(...n.children.values());
+		}
+		paths.sort((a, b) => a.node.uri.localeCompare(b.node.uri));
+		return {
+			node,
+			paths,
+			uri: node.uri
+		};
+	}
+	addFolder(folder) {
+		folder.uri = normUri(folder.uri, true);
+		folder.path = normPath(folder.path).replace(/\/?$/, "/");
+		const folders = this.#folders.get(folder.path) ?? [];
+		this.#folders.set(folder.path, [...folders, folder]);
+		this.getNode(folder.uri, true).paths.push(folder.path);
+	}
+	matcher(opts) {
+		const pattern = opts?.pattern ?? "";
+		if (!pattern.length) return () => true;
+		const ignoreCase = opts?.ignoreCase ?? !/[A-Z]/.test(pattern);
+		const re = new RegExp(pattern, ignoreCase ? "i" : "");
+		return (uri) => re.test(uri);
+	}
+	async *find(opts = {}) {
+		const { glob } = await import("./glob.mjs");
+		const uri = normUri(opts.uri ?? "rekal://", true);
+		const root = this.resolve(uri);
+		const maxDepth = root.node.depth + (opts.depth ?? Infinity);
+		const visited = /* @__PURE__ */ new Set();
+		const filter = this.matcher(opts);
+		const stop = () => opts.limit !== void 0 && visited.size >= opts.limit;
+		const use = (p) => {
+			if (p.uri.endsWith("/") && opts.type === "file") return false;
+			if (!filter(p.uri)) return false;
+			if (stop()) return false;
+			const key = `${p.uri}:${p.path ?? ""}`;
+			if (visited.has(key)) return false;
+			visited.add(key);
+			return true;
+		};
+		function* yieldVirtual(p) {
+			if (p.node === root.node || opts.type === "file") return;
+			let parent = p.node.parent;
+			while (parent && parent !== root.node && !stop()) {
+				if (parent.depth <= maxDepth && !root.paths.some((rp) => parent?.uri.startsWith(rp.node.uri)) && use({ uri: parent.uri })) yield { uri: parent.uri };
+				parent = parent.parent;
+			}
+		}
+		scan: for (const p of root.paths) {
+			yield* yieldVirtual(p);
+			if (p.node.depth > maxDepth) continue;
+			const e = {
+				path: p.path,
+				uri: p.node.uri
+			};
+			if (p.node !== root.node && use(e)) yield e;
+			const cwd = p.path;
+			const todo = glob({
+				cwd,
+				depth: maxDepth - p.node.depth,
+				empty: false,
+				exclude: DEFAULT_EXCLUDE,
+				glob: ["**/*.md"],
+				type: opts.type
+			});
+			for await (const childPath of todo) {
+				if (basename(childPath) === "index.md") continue;
+				const path = {
+					path: join(cwd, childPath),
+					uri: p.node.uri + childPath
+				};
+				if (use(path)) yield path;
+				if (stop()) break scan;
+			}
+		}
+	}
+	async *ls(opts) {
+		yield* this.find({
+			...opts,
+			depth: 1
+		});
+	}
+	/** Normalizes the URI and path to a real path if it exists **/
+	normPath(p) {
+		const transforms = [
+			{
+				from: /\/index\.md$/,
+				to: "/"
+			},
+			{
+				from: /\/index$/,
+				to: "/"
+			},
+			{
+				from: /\.md$/,
+				to: ""
+			},
+			{
+				from: /(?!\.md)$/,
+				to: ".md"
+			}
+		];
+		if (!sstat(p.path) || basename(p.path) === "index.md") {
+			const root = this.getScope();
+			for (const t of transforms) {
+				const path = p.path.replace(t.from, t.to);
+				const uri = root.map(path);
+				if (sstat(path) && uri) return {
+					path,
+					uri
+				};
+			}
+			return { uri: p.node.uri };
+		}
+		return {
+			path: p.path,
+			uri: p.node.uri
+		};
+	}
+};
+//#endregion
+export { Vfs };

package/foo.ts

@@ -0,0 +1,3 @@
+import { stem } from "./src/stem.ts"
+
+console.log(stem("running"))

package/foo2.ts

@@ -0,0 +1,20 @@
+import { getLlama, resolveModelFile } from "node-llama-cpp"
+import path from "path"
+import { fileURLToPath } from "url"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+const m = "hf:mradermacher/snowflake-arctic-embed-s-GGUF:Q4_K_M"
+// resolve a model from Hugging Face to the models directory
+const modelPath = await resolveModelFile(m, path.join(__dirname, "models"))
+
+const llama = await getLlama()
+const model = await llama.loadModel({ modelPath })
+
+const context = await model.createEmbeddingContext()
+
+const text = "Hello world"
+console.log("Text:", text)
+
+const embedding = await context.getEmbeddingFor(text)
+console.log("Embedding vector:", embedding.vector)

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@rekal/mem",
+  "version": "0.0.0",
+  "type": "module",
+  "imports": {
+    "#runtime": {
+      "bun": "./src/runtime.bun.ts",
+      "default": "./src/runtime.node.ts"
+    }
+  },
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "default": "./dist/index.mjs"
+    },
+    "./glob": {
+      "bun": "./src/glob.ts",
+      "default": "./dist/glob.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "exports": {
+      ".": "./dist/index.mjs",
+      "./glob": "./dist/glob.mjs",
+      "./package.json": "./package.json"
+    }
+  },
+  "scripts": {
+    "build": "tsdown --cwd ../../ --filter @rekal/mem",
+    "test": "bun test:node && bun test:bun",
+    "test:node": "cd ../../; vitest --project @rekal/mem run",
+    "test:bun": "bun test --only-failures"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.0.0",
+    "better-sqlite3": "^12.8.0",
+    "defu": "^6.1.6",
+    "gpt-tokenizer": "^3.4.0",
+    "ignore": "^7.0.5",
+    "js-yaml": "^4.1.1",
+    "node-llama-cpp": "^3.18.1",
+    "pathe": "^2.0.3",
+    "sqlite-vec": "^0.1.9"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.18.2",
+    "@types/better-sqlite3": "^7.6.13",
+    "publint": "^0.3.18",
+    "tsdown": "^0.21.7",
+    "type-fest": "^5.5.0",
+    "typescript": "^6.0.2"
+  },
+  "inlinedDependencies": {
+    "type-fest": "5.5.0"
+  },
+  "trustedDependencies": [
+    "sqlite-vec",
+    "better-sqlite3"
+  ]
+}

package/src/context.ts

@@ -0,0 +1,77 @@
+import type { Db } from "./db.ts"
+import type { Embedder, EmbedderOptions } from "./embed/index.ts"
+import type { LogLevel } from "./log.ts"
+import type { Progress } from "./progress.ts"
+import type { Search } from "./search.ts"
+import type { Store } from "./store.ts"
+import type { TypedEmitter } from "./util.ts"
+import type { Vfs, VfsFolder } from "./vfs.ts"
+
+import { EventEmitter } from "node:events"
+import { mkdirSync } from "node:fs"
+import { join } from "pathe"
+import { normPath } from "./fs.ts"
+import { LoggerBase } from "./log.ts"
+
+type ContextEvents = {
+  log: [level: LogLevel, ...msg: unknown[]]
+  progress: [progress: Progress]
+}
+
+export type ContextOptions = {
+  embedder?: EmbedderOptions
+  folders?: VfsFolder[]
+  root?: string // path to data folder, defaults to `~/.rekal-data/`
+}
+
+export class Context extends LoggerBase {
+  #embedder?: Embedder
+  #root: string
+  #db?: Db
+  #search?: Search
+  #store?: Store
+  #vfs?: Vfs
+  events = new EventEmitter() as TypedEmitter<ContextEvents> & EventEmitter
+
+  constructor(public opts: ContextOptions = {}) {
+    super()
+    this.opts.embedder ??= {}
+    this.opts.embedder.onProgress ??= (progress) => this.events.emit("progress", progress)
+    this.#root = normPath(this.opts.root ?? "~/.rekal-data")
+    mkdirSync(this.#root, { recursive: true })
+  }
+
+  // Emit log events instead of logging directly
+  protected _log(level: LogLevel, ...msg: unknown[]) {
+    this.events.emit("log", level, ...msg)
+  }
+
+  get root() {
+    return this.#root
+  }
+
+  async db() {
+    const { Db } = await import("./db.ts")
+    return (this.#db ??= await Db.load(join(this.root, "index.sqlite3")))
+  }
+
+  async search() {
+    const { Search } = await import("./search.ts")
+    return (this.#search ??= await Search.load(this))
+  }
+
+  async store() {
+    const { Store } = await import("./store.ts")
+    return (this.#store ??= await Store.load(this))
+  }
+
+  async embedder() {
+    const { Embedder } = await import("./embed/index.ts")
+    return (this.#embedder ??= new Embedder(this))
+  }
+
+  async vfs() {
+    const { Vfs } = await import("./vfs.ts")
+    return (this.#vfs ??= new Vfs(this))
+  }
+}