@rekal/mem 0.0.0

package/dist/index.mjs

@@ -0,0 +1,351 @@
+import { a as sstat, i as normPath, n as findUp, r as gitRoot, t as astat } from "./fs-DMp26Byo.mjs";
+import { n as parseYaml } from "./runtime.node-DlQPaGrV.mjs";
+import { n as toError, t as hash } from "./util-DNyrmcA3.mjs";
+import { a as parseMarkdown, i as parseFrontmatter, n as chunkMarkdown, o as parseSections, r as chunkText, t as Progress } from "./progress-B1JdNapX.mjs";
+import { i as parentUri, n as assertUri, r as normUri, t as URI_PREFIX } from "./uri-CehXVDGB.mjs";
+import { t as Doc } from "./doc-DnYN4jAU.mjs";
+import { n as tokenize, t as toFts } from "./query-VFSpErTB.mjs";
+import { mkdirSync } from "node:fs";
+import { join } from "pathe";
+import { EventEmitter } from "node:events";
+//#region src/log.ts
+const LOG_LEVELS = [
+	"cancel",
+	"info",
+	"success",
+	"warn",
+	"error",
+	"debug",
+	"fatal",
+	"prompt",
+	"log",
+	"trace"
+];
+const LOG_PRIORITY = {
+	trace: 0,
+	debug: 1,
+	log: 2,
+	info: 2,
+	prompt: 2,
+	success: 2,
+	cancel: 2,
+	warn: 3,
+	error: 4,
+	fatal: 5
+};
+function isLogLevel(level) {
+	return LOG_LEVELS.includes(level);
+}
+function shouldLog(level, minLevel) {
+	if (!isLogLevel(level)) return true;
+	return LOG_PRIORITY[level] >= LOG_PRIORITY[minLevel ?? "log"];
+}
+var LoggerBase = class {
+	cancel;
+	info;
+	success;
+	warn;
+	error;
+	debug;
+	fatal;
+	prompt;
+	log;
+	trace;
+	constructor() {
+		for (const level of LOG_LEVELS) this[level] = (...msg) => this._log(level, ...msg);
+	}
+};
+//#endregion
+//#region src/context.ts
+var Context = class extends LoggerBase {
+	#embedder;
+	#root;
+	#db;
+	#search;
+	#store;
+	#vfs;
+	events = new EventEmitter();
+	constructor(opts = {}) {
+		super();
+		this.opts = opts;
+		this.opts.embedder ??= {};
+		this.opts.embedder.onProgress ??= (progress) => this.events.emit("progress", progress);
+		this.#root = normPath(this.opts.root ?? "~/.rekal-data");
+		mkdirSync(this.#root, { recursive: true });
+	}
+	_log(level, ...msg) {
+		this.events.emit("log", level, ...msg);
+	}
+	get root() {
+		return this.#root;
+	}
+	async db() {
+		const { Db } = await import("./db-BMh1OP4b.mjs");
+		return this.#db ??= await Db.load(join(this.root, "index.sqlite3"));
+	}
+	async search() {
+		const { Search } = await import("./search-BllHWtZF.mjs");
+		return this.#search ??= await Search.load(this);
+	}
+	async store() {
+		const { Store } = await import("./store-DE7S35SS.mjs");
+		return this.#store ??= await Store.load(this);
+	}
+	async embedder() {
+		const { Embedder } = await import("./embed-rUMZxqed.mjs");
+		return this.#embedder ??= new Embedder(this);
+	}
+	async vfs() {
+		const { Vfs } = await import("./vfs-CNQbkhsf.mjs");
+		return this.#vfs ??= new Vfs(this);
+	}
+};
+//#endregion
+//#region src/snippet.ts
+const WORD_REGEX = /[\p{L}\p{N}]+/gu;
+const SCORE_EXACT = 3;
+const SCORE_LOWER = 2;
+const SCORE_QUERY_PREFIX = 1.5;
+const SCORE_DOC_PREFIX = 1;
+const SCORE_OVERLAP = 1;
+const MIN_PREFIX_LENGTH = 3;
+const STOPWORD_MIN = 0;
+const STOPWORD_MAX = 1;
+const HEAT_SPREAD = 5;
+const WEIGHT_EMPTY = .1;
+const WEIGHT_NONWORD = .3;
+const WEIGHT_REPETITION = .7;
+const decayLinear = (d, r) => Math.max(0, 1 - d / r);
+const STOP_WORDS = new Set([
+	"a",
+	"an",
+	"and",
+	"are",
+	"as",
+	"at",
+	"be",
+	"but",
+	"by",
+	"do",
+	"for",
+	"from",
+	"had",
+	"has",
+	"have",
+	"he",
+	"her",
+	"his",
+	"how",
+	"i",
+	"if",
+	"in",
+	"is",
+	"it",
+	"its",
+	"my",
+	"no",
+	"not",
+	"of",
+	"on",
+	"or",
+	"our",
+	"she",
+	"so",
+	"than",
+	"that",
+	"the",
+	"their",
+	"them",
+	"then",
+	"there",
+	"these",
+	"they",
+	"this",
+	"to",
+	"up",
+	"us",
+	"was",
+	"we",
+	"what",
+	"when",
+	"which",
+	"who",
+	"will",
+	"with",
+	"you",
+	"your"
+]);
+function isStopWord(word) {
+	return STOP_WORDS.has(word.toLowerCase());
+}
+var Snippet = class {
+	query = [];
+	prefixes = /* @__PURE__ */ new Set();
+	prefixRegex;
+	opts;
+	constructor(opts) {
+		this.opts = {
+			...opts,
+			lines: 5,
+			stopWords: /* @__PURE__ */ new Map()
+		};
+		const stopwords = new Map([...STOP_WORDS].map((w) => [w, 0]));
+		this.opts.stopWords.forEach((s, w) => stopwords.set(w, Math.max(STOPWORD_MIN, Math.min(STOPWORD_MAX, s))));
+		const tokens = this.tokenize(opts.query, false);
+		for (const tok of tokens) {
+			const score = stopwords.get(tok.lower) ?? 2;
+			if (score === 0) continue;
+			this.prefixes.add(tok.lower.slice(0, 2));
+			this.query.push({
+				...tok,
+				score
+			});
+		}
+		this.prefixRegex = this.prefixes.size > 0 ? new RegExp(`(${[...this.prefixes].join("|")})`, "i") : /(?!)/;
+	}
+	normalize(text) {
+		text = text.normalize("NFD").replace(/\p{M}/gu, "");
+		text = text.replace(/['’]s\b/gi, "");
+		text = text.replace(/['’]/g, " ");
+		return text;
+	}
+	tokenize(text, queryOnly = true) {
+		if (queryOnly && !this.prefixRegex.test(text)) return [];
+		const tokens = this.normalize(text).match(WORD_REGEX) ?? [];
+		const ret = [];
+		for (const token of tokens) {
+			const lower = token.toLowerCase();
+			ret.push({
+				lower,
+				text: token
+			});
+		}
+		return ret;
+	}
+	score(token, queryToken) {
+		const tl = token.lower.length;
+		if (token.text === queryToken.text) return SCORE_EXACT;
+		if (token.lower === queryToken.lower) return SCORE_LOWER;
+		if (token.lower.startsWith(queryToken.lower)) return SCORE_QUERY_PREFIX;
+		if (queryToken.lower.startsWith(token.lower) && tl >= MIN_PREFIX_LENGTH) return SCORE_DOC_PREFIX;
+		let prefix = 0;
+		for (let i = 0; i < token.lower.length; i++) {
+			if (token.lower[i] !== queryToken.lower[i]) break;
+			prefix++;
+		}
+		return prefix >= MIN_PREFIX_LENGTH ? SCORE_OVERLAP * (prefix / Math.max(token.lower.length, queryToken.lower.length)) : 0;
+	}
+	match(input) {
+		let token;
+		if (typeof input === "string") token = {
+			lower: this.normalize(input).toLowerCase(),
+			text: input
+		};
+		else token = input;
+		let [bestScore, bestTok] = [0, this.query[0]];
+		for (let t = 0; t < this.query.length; t++) {
+			const queryTok = this.query[t];
+			const s = this.score(token, queryTok);
+			if (s > bestScore) [bestScore, bestTok] = [s, queryTok];
+		}
+		if (bestScore > 0) return {
+			...bestTok,
+			score: bestTok.score * bestScore
+		};
+	}
+	scores(tokens) {
+		const coverage = tokens.map(() => /* @__PURE__ */ new Set());
+		return {
+			coverage,
+			scores: tokens.map((line, l) => {
+				let lineScore = 0;
+				for (const token of line) {
+					const queryTok = this.match(token);
+					if (!queryTok) continue;
+					let score = queryTok.score;
+					if (coverage[l].has(queryTok.lower)) score *= WEIGHT_REPETITION;
+					coverage[l].add(queryTok.lower);
+					lineScore += score;
+				}
+				return lineScore;
+			})
+		};
+	}
+	heat(lines, scores) {
+		const spread = Math.max(HEAT_SPREAD, Math.ceil(this.opts.lines / 2));
+		const heat = new Float64Array(scores.length);
+		for (let i = 0; i < scores.length; i++) {
+			if (scores[i] === 0) continue;
+			const spreadStart = Math.max(0, i - spread);
+			const spreadEnd = Math.min(scores.length - 1, i + spread);
+			for (let j = spreadStart; j <= spreadEnd; j++) {
+				let weight = 1;
+				if (!lines[j].trim()) weight *= WEIGHT_EMPTY;
+				else if (!lines[j].match(/\p{L}/u)) weight *= WEIGHT_NONWORD;
+				weight *= decayLinear(Math.abs(i - j), spread);
+				heat[j] += scores[i] * weight;
+			}
+		}
+		return [...heat];
+	}
+	extract(text) {
+		const lines = text.split("\n");
+		const radius = Math.min(this.opts.lines, lines.length);
+		const tokens = lines.map((line) => this.tokenize(line));
+		const { scores, coverage } = this.scores(tokens);
+		const heat = this.heat(lines, scores);
+		const windows = [];
+		for (let i = 0; i <= scores.length - radius; i++) {
+			if (heat[i] === 0 && windows.length > 0) continue;
+			let heatSum = 0;
+			const covered = /* @__PURE__ */ new Set();
+			for (let j = i; j < i + radius; j++) {
+				heatSum += heat[j];
+				coverage[j].forEach((t) => covered.add(t));
+			}
+			const cov = this.query.length === 0 ? 1 : covered.size / this.query.length;
+			windows.push({
+				coverage: cov,
+				heat: heatSum,
+				score: heatSum * cov,
+				start: i
+			});
+		}
+		let best = windows[0] ?? {
+			coverage: 0,
+			heat: 0,
+			score: 0,
+			start: 0
+		};
+		for (let i = 1; i < windows.length; i++) if (windows[i].score > best.score) best = windows[i];
+		const snippet = lines.slice(best.start, best.start + radius);
+		return {
+			best,
+			heat: [...heat],
+			lines,
+			scores,
+			snippet,
+			tokens,
+			windows
+		};
+	}
+	debug(result) {
+		console.info("Options:", this.opts);
+		console.info("Query :", this.query);
+		const score = (n, f = 1) => n !== void 0 ? n.toFixed(f).padEnd(4) : " ".repeat(4);
+		result.lines.forEach((line, i) => {
+			const isBest = (l) => l >= result.best.start && l < result.best.start + this.opts.lines;
+			const lineScore = score(result.scores[i]);
+			const lineHeat = score(result.heat[i]);
+			const windowHeat = score(result.windows[i]?.heat ?? 0);
+			const windowScore = score(result.windows[i]?.score ?? 0);
+			const coverage = ((result.windows[i]?.coverage ?? 0) * 100).toFixed(0).padStart(3);
+			console.log(`s:${lineScore} h:${lineHeat} wh:${windowHeat} ws:${windowScore} ${coverage}% ${isBest(i) ? ">" : " "} ${line}`);
+		});
+	}
+	highlight(text, hl) {
+		return text.replace(WORD_REGEX, (word, offset) => this.match(word) ? hl(word, offset) : word);
+	}
+};
+//#endregion
+export { Context, Doc, LOG_LEVELS, LoggerBase, Progress, Snippet, URI_PREFIX, WORD_REGEX, assertUri, astat, chunkMarkdown, chunkText, findUp, gitRoot, hash, isLogLevel, isStopWord, normPath, normUri, parentUri, parseFrontmatter, parseMarkdown, parseSections, parseYaml, shouldLog, sstat, toError, toFts, tokenize };

package/dist/llama-CT3dc9Cn.mjs

@@ -0,0 +1,75 @@
+import { n as parseModelUri } from "./models-DFQSgBNr.mjs";
+import { join } from "pathe";
+import { availableParallelism } from "node:os";
+import { LlamaLogLevel } from "node-llama-cpp";
+//#region src/embed/llama.ts
+var LlamaBackend = class LlamaBackend {
+	static llama;
+	device;
+	maxTokens;
+	dims;
+	#contexts = [];
+	#model;
+	#poolSize;
+	#threadsPerCtx;
+	#ctx;
+	constructor(llama, model, ctx) {
+		this.#model = model;
+		this.#ctx = ctx;
+		this.maxTokens = model.trainContextSize;
+		this.dims = model.embeddingVectorSize;
+		this.device = llama.gpu ? "gpu" : "cpu";
+		const useGpu = llama.gpu && ctx.opts.useGpu;
+		const cores = availableParallelism();
+		const poolSize = useGpu ? Math.min(8, Math.max(1, Math.floor(cores / 4))) : Math.min(8, Math.max(1, Math.floor(cores / 4)));
+		this.#threadsPerCtx = useGpu ? 0 : Math.max(1, Math.floor(ctx.opts.threads / poolSize));
+		this.#poolSize = poolSize;
+		ctx.logger.debug({
+			cores,
+			gpu: llama.gpu,
+			poolSize,
+			threadsPerCtx: this.#threadsPerCtx
+		});
+	}
+	static async load(ctx) {
+		const { model, variant } = parseModelUri(ctx.opts.model.uri);
+		const { getLlama, resolveModelFile } = await import("node-llama-cpp");
+		const modelsDir = join(ctx.root, "models");
+		const modelPath = await resolveModelFile(`hf:${model}${variant ? `:${variant}` : ""}`, modelsDir);
+		LlamaBackend.llama ??= await getLlama({
+			gpu: ctx.opts.useGpu ? "auto" : false,
+			logLevel: LlamaLogLevel.error,
+			logger: (level, message) => {
+				const l = level;
+				(ctx.logger[l] ?? ctx.logger.log)(level.toString(), message);
+			}
+		});
+		const lm = await LlamaBackend.llama.loadModel({ modelPath });
+		return new LlamaBackend(LlamaBackend.llama, lm, ctx);
+	}
+	/** Get or create up to `count` embedding contexts */
+	async acquire(count) {
+		const needed = Math.min(count, this.#poolSize) - this.#contexts.length;
+		for (let i = 0; i < needed; i++) try {
+			this.#ctx.logger.debug(`Creating embedding context ${this.#contexts.length + 1}/${this.#poolSize}...`);
+			this.#contexts.push(await this.#model.createEmbeddingContext({
+				contextSize: this.#ctx.opts.maxTokens,
+				threads: this.#threadsPerCtx
+			}));
+		} catch {
+			this.#ctx.logger.warn(`Failed to create embedding context ${this.#contexts.length + 1}. Adjusting pool size down to ${this.#contexts.length}.`);
+			this.#poolSize = this.#contexts.length;
+			break;
+		}
+		return this.#contexts;
+	}
+	async embed(texts) {
+		const contexts = await this.acquire(texts.length);
+		return Promise.all(texts.map((text, idx) => contexts[idx % contexts.length].getEmbeddingFor(text).then((embedding) => [...embedding.vector])));
+	}
+	toks(input) {
+		return this.#model.tokenize(input).length;
+	}
+};
+//#endregion
+export { LlamaBackend };

package/dist/models-DFQSgBNr.mjs

@@ -0,0 +1,77 @@
+import { defu } from "defu";
+//#region src/embed/models.ts
+function parseModelUri(uri) {
+	const [backend, model, variant] = uri.split(":");
+	if (!backend || !model) throw new Error(`Invalid model URI: ${uri}. Expected format "provider:model[:variant]"`);
+	return {
+		backend,
+		model,
+		variant
+	};
+}
+function resolveModel(opts) {
+	const uri = typeof opts === "string" ? opts : opts?.uri ?? DEFAULTS.uri;
+	const options = typeof opts === "string" ? { uri } : opts ?? { uri };
+	const base = MODELS[options.base ?? MODELS[uri]?.base ?? ""] ?? {};
+	return defu(options, MODELS[uri] ?? {}, base, DEFAULTS);
+}
+function loadModel(ctx) {
+	const { backend } = parseModelUri(ctx.opts.model.uri);
+	if (backend === "transformers") return import("./transformers-CJ3QA2PK.mjs").then(({ TransformersBackend }) => TransformersBackend.load(ctx));
+	else if (backend === "llama") return import("./llama-CT3dc9Cn.mjs").then(({ LlamaBackend }) => LlamaBackend.load(ctx));
+	else if (backend === "openai") return import("./openai-j2_2GM4J.mjs").then(({ OpenAIBackend }) => OpenAIBackend.load(ctx));
+	else throw new Error(`Unsupported model backend: ${String(backend)}`);
+}
+const DEFAULTS = {
+	prompt: {
+		document: (doc) => {
+			const title = doc.title?.trim();
+			if (title?.length) {
+				const idx = doc.text.indexOf(title);
+				if (idx === -1 || idx > 10) return `${title}\n\n${doc.text}`;
+			}
+			return doc.text;
+		},
+		query: (query) => query
+	},
+	uri: "transformers:Snowflake/snowflake-arctic-embed-s:q8"
+};
+const MODELS = {
+	BAAI: {
+		models: [
+			"transformers:BAAI/bge-large-en-v1.5",
+			"transformers:BAAI/bge-base-en-v1.5",
+			"transformers:BAAI/bge-small-en-v1.5"
+		],
+		pooling: "cls",
+		prompt: { query: (query) => `Represent this sentence for searching relevant passages: ${query}` }
+	},
+	openai: { models: [
+		"openai:text-embedding-3-small",
+		"openai:text-embedding-3-large",
+		"openai:text-embedding-ada-002"
+	] },
+	snowflake: {
+		models: [
+			"transformers:Snowflake/snowflake-arctic-embed-m-v2.0:q8",
+			"transformers:Snowflake/snowflake-arctic-embed-s:q8",
+			"transformers:Snowflake/snowflake-arctic-embed-xs:q8",
+			"llama:mradermacher/snowflake-arctic-embed-s-GGUF:Q4_K_M"
+		],
+		pooling: "cls",
+		prompt: { query: (query) => `Represent this sentence for searching relevant passages: ${query}` }
+	},
+	"transformers:onnx-community/embeddinggemma-300m-ONNX:q8": {
+		pooling: "mean",
+		prompt: {
+			document: (doc) => `title: ${doc.title ?? "none"} | ${doc.text}`,
+			query: (query) => `task: search result | query: ${query}`
+		}
+	}
+};
+for (const [base, options] of Object.entries(MODELS)) for (const model of options.models ?? []) MODELS[model] = {
+	base,
+	uri: model
+};
+//#endregion
+export { parseModelUri as n, resolveModel as r, loadModel as t };

package/dist/openai-j2_2GM4J.mjs

@@ -0,0 +1,76 @@
+import { n as parseModelUri } from "./models-DFQSgBNr.mjs";
+//#region src/embed/openai.ts
+const OPENAI_EMBEDDING_URL = "https://api.openai.com/v1/embeddings";
+const MODEL_INFO = {
+	"text-embedding-3-large": {
+		contextSize: 8191,
+		vectorSize: 3072
+	},
+	"text-embedding-3-small": {
+		contextSize: 8191,
+		vectorSize: 1536
+	},
+	"text-embedding-ada-002": {
+		contextSize: 8191,
+		vectorSize: 1536
+	}
+};
+var OpenAIBackend = class OpenAIBackend {
+	device = "api";
+	maxTokens;
+	dims;
+	#model;
+	#apiKey;
+	#tokenizer;
+	#ctx;
+	constructor(model, apiKey, info, tokenizer, ctx) {
+		this.#model = model;
+		this.#apiKey = apiKey;
+		this.maxTokens = info.contextSize;
+		this.dims = info.vectorSize;
+		this.#tokenizer = tokenizer;
+		this.#ctx = ctx;
+	}
+	static async load(ctx) {
+		const { model } = parseModelUri(ctx.opts.model.uri);
+		const apiKey = process.env.OPENAI_API_KEY;
+		if (!apiKey) throw new Error("Missing `OPENAI_API_KEY` environment variable for OpenAI embeddings.");
+		const { encode } = await import(`gpt-tokenizer/model/${model}`);
+		let info = MODEL_INFO[model];
+		if (!info) {
+			ctx.status.status = "probing model dimensions...";
+			info = {
+				contextSize: 8191,
+				vectorSize: (await new OpenAIBackend(model, apiKey, {
+					contextSize: 8191,
+					vectorSize: 0
+				}, encode, ctx).embed(["test"]))[0].length
+			};
+		}
+		return new OpenAIBackend(model, apiKey, info, encode, ctx);
+	}
+	async embed(texts) {
+		const response = await fetch(OPENAI_EMBEDDING_URL, {
+			body: JSON.stringify({
+				dimensions: this.#ctx.opts.maxDims,
+				input: texts,
+				model: this.#model
+			}),
+			headers: {
+				Authorization: `Bearer ${this.#apiKey}`,
+				"Content-Type": "application/json"
+			},
+			method: "POST"
+		});
+		if (!response.ok) {
+			const error = await response.text();
+			throw new Error(`OpenAI embedding API error (${response.status}): ${error}`);
+		}
+		return (await response.json()).data.toSorted((a, b) => a.index - b.index).map((d) => d.embedding);
+	}
+	toks(input) {
+		return this.#tokenizer(input).length;
+	}
+};
+//#endregion
+export { OpenAIBackend };