@rekal/mem 0.0.0

package/dist/progress-B1JdNapX.mjs

@@ -0,0 +1,263 @@
+import { n as parseYaml } from "./runtime.node-DlQPaGrV.mjs";
+import { EventEmitter } from "node:events";
+import { inspect } from "node:util";
+//#region src/md.ts
+const CHARS_PER_TOKEN = 3;
+function parseFrontmatter(text) {
+	const match = text.match(/^---\n([\s\S]*?)\n---\n?/);
+	return {
+		body: match ? text.slice(match[0].length) : text,
+		bodyOffset: match?.[0].trim().split("\n").length ?? 0,
+		frontmatter: match ? parseYaml(match[1]) : {},
+		frontmatterText: match?.[0],
+		text
+	};
+}
+function parseMarkdown(text) {
+	const ret = parseFrontmatter(text);
+	return {
+		...ret,
+		sections: parseSections(ret.body)
+	};
+}
+function parseSections(md) {
+	const lines = md.split(/\n/);
+	let current = {
+		content: [],
+		context: [],
+		heading: "",
+		headingText: "",
+		level: 0,
+		offset: 0
+	};
+	const sections = [current];
+	let codeBlock = void 0;
+	for (const [i, line] of lines.entries()) {
+		const match = line.match(/^(#+)\s+(.*)/);
+		const fenceMatch = line.match(/^\s*(`{3,}|~{3,})/);
+		if (codeBlock && line.startsWith(codeBlock)) codeBlock = void 0;
+		else if (!codeBlock && fenceMatch) codeBlock = fenceMatch[1];
+		if (!codeBlock && match) {
+			if (current.content.length === 0) sections.pop();
+			const level = match[1].length;
+			current = {
+				content: [line],
+				context: [],
+				heading: match[2].trim(),
+				headingText: match[0].trim(),
+				level,
+				offset: i
+			};
+			sections.push(current);
+		} else current.content.push(line);
+	}
+	const stack = [];
+	for (const section of sections) {
+		while ((stack.at(-1)?.level ?? -1) >= section.level) stack.pop();
+		section.context = stack.map((s) => s.headingText);
+		if (section.level > 0) stack.push(section);
+	}
+	return sections;
+}
+function findSplit(slice) {
+	for (const sub of [
+		"\n\n",
+		"\n",
+		"  ",
+		"	",
+		" "
+	]) {
+		const i = slice.lastIndexOf(sub);
+		if (i > slice.length * .8) return i;
+	}
+	return slice.length;
+}
+var SafeCounter = class SafeCounter {
+	static #chars = 0;
+	static #toks = 0;
+	constructor(tok, maxTokens = 500) {
+		this.tok = tok;
+		this.maxTokens = maxTokens;
+	}
+	get charsPerToken() {
+		return SafeCounter.#toks > this.maxTokens * 2 ? SafeCounter.#chars / SafeCounter.#toks : CHARS_PER_TOKEN;
+	}
+	estimate(text) {
+		return Math.ceil(text.length / this.charsPerToken);
+	}
+	toks(text) {
+		if (text.length === 0) return {
+			count: 0,
+			estimated: false
+		};
+		let count = this.estimate(text) * .9;
+		if (count > this.maxTokens) return {
+			count,
+			estimated: true
+		};
+		count = this.tok.toks(text);
+		SafeCounter.#chars += text.length;
+		SafeCounter.#toks += count;
+		return {
+			count,
+			estimated: false
+		};
+	}
+};
+function chunkText(text, tok, size = 500) {
+	const counter = new SafeCounter(tok, size);
+	const chunks = [];
+	while (text.length) {
+		let next = text;
+		let toks = counter.toks(next);
+		if (toks.count <= size) {
+			chunks.push(next);
+			break;
+		}
+		let maxChars = size * counter.charsPerToken * .8;
+		while (true) {
+			maxChars = Math.min(maxChars, next.length);
+			const split = findSplit(next.slice(0, maxChars));
+			next = next.slice(0, split);
+			toks = counter.toks(next);
+			if (toks.count <= size) break;
+			maxChars *= size / toks.count * .8;
+		}
+		chunks.push(next);
+		text = text.slice(next.length);
+	}
+	return chunks;
+}
+function chunkMarkdown(md, tok, size = 500) {
+	const sections = parseSections(md);
+	const chunks = [{
+		content: [],
+		context: [],
+		tokens: 0
+	}];
+	const counter = new SafeCounter(tok, size);
+	for (const section of sections) {
+		const chunk = chunks.at(-1);
+		const content = [...section.context, ...section.content];
+		const text = content.join("\n");
+		const toks = counter.toks(text).count;
+		if (chunk.tokens + toks <= size) {
+			const context = section.context.filter((h, c) => chunk.context[c] !== h);
+			chunk.content.push(...context);
+			chunk.content.push(...section.content);
+			chunk.context = [...section.context, section.headingText];
+			chunk.tokens += toks;
+		} else if (toks <= size) chunks.push({
+			content,
+			context: [...section.context, section.headingText],
+			tokens: toks
+		});
+		else {
+			const context = section.context.join("\n");
+			const toksCtx = counter.toks(context);
+			chunks.push(...chunkText(section.content.join("\n"), tok, size - toksCtx.count).map((c) => ({
+				content: (context.length ? `${context}\n${c}` : c).split("\n"),
+				context: [],
+				tokens: 0
+			})));
+			chunks.push({
+				content: [],
+				context: [],
+				tokens: 0
+			});
+		}
+	}
+	return chunks.map((c) => c.content.join("\n").trim()).filter(Boolean);
+}
+//#endregion
+//#region src/progress.ts
+var Progress = class Progress extends EventEmitter {
+	#max = 100;
+	#value = 0;
+	#children = /* @__PURE__ */ new Map();
+	#status;
+	#done = false;
+	constructor(name, opts = {}) {
+		super();
+		this.name = name;
+		this.set(opts);
+	}
+	get group() {
+		return this.#children.size > 0;
+	}
+	set(opts) {
+		if (this.#done) return this;
+		if (typeof opts === "number") this.#value = opts;
+		else {
+			this.#max = opts.max ?? this.#max;
+			this.#status = opts.status ?? this.#status;
+			this.#value = opts.value ?? this.#value;
+		}
+		this.emit("update", this);
+		if (this.#value >= this.#max) this.stop();
+		return this;
+	}
+	get status() {
+		return this.#status ?? this.name;
+	}
+	set status(status) {
+		this.set({ status });
+	}
+	set value(value) {
+		this.set(value);
+	}
+	get value() {
+		return !this.group ? this.#value : this.#children.values().reduce((sum, c) => sum + c.value, 0);
+	}
+	set max(max) {
+		this.set({ max });
+	}
+	get max() {
+		return !this.group ? this.#max : this.#children.values().reduce((sum, c) => sum + c.max, 0);
+	}
+	get done() {
+		return this.#done;
+	}
+	get ratio() {
+		return this.max === 0 ? 0 : Math.min(1, this.value / this.max);
+	}
+	get pct() {
+		return this.ratio * 100;
+	}
+	stop() {
+		if (this.#done) return;
+		this.#done = true;
+		if (!this.group) this.#value = this.#max;
+		this.#children.forEach((c) => c.stop());
+		this.emit("done", this);
+	}
+	children() {
+		return [...this.#children.values()];
+	}
+	child(name, opts = {}) {
+		if (this.#value > 0) throw new Error("Cannot add child to Progress that has already made progress");
+		let child = this.#children.get(name);
+		if (!child) {
+			child = new Progress(name, opts);
+			child.on("update", () => this.emit("update", this));
+			child.on("done", () => {
+				if (!this.done && this.children().every((c) => c.done)) this.stop();
+			});
+			this.#children.set(name, child);
+		}
+		return child;
+	}
+	[inspect.custom](_depth, _options) {
+		return this.toString();
+	}
+	toString(indent = 0) {
+		const pad = "  ".repeat(indent);
+		const pct = `${this.pct.toFixed(0)}%`.padStart(4);
+		const status = this.#status ? ` ${this.#status}` : "";
+		const line = `${pad}${pct} ${this.name}${status}`;
+		if (!this.group) return line;
+		return [line, ...[...this.#children.values()].map((c) => c.toString(indent + 1))].join("\n");
+	}
+};
+//#endregion
+export { parseMarkdown as a, parseFrontmatter as i, chunkMarkdown as n, parseSections as o, chunkText as r, Progress as t };

package/dist/query-VFSpErTB.mjs

@@ -0,0 +1,125 @@
+//#region src/query.ts
+const FTS_FIELDS = new Set([
+	"entities",
+	"tags",
+	"description",
+	"title",
+	"body"
+]);
+function tokenize(input) {
+	const tokens = [];
+	let i = 0;
+	while (i < input.length) {
+		while (i < input.length && input[i] === " ") i++;
+		if (i >= input.length) break;
+		const ch = input[i];
+		if (ch === "(" || ch === ")") {
+			tokens.push({
+				type: "paren",
+				value: ch
+			});
+			i++;
+		} else if (ch === "|") {
+			tokens.push({
+				type: "op",
+				value: "OR"
+			});
+			i++;
+		} else if ((ch === "\"" || ch === "'") && (i === 0 || input[i - 1] === " ")) {
+			const quote = ch;
+			i++;
+			const start = i;
+			while (i < input.length && input[i] !== quote) i++;
+			if (start < i) tokens.push({
+				type: "term",
+				value: input.slice(start, i)
+			});
+			if (i < input.length) i++;
+		} else {
+			const neg = ch === "-";
+			const req = ch === "+";
+			if (neg || req) i++;
+			const start = i;
+			while (i < input.length && !" \"()|".includes(input[i])) i++;
+			if (start < i) {
+				const raw = input.slice(start, i);
+				const colon = raw.indexOf(":");
+				if (colon > 0 && FTS_FIELDS.has(raw.slice(0, colon))) tokens.push({
+					field: raw.slice(0, colon),
+					neg: neg || void 0,
+					req: req || void 0,
+					type: "term",
+					value: raw.slice(colon + 1)
+				});
+				else tokens.push({
+					neg: neg || void 0,
+					req: req || void 0,
+					type: "term",
+					value: raw
+				});
+			}
+		}
+	}
+	return tokens;
+}
+/** Sanitize a term for FTS5 — strip non-word/non-apostrophe chars, preserve colons */
+function sanitize(term) {
+	return term.replace(/[^\p{L}\p{N}\s':]/gu, "").trim();
+}
+function buildTerm(token) {
+	const clean = sanitize(token.value);
+	if (!clean) return;
+	const phrase = `"${clean}"${token.value.endsWith("*") ? "*" : ""}`;
+	const scoped = token.field ? `${token.field} : ${phrase}` : phrase;
+	return token.neg ? `NOT ${scoped}` : scoped;
+}
+function joinParts(parts, op) {
+	return parts.join(` ${op} `);
+}
+/** Build an FTS5 query string from user input */
+function toFts(input, defaultOp = "OR") {
+	const tokens = tokenize(input);
+	if (!tokens.some((t) => t.type === "term" && t.req)) {
+		const parts = [];
+		let needsOp = false;
+		for (const token of tokens) {
+			if (token.type === "paren") {
+				if (token.value === "(") {
+					if (needsOp) parts.push(defaultOp);
+					parts.push(token.value);
+					needsOp = false;
+				} else {
+					parts.push(token.value);
+					needsOp = true;
+				}
+				continue;
+			}
+			if (token.type === "op") {
+				parts.push(token.value);
+				needsOp = false;
+				continue;
+			}
+			const term = buildTerm(token);
+			if (!term) continue;
+			if (needsOp) parts.push(defaultOp);
+			parts.push(term);
+			needsOp = true;
+		}
+		return parts.join(" ").replace(/\( /g, "(").replace(/ \)/g, ")");
+	}
+	const required = [];
+	const all = [];
+	for (const token of tokens) {
+		if (token.type !== "term") continue;
+		const term = buildTerm(token);
+		if (!term) continue;
+		all.push(term);
+		if (token.req) required.push(term);
+	}
+	const requiredPart = joinParts(required, "AND");
+	const allPart = joinParts(all, "OR");
+	if (required.length === all.length) return requiredPart;
+	return `${requiredPart} AND (${allPart})`;
+}
+//#endregion
+export { tokenize as n, toFts as t };

package/dist/runtime.node-DlQPaGrV.mjs

@@ -0,0 +1,35 @@
+//#region src/runtime.node.ts
+let DB;
+async function dbInit() {
+	const { default: BetterDatabase } = await import("better-sqlite3");
+	return class extends BetterDatabase {
+		prepareCache = /* @__PURE__ */ new Map();
+		constructor(filename) {
+			super(filename);
+		}
+		run(...args) {
+			return this.exec(...args);
+		}
+		query(source) {
+			let ret = this.prepareCache.get(source);
+			if (!ret) {
+				ret = this.prepare(source);
+				this.prepareCache.set(source, ret);
+			}
+			return ret;
+		}
+	};
+}
+async function openDatabase(path) {
+	DB ??= await dbInit();
+	const { load: sqliteVec } = await import("sqlite-vec");
+	const db = new DB(path, { strict: true });
+	sqliteVec(db);
+	return db;
+}
+const { load: loadYaml } = await import("js-yaml");
+function parseYaml(content) {
+	return loadYaml(content);
+}
+//#endregion
+export { parseYaml as n, openDatabase as t };

package/dist/search-BllHWtZF.mjs

@@ -0,0 +1,166 @@
+import { t as hash } from "./util-DNyrmcA3.mjs";
+import { i as parentUri } from "./uri-CehXVDGB.mjs";
+import { t as toFts } from "./query-VFSpErTB.mjs";
+//#region src/search.ts
+const DESC_BOOST = .2;
+const PARENT_BOOST = .3;
+const RRF_K = 60;
+const RRF_LIMIT = 50;
+const VEC_OVERSAMPLE = 4;
+var Search = class Search {
+	constructor(db, ctx) {
+		this.db = db;
+		this.ctx = ctx;
+	}
+	static async load(ctx) {
+		return new Search(await ctx.db(), ctx);
+	}
+	async search(query, opts = {}) {
+		const mode = opts.mode ?? "hybrid";
+		if (mode === "fts") return this.searchFts(query, opts);
+		if (mode === "vec") return this.searchVec(query, opts);
+		const limit = opts.limit ?? 20;
+		const subLimit = Math.max(RRF_LIMIT, limit * 2);
+		const [fts, vec] = await Promise.all([this.searchFts(query, {
+			...opts,
+			limit: subLimit
+		}), this.searchVec(query, {
+			...opts,
+			limit: subLimit,
+			slice: false
+		})]);
+		return this.fuse(fts, vec, limit);
+	}
+	async searchVec(query, opts = {}) {
+		const cacheKey = hash(`embed:${query}`);
+		const embedder = await this.ctx.embedder();
+		const vfs = await this.ctx.vfs();
+		const embedding = this.db.cacheGet(cacheKey) ?? this.db.cacheSet(cacheKey, await embedder.embed(query));
+		const scope = vfs.getScope(opts.uri);
+		const limit = opts.limit ?? 20;
+		const results = this.db.searchVec(embedding, { limit: Math.max(limit, RRF_LIMIT) * VEC_OVERSAMPLE });
+		const best = /* @__PURE__ */ new Map();
+		for (const vec of results) {
+			const uri = scope.map(vec.path);
+			if (!uri) continue;
+			vec.score = vec.seq === 0 ? vec.score + DESC_BOOST * (1 - vec.score) : vec.score;
+			const existing = best.get(vec.doc_id)?.score ?? -Infinity;
+			if (vec.score > existing) best.set(vec.doc_id, Object.assign(vec, {
+				hiscore: 0,
+				uri
+			}));
+		}
+		const scores = new Map(best.values().map((vec) => [vec.uri, vec.score]));
+		const parentScores = /* @__PURE__ */ new Map();
+		const getParentScore = (uri) => {
+			const parent = parentUri(uri);
+			if (!parent) return 0;
+			let score = parentScores.get(parent);
+			if (score !== void 0) return score;
+			score = (scores.get(parent) ?? 0) * .5 + getParentScore(parent) * .5;
+			parentScores.set(parent, score);
+			return score;
+		};
+		for (const vec of best.values()) {
+			const parentScore = getParentScore(vec.uri);
+			vec.score += PARENT_BOOST * parentScore * (1 - vec.score);
+		}
+		let bestResults = [...best.values()].toSorted((a, b) => b.score - a.score);
+		bestResults = opts.slice === false ? bestResults : bestResults.slice(0, limit);
+		const docs = this.db.getDocs(bestResults.map((r) => r.doc_id));
+		const ret = [];
+		for (const vec of bestResults) {
+			const doc = docs.get(vec.doc_id);
+			if (doc) ret.push({
+				doc,
+				match: { vec },
+				path: vec.path,
+				scores: { vec: {
+					rank: 0,
+					score: vec.score
+				} },
+				uri: vec.uri
+			});
+		}
+		return this.rank("vec", ret);
+	}
+	async searchFts(query, opts = {}) {
+		const scope = (await this.ctx.vfs()).getScope(opts.uri);
+		const results = this.db.searchFts(toFts(query, opts.op ?? "OR"), {
+			limit: opts.limit ?? 20,
+			scope: scope.paths.map((p) => p.path)
+		});
+		const docs = this.db.getDocs(results.map((r) => r.rowid));
+		const ret = [];
+		for (const fts of results) {
+			fts.score = Math.abs(fts.score) / (1 + Math.abs(fts.score));
+			const doc = docs.get(fts.rowid);
+			const uri = scope.map(doc?.path ?? "");
+			if (doc && uri) ret.push({
+				doc,
+				match: { fts },
+				path: doc.path,
+				scores: { fts: {
+					rank: 0,
+					score: fts.score
+				} },
+				uri
+			});
+		}
+		return this.rank("fts", ret);
+	}
+	rank(mode, results) {
+		const score = (r) => r.scores[mode];
+		return results.toSorted((a, b) => score(b).score - score(a).score || (score(b).display_score ?? 0) - (score(a).display_score ?? 0)).map((r, i) => {
+			score(r).rank = i + 1;
+			return r;
+		});
+	}
+	/** Reciprocal Rank Fusion: merge FTS and vector results */
+	fuse(ftsResults, vecResults, limit) {
+		const merged = /* @__PURE__ */ new Map();
+		const minVecScore = vecResults.length ? vecResults[vecResults.length - 1]?.scores.vec.score : void 0;
+		const minFtsScore = ftsResults.length ? ftsResults[ftsResults.length - 1]?.scores.fts.score : void 0;
+		const minScore = Math.min(minVecScore ?? 1, minFtsScore ?? 1);
+		for (const fts of ftsResults) merged.set(fts.doc.id, {
+			fts,
+			uri: fts.uri
+		});
+		for (const vec of vecResults) merged.set(vec.doc.id, {
+			...merged.get(vec.doc.id),
+			uri: vec.uri,
+			vec
+		});
+		let ret = [...merged.values()].map(({ uri, fts, vec }) => {
+			const ftsScore = fts?.scores.fts;
+			const vecScore = vec?.scores.vec;
+			const score = (ftsScore?.rank !== void 0 ? 1 / (RRF_K + ftsScore.rank) : 0) + (vecScore?.rank !== void 0 ? 1 / (RRF_K + vecScore.rank) : 0);
+			const display_score = .6 * (vecScore?.score ?? minScore) + .4 * (ftsScore?.score ?? minScore);
+			const doc = fts?.doc ?? vec?.doc;
+			return {
+				doc,
+				match: {
+					...fts?.match,
+					...vec?.match
+				},
+				path: doc.path,
+				scores: {
+					...fts?.scores,
+					...vec?.scores,
+					hybrid: {
+						display_score,
+						rank: 0,
+						score
+					}
+				},
+				uri
+			};
+		});
+		ret = this.rank("hybrid", ret).slice(0, limit);
+		const bestScore = ret[0]?.scores.hybrid.score ?? 1;
+		for (const r of ret) r.scores.hybrid.score /= bestScore;
+		return ret;
+	}
+};
+//#endregion
+export { Search };