@tekmidian/pai 0.66.1 → 0.66.2

package/dist/{postgres-Ceqsa64C.mjs → postgres-D3xc2RB4.mjs}

@@ -1,12 +1,50 @@
-import { o as sha256 } from "./chunker-BH4i-F2b.mjs";
-import "./helpers-BRCJg0G3.mjs";
-import { t as STOP_WORDS } from "./stop-words-BdQuaE9K.mjs";
-import { n as resolveFromModule } from "./module-paths-DdRzbkUI.mjs";
-import { n as indexAllWithBackend } from "./async-C2Bm_Lal.mjs";
-import { t as entityContentId } from "./kg-entity-DCOcsVFD.mjs";
-import { readFileSync } from "node:fs";
+import { a as parseSessionTitleChunk, c as yieldToEventLoop, f as fileContentHash, h as sha256File, i as isPathTooBroadForContentScan, m as sha256, n as chunkId, o as walkContentFiles, r as detectTier, s as walkMdFiles, t as INDEX_YIELD_EVERY, u as chunkMarkdown } from "./helpers-CZsi_49C.mjs";
+import { t as STOP_WORDS } from "./stop-words-DtxaTWU_.mjs";
+import { a as generateEmbeddings, o as serializeEmbedding } from "./embeddings-DOLZnT1X.mjs";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename, dirname, join, relative } from "node:path";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
 import pg from "pg";
 
+//#region src/module-paths.ts
+/**
+* module-paths.ts — locate a file relative to the package root from a running module.
+*
+* tsdown bundles shared code into chunks emitted directly under dist/ (e.g.
+* dist/program-*.mjs) rather than always alongside the entry that needs them
+* (dist/cli/index.mjs, dist/daemon/index.mjs, ...). A fixed "../daemon/index.mjs"
+* relative to import.meta.url assumes one specific chunk depth and silently
+* resolves to the wrong file — or a file under the source tree instead of
+* dist — whenever the bundler's chunking changes. Walking up from the running
+* module's own directory until `dir/rel` exists is depth-independent, and works
+* whether `rel` lives under dist/ (built entries) or is a sibling of dist/ at
+* the package root (files excluded from the bundle, like docker/migrate-sqlite.ts).
+*/
+/**
+* Resolve `rel` by walking up from the directory containing `moduleUrl`,
+* returning the first `dir/rel` that exists on disk.
+*
+* Throws rather than returning a guessed path, so a caller writing a service
+* file or spawning a process fails loudly instead of pointing at nothing.
+*/
+function resolveFromModule(moduleUrl, rel, maxLevels = 8) {
+	let dir = dirname(fileURLToPath(moduleUrl));
+	for (let i = 0; i < maxLevels; i++) {
+		const candidate = join(dir, rel);
+		if (existsSync(candidate)) return candidate;
+		const parent = dirname(dir);
+		if (parent === dir) break;
+		dir = parent;
+	}
+	throw new Error(`cannot locate "${rel}" above ${dirname(fileURLToPath(moduleUrl))} — is the project built? (bun run build)`);
+}
+/** Package root (the directory holding package.json) for a running module, at any bundle depth. */
+function packageRoot(moduleUrl) {
+	return dirname(resolveFromModule(moduleUrl, "package.json"));
+}
+
+//#endregion
 //#region src/storage/postgres/helpers.ts
 /**
 * Internal helper utilities for the Postgres storage backend.
@@ -508,6 +546,323 @@ async function searchVaultNameIndex(pool, query, limit = 100) {
 	return (await pool.query("SELECT DISTINCT vault_path FROM vault_name_index WHERE lower(name) LIKE lower($1) LIMIT $2", [`%${query}%`, limit])).rows.map((row) => row.vault_path);
 }
 
+//#endregion
+//#region src/memory/kg-entity.ts
+/**
+* kg-entity.ts — Entity content-addressing with multi-tenant support.
+*
+* Provides UUID5-style deterministic content hashes for KG entities and edges,
+* ensuring that the same entity name always maps to the same ID within a tenant.
+* This enables idempotent upserts and stable foreign keys for kg_triples.
+*
+* Multi-tenant support: each tenant namespace gets its own entity ID space.
+* The default tenant is "default" for single-user deployments.
+*/
+/**
+* Generate a deterministic entity ID (UUID5-style) for a given name and tenant.
+*
+* The ID is a hex digest derived from "tenant_id:name" so the same entity
+* always receives the same ID within a tenant namespace.
+*
+* @param name      Entity name (case-preserved)
+* @param tenantId  Tenant namespace (default: "default")
+*/
+function entityContentId(name, tenantId = "default") {
+	return createHash("sha256").update(`entity:${tenantId}:${name}`).digest("hex").slice(0, 32);
+}
+
+//#endregion
+//#region src/memory/indexer/async.ts
+/**
+* Backend-aware async indexer for PAI federation memory.
+*
+* Provides the same functionality as sync.ts but writes through the
+* StorageBackend interface instead of directly to the SQLite driver.
+* Used when the daemon is configured with the Postgres backend.
+*
+* The SQLite path still uses sync.ts directly (which is faster for SQLite
+* due to synchronous transactions).
+*/
+/**
+* Index a single file through the StorageBackend interface.
+* Returns true if the file was re-indexed (changed or new), false if skipped.
+*/
+async function indexFileWithBackend(backend, projectId, rootPath, relativePath, source, tier) {
+	const absPath = join(rootPath, relativePath);
+	let content;
+	let stat;
+	try {
+		content = readFileSync(absPath, "utf8");
+		stat = statSync(absPath);
+	} catch {
+		return false;
+	}
+	const hash = fileContentHash(content);
+	const mtime = Math.floor(stat.mtimeMs);
+	const size = stat.size;
+	if (await backend.getFileHash(projectId, relativePath) === hash) return false;
+	await backend.deleteChunksForFile(projectId, relativePath);
+	const rawChunks = chunkMarkdown(content);
+	const updatedAt = Date.now();
+	const chunks = rawChunks.map((c, i) => ({
+		id: chunkId(projectId, relativePath, i, c.startLine, c.endLine),
+		projectId,
+		source,
+		tier,
+		path: relativePath,
+		startLine: c.startLine,
+		endLine: c.endLine,
+		hash: c.hash,
+		text: c.text,
+		updatedAt,
+		embedding: null
+	}));
+	await backend.insertChunks(chunks);
+	await backend.upsertFile({
+		projectId,
+		path: relativePath,
+		source,
+		tier,
+		hash,
+		mtime,
+		size
+	});
+	return true;
+}
+async function indexProjectWithBackend(backend, projectId, rootPath, claudeNotesDir) {
+	const result = {
+		filesProcessed: 0,
+		chunksCreated: 0,
+		filesSkipped: 0
+	};
+	const filesToIndex = [];
+	const rootMemoryMd = join(rootPath, "MEMORY.md");
+	if (existsSync(rootMemoryMd)) filesToIndex.push({
+		absPath: rootMemoryMd,
+		rootBase: rootPath,
+		source: "memory",
+		tier: "evergreen"
+	});
+	const memoryDir = join(rootPath, "memory");
+	for (const absPath of walkMdFiles(memoryDir)) {
+		const tier = detectTier(relative(rootPath, absPath));
+		filesToIndex.push({
+			absPath,
+			rootBase: rootPath,
+			source: "memory",
+			tier
+		});
+	}
+	const notesDir = join(rootPath, "Notes");
+	for (const absPath of walkMdFiles(notesDir)) filesToIndex.push({
+		absPath,
+		rootBase: rootPath,
+		source: "notes",
+		tier: "session"
+	});
+	{
+		const updatedAt = Date.now();
+		for (const absPath of walkMdFiles(notesDir)) {
+			const text = parseSessionTitleChunk(basename(absPath));
+			if (!text) continue;
+			const syntheticPath = `${relative(rootPath, absPath)}::title`;
+			const titleChunk = {
+				id: chunkId(projectId, syntheticPath, 0, 0, 0),
+				projectId,
+				source: "notes",
+				tier: "session",
+				path: syntheticPath,
+				startLine: 0,
+				endLine: 0,
+				hash: sha256File(text),
+				text,
+				updatedAt,
+				embedding: null
+			};
+			try {
+				await backend.insertChunks([titleChunk]);
+			} catch {}
+		}
+	}
+	if (!isPathTooBroadForContentScan(rootPath)) for (const absPath of walkContentFiles(rootPath)) filesToIndex.push({
+		absPath,
+		rootBase: rootPath,
+		source: "content",
+		tier: "topic"
+	});
+	if (claudeNotesDir && claudeNotesDir !== notesDir) {
+		for (const absPath of walkMdFiles(claudeNotesDir)) filesToIndex.push({
+			absPath,
+			rootBase: claudeNotesDir,
+			source: "notes",
+			tier: "session"
+		});
+		{
+			const updatedAt = Date.now();
+			for (const absPath of walkMdFiles(claudeNotesDir)) {
+				const text = parseSessionTitleChunk(basename(absPath));
+				if (!text) continue;
+				const syntheticPath = `${relative(claudeNotesDir, absPath)}::title`;
+				const titleChunk = {
+					id: chunkId(projectId, syntheticPath, 0, 0, 0),
+					projectId,
+					source: "notes",
+					tier: "session",
+					path: syntheticPath,
+					startLine: 0,
+					endLine: 0,
+					hash: sha256File(text),
+					text,
+					updatedAt,
+					embedding: null
+				};
+				try {
+					await backend.insertChunks([titleChunk]);
+				} catch {}
+			}
+		}
+		if (claudeNotesDir.endsWith("/Notes")) {
+			const claudeProjectDir = claudeNotesDir.slice(0, -6);
+			const claudeMemoryMd = join(claudeProjectDir, "MEMORY.md");
+			if (existsSync(claudeMemoryMd)) filesToIndex.push({
+				absPath: claudeMemoryMd,
+				rootBase: claudeProjectDir,
+				source: "memory",
+				tier: "evergreen"
+			});
+			const claudeMemoryDir = join(claudeProjectDir, "memory");
+			for (const absPath of walkMdFiles(claudeMemoryDir)) {
+				const tier = detectTier(relative(claudeProjectDir, absPath));
+				filesToIndex.push({
+					absPath,
+					rootBase: claudeProjectDir,
+					source: "memory",
+					tier
+				});
+			}
+		}
+	}
+	await yieldToEventLoop();
+	let filesSinceYield = 0;
+	for (const { absPath, rootBase, source, tier } of filesToIndex) {
+		if (filesSinceYield >= INDEX_YIELD_EVERY) {
+			await yieldToEventLoop();
+			filesSinceYield = 0;
+		}
+		filesSinceYield++;
+		const relPath = relative(rootBase, absPath);
+		try {
+			if (await indexFileWithBackend(backend, projectId, rootBase, relPath, source, tier)) {
+				const ids = await backend.getChunkIds(projectId, relPath);
+				result.filesProcessed++;
+				result.chunksCreated += ids.length;
+			} else result.filesSkipped++;
+		} catch {
+			result.filesSkipped++;
+		}
+	}
+	const livePaths = /* @__PURE__ */ new Set();
+	for (const { absPath, rootBase } of filesToIndex) livePaths.add(relative(rootBase, absPath));
+	const dbChunkPaths = await backend.getDistinctChunkPaths(projectId);
+	const stalePaths = [];
+	for (const p of dbChunkPaths) {
+		const basePath = p.endsWith("::title") ? p.slice(0, -7) : p;
+		if (!livePaths.has(basePath)) stalePaths.push(p);
+	}
+	if (stalePaths.length > 0) await backend.deletePaths(projectId, stalePaths);
+	return result;
+}
+const EMBED_BATCH_SIZE = 50;
+/** Default ceiling on one pass. See EmbedPassOptions. */
+const DEFAULT_MAX_CHUNKS_PER_PASS = 5e3;
+/** Default wall-clock ceiling on one pass, in milliseconds. */
+const DEFAULT_MAX_MILLIS_PER_PASS = 12e4;
+/**
+* Generate and store embeddings for unembedded chunks via the StorageBackend.
+*
+* The pass is deliberately bounded (see EmbedPassOptions) and resumable: it
+* takes a slice of the backlog, embeds it, and returns so the scheduler can run
+* an index pass before coming back. Draining the whole backlog in one call
+* starves indexing for as long as the call runs.
+*
+* The optional `shouldStop` callback is checked between every batch. When it
+* returns true the embed loop exits early so the caller (e.g. the daemon
+* shutdown handler) can close the pool without racing against active queries.
+*
+* Returns the number of newly embedded chunks.
+*/
+async function embedChunksWithBackend(backend, shouldStop, projectNames, options) {
+	const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS_PER_PASS;
+	const maxMillis = options?.maxMillis ?? DEFAULT_MAX_MILLIS_PER_PASS;
+	const deadline = Date.now() + maxMillis;
+	const rows = await backend.getUnembeddedChunkIds(void 0, maxChunks);
+	if (rows.length === 0) return 0;
+	const total = rows.length;
+	let embedded = 0;
+	const projectChunkCounts = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const entry = projectChunkCounts.get(row.project_id);
+		if (entry) entry.count++;
+		else projectChunkCounts.set(row.project_id, {
+			count: 1,
+			samplePath: row.path
+		});
+	}
+	const pName = (pid) => projectNames?.get(pid) ?? `project ${pid}`;
+	const projectSummary = Array.from(projectChunkCounts.entries()).map(([pid, { count, samplePath }]) => `  ${pName(pid)}: ${count} chunks (e.g. ${samplePath})`).join("\n");
+	process.stderr.write(`[pai-daemon] Embed pass: ${total} unembedded chunks across ${projectChunkCounts.size} project(s)\n${projectSummary}\n`);
+	let currentProjectId = -1;
+	let projectEmbedded = 0;
+	for (let i = 0; i < rows.length; i += EMBED_BATCH_SIZE) {
+		if (shouldStop?.()) {
+			process.stderr.write(`[pai-daemon] Embed pass cancelled after ${embedded}/${total} chunks (shutdown requested)\n`);
+			break;
+		}
+		if (Date.now() >= deadline) {
+			process.stderr.write(`[pai-daemon] Embed pass yielding after ${embedded}/${total} chunks (${maxMillis}ms budget spent); resuming next pass\n`);
+			break;
+		}
+		const batch = rows.slice(i, i + EMBED_BATCH_SIZE);
+		await yieldToEventLoop();
+		const vecs = await generateEmbeddings(batch.map((r) => r.text));
+		await Promise.all(batch.map((row, j) => backend.updateEmbedding(row.id, serializeEmbedding(vecs[j]))));
+		for (const { project_id, path } of batch) {
+			if (project_id !== currentProjectId) {
+				if (currentProjectId !== -1) process.stderr.write(`[pai-daemon] Finished ${pName(currentProjectId)}: ${projectEmbedded} chunks embedded\n`);
+				const info = projectChunkCounts.get(project_id);
+				process.stderr.write(`[pai-daemon] Embedding ${pName(project_id)} (${info?.count ?? "?"} chunks, starting at ${path})\n`);
+				currentProjectId = project_id;
+				projectEmbedded = 0;
+			}
+			projectEmbedded++;
+		}
+		embedded += batch.length;
+		const lastChunk = batch[batch.length - 1];
+		process.stderr.write(`[pai-daemon] Embedded ${embedded}/${total} chunks (${pName(lastChunk.project_id)}: ${lastChunk.path})\n`);
+	}
+	if (currentProjectId !== -1) process.stderr.write(`[pai-daemon] Finished ${pName(currentProjectId)}: ${projectEmbedded} chunks embedded\n`);
+	return embedded;
+}
+async function indexAllWithBackend(backend, registry) {
+	const projects = await registry.listProjects({ status: "active" });
+	const totals = {
+		filesProcessed: 0,
+		chunksCreated: 0,
+		filesSkipped: 0
+	};
+	for (const project of projects) {
+		await yieldToEventLoop();
+		const r = await indexProjectWithBackend(backend, project.id, project.root_path, project.claude_notes_dir);
+		totals.filesProcessed += r.filesProcessed;
+		totals.chunksCreated += r.chunksCreated;
+		totals.filesSkipped += r.filesSkipped;
+	}
+	return {
+		projects: projects.length,
+		result: totals
+	};
+}
+
 //#endregion
 //#region src/storage/postgres/tunnels.ts
 /**
@@ -1765,5 +2120,5 @@ function rowToKgEntity(row) {
 }
 
 //#endregion
-export { PostgresBackend };
-//# sourceMappingURL=postgres-Ceqsa64C.mjs.map
+export { buildPgTsQuery as a, entityContentId as i, embedChunksWithBackend as n, packageRoot as o, indexProjectWithBackend as r, resolveFromModule as s, PostgresBackend as t };
+//# sourceMappingURL=postgres-D3xc2RB4.mjs.map