@snoai/memory 0.0.0-stage → 1.0.1

package/dist/connection-C9YD5sgm.js

@@ -0,0 +1,297 @@
+import { StorageError } from "./internal/engine/shared/errors.js";
+import { a as schema_exports, i as migrateLegacyDatabaseNamespace, n as loadSqliteVecExtension, o as applyEntityNameKeyMigration, r as resolveSimpleTokenizerPath, t as migrateUnplacedCandidates } from "./unplaced-candidate-migration-CbQHzcEM.js";
+import { SqliteFileMissingError, openSqliteDatabase } from "./internal/store/sqlite-runtime.js";
+import { createLogger } from "@snoai/utils/logger";
+import { dirname, join } from "node:path";
+import { accessSync, constants, existsSync, lstatSync, mkdirSync, realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+import { migrate } from "drizzle-orm/better-sqlite3/migrator";
+//#region src/store/path-validation.ts
+/**
+* Validate and prepare the storage directory for the SQLite database file.
+* Resolves symlinks, creates missing directories, and checks write permissions.
+* Returns the resolved parent directory path on success, or throws a descriptive error.
+*/
+function validateStoragePath(dbPath) {
+	const dirPath = dirname(dbPath);
+	let resolvedDir = dirPath;
+	try {
+		if (lstatSync(dirPath).isSymbolicLink()) try {
+			resolvedDir = realpathSync(dirPath);
+		} catch (err) {
+			const e = err;
+			throw new Error(`dbPath parent "${dirPath}" is a symlink whose target does not exist.\n  Fix: Create the target directory, or update the symlink to a valid path.\n  Details: ${e.code ?? ""} ${e.message}`);
+		}
+	} catch (err) {
+		const e = err;
+		if (e.code === "ENOENT") {} else if (typeof e.message === "string" && e.message.includes("symlink whose target does not exist")) throw err;
+	}
+	if (!existsSync(resolvedDir)) try {
+		mkdirSync(resolvedDir, { recursive: true });
+	} catch (err) {
+		const e = err;
+		throw new Error(`Failed to create dbPath directory "${resolvedDir}".\n  Fix: Ensure the parent directory "${dirname(resolvedDir)}" exists and is writable,\n       or create it manually: mkdir -p "${resolvedDir}"\n  Details: ${e.code ?? ""} ${e.message}`);
+	}
+	try {
+		accessSync(resolvedDir, constants.W_OK);
+	} catch (err) {
+		const e = err;
+		throw new Error(`dbPath directory "${resolvedDir}" is not writable.\n  Fix: Check permissions with: ls -la "${dirname(resolvedDir)}"\n       Or grant write access: chmod u+w "${resolvedDir}"\n  Details: ${e.code ?? ""} ${e.message}`);
+	}
+	return resolvedDir;
+}
+//#endregion
+//#region src/store/connection.ts
+/** @file connection.ts
+* @purpose Creates and validates SQLite connections used by the memory store.
+* @boundary Filesystem safety, database pragmas, migrations, and runtime loading.
+* @see sqlite-runtime.ts, schema.ts, store.ts.
+*/
+const log = createLogger("sno-station-mem:db");
+/**
+* Opens the Node SQLite runtime and applies SQLite connection setup fallback behavior for missing data.
+*/
+/**
+* Reads the locked embedding dimension from the existing `nodix_memory_chunk_vectors`
+* DDL. Returns `undefined` if the DB file does not exist or the table has not
+* been created yet; throws on a corrupt-state DDL.
+*
+* This lets out-of-band tooling (CLI preset switching, diagnostics) detect
+* dim mismatches *before* the gateway boots, without paying the cost of
+* spinning up a real `MemoryStore`. Keeping the regex here means CLI,
+* diagnostics, and store startup agree on one canonical parser.
+*/
+function readChunkVecTableDimension(dbPath) {
+	if (!existsSync(dbPath)) return void 0;
+	let sqlite;
+	try {
+		sqlite = openSqliteDatabase(dbPath, {
+			readonly: true,
+			fileMustExist: true
+		});
+	} catch (error) {
+		if (error instanceof SqliteFileMissingError) return void 0;
+		throw error;
+	}
+	try {
+		const row = sqlite.db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'nodix_memory_chunk_vectors' LIMIT 1").get();
+		if (!row?.sql) return void 0;
+		return parseVecTableDimension(row.sql);
+	} finally {
+		try {
+			sqlite.db.close();
+		} catch {}
+	}
+}
+function parseVecTableDimension(sql) {
+	if (!/create\s+virtual\s+table/i.test(sql) || !/using\s+vec0/i.test(sql)) throw new StorageError(`nodix_memory_chunk_vectors exists but is not a sqlite-vec virtual table: ${sql}`);
+	const token = sql.match(/embedding\s+float\[(\d+)\]/i)?.[1];
+	if (!token) throw new StorageError(`Could not parse embedding dimension from nodix_memory_chunk_vectors DDL: ${sql}`);
+	const dim = Number.parseInt(token, 10);
+	if (!Number.isFinite(dim) || dim <= 0) throw new StorageError(`Invalid persisted vector dimension '${token}' in nodix_memory_chunk_vectors DDL`);
+	return dim;
+}
+/** Phase D: project_id PARTITION KEY on nodix_memory_chunk_vectors. */
+function vecTableDdlHasPartitionKey(sql) {
+	return /partition\s+key/i.test(sql);
+}
+function readChunkVecTableState(db) {
+	const row = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'nodix_memory_chunk_vectors' LIMIT 1").get();
+	if (!row?.sql) return void 0;
+	const rawCount = db.prepare("SELECT COUNT(*) AS count FROM nodix_memory_chunk_vectors").get()?.count ?? 0;
+	const rowCount = typeof rawCount === "bigint" ? Number(rawCount) : Number(rawCount);
+	return {
+		dimension: parseVecTableDimension(row.sql),
+		rowCount: Number.isFinite(rowCount) ? rowCount : 0,
+		hasPartitionKey: vecTableDdlHasPartitionKey(row.sql)
+	};
+}
+/**
+* Creates `nodix_memory_chunk_vectors` with the configured embedding dim and the
+* `project_id` PARTITION KEY (Phase D) if it does not already exist. Round A
+* migration 0004 creates the table at a placeholder 1024-d. vec0 virtual
+* tables cannot be ALTERed, so any pre-existing table that doesn't match the
+* current shape — wrong dimension, or predates the partition key — is
+* dropped and recreated. Nothing is shipped until npm publish (see
+* apps/sno-station-mem/host.md), so no installed database has vectors worth
+* migrating in place; a non-empty mismatch preserves the table and disables vector search for this open.
+*/
+function ensureChunkVecTable(db, vectorDim) {
+	if (!Number.isInteger(vectorDim) || vectorDim <= 0) throw new StorageError(`Invalid vectorDim ${vectorDim}; must be a positive integer`);
+	const existing = readChunkVecTableState(db);
+	if (existing && (existing.dimension !== vectorDim || !existing.hasPartitionKey)) {
+		if (existing.rowCount > 0) {
+			log.error("storage.vector.configuration.mismatch", {
+				vectorDim,
+				existing
+			}, {
+				event_name: "storage.vector.configuration.mismatch",
+				file: "packages/memory/src/store/connection.ts",
+				function: "ensureChunkVecTable",
+				site_id: "storage.vector.configuration.mismatch"
+			});
+			return;
+		}
+		db.exec("DROP TABLE nodix_memory_chunk_vectors");
+	}
+	db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS nodix_memory_chunk_vectors USING vec0(
+			id TEXT PRIMARY KEY,
+			project_id TEXT PARTITION KEY,
+			embedding float[${vectorDim}]
+		)`);
+	const verified = readChunkVecTableState(db);
+	if (!verified) throw new StorageError("nodix_memory_chunk_vectors table is missing after initialization; cannot verify vector dimension");
+	if (verified.dimension !== vectorDim) throw new StorageError(`Vector dimension mismatch between configuration (${vectorDim}) and database (${verified.dimension}); wipe or re-import vectors before switching dimensions`);
+	if (!verified.hasPartitionKey) throw new StorageError("nodix_memory_chunk_vectors is missing the project_id partition key after initialization");
+	return verified.dimension;
+}
+/**
+* Loads the wangfenjin/simple SQLite
+* tokenizer extension and registers its jieba dictionary path. Called once
+* per connection, BEFORE migrations run, so 0006 can create the
+* `nodix_memory_chunks_fts` virtual table with `tokenize='simple 0'`.
+*
+* The trailing smoke test fails fast if the extension load silently no-op'd:
+* creating an FTS5 table with an unregistered tokenizer name surfaces the
+* exact "no such tokenizer" error from sqlite-vec, instead of letting the
+* gateway start and silently fall back to unicode61 on first ingest.
+*/
+function loadSimpleTokenizer(db) {
+	const { platform, extensionPath, dictPath } = resolveSimpleTokenizerPath();
+	db.loadExtension(extensionPath);
+	db.prepare("SELECT jieba_dict(?)").get(dictPath);
+	db.exec("CREATE VIRTUAL TABLE temp.nodix_simple_tokenizer_smoke USING fts5(x, tokenize='simple 0'); DROP TABLE temp.nodix_simple_tokenizer_smoke;");
+	log.info("simple-tokenizer loaded", {
+		platform,
+		extensionPath
+	}, {
+		event_name: "sno_station_mem.connection.simple.tokenizer.loaded",
+		file: "packages/memory/src/store/connection.ts",
+		function: "loadSimpleTokenizer",
+		site_id: "connection.loadSimpleTokenizer.778ee7da6c"
+	});
+}
+function loadStorageExtensions(sqlite) {
+	loadSqliteVecExtension(sqlite.raw);
+	loadSimpleTokenizer(sqlite.db);
+}
+/**
+* Startup invariant: the persisted DDL
+* for `nodix_memory_chunks_fts` must declare `tokenize='simple ...'`. Catches DBs
+* created against an older plugin build (still on `unicode61`) so a stale
+* DB plus a fresh plugin doesn't silently degrade CJK recall to char-level.
+*
+* Mirrors the regex pattern used by `parseVecTableDimension` for
+* `nodix_memory_chunk_vectors` — single SELECT against `sqlite_master`, regex match.
+*/
+/**
+* Round A invariant: detect legacy DBs that pre-date migration 0004
+* (chunk-table swap). If the parent vec/FTS objects still exist, the DB
+* was opened against a build that never ran the new migration — fail loud
+* so the dev wipes the file.
+*/
+function assertNoLegacyBoundarySchema(db) {
+	const rows = db.prepare("PRAGMA table_info(nodix_memories)").all();
+	if (rows.length === 0) return;
+	const columnNames = new Set(rows.map((row) => row.name).filter((name) => !!name));
+	const legacyBoundaryColumn = [
+		"s",
+		"c",
+		"o",
+		"p",
+		"e"
+	].join("");
+	if (columnNames.has(legacyBoundaryColumn) && !columnNames.has("project_id")) throw new StorageError("legacy boundary schema is unsupported for provider v1");
+}
+function ensureMemoryKindIndexes(db) {
+	db.exec("CREATE INDEX IF NOT EXISTS nodix_idx_memories_project_fact_key_active ON nodix_memories(project_id, json_extract(metadata, '$.fact_key')) WHERE json_valid(metadata) AND json_extract(metadata, '$.fact_key') IS NOT NULL AND json_extract(metadata, '$.invalidated_at') IS NULL;");
+}
+const CONNECTION_DIR = dirname(fileURLToPath(import.meta.url));
+function resolveMigrationsDir() {
+	const candidates = [
+		join(CONNECTION_DIR, "../../drizzle"),
+		join(CONNECTION_DIR, "../drizzle"),
+		join(CONNECTION_DIR, "../../../drizzle")
+	];
+	for (const dir of candidates) if (existsSync(join(dir, "meta/_journal.json"))) return dir;
+	throw new Error(`drizzle migrations folder not found; checked: ${candidates.join(", ")}`);
+}
+const MIGRATIONS_DIR = resolveMigrationsDir();
+/** Implements init db as the local SQLite connection setup operation. */
+function initDb(dbPath, vectorDim) {
+	log.info("initializing database", {
+		dbPath,
+		vectorDim
+	}, {
+		event_name: "sno_station_mem.connection.initializing.database",
+		file: "packages/memory/src/store/connection.ts",
+		function: "initDb",
+		site_id: "connection.initDb.ee37a0a971"
+	});
+	mkdirSync(dirname(dbPath), { recursive: true });
+	const sqlite = openSqliteDatabase(dbPath);
+	const pending = /* @__PURE__ */ new Map();
+	const retrySetup = () => {
+		for (const [step, run] of pending) try {
+			run();
+			pending.delete(step);
+		} catch (error) {
+			log.error("storage.setup.failed", {
+				step,
+				error
+			}, {
+				event_name: "storage.setup.failed",
+				file: "packages/memory/src/store/connection.ts",
+				function: "retrySetup",
+				site_id: "storage.setup.failed"
+			});
+			break;
+		}
+	};
+	const db = Object.assign(drizzle(sqlite.raw, { schema: schema_exports }), {
+		vectorDimension: vectorDim,
+		vectorSearchAvailable: false,
+		chokepoint: sqlite.db,
+		retrySetup
+	});
+	for (const [step, run] of [
+		["extensions", () => loadStorageExtensions(sqlite)],
+		["pragmas", () => sqlite.db.exec("PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL; PRAGMA cache_size=-65536; PRAGMA temp_store=MEMORY; PRAGMA journal_size_limit=67108864;")],
+		["namespace", () => migrateLegacyDatabaseNamespace(sqlite.db, vectorDim)],
+		["rem-columns", () => ensureRemMigrationCompatibilityColumns(sqlite.db)],
+		["migrations", () => migrateCompat(db, { migrationsFolder: MIGRATIONS_DIR })],
+		["entity-keys", () => applyEntityNameKeyMigration(sqlite.db)],
+		["unplaced-candidates", () => migrateUnplacedCandidates(sqlite.db)],
+		["memory-indexes", () => ensureMemoryKindIndexes(sqlite.db)],
+		["vectors", () => {
+			const dimension = ensureChunkVecTable(sqlite.db, vectorDim);
+			db.vectorSearchAvailable = dimension !== void 0;
+			if (dimension !== void 0) db.vectorDimension = dimension;
+		}]
+	]) pending.set(step, run);
+	retrySetup();
+	return db;
+}
+/** Implements close db as the local SQLite connection setup operation. */
+function closeDb(db) {
+	db.$client.close();
+}
+/** Implements migrate compat as the local SQLite connection setup operation. */
+function migrateCompat(db, options) {
+	migrate(db, options);
+}
+function ensureRemMigrationCompatibilityColumns(database) {
+	ensureColumn(database, "nodix_rem_scan_pairs", "budget_invocation_id", "TEXT");
+	ensureColumn(database, "nodix_rem_scan_pairs", "attempt_count", "INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0)");
+	ensureColumn(database, "nodix_rem_scan_pairs", "progress_state", "TEXT NOT NULL DEFAULT 'pending' CHECK (progress_state IN ('pending', 'refused', 'closed', 'exhausted'))");
+	ensureColumn(database, "nodix_rem_scan_pairs", "refusal_reason", "TEXT");
+	ensureColumn(database, "nodix_rem_scan_pairs", "inherited_from_generation_id", "TEXT");
+}
+function ensureColumn(database, tableName, columnName, declaration) {
+	if (database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName) === void 0) return;
+	if (database.prepare(`PRAGMA table_info(${tableName})`).all().some(({ name }) => name === columnName)) return;
+	database.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${declaration}`);
+}
+//#endregion
+export { parseVecTableDimension as a, vecTableDdlHasPartitionKey as c, loadStorageExtensions as i, validateStoragePath as l, closeDb as n, readChunkVecTableDimension as o, initDb as r, readChunkVecTableState as s, assertNoLegacyBoundarySchema as t };