@mulmoclaude/core 0.25.0 → 0.26.0

package/dist/{discovery-3K_Wtkvn.js → discovery-QUi_7aNG.js}

@@ -1,9 +1,12 @@
-import { a as AGENT_INGEST_KIND, c as INGEST_KINDS, i as isSafeSlug, n as SAFE_SLUG_PATTERN, o as COMPUTED_TYPES, r as isSafeRecordId, s as FEED_SCHEDULES, t as SAFE_RECORD_ID_PATTERN } from "./ids-D4e10971.js";
+import { a as AGENT_INGEST_KIND, c as INGEST_KINDS, d as isReadOnlySchema, f as storageKindFor, i as isSafeSlug, n as SAFE_SLUG_PATTERN, o as COMPUTED_TYPES, r as isSafeRecordId, s as FEED_SCHEDULES, t as SAFE_RECORD_ID_PATTERN } from "./ids-P2T89Gou.js";
+import { t as projectRecordFields } from "./project-bU98ycsy.js";
 import { n as isSafeCustomViewI18nPath, r as isSafeCustomViewPath, t as isSafeActionTemplatePath } from "./templatePath-k_WNbL_Q.js";
 import path from "node:path";
 import { promises, realpathSync } from "node:fs";
-import { lstat, mkdir, open, readFile, readdir, stat, unlink } from "node:fs/promises";
-import { randomBytes } from "node:crypto";
+import { lstat, mkdir, open, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
+import { createHash, randomBytes } from "node:crypto";
+import { tmpdir } from "node:os";
+import iconv from "iconv-lite";
 import { z } from "zod";
 //#region src/collection/server/host.ts
 var current = null;
@@ -633,6 +636,843 @@ ${dataJson}
 
 ${templateText}`;
 }
+//#endregion
+//#region src/collection/core/queryZ.ts
+/** Result-column aliases double as SQL identifiers and JSON keys — keep
+*  them to a conservative identifier charset so neither side needs
+*  escaping gymnastics. */
+var SAFE_ALIAS_PATTERN = /^[A-Za-z_]\w{0,63}$/;
+/** Hard ceiling on returned rows; `limit` clamps below it. A group-by on
+*  a near-unique column would otherwise return one row per source row —
+*  the exact materialization the aggregate path exists to avoid. */
+var MAX_QUERY_ROWS = 1e4;
+/** Default row cap when the query declares no `limit`. */
+var DEFAULT_QUERY_ROWS = 1e3;
+/** One aggregate column: `count` (rows; `column` optional to count
+*  non-null cells) or `sum`/`avg`/`min`/`max` over a named CSV column. */
+var QueryAggregateZ = z.object({
+	op: z.enum([
+		"count",
+		"sum",
+		"avg",
+		"min",
+		"max"
+	]),
+	column: z.string().min(1).optional()
+}).refine((aggregate) => aggregate.op === "count" || aggregate.column !== void 0, {
+	message: "`column` is required for every aggregate op except `count`",
+	path: ["column"]
+});
+/** One filter condition. Same op vocabulary as the schema-level `where`
+*  (`core/where.ts`) so authors learn one set; values may be typed
+*  (number / boolean) since CSV columns are. `in` requires an array
+*  value, every other op a scalar. */
+var QueryWhereZ = z.object({
+	field: z.string().min(1),
+	op: z.enum([
+		"eq",
+		"ne",
+		"in",
+		"gt",
+		"gte",
+		"lt",
+		"lte",
+		"contains"
+	]),
+	value: z.union([
+		z.string(),
+		z.number(),
+		z.boolean(),
+		z.array(z.union([
+			z.string(),
+			z.number(),
+			z.boolean()
+		])).min(1).max(100)
+	])
+}).refine((cond) => cond.op === "in" === Array.isArray(cond.value), {
+	message: "`in` requires an array value (the allowed set); every other op requires a scalar value",
+	path: ["value"]
+});
+var QueryOrderZ = z.object({
+	/** A `groupBy` column or an aggregate alias — membership enforced by
+	*  the whole-query refine below. */
+	field: z.string().min(1),
+	dir: z.enum(["asc", "desc"]).optional()
+});
+/** The whole query. At least one of `groupBy` / `aggregates` must be
+*  present: bare `groupBy` is a DISTINCT listing, bare `aggregates` a
+*  whole-file scalar row, together a grouped aggregation. */
+var CollectionQueryZ = z.object({
+	groupBy: z.array(z.string().min(1)).max(8).refine((columns) => new Set(columns.map((column) => column.toLowerCase())).size === columns.length, { message: "`groupBy` columns must be unique (case-insensitively — SQL identifiers ignore case)" }).optional(),
+	aggregates: z.record(z.string().regex(SAFE_ALIAS_PATTERN, "aggregate aliases must be simple identifiers (letters/digits/underscore)"), QueryAggregateZ).optional(),
+	where: z.array(QueryWhereZ).max(16).optional(),
+	orderBy: z.array(QueryOrderZ).max(4).optional(),
+	limit: z.number().int().min(1).max(MAX_QUERY_ROWS).optional()
+}).refine((query) => (query.groupBy?.length ?? 0) > 0 || Object.keys(query.aggregates ?? {}).length > 0, {
+	message: "declare at least one of `groupBy` (columns to bucket by) or `aggregates` (values to compute)",
+	path: ["groupBy"]
+}).refine((query) => Object.keys(query.aggregates ?? {}).length <= 32, {
+	message: `\`aggregates\` supports at most 32 entries`,
+	path: ["aggregates"]
+}).refine((query) => {
+	const groupLower = new Set((query.groupBy ?? []).map((column) => column.toLowerCase()));
+	const seen = /* @__PURE__ */ new Set();
+	return Object.keys(query.aggregates ?? {}).every((alias) => {
+		const lower = alias.toLowerCase();
+		if (groupLower.has(lower) || seen.has(lower)) return false;
+		seen.add(lower);
+		return true;
+	});
+}, {
+	message: "aggregate aliases must be unique and must not collide with `groupBy` column names (case-insensitively — SQL identifiers ignore case)",
+	path: ["aggregates"]
+}).refine((query) => {
+	const sortable = /* @__PURE__ */ new Set([...query.groupBy ?? [], ...Object.keys(query.aggregates ?? {})]);
+	return (query.orderBy ?? []).every((order) => sortable.has(order.field));
+}, {
+	message: "every `orderBy.field` must be a `groupBy` column or an aggregate alias",
+	path: ["orderBy"]
+});
+//#endregion
+//#region src/collection/server/csvQuery.ts
+/** Double-quote a SQL identifier (CSV column name / result alias). */
+function quoteIdent(name) {
+	return `"${name.replaceAll("\"", "\"\"")}"`;
+}
+/** Single-quote a SQL string literal (a `types={...}` struct key). */
+function quoteLiteral(value) {
+	return `'${value.replaceAll("'", "''")}'`;
+}
+/** The `read_csv` argument list shared by every CSV query: the (prepared)
+*  path plus a `types` pin forcing the key column to VARCHAR — without it
+*  DuckDB's sniffer turns `001` into BIGINT 1, so leading zeros vanish
+*  and distinct keys collapse. */
+function readCsvArgs(primaryKey) {
+	return `?, types={${quoteLiteral(primaryKey)}: 'VARCHAR'}`;
+}
+/** One aggregate's SQL expression. `sum`/`avg` TRY_CAST to DOUBLE so a
+*  column the sniffer kept as VARCHAR (mixed values) aggregates over its
+*  numeric cells instead of erroring; non-numeric cells become NULL and
+*  are skipped — standard BI tolerance. `min`/`max` stay native (they are
+*  meaningful on strings and dates too). */
+function aggregateExpr(aggregate) {
+	const { op, column } = aggregate;
+	if (op === "count") return column === void 0 ? "count(*)" : `count(${quoteIdent(column)})`;
+	if (op === "sum" || op === "avg") return `${op}(TRY_CAST(${quoteIdent(column ?? "")} AS DOUBLE))`;
+	return `${op}(${quoteIdent(column ?? "")})`;
+}
+/** One where condition → SQL fragment + its bound parameters. String
+*  equality compares against `CAST(col AS VARCHAR)` so a sniffer-typed
+*  column still matches its textual value; numeric/boolean values compare
+*  natively (DuckDB coerces the column side). */
+function whereFragment(cond) {
+	const column = quoteIdent(cond.field);
+	const asText = `CAST(${column} AS VARCHAR)`;
+	if (cond.op === "in") {
+		const values = cond.value;
+		return {
+			sql: `${values.every((value) => typeof value === "string") ? asText : column} IN (${values.map(() => "?").join(", ")})`,
+			params: values
+		};
+	}
+	if (cond.op === "contains") return {
+		sql: `contains(${asText}, ?)`,
+		params: [String(cond.value)]
+	};
+	const operator = {
+		eq: "=",
+		ne: "<>",
+		gt: ">",
+		gte: ">=",
+		lt: "<",
+		lte: "<="
+	}[cond.op];
+	return {
+		sql: `${typeof cond.value === "string" && (cond.op === "eq" || cond.op === "ne") ? asText : column} ${operator} ?`,
+		params: [cond.value]
+	};
+}
+/** Compile a validated query against `fromSql` (a table-function call
+*  whose FIRST placeholder is the source path — the executor binds it).
+*  Returns the SQL and the where-value parameters that follow the path.
+*  Callers MUST have run `CollectionQueryZ` first; this function trusts
+*  the shape (aliases already charset-checked, orderBy membership already
+*  enforced). */
+function compileQuery(query, fromSql) {
+	const groupBy = query.groupBy ?? [];
+	const aggregates = Object.entries(query.aggregates ?? {});
+	const selectList = [...groupBy.map(quoteIdent), ...aggregates.map(([alias, aggregate]) => `${aggregateExpr(aggregate)} AS ${quoteIdent(alias)}`)];
+	const where = (query.where ?? []).map(whereFragment);
+	const clauses = [`SELECT ${selectList.join(", ")}`, `FROM ${fromSql}`];
+	if (where.length > 0) clauses.push(`WHERE ${where.map((fragment) => fragment.sql).join(" AND ")}`);
+	if (groupBy.length > 0) clauses.push(`GROUP BY ${groupBy.map(quoteIdent).join(", ")}`);
+	const orderBy = (query.orderBy ?? []).map((order) => quoteIdent(order.field) + (order.dir === "desc" ? " DESC" : " ASC"));
+	if (orderBy.length > 0) clauses.push(`ORDER BY ${orderBy.join(", ")}`);
+	clauses.push(`LIMIT ${query.limit ?? 1e3}`);
+	return {
+		sql: clauses.join(" "),
+		params: where.flatMap((fragment) => fragment.params)
+	};
+}
+/** Compile against a CSV file (the dataSource store's engine). */
+function compileCsvQuery(query, primaryKey) {
+	return compileQuery(query, `read_csv(${readCsvArgs(primaryKey)})`);
+}
+/** Compile against a JSONL file of ENRICHED records — the file-backed
+*  collections' engine (see `jsonlQuery.ts`). No VARCHAR key pin needed:
+*  enriched record ids are already strings. `sample_size=-1` makes the
+*  schema inference scan EVERY line — with the default sample, a sparse
+*  optional/derived field first appearing past the sample would not be
+*  inferred as a column and the query would binder-error on it (Codex P2
+*  on #2165). The full scan costs nothing extra here: aggregation reads
+*  the whole file anyway. */
+function compileJsonlQuery(query) {
+	return compileQuery(query, `read_json(?, format='newline_delimited', sample_size=-1)`);
+}
+//#endregion
+//#region src/collection/server/csvStore.ts
+/** `list()` row cap. Over-cap files are truncated with a warn — the v1
+*  contract is "browse + per-record views", not full-table analytics. */
+var MAX_CSV_ROWS = 5e3;
+/** Record ids minted from non-safe key values: `id0x` + utf-8 hex. Raw key
+*  values that themselves match this pattern are ALSO encoded, so the
+*  encoded namespace never collides with a raw value (injective mapping). */
+var ENCODED_ID_PATTERN = /^id0x([0-9a-f]+)$/;
+/** A CSV key value → the record id it's addressed by. Safe values pass
+*  through untouched; everything else (and anything shaped like an encoded
+*  id) becomes `id0x<hex>`. Pure + exported for unit tests. */
+function encodeCsvRecordId(rawKey) {
+	if (safeRecordId(rawKey) === rawKey && !ENCODED_ID_PATTERN.test(rawKey)) return rawKey;
+	return `id0x${Buffer.from(rawKey, "utf-8").toString("hex")}`;
+}
+/** A record id → the CSV key value to look up. Inverse of
+*  `encodeCsvRecordId` for encoded ids; anything else is already the raw
+*  value. Pure + exported for unit tests. */
+function decodeCsvRecordId(itemId) {
+	const match = ENCODED_ID_PATTERN.exec(itemId);
+	if (!match) return itemId;
+	return Buffer.from(match[1], "hex").toString("utf-8");
+}
+/** Normalize one DuckDB JS value into a JSON-safe record value: BigInt →
+*  number (string beyond the safe range), DATE/TIMESTAMP → ISO string
+*  (date-only when the clock is exactly UTC midnight, matching the `date`
+*  field contract), exotic DuckDB values → their string form. Pure +
+*  exported for unit tests. */
+function normalizeCsvValue(value) {
+	if (typeof value === "bigint") return value <= BigInt(Number.MAX_SAFE_INTEGER) && value >= BigInt(-Number.MAX_SAFE_INTEGER) ? Number(value) : value.toString();
+	if (value instanceof Date) {
+		const iso = value.toISOString();
+		return iso.endsWith("T00:00:00.000Z") ? iso.slice(0, 10) : iso;
+	}
+	if (value !== null && typeof value === "object") return String(value);
+	return value;
+}
+/** One raw DuckDB row → a CollectionItem, or null when the key cell is
+*  missing/empty (the row can't be addressed). The primaryKey field is
+*  OVERWRITTEN with the (possibly encoded) record id so `item[primaryKey]`
+*  and the record's address never drift — same invariant the file store's
+*  write path enforces. Pure + exported for unit tests. */
+function csvRowToItem(row, primaryKey) {
+	const normalized = Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)]));
+	const rawKey = normalized[primaryKey];
+	if (rawKey === null || rawKey === void 0 || String(rawKey) === "") return null;
+	return {
+		...normalized,
+		[primaryKey]: encodeCsvRecordId(String(rawKey))
+	};
+}
+/** Dedupe by record id, LAST row wins (matches `csvRead`'s last-match
+*  pick). Returns the surviving items in first-seen order. Pure +
+*  exported for unit tests. */
+function dedupeByRecordId(items, primaryKey) {
+	const byId = /* @__PURE__ */ new Map();
+	for (const item of items) byId.set(String(item[primaryKey]), item);
+	return {
+		items: [...byId.values()],
+		duplicates: items.length - byId.size
+	};
+}
+/** True when a thrown DuckDB error is the `types` pin naming a column the
+*  CSV doesn't have — the schema/file-mismatch case the caller downgrades
+*  to "empty collection + warn" instead of a 500. */
+function isMissingKeyColumnError(err) {
+	return String(err).includes("do not exist in the CSV");
+}
+/** Bytes sniffed for UTF-8 validity. The trailing 3 bytes of the sample
+*  are dropped so a multibyte char split at the boundary can't produce a
+*  false negative on a valid file. */
+var SNIFF_BYTES = 1024 * 1024;
+function isValidUtf8(buf) {
+	try {
+		new TextDecoder("utf-8", { fatal: true }).decode(buf);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/** Detect the (best-effort) encoding of a non-UTF-8 buffer. BOMs decide
+*  UTF-16; otherwise cp932 (the Shift_JIS superset — Excel-exported
+*  Japanese CSVs are the primary non-UTF-8 case this feature serves). */
+function fallbackEncoding(buf) {
+	if (buf.length >= 2 && buf[0] === 255 && buf[1] === 254) return "utf-16le";
+	if (buf.length >= 2 && buf[0] === 254 && buf[1] === 255) return "utf-16be";
+	return "cp932";
+}
+function cacheDir() {
+	return path.join(tmpdir(), "mulmoclaude-csv-utf8");
+}
+/** Read only the first `bytes` of a file — the encoding sniff must not
+*  pull a multi-hundred-MB CSV into memory on the (common) UTF-8 path. */
+async function readHead(absPath, bytes) {
+	const handle = await open(absPath, "r");
+	try {
+		const { size } = await handle.stat();
+		const buf = Buffer.alloc(Math.min(bytes, size));
+		await handle.read(buf, 0, buf.length, 0);
+		return buf;
+	} finally {
+		await handle.close();
+	}
+}
+/** Decode the whole file into a UTF-8 cache copy and return its path.
+*  Cache key = (path, mtime, size), so a replaced CSV re-decodes and an
+*  unchanged one never does. */
+async function pathExists(target) {
+	try {
+		await stat(target);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/** Best-effort removal of older decode-cache entries for the same source
+*  path — a frequently-replaced large CSV would otherwise accumulate one
+*  full copy per (mtime, size) forever. Runs AFTER the current copy is
+*  published; a concurrent reader holding an old fd is unaffected
+*  (unlink-while-open is safe on POSIX). */
+async function evictSupersededCache(key, keepBasename) {
+	try {
+		const entries = await readdir(cacheDir());
+		await Promise.all(entries.filter((name) => name.startsWith(`${key}-`) && name !== keepBasename).map((name) => unlink(path.join(cacheDir(), name)).catch(() => void 0)));
+	} catch {}
+}
+/** Decode the whole file into a UTF-8 cache copy and return its path.
+*  Cache key = (path, mtime, size), so a replaced CSV re-decodes and an
+*  unchanged one never does; superseded copies are evicted. The cache
+*  lives in the SHARED OS tmpdir, so the dir is 0700 and files 0600 —
+*  decoded rows must not be readable by other local users. */
+async function decodeToCache(absPath, info) {
+	const key = createHash("sha256").update(absPath).digest("hex").slice(0, 16);
+	const cached = path.join(cacheDir(), `${key}-${Math.trunc(info.mtimeMs)}-${info.size}.csv`);
+	if (!await pathExists(cached)) {
+		const whole = await readFile(absPath);
+		const encoding = fallbackEncoding(whole);
+		const text = iconv.decode(whole, encoding);
+		await mkdir(cacheDir(), {
+			recursive: true,
+			mode: 448
+		});
+		const tmp = `${cached}.${randomBytes(4).toString("hex")}.tmp`;
+		await writeFile(tmp, text, {
+			encoding: "utf-8",
+			mode: 384
+		});
+		await rename(tmp, cached);
+		log.info("collections", "decoded non-UTF-8 dataSource file to cache", {
+			path: absPath,
+			encoding
+		});
+		await evictSupersededCache(key, path.basename(cached));
+	}
+	return cached;
+}
+/** Re-validate the dataSource file at READ time, mirroring the JSON
+*  store's per-read defenses: realpath containment (a symlink swapped in
+*  after discovery must not walk out of the workspace) and an lstat
+*  regular-file check (a symlink leaf is refused outright, even one
+*  pointing inside the workspace — same rule as `isRegularFile` on
+*  record files). Returns the stat info, or null for "no readable file"
+*  (ENOENT / refused), which callers render as an empty collection. */
+async function safeCsvStat(absPath, workspaceRoot) {
+	if (!isContainedInRoot(absPath, workspaceRoot)) {
+		log.warn("collections", "dataSource read refused: path escapes workspace", { path: absPath });
+		return null;
+	}
+	let info;
+	try {
+		info = await lstat(absPath);
+	} catch (err) {
+		if (err.code === "ENOENT") return null;
+		throw err;
+	}
+	if (!info.isFile()) {
+		log.warn("collections", "dataSource read refused: not a regular file (symlink?)", { path: absPath });
+		return null;
+	}
+	return info;
+}
+/** Return a path DuckDB can read as UTF-8: the original file when it
+*  already is UTF-8 (the cheap, common case — only the head is sniffed),
+*  else a decoded cache copy (see `decodeToCache`). Returns null when
+*  there is no readable file (missing, symlink, or containment-refused —
+*  see `safeCsvStat`), which callers render as an empty collection. */
+async function ensureUtf8CsvPath(absPath, workspaceRoot) {
+	const info = await safeCsvStat(absPath, workspaceRoot);
+	if (info === null) return null;
+	const head = await readHead(absPath, SNIFF_BYTES);
+	const sample = head.length === SNIFF_BYTES ? head.subarray(0, SNIFF_BYTES - 3) : head;
+	if (!(head.length >= 2 && (head[0] === 255 && head[1] === 254 || head[0] === 254 && head[1] === 255)) && isValidUtf8(sample)) return absPath;
+	return decodeToCache(absPath, info);
+}
+var instancePromise = null;
+/** Lazily create one shared in-memory DuckDB instance. The dynamic import
+*  keeps the native module OUT of core's load path — a platform where the
+*  prebuilt binding is missing degrades to a per-query error on dataSource
+*  collections only, never a broken core. A failed init is retried on the
+*  next call (the promise is reset). */
+async function duckDbInstance() {
+	if (instancePromise === null) instancePromise = import("@duckdb/node-api").then((mod) => mod.DuckDBInstance.create(":memory:"));
+	try {
+		return await instancePromise;
+	} catch (err) {
+		instancePromise = null;
+		throw new Error(`DuckDB is unavailable on this host (@duckdb/node-api failed to load: ${String(err)}) — dataSource collections cannot be read`);
+	}
+}
+async function queryCsv(sql, params) {
+	const connection = await (await duckDbInstance()).connect();
+	try {
+		return (await connection.runAndReadAll(sql, params)).getRowObjectsJS();
+	} finally {
+		connection.disconnectSync();
+	}
+}
+async function csvList(absPath, primaryKey, workspaceRoot) {
+	const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
+	if (utf8Path === null) return {
+		items: [],
+		truncated: false
+	};
+	let rows;
+	try {
+		rows = await queryCsv(`SELECT * FROM read_csv(${readCsvArgs(primaryKey)}) LIMIT 5001`, [utf8Path]);
+	} catch (err) {
+		if (!isMissingKeyColumnError(err)) throw err;
+		log.warn("collections", "dataSource CSV has no primaryKey column — every row is skipped", {
+			path: absPath,
+			primaryKey
+		});
+		return {
+			items: [],
+			truncated: false
+		};
+	}
+	const truncated = rows.length > MAX_CSV_ROWS;
+	if (truncated) {
+		log.warn("collections", "dataSource CSV truncated to row cap", {
+			path: absPath,
+			cap: MAX_CSV_ROWS
+		});
+		rows.length = MAX_CSV_ROWS;
+	}
+	const items = rows.map((row) => csvRowToItem(row, primaryKey)).filter((item) => item !== null);
+	const skipped = rows.length - items.length;
+	if (skipped > 0) log.warn("collections", "dataSource CSV rows skipped (empty key cell)", {
+		path: absPath,
+		skipped
+	});
+	const deduped = dedupeByRecordId(items, primaryKey);
+	if (deduped.duplicates > 0) log.warn("collections", "dataSource CSV has duplicate key values (last row wins)", {
+		path: absPath,
+		duplicates: deduped.duplicates
+	});
+	return {
+		items: deduped.items,
+		truncated
+	};
+}
+/** The scan-order ordinal column the last-match read adds. Underscore
+*  prefix keeps it out of any plausible CSV header namespace; it is
+*  stripped from the returned record either way. */
+var ROW_ORDINAL = "__mc_row";
+/** One record by id. The comparison value rides as a prepared-statement
+*  parameter, and the LAST matching row is selected IN DuckDB (scan-order
+*  ordinal + LIMIT 1) — a CSV with thousands of duplicate keys must not
+*  materialize them all for one detail read. Consistent with csvList's
+*  last-wins dedupe. */
+async function csvRead(absPath, primaryKey, itemId, workspaceRoot) {
+	const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
+	if (utf8Path === null) return null;
+	const rawKey = decodeCsvRecordId(itemId);
+	const last = (await queryCsv(`SELECT * FROM (SELECT *, row_number() OVER () AS ${quoteIdent(ROW_ORDINAL)} FROM read_csv(${readCsvArgs(primaryKey)})) WHERE CAST(${quoteIdent(primaryKey)} AS VARCHAR) = ? ORDER BY ${quoteIdent(ROW_ORDINAL)} DESC LIMIT 1`, [utf8Path, rawKey])).at(0);
+	if (last === void 0) return null;
+	const { [ROW_ORDINAL]: __ordinal, ...record } = last;
+	return csvRowToItem(record, primaryKey);
+}
+/** Run a validated aggregation query (the structured DSL — see
+*  `core/queryZ.ts`) over the WHOLE file: no row cap on the scan (a
+*  capped aggregate would be a wrong number), only the result-row LIMIT
+*  the compiler emits. Values are normalized like list/read rows so a
+*  chart consumer gets plain JSON scalars. */
+async function csvRunQuery(absPath, primaryKey, query, workspaceRoot) {
+	const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
+	if (utf8Path === null) return [];
+	const { sql, params } = compileCsvQuery(query, primaryKey);
+	return (await queryCsv(sql, [utf8Path, ...params])).map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)])));
+}
+//#endregion
+//#region src/collection/server/storePage.ts
+/** Project `fields` (+ the primary key, always) out of each record. Thin
+*  server-typed alias over the shared isomorphic `projectRecordFields`
+*  (../core/project.ts) — kept as the store layer's exported name. */
+function projectItemFields(items, fields, primaryKey) {
+	return projectRecordFields(items, fields, primaryKey);
+}
+/** Slice + project an already-ordered full read into a `ListPage` — the
+*  shared emulation for stores without native paging. Pure, exported for
+*  tests. `limit: 0` is a valid "count only" page. */
+function pageFromFullRead(items, opts, primaryKey, truncated) {
+	const offset = Math.max(0, opts.offset ?? 0);
+	const end = opts.limit === void 0 ? void 0 : offset + Math.max(0, opts.limit);
+	return {
+		items: projectItemFields(items.slice(offset, end), opts.fields, primaryKey),
+		total: items.length,
+		truncated
+	};
+}
+//#endregion
+//#region src/collection/server/sqliteStore.ts
+var sqliteModule = null;
+/** Lazy-load node:sqlite once. A runtime without it (Node < 22.5) throws a
+*  clearly-worded error the caller surfaces — never a bare MODULE_NOT_FOUND. */
+function loadSqlite() {
+	sqliteModule ??= import("node:sqlite").then((mod) => mod, (err) => {
+		sqliteModule = null;
+		throw new Error(`sqlite storage needs the node:sqlite module (Node.js >= 22.5) — this runtime cannot load it: ${String(err)}`);
+	});
+	return sqliteModule;
+}
+/** The db file's on-disk state. A symlink or non-regular file is refused
+*  (file-disclosure defense, same rule as io.ts record files); ENOENT is
+*  just "no records yet". Any OTHER lstat failure (EACCES, EIO, …) is
+*  rethrown so reads surface a real filesystem problem instead of
+*  silently reporting an empty collection. */
+async function dbFileState(absPath) {
+	try {
+		return (await lstat(absPath)).isFile() ? "file" : "refused";
+	} catch (err) {
+		if (err.code === "ENOENT") return "missing";
+		throw err;
+	}
+}
+var CREATE_TABLE = "CREATE TABLE IF NOT EXISTS records (id TEXT PRIMARY KEY, record TEXT NOT NULL)";
+/** Open the database for one operation, classifying the two unavailable
+*  states so callers can map them honestly (`refused` ⇒ path-escape,
+*  `missing` ⇒ empty / not-found — conflating them would misreport a
+*  containment escape as "item not found"). The containment pre-check runs
+*  BEFORE mkdir even when the file is missing — `isContainedInRoot`
+*  resolves through the closest existing ancestor, so a symlinked-away
+*  parent can never make the recursive mkdir create directories outside
+*  the workspace (same pre/post belt-and-suspenders as io.ts writes). */
+async function openDb(absPath, workspaceRoot, mode) {
+	const state = await dbFileState(absPath);
+	if (state === "refused") {
+		log.warn("collections", "sqlite database refused: not a regular file", { path: absPath });
+		return { kind: "refused" };
+	}
+	if (!isContainedInRoot(path.dirname(absPath), workspaceRoot)) {
+		log.warn("collections", "sqlite refused: database dir escapes workspace via symlink", { path: absPath });
+		return { kind: "refused" };
+	}
+	if (mode === "read" && state === "missing") return { kind: "missing" };
+	if (mode === "write") {
+		await mkdir(path.dirname(absPath), { recursive: true });
+		if (!isContainedInRoot(path.dirname(absPath), workspaceRoot)) {
+			log.warn("collections", "sqlite write refused: database dir escapes workspace via symlink (post-mkdir)", { path: absPath });
+			return { kind: "refused" };
+		}
+	}
+	const { DatabaseSync } = await loadSqlite();
+	const database = new DatabaseSync(absPath);
+	database.exec("PRAGMA busy_timeout = 5000");
+	database.exec(CREATE_TABLE);
+	return {
+		kind: "ok",
+		database
+	};
+}
+/** Run `operation` against the database and always close it; unavailable
+*  states resolve through `onUnavailable` so each caller maps `missing`
+*  vs `refused` to its own result kind. */
+async function withDb(absPath, workspaceRoot, mode, onUnavailable, operation) {
+	const handle = await openDb(absPath, workspaceRoot, mode);
+	if (handle.kind !== "ok") return onUnavailable(handle.kind);
+	try {
+		return await operation(handle.database);
+	} finally {
+		handle.database.close();
+	}
+}
+/** node:sqlite throws ERR_SQLITE_ERROR with the SQLite extended result
+*  code on `errcode`. Our INSERT's duplicate-id failure is
+*  SQLITE_CONSTRAINT_PRIMARYKEY (1555); SQLITE_CONSTRAINT_UNIQUE (2067)
+*  covers a plain UNIQUE index. Checked structurally (message text kept
+*  only as a fallback for runtimes that don't expose `errcode`). */
+function isUniqueConstraintError(err) {
+	const { errcode } = err;
+	if (errcode !== void 0) return errcode === 1555 || errcode === 2067;
+	return String(err).includes("UNIQUE constraint");
+}
+function parseRow(raw) {
+	if (typeof raw !== "string") return null;
+	try {
+		const parsed = JSON.parse(raw);
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
+	} catch {
+		return null;
+	}
+}
+function rowsToItems(rows) {
+	return rows.map((row) => parseRow(row.record)).filter((item) => item !== null);
+}
+async function sqliteList(absPath, workspaceRoot) {
+	return withDb(absPath, workspaceRoot, "read", () => [], (database) => rowsToItems(database.prepare("SELECT record FROM records ORDER BY id").all()));
+}
+async function sqlitePage(absPath, primaryKey, opts, workspaceRoot) {
+	const emptyPage = {
+		items: [],
+		total: 0,
+		truncated: false
+	};
+	return withDb(absPath, workspaceRoot, "read", () => emptyPage, (database) => {
+		const total = Number(database.prepare("SELECT COUNT(*) AS n FROM records").get().n);
+		const offset = Math.max(0, opts.offset ?? 0);
+		const limit = opts.limit === void 0 ? -1 : Math.max(0, opts.limit);
+		return {
+			items: projectItemFields(rowsToItems(database.prepare("SELECT record FROM records ORDER BY id LIMIT ? OFFSET ?").all(limit, offset)), opts.fields, primaryKey),
+			total,
+			truncated: false
+		};
+	});
+}
+async function sqliteRead(absPath, itemId, workspaceRoot) {
+	const safeId = safeRecordId(itemId);
+	if (safeId === null) return null;
+	return withDb(absPath, workspaceRoot, "read", () => null, (database) => {
+		const row = database.prepare("SELECT record FROM records WHERE id = ?").get(safeId);
+		return row === void 0 ? null : parseRow(row.record);
+	});
+}
+async function sqliteWrite(absPath, itemId, item, opts) {
+	const safeId = safeRecordId(itemId);
+	if (safeId === null) return {
+		kind: "invalid-id",
+		itemId
+	};
+	const outcome = await withDb(absPath, opts.workspaceRoot, "write", () => ({
+		kind: "path-escape",
+		itemId: safeId
+	}), (database) => {
+		const payload = JSON.stringify(item);
+		if (opts.refuseOverwrite) try {
+			database.prepare("INSERT INTO records (id, record) VALUES (?, ?)").run(safeId, payload);
+		} catch (err) {
+			if (isUniqueConstraintError(err)) return {
+				kind: "conflict",
+				itemId: safeId
+			};
+			throw err;
+		}
+		else database.prepare("INSERT INTO records (id, record) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET record = excluded.record").run(safeId, payload);
+		return {
+			kind: "ok",
+			itemId: safeId,
+			item
+		};
+	});
+	if (outcome.kind === "ok" && opts.slug) publishCollectionChange({
+		slug: opts.slug,
+		ids: [safeId],
+		op: "upsert"
+	});
+	return outcome;
+}
+async function sqliteDelete(absPath, itemId, opts) {
+	const safeId = safeRecordId(itemId);
+	if (safeId === null) return {
+		kind: "invalid-id",
+		itemId
+	};
+	const outcome = await withDb(absPath, opts.workspaceRoot, "read", (reason) => reason === "refused" ? {
+		kind: "path-escape",
+		itemId: safeId
+	} : {
+		kind: "not-found",
+		itemId: safeId
+	}, (database) => {
+		const { changes } = database.prepare("DELETE FROM records WHERE id = ?").run(safeId);
+		return Number(changes) === 0 ? {
+			kind: "not-found",
+			itemId: safeId
+		} : {
+			kind: "ok",
+			itemId: safeId
+		};
+	});
+	if (outcome.kind === "ok" && opts.slug) publishCollectionChange({
+		slug: opts.slug,
+		ids: [safeId],
+		op: "delete"
+	});
+	return outcome;
+}
+/** Best-effort full WAL checkpoint so the MAIN db file alone is a
+*  complete snapshot (committed pages in `<db>-wal` are folded in and the
+*  WAL truncated). Used by `deleteCollection` before archiving. Returns
+*  false on any failure (runtime without node:sqlite, locked db, missing
+*  file) — the caller then archives the sidecar files alongside the db so
+*  no committed data is lost either way. */
+async function checkpointSqliteDatabase(absPath) {
+	try {
+		const { DatabaseSync } = await loadSqlite();
+		const database = new DatabaseSync(absPath);
+		try {
+			database.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+		} finally {
+			database.close();
+		}
+		return true;
+	} catch {
+		return false;
+	}
+}
+/** A `storage: sqlite` store over `collection.storageFile`. A schema whose
+*  `storageFile` failed to resolve yields a read-only EMPTY store rather
+*  than a writable one — same fail-closed rule as the CSV store. */
+function sqliteStoreFor(collection, opts) {
+	const file = collection.storageFile;
+	const key = collection.schema.primaryKey;
+	const slug = opts.slug ?? collection.slug;
+	const root = () => opts.workspaceRoot ?? getWorkspaceRoot();
+	if (file === void 0) return {
+		capabilities: {
+			writable: false,
+			nativeQuery: false,
+			nativePaging: false
+		},
+		list: () => Promise.resolve([]),
+		page: () => Promise.resolve({
+			items: [],
+			total: 0,
+			truncated: false
+		}),
+		read: () => Promise.resolve(null)
+	};
+	return {
+		capabilities: {
+			writable: true,
+			nativeQuery: false,
+			nativePaging: true
+		},
+		list: () => sqliteList(file, root()),
+		page: (pageOpts = {}) => sqlitePage(file, key, pageOpts, root()),
+		read: (itemId) => sqliteRead(file, itemId, root()),
+		write: (itemId, item, writeOpts = {}) => sqliteWrite(file, itemId, item, {
+			workspaceRoot: root(),
+			slug,
+			refuseOverwrite: writeOpts.refuseOverwrite
+		}),
+		delete: (itemId) => sqliteDelete(file, itemId, {
+			workspaceRoot: root(),
+			slug
+		})
+	};
+}
+//#endregion
+//#region src/collection/server/store.ts
+/** The file store's stable order: lexicographic by record id (codepoint
+*  compare — locale-independent). `listItems` returns readdir order, which
+*  is filesystem-dependent; paging needs determinism. */
+function sortByRecordId(items, primaryKey) {
+	return [...items].sort((left, right) => {
+		const leftId = String(left[primaryKey] ?? "");
+		const rightId = String(right[primaryKey] ?? "");
+		if (leftId < rightId) return -1;
+		return leftId > rightId ? 1 : 0;
+	});
+}
+/** True when the collection accepts UI/tool writes. A `dataSource`
+*  collection is read-only: updates happen by editing/replacing the
+*  data file itself. Every write entry point checks this BEFORE calling
+*  `writeItem`/`deleteItem` — server-enforced, not just UI-hidden. */
+function collectionWritable(collection) {
+	return !isReadOnlySchema(collection.schema);
+}
+/** The one-line refusal write paths surface (HTTP 405 / MCP error text). */
+function readOnlyRefusal(slug) {
+	return `collection '${slug}' is read-only (backed by an external dataSource) — update the data file itself instead`;
+}
+/** A `dataSource` store over `file` (CSV row order; DuckDB-native query).
+*  A schema whose `dataSourceFile` failed to resolve yields a read-only
+*  EMPTY store rather than falling back to the (writable) file store — a
+*  half-loaded read-only collection must never become writable. */
+function csvStoreFor(collection, opts) {
+	const file = collection.dataSourceFile;
+	const key = collection.schema.primaryKey;
+	const listAll = () => file === void 0 ? Promise.resolve({
+		items: [],
+		truncated: false
+	}) : csvList(file, key, opts.workspaceRoot);
+	return {
+		capabilities: {
+			writable: false,
+			nativeQuery: true,
+			nativePaging: false
+		},
+		list: () => listAll().then((result) => result.items),
+		page: (pageOpts = {}) => listAll().then((result) => pageFromFullRead(result.items, pageOpts, key, result.truncated)),
+		read: (itemId) => file === void 0 ? Promise.resolve(null) : csvRead(file, key, itemId, opts.workspaceRoot),
+		query: (query) => file === void 0 ? Promise.resolve([]) : csvRunQuery(file, key, query, opts.workspaceRoot)
+	};
+}
+/** The classic file store over `<dataDir>/<itemId>.json` records. */
+function fileStoreFor(collection, opts) {
+	const key = collection.schema.primaryKey;
+	const ioOpts = {
+		...opts,
+		slug: opts.slug ?? collection.slug
+	};
+	return {
+		capabilities: {
+			writable: true,
+			nativeQuery: false,
+			nativePaging: false
+		},
+		list: () => listItems(collection.dataDir, opts),
+		page: async (pageOpts = {}) => pageFromFullRead(sortByRecordId(await listItems(collection.dataDir, opts), key), pageOpts, key, false),
+		read: (itemId) => readItem(collection.dataDir, itemId, opts),
+		write: (itemId, item, writeOpts = {}) => writeItem(collection.dataDir, itemId, item, {
+			...ioOpts,
+			refuseOverwrite: writeOpts.refuseOverwrite
+		}),
+		delete: (itemId) => deleteItem(collection.dataDir, itemId, ioOpts)
+	};
+}
+var storeFactories = /* @__PURE__ */ new Map([
+	["file", fileStoreFor],
+	["csv", csvStoreFor],
+	["sqlite", sqliteStoreFor]
+]);
+/** Pick the store implementation for a discovered collection via the
+*  factory registry. An unknown kind cannot normally reach here (the
+*  schema's `StorageZ` union gates it), so the throw is a loud invariant
+*  breach, not a user-facing path. */
+function storeFor(collection, opts = {}) {
+	const kind = storageKindFor(collection.schema);
+	const factory = storeFactories.get(kind);
+	if (!factory) throw new Error(`no store factory registered for storage kind '${kind}'`);
+	return factory(collection, opts);
+}
 /** The param name a `set` value references, or null when the value is a
 *  literal (non-strings can never be references). A bare/empty prefix
 *  (`"$params."`) returns the empty string — the schema refine rejects
@@ -1208,11 +2048,24 @@ var DataSourceZ = z.object({
 	type: z.literal("csv"),
 	path: z.string().min(1)
 });
+/** Alternative WRITABLE storage backend for a collection's records —
+*  unlike `dataSource` (external read-only file), a `storage` collection
+*  behaves like a normal writable collection; only where the rows live
+*  changes. v1: `sqlite` — records in a single SQLite database file
+*  (`node:sqlite`, one JSON record per row keyed by the primaryKey).
+*  `path` is workspace-relative and containment-checked exactly like
+*  `dataPath`. The store factory registry (`server/store.ts`) picks the
+*  implementation by `type` (plans/refactor-storage-virtualization.md). */
+var StorageZ = z.object({
+	type: z.literal("sqlite"),
+	path: z.string().min(1)
+});
 var BareCollectionSchemaZ = z.object({
 	title: z.string().min(1),
 	icon: z.string().min(1),
 	dataPath: z.string().min(1).optional(),
 	dataSource: DataSourceZ.optional(),
+	storage: StorageZ.optional(),
 	primaryKey: z.string().min(1),
 	singleton: z.string().trim().min(1).optional(),
 	fields: z.record(z.string(), FieldSpecZ),
@@ -1233,8 +2086,12 @@ var BareCollectionSchemaZ = z.object({
 	ingest: IngestZ.optional(),
 	googleCalendar: GoogleCalendarSyncZ.optional(),
 	dynamicIcon: DynamicIconSpecZ.optional()
-}).refine((schema) => schema.dataPath !== void 0 !== (schema.dataSource !== void 0), {
-	message: "declare exactly one of `dataPath` (native JSON records) or `dataSource` (external read-only data file), never both or neither",
+}).refine((schema) => [
+	schema.dataPath,
+	schema.dataSource,
+	schema.storage
+].filter((declared) => declared !== void 0).length === 1, {
+	message: "declare exactly one of `dataPath` (native JSON records), `dataSource` (external read-only data file), or `storage` (alternative writable backend)",
 	path: ["dataPath"]
 }).refine((schema) => schema.dataSource === void 0 || schema.singleton === void 0 && schema.ingest === void 0 && schema.spawn === void 0 && schema.googleCalendar === void 0, {
 	message: "a `dataSource` collection is read-only — it cannot declare `singleton`, `ingest`, `spawn`, or `googleCalendar` (all of them write records)",
@@ -1485,6 +2342,23 @@ function acceptParsedSchema(schema, opts) {
 			dataSourceFile
 		};
 	}
+	if (schema.storage !== void 0) {
+		const storageFile = resolveDataDir(schema.storage.path, opts.workspaceRoot);
+		if (storageFile === null) return {
+			ok: false,
+			reason: `storage.path '${schema.storage.path}' escapes the workspace`
+		};
+		const dataDir = resolveDataDir(conventionalDataPath(opts.slug), opts.workspaceRoot);
+		if (dataDir === null) return {
+			ok: false,
+			reason: `slug '${opts.slug}' yields no workspace-contained data dir`
+		};
+		return {
+			ok: true,
+			dataDir,
+			storageFile
+		};
+	}
 	const dataDir = resolveDataDir(schema.dataPath ?? "", opts.workspaceRoot);
 	if (dataDir === null) return {
 		ok: false,
@@ -1549,6 +2423,7 @@ async function loadOneCollection(skillsRoot, slug, source, workspaceRoot) {
 		schema,
 		dataDir: acceptance.dataDir,
 		...acceptance.dataSourceFile !== void 0 ? { dataSourceFile: acceptance.dataSourceFile } : {},
+		...acceptance.storageFile !== void 0 ? { storageFile: acceptance.storageFile } : {},
 		skillDir: path.join(skillsRoot, safeName)
 	};
 }
@@ -1631,6 +2506,6 @@ function toDetail(collection) {
 	};
 }
 //#endregion
-export { archiveDir as A, isContainedInRoot as C, resolveTemplatePath as D, resolveDataDir as E, log as F, publishCollectionChange as I, setCollectionChangePublisher as L, configureCollectionHost as M, getWorkspaceRoot as N, safeRecordId as O, isPresetSlug as P, skillsStagingDir as R, SCHEMA_FILE as S, itemFilePath as T, readItem as _, toSummary as a, writeItem as b, buildActionSeedPrompt as c, generateItemId as d, isRegularFile as f, readCustomViewI18n as g, readCustomViewHtml as h, toDetail as i, collectionsRegistriesConfigPath as j, safeSlugName as k, buildCollectionActionSeedPrompt as l, promptPathsFor as m, discoverCollections as n, CollectionSchemaZ as o, listItems as p, loadCollection as r, resolveMutateSet as s, acceptParsedSchema as t, deleteItem as u, readSkillTemplate as v, isContainedInWorkspace as w, writeFileAtomic as x, resolveCreateItemId as y };
+export { isPresetSlug as $, generateItemId as A, writeFileAtomic as B, compileJsonlQuery as C, buildActionSeedPrompt as D, MAX_QUERY_ROWS as E, readCustomViewI18n as F, resolveDataDir as G, isContainedInRoot as H, readItem as I, safeSlugName as J, resolveTemplatePath as K, readSkillTemplate as L, listItems as M, promptPathsFor as N, buildCollectionActionSeedPrompt as O, readCustomViewHtml as P, getWorkspaceRoot as Q, resolveCreateItemId as R, compileCsvQuery as S, DEFAULT_QUERY_ROWS as T, isContainedInWorkspace as U, SCHEMA_FILE as V, itemFilePath as W, collectionsRegistriesConfigPath as X, archiveDir as Y, configureCollectionHost as Z, decodeCsvRecordId as _, toSummary as a, normalizeCsvValue as b, collectionWritable as c, checkpointSqliteDatabase as d, log as et, pageFromFullRead as f, csvRowToItem as g, cacheDir as h, toDetail as i, isRegularFile as j, deleteItem as k, readOnlyRefusal as l, MAX_CSV_ROWS as m, discoverCollections as n, setCollectionChangePublisher as nt, CollectionSchemaZ as o, projectItemFields as p, safeRecordId as q, loadCollection as r, skillsStagingDir as rt, resolveMutateSet as s, acceptParsedSchema as t, publishCollectionChange as tt, storeFor as u, dedupeByRecordId as v, CollectionQueryZ as w, queryCsv as x, encodeCsvRecordId as y, writeItem as z };
 
-//# sourceMappingURL=discovery-3K_Wtkvn.js.map
+//# sourceMappingURL=discovery-QUi_7aNG.js.map