@mulmoclaude/core 0.23.0 → 0.23.1

package/dist/{server-Q7ld-FlZ.cjs → server-Jaogu3Ky.cjs}

@@ -795,17 +795,18 @@ function whereFragment(cond) {
 		params: [cond.value]
 	};
 }
-/** Compile a validated query. Returns the SQL (whose FIRST placeholder is
-*  the CSV path — the executor binds it) and the where-value parameters
-*  that follow it. Callers MUST have run `CollectionQueryZ` first; this
-*  function trusts the shape (aliases already charset-checked, orderBy
-*  membership already enforced). */
-function compileCsvQuery(query, primaryKey) {
+/** Compile a validated query against `fromSql` (a table-function call
+*  whose FIRST placeholder is the source path — the executor binds it).
+*  Returns the SQL and the where-value parameters that follow the path.
+*  Callers MUST have run `CollectionQueryZ` first; this function trusts
+*  the shape (aliases already charset-checked, orderBy membership already
+*  enforced). */
+function compileQuery(query, fromSql) {
 	const groupBy = query.groupBy ?? [];
 	const aggregates = Object.entries(query.aggregates ?? {});
 	const selectList = [...groupBy.map(quoteIdent), ...aggregates.map(([alias, aggregate]) => `${aggregateExpr(aggregate)} AS ${quoteIdent(alias)}`)];
 	const where = (query.where ?? []).map(whereFragment);
-	const clauses = [`SELECT ${selectList.join(", ")}`, `FROM read_csv(${readCsvArgs(primaryKey)})`];
+	const clauses = [`SELECT ${selectList.join(", ")}`, `FROM ${fromSql}`];
 	if (where.length > 0) clauses.push(`WHERE ${where.map((fragment) => fragment.sql).join(" AND ")}`);
 	if (groupBy.length > 0) clauses.push(`GROUP BY ${groupBy.map(quoteIdent).join(", ")}`);
 	const orderBy = (query.orderBy ?? []).map((order) => quoteIdent(order.field) + (order.dir === "desc" ? " DESC" : " ASC"));
@@ -816,6 +817,21 @@ function compileCsvQuery(query, primaryKey) {
 		params: where.flatMap((fragment) => fragment.params)
 	};
 }
+/** Compile against a CSV file (the dataSource store's engine). */
+function compileCsvQuery(query, primaryKey) {
+	return compileQuery(query, `read_csv(${readCsvArgs(primaryKey)})`);
+}
+/** Compile against a JSONL file of ENRICHED records — the file-backed
+*  collections' engine (see `jsonlQuery.ts`). No VARCHAR key pin needed:
+*  enriched record ids are already strings. `sample_size=-1` makes the
+*  schema inference scan EVERY line — with the default sample, a sparse
+*  optional/derived field first appearing past the sample would not be
+*  inferred as a column and the query would binder-error on it (Codex P2
+*  on #2165). The full scan costs nothing extra here: aggregation reads
+*  the whole file anyway. */
+function compileJsonlQuery(query) {
+	return compileQuery(query, `read_json(?, format='newline_delimited', sample_size=-1)`);
+}
 //#endregion
 //#region src/collection/server/csvStore.ts
 /** `list()` row cap. Over-cap files are truncated with a warn — the v1
@@ -1136,6 +1152,74 @@ function storeFor(collection, opts = {}) {
 	};
 }
 //#endregion
+//#region src/collection/server/jsonlQuery.ts
+/** SQL semantics for an aggregate-only query over ZERO rows: one scalar
+*  row (`count` = 0, everything else NULL) — the same shape the CSV path
+*  produces for a header-only file, so callers reading `rows[0]` never
+*  break on storage kind or emptiness. A grouped query over zero rows is
+*  zero groups (`[]`), also matching SQL. Synthesized here because DuckDB
+*  has no empty file to infer a schema from. */
+function emptyCollectionResult(query) {
+	if ((query.groupBy?.length ?? 0) > 0) return [];
+	const aggregates = Object.entries(query.aggregates ?? {});
+	return [Object.fromEntries(aggregates.map(([alias, aggregate]) => [alias, aggregate.op === "count" ? 0 : null]))];
+}
+/** The SOURCE columns a query reads: groupBy columns, aggregate input
+*  columns, and where fields. (orderBy resolves to groupBy columns or
+*  aggregate ALIASES — never to a new source column — so it adds none.) */
+function referencedSourceColumns(query) {
+	const columns = new Set(query.groupBy ?? []);
+	for (const aggregate of Object.values(query.aggregates ?? {})) if (aggregate.column !== void 0) columns.add(aggregate.column);
+	for (const cond of query.where ?? []) columns.add(cond.field);
+	return [...columns];
+}
+/** Referenced columns that appear in NO row — a freshly-added optional
+*  field, or a derived field whose inputs no record has yet. DuckDB only
+*  infers columns from keys that occur in the JSONL, so without help a
+*  valid query on such a field binder-errors here while the CSV path
+*  (whose header always declares the column) returns NULLs. Padding the
+*  FIRST line with `col: null` makes the column exist (full-scan
+*  inference unions keys across lines) with matching NULL semantics. */
+function absentReferencedColumns(rows, query) {
+	const present = /* @__PURE__ */ new Set();
+	for (const row of rows) for (const key of Object.keys(row)) present.add(key);
+	return referencedSourceColumns(query).filter((column) => !present.has(column));
+}
+/** Run a validated query over enriched collection rows. */
+async function runQueryOverRows(rows, query) {
+	if (rows.length === 0) return emptyCollectionResult(query);
+	await (0, node_fs_promises.mkdir)(cacheDir(), {
+		recursive: true,
+		mode: 448
+	});
+	const jsonlPath = node_path.default.join(cacheDir(), `q-${(0, node_crypto.randomBytes)(8).toString("hex")}.jsonl`);
+	try {
+		const nullPads = Object.fromEntries(absentReferencedColumns(rows, query).map((column) => [column, null]));
+		const handle = await (0, node_fs_promises.open)(jsonlPath, "wx", 384);
+		try {
+			let first = true;
+			for (const row of rows) {
+				await handle.write(`${JSON.stringify(first ? {
+					...nullPads,
+					...row
+				} : row)}\n`);
+				first = false;
+			}
+		} finally {
+			await handle.close();
+		}
+		const { sql, params } = compileJsonlQuery(query);
+		return (await queryCsv(sql, [jsonlPath, ...params])).map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)])));
+	} finally {
+		await (0, node_fs_promises.unlink)(jsonlPath).catch((err) => {
+			if (err.code !== "ENOENT") log.warn("collections", "temp JSONL cleanup failed", {
+				path: jsonlPath,
+				error: String(err)
+			});
+		});
+	}
+}
+//#endregion
 //#region src/collection/core/recordZ.ts
 /** The emptiness rule shared by `required` and the "only check present
 *  values" gate. NOT a truthiness check — `0` and `false` are filled. */
@@ -3176,17 +3260,27 @@ async function putOneItem(collection, record, mode, deps) {
 	if (result.kind === "conflict") return reject(itemId, `'${itemId}' already exists — mode "create" refuses overwrite; use "upsert" to update it`);
 	return reject(itemId, "write refused: the collection's data dir escapes the workspace");
 }
-/** Aggregation over a dataSource collection's WHOLE file via the
-*  structured query DSL (`core/queryZ.ts`) — the paved road for counts /
-*  sums / group-bys that `getItems` (row-capped, unaggregated) can't
-*  answer honestly. File-backed collections have no query engine yet:
-*  refuse with a pointer instead of silently emulating. */
+/** Aggregation over a collection via the structured query DSL
+*  (`core/queryZ.ts`) — the paved road for counts / sums / group-bys
+*  that a row listing can't answer honestly. Two engines behind one
+*  shape: a dataSource collection queries its CSV natively through the
+*  store (`store.query`, uncapped whole-file scan); a file-backed
+*  collection aggregates its ENRICHED records (computed fields —
+*  `derived` / `rollup` / `toggle` — are real columns) through the same
+*  compiled SQL over a temp JSONL (`runQueryOverRows`). */
 async function handleQueryItems(collection, queryArg, deps) {
-	const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
-	if (!store.query) return `manageCollection: '${collection.slug}' is file-backed — queryItems currently supports only dataSource (CSV) collections; use getItems (with \`fields\`) instead.`;
 	const parsed = CollectionQueryZ.safeParse(queryArg);
 	if (!parsed.success) return `manageCollection: \`query\` rejected — fix and retry:\n${parsed.error.issues.slice(0, 20).map((issue) => `- ${issue.path.map(String).join(".") || "(root)"}: ${require_promptSafety.defangForPrompt(issue.message)}`).join("\n")}`;
-	const rows = await store.query(parsed.data);
+	const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
+	if (store.query) {
+		const rows = await store.query(parsed.data);
+		return JSON.stringify({
+			collection: collection.slug,
+			count: rows.length,
+			rows
+		});
+	}
+	const rows = await runQueryOverRows(await enrichItems(collection, await store.list(), deps), parsed.data);
 	return JSON.stringify({
 		collection: collection.slug,
 		count: rows.length,
@@ -3316,7 +3410,7 @@ async function handlePutSchema(slug, schemaArg, deps) {
 		written: true
 	});
 }
-var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits. On a dataSource (CSV) collection, answer aggregation questions (counts, sums, averages, group-bys) with `queryItems` — it scans the WHOLE file, while getItems is row-capped, so an aggregate computed from getItems output can be silently wrong on large files.";
+var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits. Answer aggregation questions (counts, sums, averages, group-bys) with `queryItems` on ANY collection — on a dataSource (CSV) collection it scans the whole file (getItems is row-capped, so aggregates computed from its output can be silently wrong on large files); on a file-backed collection it aggregates the enriched records, so computed fields (derived/rollup/toggle) are queryable columns.";
 /** Validate getItems' optional `ids`/`fields` args, then delegate. */
 async function dispatchGetItems(collection, args, deps) {
 	const ids = optionalStringArray(args.ids, "ids");
@@ -3399,7 +3493,7 @@ var MANAGE_COLLECTION_DEFINITION = {
 			},
 			query: {
 				type: "object",
-				description: "queryItems (dataSource/CSV collections only): a structured aggregation query over the WHOLE file — `{ groupBy?: [\"col\"], aggregates?: { alias: { op: \"count\"|\"sum\"|\"avg\"|\"min\"|\"max\", column? } }, where?: [{ field, op, value }], orderBy?: [{ field, dir? }], limit? }`. At least one of groupBy/aggregates. Runs uncapped over the full file (unlike getItems), so use it for counts / sums / group-bys on large CSVs."
+				description: "queryItems: a structured aggregation query — `{ groupBy?: [\"col\"], aggregates?: { alias: { op: \"count\"|\"sum\"|\"avg\"|\"min\"|\"max\", column? } }, where?: [{ field, op, value }], orderBy?: [{ field, dir? }], limit? }`. At least one of groupBy/aggregates. On a dataSource (CSV) collection it scans the WHOLE file uncapped (unlike getItems); on a file-backed collection it aggregates the ENRICHED records, so computed fields (derived/rollup/toggle) are queryable columns. Use it for counts / sums / averages / group-bys instead of arithmetic over getItems output."
 			}
 		},
 		required: ["action"]
@@ -3522,6 +3616,12 @@ Object.defineProperty(exports, "compileCsvQuery", {
 		return compileCsvQuery;
 	}
 });
+Object.defineProperty(exports, "compileJsonlQuery", {
+	enumerable: true,
+	get: function() {
+		return compileJsonlQuery;
+	}
+});
 Object.defineProperty(exports, "compileRecordZ", {
 	enumerable: true,
 	get: function() {
@@ -3786,6 +3886,12 @@ Object.defineProperty(exports, "resolveTemplatePath", {
 		return resolveTemplatePath;
 	}
 });
+Object.defineProperty(exports, "runQueryOverRows", {
+	enumerable: true,
+	get: function() {
+		return runQueryOverRows;
+	}
+});
 Object.defineProperty(exports, "safeRecordId", {
 	enumerable: true,
 	get: function() {
@@ -3859,4 +3965,4 @@ Object.defineProperty(exports, "writeItem", {
 	}
 });
 
-//# sourceMappingURL=server-Q7ld-FlZ.cjs.map
+//# sourceMappingURL=server-Jaogu3Ky.cjs.map