@mulmoclaude/core 0.23.0 → 0.23.1

package/dist/{server--FgDORd3.js → server-D2ibbixF.js}

@@ -792,17 +792,18 @@ function whereFragment(cond) {
 		params: [cond.value]
 	};
 }
-/** Compile a validated query. Returns the SQL (whose FIRST placeholder is
-*  the CSV path — the executor binds it) and the where-value parameters
-*  that follow it. Callers MUST have run `CollectionQueryZ` first; this
-*  function trusts the shape (aliases already charset-checked, orderBy
-*  membership already enforced). */
-function compileCsvQuery(query, primaryKey) {
+/** Compile a validated query against `fromSql` (a table-function call
+*  whose FIRST placeholder is the source path — the executor binds it).
+*  Returns the SQL and the where-value parameters that follow the path.
+*  Callers MUST have run `CollectionQueryZ` first; this function trusts
+*  the shape (aliases already charset-checked, orderBy membership already
+*  enforced). */
+function compileQuery(query, fromSql) {
 	const groupBy = query.groupBy ?? [];
 	const aggregates = Object.entries(query.aggregates ?? {});
 	const selectList = [...groupBy.map(quoteIdent), ...aggregates.map(([alias, aggregate]) => `${aggregateExpr(aggregate)} AS ${quoteIdent(alias)}`)];
 	const where = (query.where ?? []).map(whereFragment);
-	const clauses = [`SELECT ${selectList.join(", ")}`, `FROM read_csv(${readCsvArgs(primaryKey)})`];
+	const clauses = [`SELECT ${selectList.join(", ")}`, `FROM ${fromSql}`];
 	if (where.length > 0) clauses.push(`WHERE ${where.map((fragment) => fragment.sql).join(" AND ")}`);
 	if (groupBy.length > 0) clauses.push(`GROUP BY ${groupBy.map(quoteIdent).join(", ")}`);
 	const orderBy = (query.orderBy ?? []).map((order) => quoteIdent(order.field) + (order.dir === "desc" ? " DESC" : " ASC"));
@@ -813,6 +814,21 @@ function compileCsvQuery(query, primaryKey) {
 		params: where.flatMap((fragment) => fragment.params)
 	};
 }
+/** Compile against a CSV file (the dataSource store's engine). */
+function compileCsvQuery(query, primaryKey) {
+	return compileQuery(query, `read_csv(${readCsvArgs(primaryKey)})`);
+}
+/** Compile against a JSONL file of ENRICHED records — the file-backed
+*  collections' engine (see `jsonlQuery.ts`). No VARCHAR key pin needed:
+*  enriched record ids are already strings. `sample_size=-1` makes the
+*  schema inference scan EVERY line — with the default sample, a sparse
+*  optional/derived field first appearing past the sample would not be
+*  inferred as a column and the query would binder-error on it (Codex P2
+*  on #2165). The full scan costs nothing extra here: aggregation reads
+*  the whole file anyway. */
+function compileJsonlQuery(query) {
+	return compileQuery(query, `read_json(?, format='newline_delimited', sample_size=-1)`);
+}
 //#endregion
 //#region src/collection/server/csvStore.ts
 /** `list()` row cap. Over-cap files are truncated with a warn — the v1
@@ -1133,6 +1149,74 @@ function storeFor(collection, opts = {}) {
 	};
 }
 //#endregion
+//#region src/collection/server/jsonlQuery.ts
+/** SQL semantics for an aggregate-only query over ZERO rows: one scalar
+*  row (`count` = 0, everything else NULL) — the same shape the CSV path
+*  produces for a header-only file, so callers reading `rows[0]` never
+*  break on storage kind or emptiness. A grouped query over zero rows is
+*  zero groups (`[]`), also matching SQL. Synthesized here because DuckDB
+*  has no empty file to infer a schema from. */
+function emptyCollectionResult(query) {
+	if ((query.groupBy?.length ?? 0) > 0) return [];
+	const aggregates = Object.entries(query.aggregates ?? {});
+	return [Object.fromEntries(aggregates.map(([alias, aggregate]) => [alias, aggregate.op === "count" ? 0 : null]))];
+}
+/** The SOURCE columns a query reads: groupBy columns, aggregate input
+*  columns, and where fields. (orderBy resolves to groupBy columns or
+*  aggregate ALIASES — never to a new source column — so it adds none.) */
+function referencedSourceColumns(query) {
+	const columns = new Set(query.groupBy ?? []);
+	for (const aggregate of Object.values(query.aggregates ?? {})) if (aggregate.column !== void 0) columns.add(aggregate.column);
+	for (const cond of query.where ?? []) columns.add(cond.field);
+	return [...columns];
+}
+/** Referenced columns that appear in NO row — a freshly-added optional
+*  field, or a derived field whose inputs no record has yet. DuckDB only
+*  infers columns from keys that occur in the JSONL, so without help a
+*  valid query on such a field binder-errors here while the CSV path
+*  (whose header always declares the column) returns NULLs. Padding the
+*  FIRST line with `col: null` makes the column exist (full-scan
+*  inference unions keys across lines) with matching NULL semantics. */
+function absentReferencedColumns(rows, query) {
+	const present = /* @__PURE__ */ new Set();
+	for (const row of rows) for (const key of Object.keys(row)) present.add(key);
+	return referencedSourceColumns(query).filter((column) => !present.has(column));
+}
+/** Run a validated query over enriched collection rows. */
+async function runQueryOverRows(rows, query) {
+	if (rows.length === 0) return emptyCollectionResult(query);
+	await mkdir(cacheDir(), {
+		recursive: true,
+		mode: 448
+	});
+	const jsonlPath = path.join(cacheDir(), `q-${randomBytes(8).toString("hex")}.jsonl`);
+	try {
+		const nullPads = Object.fromEntries(absentReferencedColumns(rows, query).map((column) => [column, null]));
+		const handle = await open(jsonlPath, "wx", 384);
+		try {
+			let first = true;
+			for (const row of rows) {
+				await handle.write(`${JSON.stringify(first ? {
+					...nullPads,
+					...row
+				} : row)}\n`);
+				first = false;
+			}
+		} finally {
+			await handle.close();
+		}
+		const { sql, params } = compileJsonlQuery(query);
+		return (await queryCsv(sql, [jsonlPath, ...params])).map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)])));
+	} finally {
+		await unlink(jsonlPath).catch((err) => {
+			if (err.code !== "ENOENT") log.warn("collections", "temp JSONL cleanup failed", {
+				path: jsonlPath,
+				error: String(err)
+			});
+		});
+	}
+}
+//#endregion
 //#region src/collection/core/recordZ.ts
 /** The emptiness rule shared by `required` and the "only check present
 *  values" gate. NOT a truthiness check — `0` and `false` are filled. */
@@ -3173,17 +3257,27 @@ async function putOneItem(collection, record, mode, deps) {
 	if (result.kind === "conflict") return reject(itemId, `'${itemId}' already exists — mode "create" refuses overwrite; use "upsert" to update it`);
 	return reject(itemId, "write refused: the collection's data dir escapes the workspace");
 }
-/** Aggregation over a dataSource collection's WHOLE file via the
-*  structured query DSL (`core/queryZ.ts`) — the paved road for counts /
-*  sums / group-bys that `getItems` (row-capped, unaggregated) can't
-*  answer honestly. File-backed collections have no query engine yet:
-*  refuse with a pointer instead of silently emulating. */
+/** Aggregation over a collection via the structured query DSL
+*  (`core/queryZ.ts`) — the paved road for counts / sums / group-bys
+*  that a row listing can't answer honestly. Two engines behind one
+*  shape: a dataSource collection queries its CSV natively through the
+*  store (`store.query`, uncapped whole-file scan); a file-backed
+*  collection aggregates its ENRICHED records (computed fields —
+*  `derived` / `rollup` / `toggle` — are real columns) through the same
+*  compiled SQL over a temp JSONL (`runQueryOverRows`). */
 async function handleQueryItems(collection, queryArg, deps) {
-	const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
-	if (!store.query) return `manageCollection: '${collection.slug}' is file-backed — queryItems currently supports only dataSource (CSV) collections; use getItems (with \`fields\`) instead.`;
 	const parsed = CollectionQueryZ.safeParse(queryArg);
 	if (!parsed.success) return `manageCollection: \`query\` rejected — fix and retry:\n${parsed.error.issues.slice(0, 20).map((issue) => `- ${issue.path.map(String).join(".") || "(root)"}: ${defangForPrompt(issue.message)}`).join("\n")}`;
-	const rows = await store.query(parsed.data);
+	const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
+	if (store.query) {
+		const rows = await store.query(parsed.data);
+		return JSON.stringify({
+			collection: collection.slug,
+			count: rows.length,
+			rows
+		});
+	}
+	const rows = await runQueryOverRows(await enrichItems(collection, await store.list(), deps), parsed.data);
 	return JSON.stringify({
 		collection: collection.slug,
 		count: rows.length,
@@ -3313,7 +3407,7 @@ async function handlePutSchema(slug, schemaArg, deps) {
 		written: true
 	});
 }
-var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits. On a dataSource (CSV) collection, answer aggregation questions (counts, sums, averages, group-bys) with `queryItems` — it scans the WHOLE file, while getItems is row-capped, so an aggregate computed from getItems output can be silently wrong on large files.";
+var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits. Answer aggregation questions (counts, sums, averages, group-bys) with `queryItems` on ANY collection — on a dataSource (CSV) collection it scans the whole file (getItems is row-capped, so aggregates computed from its output can be silently wrong on large files); on a file-backed collection it aggregates the enriched records, so computed fields (derived/rollup/toggle) are queryable columns.";
 /** Validate getItems' optional `ids`/`fields` args, then delegate. */
 async function dispatchGetItems(collection, args, deps) {
 	const ids = optionalStringArray(args.ids, "ids");
@@ -3396,7 +3490,7 @@ var MANAGE_COLLECTION_DEFINITION = {
 			},
 			query: {
 				type: "object",
-				description: "queryItems (dataSource/CSV collections only): a structured aggregation query over the WHOLE file — `{ groupBy?: [\"col\"], aggregates?: { alias: { op: \"count\"|\"sum\"|\"avg\"|\"min\"|\"max\", column? } }, where?: [{ field, op, value }], orderBy?: [{ field, dir? }], limit? }`. At least one of groupBy/aggregates. Runs uncapped over the full file (unlike getItems), so use it for counts / sums / group-bys on large CSVs."
+				description: "queryItems: a structured aggregation query — `{ groupBy?: [\"col\"], aggregates?: { alias: { op: \"count\"|\"sum\"|\"avg\"|\"min\"|\"max\", column? } }, where?: [{ field, op, value }], orderBy?: [{ field, dir? }], limit? }`. At least one of groupBy/aggregates. On a dataSource (CSV) collection it scans the WHOLE file uncapped (unlike getItems); on a file-backed collection it aggregates the ENRICHED records, so computed fields (derived/rollup/toggle) are queryable columns. Use it for counts / sums / averages / group-bys instead of arithmetic over getItems output."
 			}
 		},
 		required: ["action"]
@@ -3411,6 +3505,6 @@ function makeManageCollectionTool(deps = {}) {
 	};
 }
 //#endregion
-export { promptPathsFor as $, validateCollectionRecords as A, dedupeByRecordId as B, discoverCollections as C, CollectionSchemaZ as D, toSummary as E, readOnlyRefusal as F, DEFAULT_QUERY_ROWS as G, normalizeCsvValue as H, storeFor as I, buildCollectionActionSeedPrompt as J, MAX_QUERY_ROWS as K, MAX_CSV_ROWS as L, compileRecordZ as M, recordFieldProblem as N, applyMutateAction as O, collectionWritable as P, listItems as Q, csvRowToItem as R, acceptParsedSchema as S, toDetail as T, compileCsvQuery as U, encodeCsvRecordId as V, CollectionQueryZ as W, generateItemId as X, deleteItem as Y, isRegularFile as Z, errorMessage as _, getWorkspaceRoot as _t, deleteCollection as a, writeItem as at, buildWorkspaceOntology as b, setCollectionChangePublisher as bt, computeSuccessor as c, isContainedInRoot as ct, isTriggerDue as d, resolveDataDir as dt, readCustomViewHtml as et, maybeSpawnSuccessor as f, resolveTemplatePath as ft, ONE_SECOND_MS as g, configureCollectionHost as gt, successorId as h, collectionsRegistriesConfigPath as ht, deleteCustomView as i, resolveCreateItemId as it, validateRecordObject as j, firstMutateParamProblem as k, daysInMonth as l, isContainedInWorkspace as lt, resolveEvery as m, safeSlugName as mt, MAX_UNSELECTIVE_ITEMS as n, readItem as nt, deleteCollectionRefusalMessage as o, writeFileAtomic as ot, parseCivil as p, safeRecordId as pt, buildActionSeedPrompt as q, makeManageCollectionTool as r, readSkillTemplate as rt, advanceTriggerDate as s, SCHEMA_FILE$1 as st, MAX_SCHEMA_ISSUES as t, readCustomViewI18n as tt, formatCivil as u, itemFilePath as ut, computeCollectionIcon as v, log as vt, loadCollection as w, schemaRelations as x, enrichItems as y, publishCollectionChange as yt, decodeCsvRecordId as z };
+export { isRegularFile as $, validateCollectionRecords as A, decodeCsvRecordId as B, discoverCollections as C, CollectionSchemaZ as D, toSummary as E, collectionWritable as F, compileJsonlQuery as G, encodeCsvRecordId as H, readOnlyRefusal as I, MAX_QUERY_ROWS as J, CollectionQueryZ as K, storeFor as L, compileRecordZ as M, recordFieldProblem as N, applyMutateAction as O, runQueryOverRows as P, generateItemId as Q, MAX_CSV_ROWS as R, acceptParsedSchema as S, setCollectionChangePublisher as St, toDetail as T, normalizeCsvValue as U, dedupeByRecordId as V, compileCsvQuery as W, buildCollectionActionSeedPrompt as X, buildActionSeedPrompt as Y, deleteItem as Z, errorMessage as _, collectionsRegistriesConfigPath as _t, deleteCollection as a, readSkillTemplate as at, buildWorkspaceOntology as b, log as bt, computeSuccessor as c, writeFileAtomic as ct, isTriggerDue as d, isContainedInWorkspace as dt, listItems as et, maybeSpawnSuccessor as f, itemFilePath as ft, ONE_SECOND_MS as g, safeSlugName as gt, successorId as h, safeRecordId as ht, deleteCustomView as i, readItem as it, validateRecordObject as j, firstMutateParamProblem as k, daysInMonth as l, SCHEMA_FILE$1 as lt, resolveEvery as m, resolveTemplatePath as mt, MAX_UNSELECTIVE_ITEMS as n, readCustomViewHtml as nt, deleteCollectionRefusalMessage as o, resolveCreateItemId as ot, parseCivil as p, resolveDataDir as pt, DEFAULT_QUERY_ROWS as q, makeManageCollectionTool as r, readCustomViewI18n as rt, advanceTriggerDate as s, writeItem as st, MAX_SCHEMA_ISSUES as t, promptPathsFor as tt, formatCivil as u, isContainedInRoot as ut, computeCollectionIcon as v, configureCollectionHost as vt, loadCollection as w, schemaRelations as x, publishCollectionChange as xt, enrichItems as y, getWorkspaceRoot as yt, csvRowToItem as z };
 
-//# sourceMappingURL=server--FgDORd3.js.map
+//# sourceMappingURL=server-D2ibbixF.js.map