@mulmoclaude/core 0.22.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/assets/helps/collection-skills.md +102 -1
  2. package/assets/helps/custom-view.md +34 -0
  3. package/assets/helps/error-recovery.md +36 -0
  4. package/dist/collection/core/queryZ.d.ts +90 -0
  5. package/dist/collection/core/schema.d.ts +14 -1
  6. package/dist/collection/core/schemaZ.d.ts +18 -1
  7. package/dist/collection/index.cjs +3 -2
  8. package/dist/collection/index.cjs.map +1 -1
  9. package/dist/collection/index.d.ts +1 -0
  10. package/dist/collection/index.js +3 -3
  11. package/dist/collection/index.js.map +1 -1
  12. package/dist/collection/registry/server/index.cjs +12 -4
  13. package/dist/collection/registry/server/index.cjs.map +1 -1
  14. package/dist/collection/registry/server/index.js +12 -4
  15. package/dist/collection/registry/server/index.js.map +1 -1
  16. package/dist/collection/server/csvQuery.d.ts +19 -0
  17. package/dist/collection/server/csvStore.d.ts +49 -0
  18. package/dist/collection/server/discoveredCollection.d.ts +9 -1
  19. package/dist/collection/server/discovery.d.ts +7 -3
  20. package/dist/collection/server/index.cjs +15 -2
  21. package/dist/collection/server/index.d.ts +5 -0
  22. package/dist/collection/server/index.js +3 -3
  23. package/dist/collection/server/manageTool.d.ts +4 -0
  24. package/dist/collection/server/store.d.ts +31 -0
  25. package/dist/collection-watchers/index.cjs +104 -3
  26. package/dist/collection-watchers/index.cjs.map +1 -1
  27. package/dist/collection-watchers/index.js +102 -3
  28. package/dist/collection-watchers/index.js.map +1 -1
  29. package/dist/feeds/index.cjs +2 -2
  30. package/dist/feeds/index.js +2 -2
  31. package/dist/feeds/server/index.cjs +3 -3
  32. package/dist/feeds/server/index.js +3 -3
  33. package/dist/google/auth.d.ts +8 -2
  34. package/dist/google/calendar.d.ts +44 -0
  35. package/dist/google/index.cjs +86 -9
  36. package/dist/google/index.cjs.map +1 -1
  37. package/dist/google/index.d.ts +2 -2
  38. package/dist/google/index.js +82 -10
  39. package/dist/google/index.js.map +1 -1
  40. package/dist/{ingestTypes-DEjpiZGM.js → ingestTypes-B-dXxUF9.js} +2 -2
  41. package/dist/{ingestTypes-DEjpiZGM.js.map → ingestTypes-B-dXxUF9.js.map} +1 -1
  42. package/dist/{ingestTypes-Dh5lniBL.cjs → ingestTypes-Cev9q77C.cjs} +2 -2
  43. package/dist/{ingestTypes-Dh5lniBL.cjs.map → ingestTypes-Cev9q77C.cjs.map} +1 -1
  44. package/dist/{promptSafety-DN5V__Ku.cjs → promptSafety-DRd15gQ6.cjs} +15 -1
  45. package/dist/promptSafety-DRd15gQ6.cjs.map +1 -0
  46. package/dist/{promptSafety-DyG3EuC7.js → promptSafety-rDWA9_JS.js} +10 -2
  47. package/dist/promptSafety-rDWA9_JS.js.map +1 -0
  48. package/dist/{server-Bd8l1bhv.js → server--FgDORd3.js} +678 -79
  49. package/dist/server--FgDORd3.js.map +1 -0
  50. package/dist/{server-Bh-DPK-H.cjs → server-Q7ld-FlZ.cjs} +754 -76
  51. package/dist/server-Q7ld-FlZ.cjs.map +1 -0
  52. package/package.json +3 -1
  53. package/dist/promptSafety-DN5V__Ku.cjs.map +0 -1
  54. package/dist/promptSafety-DyG3EuC7.js.map +0 -1
  55. package/dist/server-Bd8l1bhv.js.map +0 -1
  56. package/dist/server-Bh-DPK-H.cjs.map +0 -1
@@ -1,10 +1,12 @@
1
- import { B as isFieldDrivenEvery, C as backlinkRows, D as actionVisible, E as rollupValue, F as AGENT_INGEST_KIND, I as COMPUTED_TYPES, L as FEED_SCHEDULES, M as SAFE_SLUG_PATTERN, N as isSafeRecordId, P as isSafeSlug, R as INGEST_KINDS, T as projectBacklinkRow, b as resolveIcon, d as parseIsoDateTime, g as deriveAll, j as SAFE_RECORD_ID_PATTERN, t as defangForPrompt, u as parseIsoDate, w as coerceNumeric, x as selectDynamicRecord, y as firstDateField, z as embedTargetId } from "./promptSafety-DyG3EuC7.js";
1
+ import { B as isFieldDrivenEvery, C as backlinkRows, D as actionVisible, E as rollupValue, F as AGENT_INGEST_KIND, I as COMPUTED_TYPES, L as FEED_SCHEDULES, M as SAFE_SLUG_PATTERN, N as isSafeRecordId, P as isSafeSlug, R as INGEST_KINDS, T as projectBacklinkRow, V as isReadOnlySchema, b as resolveIcon, d as parseIsoDateTime, g as deriveAll, j as SAFE_RECORD_ID_PATTERN, t as defangForPrompt, u as parseIsoDate, w as coerceNumeric, x as selectDynamicRecord, y as firstDateField, z as embedTargetId } from "./promptSafety-rDWA9_JS.js";
2
2
  import { isSafeActionTemplatePath, isSafeCustomViewI18nPath, isSafeCustomViewPath } from "./collection/paths.js";
3
3
  import { dataSkillDir, mirrorSkillWrite } from "./skill-bridge/index.js";
4
4
  import path from "node:path";
5
5
  import { promises, realpathSync } from "node:fs";
6
- import { cp, lstat, mkdir, open, readFile, readdir, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
7
- import { randomBytes, randomUUID } from "node:crypto";
6
+ import { cp, lstat, mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
7
+ import { createHash, randomBytes, randomUUID } from "node:crypto";
8
+ import { tmpdir } from "node:os";
9
+ import iconv from "iconv-lite";
8
10
  import { z } from "zod";
9
11
  //#region src/collection/server/host.ts
10
12
  var current = null;
@@ -572,7 +574,7 @@ function promptPathsFor(collection, workspaceRoot) {
572
574
  const skillDir = (rel === "" || rel.startsWith("..") ? collection.skillDir : rel).split(path.sep).join("/");
573
575
  return {
574
576
  slug: collection.slug,
575
- dataPath: collection.schema.dataPath,
577
+ dataPath: collection.schema.dataPath ?? collection.schema.dataSource?.path ?? "",
576
578
  skillDir
577
579
  };
578
580
  }
@@ -635,6 +637,502 @@ ${dataJson}
635
637
  ${templateText}`;
636
638
  }
637
639
  //#endregion
640
+ //#region src/collection/core/queryZ.ts
641
+ /** Result-column aliases double as SQL identifiers and JSON keys — keep
642
+ * them to a conservative identifier charset so neither side needs
643
+ * escaping gymnastics. */
644
+ var SAFE_ALIAS_PATTERN = /^[A-Za-z_]\w{0,63}$/;
645
+ /** Hard ceiling on returned rows; `limit` clamps below it. A group-by on
646
+ * a near-unique column would otherwise return one row per source row —
647
+ * the exact materialization the aggregate path exists to avoid. */
648
+ var MAX_QUERY_ROWS = 1e4;
649
+ /** Default row cap when the query declares no `limit`. */
650
+ var DEFAULT_QUERY_ROWS = 1e3;
651
+ /** One aggregate column: `count` (rows; `column` optional to count
652
+ * non-null cells) or `sum`/`avg`/`min`/`max` over a named CSV column. */
653
+ var QueryAggregateZ = z.object({
654
+ op: z.enum([
655
+ "count",
656
+ "sum",
657
+ "avg",
658
+ "min",
659
+ "max"
660
+ ]),
661
+ column: z.string().min(1).optional()
662
+ }).refine((aggregate) => aggregate.op === "count" || aggregate.column !== void 0, {
663
+ message: "`column` is required for every aggregate op except `count`",
664
+ path: ["column"]
665
+ });
666
+ /** One filter condition. Same op vocabulary as the schema-level `where`
667
+ * (`core/where.ts`) so authors learn one set; values may be typed
668
+ * (number / boolean) since CSV columns are. `in` requires an array
669
+ * value, every other op a scalar. */
670
+ var QueryWhereZ = z.object({
671
+ field: z.string().min(1),
672
+ op: z.enum([
673
+ "eq",
674
+ "ne",
675
+ "in",
676
+ "gt",
677
+ "gte",
678
+ "lt",
679
+ "lte",
680
+ "contains"
681
+ ]),
682
+ value: z.union([
683
+ z.string(),
684
+ z.number(),
685
+ z.boolean(),
686
+ z.array(z.union([
687
+ z.string(),
688
+ z.number(),
689
+ z.boolean()
690
+ ])).min(1).max(100)
691
+ ])
692
+ }).refine((cond) => cond.op === "in" === Array.isArray(cond.value), {
693
+ message: "`in` requires an array value (the allowed set); every other op requires a scalar value",
694
+ path: ["value"]
695
+ });
696
+ var QueryOrderZ = z.object({
697
+ /** A `groupBy` column or an aggregate alias — membership enforced by
698
+ * the whole-query refine below. */
699
+ field: z.string().min(1),
700
+ dir: z.enum(["asc", "desc"]).optional()
701
+ });
702
+ /** The whole query. At least one of `groupBy` / `aggregates` must be
703
+ * present: bare `groupBy` is a DISTINCT listing, bare `aggregates` a
704
+ * whole-file scalar row, together a grouped aggregation. */
705
+ var CollectionQueryZ = z.object({
706
+ groupBy: z.array(z.string().min(1)).max(8).refine((columns) => new Set(columns.map((column) => column.toLowerCase())).size === columns.length, { message: "`groupBy` columns must be unique (case-insensitively — SQL identifiers ignore case)" }).optional(),
707
+ aggregates: z.record(z.string().regex(SAFE_ALIAS_PATTERN, "aggregate aliases must be simple identifiers (letters/digits/underscore)"), QueryAggregateZ).optional(),
708
+ where: z.array(QueryWhereZ).max(16).optional(),
709
+ orderBy: z.array(QueryOrderZ).max(4).optional(),
710
+ limit: z.number().int().min(1).max(MAX_QUERY_ROWS).optional()
711
+ }).refine((query) => (query.groupBy?.length ?? 0) > 0 || Object.keys(query.aggregates ?? {}).length > 0, {
712
+ message: "declare at least one of `groupBy` (columns to bucket by) or `aggregates` (values to compute)",
713
+ path: ["groupBy"]
714
+ }).refine((query) => Object.keys(query.aggregates ?? {}).length <= 32, {
715
+ message: `\`aggregates\` supports at most 32 entries`,
716
+ path: ["aggregates"]
717
+ }).refine((query) => {
718
+ const groupLower = new Set((query.groupBy ?? []).map((column) => column.toLowerCase()));
719
+ const seen = /* @__PURE__ */ new Set();
720
+ return Object.keys(query.aggregates ?? {}).every((alias) => {
721
+ const lower = alias.toLowerCase();
722
+ if (groupLower.has(lower) || seen.has(lower)) return false;
723
+ seen.add(lower);
724
+ return true;
725
+ });
726
+ }, {
727
+ message: "aggregate aliases must be unique and must not collide with `groupBy` column names (case-insensitively — SQL identifiers ignore case)",
728
+ path: ["aggregates"]
729
+ }).refine((query) => {
730
+ const sortable = /* @__PURE__ */ new Set([...query.groupBy ?? [], ...Object.keys(query.aggregates ?? {})]);
731
+ return (query.orderBy ?? []).every((order) => sortable.has(order.field));
732
+ }, {
733
+ message: "every `orderBy.field` must be a `groupBy` column or an aggregate alias",
734
+ path: ["orderBy"]
735
+ });
736
+ //#endregion
737
+ //#region src/collection/server/csvQuery.ts
738
+ /** Double-quote a SQL identifier (CSV column name / result alias). */
739
+ function quoteIdent(name) {
740
+ return `"${name.replaceAll("\"", "\"\"")}"`;
741
+ }
742
+ /** Single-quote a SQL string literal (a `types={...}` struct key). */
743
+ function quoteLiteral(value) {
744
+ return `'${value.replaceAll("'", "''")}'`;
745
+ }
746
+ /** The `read_csv` argument list shared by every CSV query: the (prepared)
747
+ * path plus a `types` pin forcing the key column to VARCHAR — without it
748
+ * DuckDB's sniffer turns `001` into BIGINT 1, so leading zeros vanish
749
+ * and distinct keys collapse. */
750
+ function readCsvArgs(primaryKey) {
751
+ return `?, types={${quoteLiteral(primaryKey)}: 'VARCHAR'}`;
752
+ }
753
+ /** One aggregate's SQL expression. `sum`/`avg` TRY_CAST to DOUBLE so a
754
+ * column the sniffer kept as VARCHAR (mixed values) aggregates over its
755
+ * numeric cells instead of erroring; non-numeric cells become NULL and
756
+ * are skipped — standard BI tolerance. `min`/`max` stay native (they are
757
+ * meaningful on strings and dates too). */
758
+ function aggregateExpr(aggregate) {
759
+ const { op, column } = aggregate;
760
+ if (op === "count") return column === void 0 ? "count(*)" : `count(${quoteIdent(column)})`;
761
+ if (op === "sum" || op === "avg") return `${op}(TRY_CAST(${quoteIdent(column ?? "")} AS DOUBLE))`;
762
+ return `${op}(${quoteIdent(column ?? "")})`;
763
+ }
764
+ /** One where condition → SQL fragment + its bound parameters. String
765
+ * equality compares against `CAST(col AS VARCHAR)` so a sniffer-typed
766
+ * column still matches its textual value; numeric/boolean values compare
767
+ * natively (DuckDB coerces the column side). */
768
+ function whereFragment(cond) {
769
+ const column = quoteIdent(cond.field);
770
+ const asText = `CAST(${column} AS VARCHAR)`;
771
+ if (cond.op === "in") {
772
+ const values = cond.value;
773
+ return {
774
+ sql: `${values.every((value) => typeof value === "string") ? asText : column} IN (${values.map(() => "?").join(", ")})`,
775
+ params: values
776
+ };
777
+ }
778
+ if (cond.op === "contains") return {
779
+ sql: `contains(${asText}, ?)`,
780
+ params: [String(cond.value)]
781
+ };
782
+ const operator = {
783
+ eq: "=",
784
+ ne: "<>",
785
+ gt: ">",
786
+ gte: ">=",
787
+ lt: "<",
788
+ lte: "<="
789
+ }[cond.op];
790
+ return {
791
+ sql: `${typeof cond.value === "string" && (cond.op === "eq" || cond.op === "ne") ? asText : column} ${operator} ?`,
792
+ params: [cond.value]
793
+ };
794
+ }
795
+ /** Compile a validated query. Returns the SQL (whose FIRST placeholder is
796
+ * the CSV path — the executor binds it) and the where-value parameters
797
+ * that follow it. Callers MUST have run `CollectionQueryZ` first; this
798
+ * function trusts the shape (aliases already charset-checked, orderBy
799
+ * membership already enforced). */
800
+ function compileCsvQuery(query, primaryKey) {
801
+ const groupBy = query.groupBy ?? [];
802
+ const aggregates = Object.entries(query.aggregates ?? {});
803
+ const selectList = [...groupBy.map(quoteIdent), ...aggregates.map(([alias, aggregate]) => `${aggregateExpr(aggregate)} AS ${quoteIdent(alias)}`)];
804
+ const where = (query.where ?? []).map(whereFragment);
805
+ const clauses = [`SELECT ${selectList.join(", ")}`, `FROM read_csv(${readCsvArgs(primaryKey)})`];
806
+ if (where.length > 0) clauses.push(`WHERE ${where.map((fragment) => fragment.sql).join(" AND ")}`);
807
+ if (groupBy.length > 0) clauses.push(`GROUP BY ${groupBy.map(quoteIdent).join(", ")}`);
808
+ const orderBy = (query.orderBy ?? []).map((order) => quoteIdent(order.field) + (order.dir === "desc" ? " DESC" : " ASC"));
809
+ if (orderBy.length > 0) clauses.push(`ORDER BY ${orderBy.join(", ")}`);
810
+ clauses.push(`LIMIT ${query.limit ?? 1e3}`);
811
+ return {
812
+ sql: clauses.join(" "),
813
+ params: where.flatMap((fragment) => fragment.params)
814
+ };
815
+ }
816
+ //#endregion
817
+ //#region src/collection/server/csvStore.ts
818
+ /** `list()` row cap. Over-cap files are truncated with a warn — the v1
819
+ * contract is "browse + per-record views", not full-table analytics. */
820
+ var MAX_CSV_ROWS = 5e3;
821
+ /** Record ids minted from non-safe key values: `id0x` + utf-8 hex. Raw key
822
+ * values that themselves match this pattern are ALSO encoded, so the
823
+ * encoded namespace never collides with a raw value (injective mapping). */
824
+ var ENCODED_ID_PATTERN = /^id0x([0-9a-f]+)$/;
825
+ /** A CSV key value → the record id it's addressed by. Safe values pass
826
+ * through untouched; everything else (and anything shaped like an encoded
827
+ * id) becomes `id0x<hex>`. Pure + exported for unit tests. */
828
+ function encodeCsvRecordId(rawKey) {
829
+ if (safeRecordId(rawKey) === rawKey && !ENCODED_ID_PATTERN.test(rawKey)) return rawKey;
830
+ return `id0x${Buffer.from(rawKey, "utf-8").toString("hex")}`;
831
+ }
832
+ /** A record id → the CSV key value to look up. Inverse of
833
+ * `encodeCsvRecordId` for encoded ids; anything else is already the raw
834
+ * value. Pure + exported for unit tests. */
835
+ function decodeCsvRecordId(itemId) {
836
+ const match = ENCODED_ID_PATTERN.exec(itemId);
837
+ if (!match) return itemId;
838
+ return Buffer.from(match[1], "hex").toString("utf-8");
839
+ }
840
+ /** Normalize one DuckDB JS value into a JSON-safe record value: BigInt →
841
+ * number (string beyond the safe range), DATE/TIMESTAMP → ISO string
842
+ * (date-only when the clock is exactly UTC midnight, matching the `date`
843
+ * field contract), exotic DuckDB values → their string form. Pure +
844
+ * exported for unit tests. */
845
+ function normalizeCsvValue(value) {
846
+ if (typeof value === "bigint") return value <= BigInt(Number.MAX_SAFE_INTEGER) && value >= BigInt(-Number.MAX_SAFE_INTEGER) ? Number(value) : value.toString();
847
+ if (value instanceof Date) {
848
+ const iso = value.toISOString();
849
+ return iso.endsWith("T00:00:00.000Z") ? iso.slice(0, 10) : iso;
850
+ }
851
+ if (value !== null && typeof value === "object") return String(value);
852
+ return value;
853
+ }
854
+ /** One raw DuckDB row → a CollectionItem, or null when the key cell is
855
+ * missing/empty (the row can't be addressed). The primaryKey field is
856
+ * OVERWRITTEN with the (possibly encoded) record id so `item[primaryKey]`
857
+ * and the record's address never drift — same invariant the file store's
858
+ * write path enforces. Pure + exported for unit tests. */
859
+ function csvRowToItem(row, primaryKey) {
860
+ const normalized = Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)]));
861
+ const rawKey = normalized[primaryKey];
862
+ if (rawKey === null || rawKey === void 0 || String(rawKey) === "") return null;
863
+ return {
864
+ ...normalized,
865
+ [primaryKey]: encodeCsvRecordId(String(rawKey))
866
+ };
867
+ }
868
+ /** Dedupe by record id, LAST row wins (matches `csvRead`'s last-match
869
+ * pick). Returns the surviving items in first-seen order. Pure +
870
+ * exported for unit tests. */
871
+ function dedupeByRecordId(items, primaryKey) {
872
+ const byId = /* @__PURE__ */ new Map();
873
+ for (const item of items) byId.set(String(item[primaryKey]), item);
874
+ return {
875
+ items: [...byId.values()],
876
+ duplicates: items.length - byId.size
877
+ };
878
+ }
879
+ /** True when a thrown DuckDB error is the `types` pin naming a column the
880
+ * CSV doesn't have — the schema/file-mismatch case the caller downgrades
881
+ * to "empty collection + warn" instead of a 500. */
882
+ function isMissingKeyColumnError(err) {
883
+ return String(err).includes("do not exist in the CSV");
884
+ }
885
+ /** Bytes sniffed for UTF-8 validity. The trailing 3 bytes of the sample
886
+ * are dropped so a multibyte char split at the boundary can't produce a
887
+ * false negative on a valid file. */
888
+ var SNIFF_BYTES = 1024 * 1024;
889
+ function isValidUtf8(buf) {
890
+ try {
891
+ new TextDecoder("utf-8", { fatal: true }).decode(buf);
892
+ return true;
893
+ } catch {
894
+ return false;
895
+ }
896
+ }
897
+ /** Detect the (best-effort) encoding of a non-UTF-8 buffer. BOMs decide
898
+ * UTF-16; otherwise cp932 (the Shift_JIS superset — Excel-exported
899
+ * Japanese CSVs are the primary non-UTF-8 case this feature serves). */
900
+ function fallbackEncoding(buf) {
901
+ if (buf.length >= 2 && buf[0] === 255 && buf[1] === 254) return "utf-16le";
902
+ if (buf.length >= 2 && buf[0] === 254 && buf[1] === 255) return "utf-16be";
903
+ return "cp932";
904
+ }
905
+ function cacheDir() {
906
+ return path.join(tmpdir(), "mulmoclaude-csv-utf8");
907
+ }
908
+ /** Read only the first `bytes` of a file — the encoding sniff must not
909
+ * pull a multi-hundred-MB CSV into memory on the (common) UTF-8 path. */
910
+ async function readHead(absPath, bytes) {
911
+ const handle = await open(absPath, "r");
912
+ try {
913
+ const { size } = await handle.stat();
914
+ const buf = Buffer.alloc(Math.min(bytes, size));
915
+ await handle.read(buf, 0, buf.length, 0);
916
+ return buf;
917
+ } finally {
918
+ await handle.close();
919
+ }
920
+ }
921
+ /** Decode the whole file into a UTF-8 cache copy and return its path.
922
+ * Cache key = (path, mtime, size), so a replaced CSV re-decodes and an
923
+ * unchanged one never does. */
924
+ async function pathExists$1(target) {
925
+ try {
926
+ await stat(target);
927
+ return true;
928
+ } catch {
929
+ return false;
930
+ }
931
+ }
932
+ /** Best-effort removal of older decode-cache entries for the same source
933
+ * path — a frequently-replaced large CSV would otherwise accumulate one
934
+ * full copy per (mtime, size) forever. Runs AFTER the current copy is
935
+ * published; a concurrent reader holding an old fd is unaffected
936
+ * (unlink-while-open is safe on POSIX). */
937
+ async function evictSupersededCache(key, keepBasename) {
938
+ try {
939
+ const entries = await readdir(cacheDir());
940
+ await Promise.all(entries.filter((name) => name.startsWith(`${key}-`) && name !== keepBasename).map((name) => unlink(path.join(cacheDir(), name)).catch(() => void 0)));
941
+ } catch {}
942
+ }
943
+ /** Decode the whole file into a UTF-8 cache copy and return its path.
944
+ * Cache key = (path, mtime, size), so a replaced CSV re-decodes and an
945
+ * unchanged one never does; superseded copies are evicted. The cache
946
+ * lives in the SHARED OS tmpdir, so the dir is 0700 and files 0600 —
947
+ * decoded rows must not be readable by other local users. */
948
+ async function decodeToCache(absPath, info) {
949
+ const key = createHash("sha256").update(absPath).digest("hex").slice(0, 16);
950
+ const cached = path.join(cacheDir(), `${key}-${Math.trunc(info.mtimeMs)}-${info.size}.csv`);
951
+ if (!await pathExists$1(cached)) {
952
+ const whole = await readFile(absPath);
953
+ const encoding = fallbackEncoding(whole);
954
+ const text = iconv.decode(whole, encoding);
955
+ await mkdir(cacheDir(), {
956
+ recursive: true,
957
+ mode: 448
958
+ });
959
+ const tmp = `${cached}.${randomBytes(4).toString("hex")}.tmp`;
960
+ await writeFile(tmp, text, {
961
+ encoding: "utf-8",
962
+ mode: 384
963
+ });
964
+ await rename(tmp, cached);
965
+ log.info("collections", "decoded non-UTF-8 dataSource file to cache", {
966
+ path: absPath,
967
+ encoding
968
+ });
969
+ await evictSupersededCache(key, path.basename(cached));
970
+ }
971
+ return cached;
972
+ }
973
+ /** Re-validate the dataSource file at READ time, mirroring the JSON
974
+ * store's per-read defenses: realpath containment (a symlink swapped in
975
+ * after discovery must not walk out of the workspace) and an lstat
976
+ * regular-file check (a symlink leaf is refused outright, even one
977
+ * pointing inside the workspace — same rule as `isRegularFile` on
978
+ * record files). Returns the stat info, or null for "no readable file"
979
+ * (ENOENT / refused), which callers render as an empty collection. */
980
+ async function safeCsvStat(absPath, workspaceRoot) {
981
+ if (!isContainedInRoot(absPath, workspaceRoot)) {
982
+ log.warn("collections", "dataSource read refused: path escapes workspace", { path: absPath });
983
+ return null;
984
+ }
985
+ let info;
986
+ try {
987
+ info = await lstat(absPath);
988
+ } catch (err) {
989
+ if (err.code === "ENOENT") return null;
990
+ throw err;
991
+ }
992
+ if (!info.isFile()) {
993
+ log.warn("collections", "dataSource read refused: not a regular file (symlink?)", { path: absPath });
994
+ return null;
995
+ }
996
+ return info;
997
+ }
998
+ /** Return a path DuckDB can read as UTF-8: the original file when it
999
+ * already is UTF-8 (the cheap, common case — only the head is sniffed),
1000
+ * else a decoded cache copy (see `decodeToCache`). Returns null when
1001
+ * there is no readable file (missing, symlink, or containment-refused —
1002
+ * see `safeCsvStat`), which callers render as an empty collection. */
1003
+ async function ensureUtf8CsvPath(absPath, workspaceRoot) {
1004
+ const info = await safeCsvStat(absPath, workspaceRoot);
1005
+ if (info === null) return null;
1006
+ const head = await readHead(absPath, SNIFF_BYTES);
1007
+ const sample = head.length === SNIFF_BYTES ? head.subarray(0, SNIFF_BYTES - 3) : head;
1008
+ if (!(head.length >= 2 && (head[0] === 255 && head[1] === 254 || head[0] === 254 && head[1] === 255)) && isValidUtf8(sample)) return absPath;
1009
+ return decodeToCache(absPath, info);
1010
+ }
1011
+ var instancePromise = null;
1012
+ /** Lazily create one shared in-memory DuckDB instance. The dynamic import
1013
+ * keeps the native module OUT of core's load path — a platform where the
1014
+ * prebuilt binding is missing degrades to a per-query error on dataSource
1015
+ * collections only, never a broken core. A failed init is retried on the
1016
+ * next call (the promise is reset). */
1017
+ async function duckDbInstance() {
1018
+ if (instancePromise === null) instancePromise = import("@duckdb/node-api").then((mod) => mod.DuckDBInstance.create(":memory:"));
1019
+ try {
1020
+ return await instancePromise;
1021
+ } catch (err) {
1022
+ instancePromise = null;
1023
+ throw new Error(`DuckDB is unavailable on this host (@duckdb/node-api failed to load: ${String(err)}) — dataSource collections cannot be read`);
1024
+ }
1025
+ }
1026
+ async function queryCsv(sql, params) {
1027
+ const connection = await (await duckDbInstance()).connect();
1028
+ try {
1029
+ return (await connection.runAndReadAll(sql, params)).getRowObjectsJS();
1030
+ } finally {
1031
+ connection.disconnectSync();
1032
+ }
1033
+ }
1034
+ /** Every row of the CSV as records — capped, deduped, id-encoded. The
1035
+ * key column is pinned to VARCHAR (see `readCsvArgs`). `workspaceRoot`
1036
+ * drives the per-read containment check; omitted, the configured host
1037
+ * root is used. */
1038
+ async function csvList(absPath, primaryKey, workspaceRoot) {
1039
+ const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
1040
+ if (utf8Path === null) return [];
1041
+ let rows;
1042
+ try {
1043
+ rows = await queryCsv(`SELECT * FROM read_csv(${readCsvArgs(primaryKey)}) LIMIT 5001`, [utf8Path]);
1044
+ } catch (err) {
1045
+ if (!isMissingKeyColumnError(err)) throw err;
1046
+ log.warn("collections", "dataSource CSV has no primaryKey column — every row is skipped", {
1047
+ path: absPath,
1048
+ primaryKey
1049
+ });
1050
+ return [];
1051
+ }
1052
+ if (rows.length > 5e3) {
1053
+ log.warn("collections", "dataSource CSV truncated to row cap", {
1054
+ path: absPath,
1055
+ cap: MAX_CSV_ROWS
1056
+ });
1057
+ rows.length = MAX_CSV_ROWS;
1058
+ }
1059
+ const items = rows.map((row) => csvRowToItem(row, primaryKey)).filter((item) => item !== null);
1060
+ const skipped = rows.length - items.length;
1061
+ if (skipped > 0) log.warn("collections", "dataSource CSV rows skipped (empty key cell)", {
1062
+ path: absPath,
1063
+ skipped
1064
+ });
1065
+ const deduped = dedupeByRecordId(items, primaryKey);
1066
+ if (deduped.duplicates > 0) log.warn("collections", "dataSource CSV has duplicate key values (last row wins)", {
1067
+ path: absPath,
1068
+ duplicates: deduped.duplicates
1069
+ });
1070
+ return deduped.items;
1071
+ }
1072
+ /** The scan-order ordinal column the last-match read adds. Underscore
1073
+ * prefix keeps it out of any plausible CSV header namespace; it is
1074
+ * stripped from the returned record either way. */
1075
+ var ROW_ORDINAL = "__mc_row";
1076
+ /** One record by id. The comparison value rides as a prepared-statement
1077
+ * parameter, and the LAST matching row is selected IN DuckDB (scan-order
1078
+ * ordinal + LIMIT 1) — a CSV with thousands of duplicate keys must not
1079
+ * materialize them all for one detail read. Consistent with csvList's
1080
+ * last-wins dedupe. */
1081
+ async function csvRead(absPath, primaryKey, itemId, workspaceRoot) {
1082
+ const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
1083
+ if (utf8Path === null) return null;
1084
+ const rawKey = decodeCsvRecordId(itemId);
1085
+ const last = (await queryCsv(`SELECT * FROM (SELECT *, row_number() OVER () AS ${quoteIdent(ROW_ORDINAL)} FROM read_csv(${readCsvArgs(primaryKey)})) WHERE CAST(${quoteIdent(primaryKey)} AS VARCHAR) = ? ORDER BY ${quoteIdent(ROW_ORDINAL)} DESC LIMIT 1`, [utf8Path, rawKey])).at(0);
1086
+ if (last === void 0) return null;
1087
+ const { [ROW_ORDINAL]: __ordinal, ...record } = last;
1088
+ return csvRowToItem(record, primaryKey);
1089
+ }
1090
+ /** Run a validated aggregation query (the structured DSL — see
1091
+ * `core/queryZ.ts`) over the WHOLE file: no row cap on the scan (a
1092
+ * capped aggregate would be a wrong number), only the result-row LIMIT
1093
+ * the compiler emits. Values are normalized like list/read rows so a
1094
+ * chart consumer gets plain JSON scalars. */
1095
+ async function csvRunQuery(absPath, primaryKey, query, workspaceRoot) {
1096
+ const utf8Path = await ensureUtf8CsvPath(absPath, workspaceRoot ?? getWorkspaceRoot());
1097
+ if (utf8Path === null) return [];
1098
+ const { sql, params } = compileCsvQuery(query, primaryKey);
1099
+ return (await queryCsv(sql, [utf8Path, ...params])).map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCsvValue(value)])));
1100
+ }
1101
+ //#endregion
1102
+ //#region src/collection/server/store.ts
1103
+ /** True when the collection accepts UI/tool writes. A `dataSource`
1104
+ * collection is read-only: updates happen by editing/replacing the
1105
+ * data file itself. Every write entry point checks this BEFORE calling
1106
+ * `writeItem`/`deleteItem` — server-enforced, not just UI-hidden. */
1107
+ function collectionWritable(collection) {
1108
+ return !isReadOnlySchema(collection.schema);
1109
+ }
1110
+ /** The one-line refusal write paths surface (HTTP 405 / MCP error text). */
1111
+ function readOnlyRefusal(slug) {
1112
+ return `collection '${slug}' is read-only (backed by an external dataSource) — update the data file itself instead`;
1113
+ }
1114
+ /** Pick the store implementation for a discovered collection. A
1115
+ * `dataSource` schema whose `dataSourceFile` failed to resolve yields a
1116
+ * read-only EMPTY store rather than falling back to the (writable) file
1117
+ * store — a half-loaded read-only collection must never become writable. */
1118
+ function storeFor(collection, opts = {}) {
1119
+ if (isReadOnlySchema(collection.schema)) {
1120
+ const file = collection.dataSourceFile;
1121
+ const key = collection.schema.primaryKey;
1122
+ return {
1123
+ capabilities: { writable: false },
1124
+ list: () => file === void 0 ? Promise.resolve([]) : csvList(file, key, opts.workspaceRoot),
1125
+ read: (itemId) => file === void 0 ? Promise.resolve(null) : csvRead(file, key, itemId, opts.workspaceRoot),
1126
+ query: (query) => file === void 0 ? Promise.resolve([]) : csvRunQuery(file, key, query, opts.workspaceRoot)
1127
+ };
1128
+ }
1129
+ return {
1130
+ capabilities: { writable: true },
1131
+ list: () => listItems(collection.dataDir, opts),
1132
+ read: (itemId) => readItem(collection.dataDir, itemId, opts)
1133
+ };
1134
+ }
1135
+ //#endregion
638
1136
  //#region src/collection/core/recordZ.ts
639
1137
  /** The emptiness rule shared by `required` and the "only check present
640
1138
  * values" gate. NOT a truthiness check — `0` and `false` are filled. */
@@ -751,6 +1249,7 @@ async function listRecordFilenames(dataDir, workspaceRoot) {
751
1249
  }
752
1250
  }
753
1251
  async function validateCollectionRecords(collection, opts = {}) {
1252
+ if (collection.schema.dataSource !== void 0) return [];
754
1253
  const workspaceRoot = opts.workspaceRoot ?? getWorkspaceRoot();
755
1254
  const entries = await listRecordFilenames(collection.dataDir, workspaceRoot);
756
1255
  const issues = [];
@@ -1400,10 +1899,24 @@ function fieldDrivenFromFieldCarried(schema) {
1400
1899
  }
1401
1900
  return (carry ?? []).includes(driven.fromField);
1402
1901
  }
1902
+ /** External-data collection: the records ARE the rows of a user-supplied
1903
+ * data file (v1: CSV), queried through DuckDB — never copied into
1904
+ * `<dataDir>/<id>.json` files. Declaring `dataSource` makes the collection
1905
+ * **read-only** in every UI/tool write path; updates happen by replacing /
1906
+ * editing the file itself (file-watch republishes the views). `path` is
1907
+ * workspace-relative and containment-checked exactly like `dataPath`. The
1908
+ * row-id column is the schema's existing `primaryKey` — there is
1909
+ * deliberately no second key concept here.
1910
+ * See plans/feat-collection-csv-duckdb-source.md. */
1911
+ var DataSourceZ = z.object({
1912
+ type: z.literal("csv"),
1913
+ path: z.string().min(1)
1914
+ });
1403
1915
  var CollectionSchemaZ = z.object({
1404
1916
  title: z.string().min(1),
1405
1917
  icon: z.string().min(1),
1406
- dataPath: z.string().min(1),
1918
+ dataPath: z.string().min(1).optional(),
1919
+ dataSource: DataSourceZ.optional(),
1407
1920
  primaryKey: z.string().min(1),
1408
1921
  singleton: z.string().trim().min(1).optional(),
1409
1922
  fields: z.record(z.string(), FieldSpecZ),
@@ -1423,6 +1936,15 @@ var CollectionSchemaZ = z.object({
1423
1936
  notifyWhen: WhenZ.optional(),
1424
1937
  ingest: IngestZ.optional(),
1425
1938
  dynamicIcon: DynamicIconSpecZ.optional()
1939
+ }).refine((schema) => schema.dataPath !== void 0 !== (schema.dataSource !== void 0), {
1940
+ message: "declare exactly one of `dataPath` (native JSON records) or `dataSource` (external read-only data file), never both or neither",
1941
+ path: ["dataPath"]
1942
+ }).refine((schema) => schema.dataSource === void 0 || schema.singleton === void 0 && schema.ingest === void 0 && schema.spawn === void 0, {
1943
+ message: "a `dataSource` collection is read-only — it cannot declare `singleton`, `ingest`, or `spawn` (all of them write records)",
1944
+ path: ["dataSource"]
1945
+ }).refine((schema) => schema.dataSource === void 0 || [...schema.actions ?? [], ...schema.collectionActions ?? []].every((action) => action.kind !== "mutate"), {
1946
+ message: "a `dataSource` collection is read-only — its actions cannot use `kind: \"mutate\"` (a host write); use `chat`/`agent` actions instead",
1947
+ path: ["dataSource"]
1426
1948
  }).refine((schema) => schema.singleton === void 0 || isSafeRecordId(schema.singleton), {
1427
1949
  message: "schema `singleton` must be a valid item id (alphanumeric / hyphen / underscore / interior dot, no `..` or path separators)",
1428
1950
  path: ["singleton"]
@@ -1548,6 +2070,12 @@ function applyFeedSchemaDefaults(parsed, slug) {
1548
2070
  dataPath: `data/feeds/${slug}`
1549
2071
  };
1550
2072
  }
2073
+ /** The conventional per-slug records dir a `dataSource` collection gets as
2074
+ * its `dataDir` (records never live there, but archive/delete paths stay
2075
+ * well-defined — same shape the registry's R3 normalization uses). */
2076
+ function conventionalDataPath(slug) {
2077
+ return `data/collections/${slug}/items`;
2078
+ }
1551
2079
  /** The acceptance gates discovery applies AFTER `CollectionSchemaZ` parses,
1552
2080
  * before a schema becomes a live collection:
1553
2081
  *
@@ -1557,7 +2085,8 @@ function applyFeedSchemaDefaults(parsed, slug) {
1557
2085
  * edit is dropped with no error;
1558
2086
  * - a `feed` schema must declare an `ingest` block (else it's a dead,
1559
2087
  * non-refreshable card);
1560
- * - `dataPath` must resolve INSIDE the workspace.
2088
+ * - `dataPath` — or a `dataSource`'s `path` — must resolve INSIDE the
2089
+ * workspace (same realpath containment for both).
1561
2090
  *
1562
2091
  * Exported so `manageCollection`'s `putSchema` can run the SAME gates before
1563
2092
  * it reports success — a schema that passes `CollectionSchemaZ` but fails one
@@ -1577,7 +2106,24 @@ function acceptParsedSchema(schema, opts) {
1577
2106
  ok: false,
1578
2107
  reason: "a feed schema must declare an `ingest` block"
1579
2108
  };
1580
- const dataDir = resolveDataDir(schema.dataPath, opts.workspaceRoot);
2109
+ if (schema.dataSource !== void 0) {
2110
+ const dataSourceFile = resolveDataDir(schema.dataSource.path, opts.workspaceRoot);
2111
+ if (dataSourceFile === null) return {
2112
+ ok: false,
2113
+ reason: `dataSource.path '${schema.dataSource.path}' escapes the workspace`
2114
+ };
2115
+ const dataDir = resolveDataDir(conventionalDataPath(opts.slug), opts.workspaceRoot);
2116
+ if (dataDir === null) return {
2117
+ ok: false,
2118
+ reason: `slug '${opts.slug}' yields no workspace-contained data dir`
2119
+ };
2120
+ return {
2121
+ ok: true,
2122
+ dataDir,
2123
+ dataSourceFile
2124
+ };
2125
+ }
2126
+ const dataDir = resolveDataDir(schema.dataPath ?? "", opts.workspaceRoot);
1581
2127
  if (dataDir === null) return {
1582
2128
  ok: false,
1583
2129
  reason: `dataPath '${schema.dataPath}' escapes the workspace`
@@ -1625,7 +2171,8 @@ async function loadOneCollection(skillsRoot, slug, source, workspaceRoot) {
1625
2171
  const schema = parsed.data;
1626
2172
  const acceptance = acceptParsedSchema(schema, {
1627
2173
  source,
1628
- workspaceRoot
2174
+ workspaceRoot,
2175
+ slug: safeName
1629
2176
  });
1630
2177
  if (!acceptance.ok) {
1631
2178
  log.warn("collections", "schema.json rejected after validation, skipping", {
@@ -1639,6 +2186,7 @@ async function loadOneCollection(skillsRoot, slug, source, workspaceRoot) {
1639
2186
  source,
1640
2187
  schema,
1641
2188
  dataDir: acceptance.dataDir,
2189
+ ...acceptance.dataSourceFile !== void 0 ? { dataSourceFile: acceptance.dataSourceFile } : {},
1642
2190
  skillDir: path.join(skillsRoot, safeName)
1643
2191
  };
1644
2192
  }
@@ -1710,7 +2258,8 @@ function toSummary(collection) {
1710
2258
  slug: collection.slug,
1711
2259
  title: collection.schema.title,
1712
2260
  icon: collection.schema.icon,
1713
- source: collection.source
2261
+ source: collection.source,
2262
+ ...collection.schema.dataSource !== void 0 ? { readonly: true } : {}
1714
2263
  };
1715
2264
  }
1716
2265
  function toDetail(collection) {
@@ -1765,6 +2314,20 @@ async function countRecordFiles(dataDir, workspaceRoot) {
1765
2314
  return 0;
1766
2315
  }
1767
2316
  }
2317
+ /** A collection's record count for the ontology. File-backed: the cheap
2318
+ * readdir count. `dataSource`-backed: the row count via the CSV store
2319
+ * (subject to its row cap) — the dataDir is a phantom there, so the
2320
+ * readdir count would always misreport 0. Fail-soft to 0 like
2321
+ * `countRecordFiles` (an unreadable file / missing DuckDB must not
2322
+ * break the whole ontology). */
2323
+ async function countRecords(collection, workspaceRoot) {
2324
+ if (!collectionWritable(collection)) try {
2325
+ return (await storeFor(collection, { workspaceRoot }).list()).length;
2326
+ } catch {
2327
+ return 0;
2328
+ }
2329
+ return countRecordFiles(collection.dataDir, workspaceRoot);
2330
+ }
1768
2331
  async function toOntologyEntry(collection, workspaceRoot) {
1769
2332
  const { schema } = collection;
1770
2333
  return {
@@ -1773,7 +2336,7 @@ async function toOntologyEntry(collection, workspaceRoot) {
1773
2336
  icon: schema.icon,
1774
2337
  primaryKey: schema.primaryKey,
1775
2338
  displayField: schema.displayField ?? schema.primaryKey,
1776
- recordCount: await countRecordFiles(collection.dataDir, workspaceRoot),
2339
+ recordCount: await countRecords(collection, workspaceRoot),
1777
2340
  relations: schemaRelations(schema)
1778
2341
  };
1779
2342
  }
@@ -1819,7 +2382,7 @@ function uniqueBacklinkSources(schema) {
1819
2382
  async function loadTarget(slug, opts) {
1820
2383
  const target = await loadCollection(slug, opts);
1821
2384
  if (!target) return null;
1822
- const items = await listItems(target.dataDir, { workspaceRoot: opts.workspaceRoot });
2385
+ const items = await storeFor(target, { workspaceRoot: opts.workspaceRoot }).list();
1823
2386
  const byId = {};
1824
2387
  for (const item of items) {
1825
2388
  const itemId = item[target.schema.primaryKey];
@@ -1937,7 +2500,7 @@ async function computeCollectionIcon(collection, opts = {}) {
1937
2500
  try {
1938
2501
  const source = await loadCollection(spec.source.collection, opts);
1939
2502
  if (!source) return spec.fallback ?? schema.icon;
1940
- const ordered = sortByPrimaryKey(await listItems(source.dataDir, { workspaceRoot: opts.workspaceRoot }), source.schema.primaryKey);
2503
+ const ordered = sortByPrimaryKey(await storeFor(source, { workspaceRoot: opts.workspaceRoot }).list(), source.schema.primaryKey);
1941
2504
  const orderBy = spec.source.orderBy ?? firstDateField(source.schema);
1942
2505
  const recordsById = buildRecordsById(ordered, source.schema.primaryKey);
1943
2506
  return resolveIcon(selectDynamicRecord(ordered, spec.source, orderBy, recordsById), spec, schema.icon, recordsById);
@@ -2235,6 +2798,20 @@ function isDataDirSafe(dataDir, slug, workspaceRoot) {
2235
2798
  const resolved = path.resolve(dataDir);
2236
2799
  return acceptableRoots.some((root) => resolved === root || resolved.startsWith(root + path.sep));
2237
2800
  }
2801
+ /** Step 2 of the restore doc — how to bring the records back. A
2802
+ * `dataSource` collection has no record files to copy (its rows live in
2803
+ * the external data file, which the delete never touches). */
2804
+ function restoreRecordsStep(schema) {
2805
+ if (schema.dataPath === void 0) return `2. Records: nothing to copy. This is a \`dataSource\` collection —
2806
+ its records are the rows of \`${schema.dataSource?.path}\`, which the
2807
+ delete never touched.`;
2808
+ return `2. Copy the item data: \`cp\` every file under \`records/\` into
2809
+ \`${schema.dataPath}/\`. The records are part of the collection and
2810
+ must be restored. They are plain data files (NOT bridged), so use
2811
+ \`cp\` — the Write-tool rule in step 1 applies ONLY to the skill
2812
+ files, not to these records (there may be many; copy them, do not
2813
+ Write them one by one).`;
2814
+ }
2238
2815
  function buildRestoreDoc(collection) {
2239
2816
  const { slug, schema } = collection;
2240
2817
  return `# Restore "${schema.title}" (collection \`${slug}\`)
@@ -2255,18 +2832,13 @@ Follow these steps to restore it.
2255
2832
  \`.claude/skills/\` directly is not an option either: that path is
2256
2833
  permission-gated.)
2257
2834
 
2258
- 2. Copy the item data: \`cp\` every file under \`records/\` into
2259
- \`${schema.dataPath}/\`. The records are part of the collection and
2260
- must be restored. They are plain data files (NOT bridged), so use
2261
- \`cp\` — the Write-tool rule in step 1 applies ONLY to the skill
2262
- files, not to these records (there may be many; copy them, do not
2263
- Write them one by one).
2835
+ ${restoreRecordsStep(schema)}
2264
2836
 
2265
2837
  3. Confirm the collection reappears at \`/collections/${slug}\`.
2266
2838
 
2267
2839
  - slug: \`${slug}\`
2268
2840
  - title: ${schema.title}
2269
- - dataPath: \`${schema.dataPath}\`
2841
+ - dataPath: \`${schema.dataPath ?? `(dataSource) ${schema.dataSource?.path}`}\`
2270
2842
  `;
2271
2843
  }
2272
2844
  /** Copy one skill copy + the records + RESTORE.md into `archiveDir`. */
@@ -2489,14 +3061,15 @@ async function recordIssuesWarning(collection, deps) {
2489
3061
  return `${issues.length} record file(s) have data problems and are missing from this result. Fix each (Read → correct → Write):\n${lines}`;
2490
3062
  }
2491
3063
  async function loadRequestedItems(collection, ids, deps) {
3064
+ const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
2492
3065
  if (!ids) return {
2493
- items: await listItems(collection.dataDir, { workspaceRoot: deps.workspaceRoot }),
3066
+ items: await store.list(),
2494
3067
  missing: []
2495
3068
  };
2496
3069
  const items = [];
2497
3070
  const missing = [];
2498
3071
  for (const recordId of ids) {
2499
- const item = await readItem(collection.dataDir, recordId, { workspaceRoot: deps.workspaceRoot }).catch(() => null);
3072
+ const item = await store.read(recordId).catch(() => null);
2500
3073
  if (item) items.push(item);
2501
3074
  else missing.push(recordId);
2502
3075
  }
@@ -2600,7 +3173,25 @@ async function putOneItem(collection, record, mode, deps) {
2600
3173
  if (result.kind === "conflict") return reject(itemId, `'${itemId}' already exists — mode "create" refuses overwrite; use "upsert" to update it`);
2601
3174
  return reject(itemId, "write refused: the collection's data dir escapes the workspace");
2602
3175
  }
3176
+ /** Aggregation over a dataSource collection's WHOLE file via the
3177
+ * structured query DSL (`core/queryZ.ts`) — the paved road for counts /
3178
+ * sums / group-bys that `getItems` (row-capped, unaggregated) can't
3179
+ * answer honestly. File-backed collections have no query engine yet:
3180
+ * refuse with a pointer instead of silently emulating. */
3181
+ async function handleQueryItems(collection, queryArg, deps) {
3182
+ const store = storeFor(collection, { workspaceRoot: deps.workspaceRoot });
3183
+ if (!store.query) return `manageCollection: '${collection.slug}' is file-backed — queryItems currently supports only dataSource (CSV) collections; use getItems (with \`fields\`) instead.`;
3184
+ const parsed = CollectionQueryZ.safeParse(queryArg);
3185
+ if (!parsed.success) return `manageCollection: \`query\` rejected — fix and retry:\n${parsed.error.issues.slice(0, 20).map((issue) => `- ${issue.path.map(String).join(".") || "(root)"}: ${defangForPrompt(issue.message)}`).join("\n")}`;
3186
+ const rows = await store.query(parsed.data);
3187
+ return JSON.stringify({
3188
+ collection: collection.slug,
3189
+ count: rows.length,
3190
+ rows
3191
+ });
3192
+ }
2603
3193
  async function handlePutItems(collection, args, deps) {
3194
+ if (!collectionWritable(collection)) return `manageCollection: ${readOnlyRefusal(collection.slug)} (its records are the rows of '${collection.schema.dataSource?.path}'; edit that file to change the data).`;
2604
3195
  const written = [];
2605
3196
  const rejected = [];
2606
3197
  for (const record of args.items) {
@@ -2699,7 +3290,8 @@ function schemaDiscoveryGate(schema, base) {
2699
3290
  const primaryField = schema.fields[schema.primaryKey];
2700
3291
  if (!primaryField) return `primaryKey '${schema.primaryKey}' is not one of the declared fields`;
2701
3292
  if (primaryField.primary !== true) return `the primaryKey field '${schema.primaryKey}' must be flagged \`primary: true\``;
2702
- if (resolveDataDir(schema.dataPath, base) === null) return `dataPath '${schema.dataPath}' escapes the workspace`;
3293
+ if (schema.dataPath !== void 0 && resolveDataDir(schema.dataPath, base) === null) return `dataPath '${schema.dataPath}' escapes the workspace`;
3294
+ if (schema.dataSource !== void 0 && resolveDataDir(schema.dataSource.path, base) === null) return `dataSource.path '${schema.dataSource.path}' escapes the workspace`;
2703
3295
  return null;
2704
3296
  }
2705
3297
  /** Validate a schema against CollectionSchemaZ and, on success, persist it.
@@ -2721,7 +3313,7 @@ async function handlePutSchema(slug, schemaArg, deps) {
2721
3313
  written: true
2722
3314
  });
2723
3315
  }
2724
- var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits.";
3316
+ var MANAGE_COLLECTION_PROMPT = "Use `manageCollection` instead of raw Read/Write/Edit when working with a collection's records OR its schema (raw file I/O stays available as the escape hatch). Before authoring or changing a collection's `schema.json`, call `schemaDocs` to load the field/DSL reference, then read with `getSchema` and write with `putSchema` — `putSchema` validates the whole schema before writing and returns actionable errors instead of silently failing discovery's validation. `getItems` is the only way to see computed values — `derived` fields (e.g. a portfolio's value), `toggle` projections, and `embed` records are host-computed and never present in the stored JSON files. On large collections pass `ids` and/or `fields` to keep the result small. For a question that spans collections (\"which clients have unpaid invoices?\"), start with `getOntology`: it lists every collection with its primaryKey, record count, and outbound `ref`/`embed` relations, so you know which collections to join before reading any records. `putItems` validates every row against the schema before writing (required fields, enum values, primaryKey = record id) and returns `{ written, rejected }`; fix each rejected row using its `problem` text and retry just those rows. Never include computed fields in a row you write. To update a few fields of an existing record, use `mode: \"merge\"` with a partial row ({ id, <changed fields> }) — the default upsert replaces the WHOLE record, so a partial upsert would silently erase every optional field it omits. On a dataSource (CSV) collection, answer aggregation questions (counts, sums, averages, group-bys) with `queryItems` — it scans the WHOLE file, while getItems is row-capped, so an aggregate computed from getItems output can be silently wrong on large files.";
2725
3317
  /** Validate getItems' optional `ids`/`fields` args, then delegate. */
2726
3318
  async function dispatchGetItems(collection, args, deps) {
2727
3319
  const ids = optionalStringArray(args.ids, "ids");
@@ -2742,76 +3334,83 @@ async function manageCollectionHandler(deps, args) {
2742
3334
  if (!slug) return "manageCollection: `slug` is required (the collection's slug).";
2743
3335
  if (action === "getSchema") return handleGetSchema(slug, deps);
2744
3336
  if (action === "putSchema") return handlePutSchema(slug, args.schema, deps);
2745
- if (action !== "getItems" && action !== "putItems") return "manageCollection: `action` must be \"getItems\", \"putItems\", \"getOntology\", \"schemaDocs\", \"getSchema\", or \"putSchema\".";
3337
+ if (action !== "getItems" && action !== "putItems" && action !== "queryItems") return "manageCollection: `action` must be \"getItems\", \"putItems\", \"queryItems\", \"getOntology\", \"schemaDocs\", \"getSchema\", or \"putSchema\".";
2746
3338
  const collection = await loadCollection(slug, deps);
2747
3339
  if (!collection) return unknownCollection(slug);
2748
3340
  if (action === "getItems") return dispatchGetItems(collection, args, deps);
3341
+ if (action === "queryItems") return handleQueryItems(collection, args.query, deps);
2749
3342
  const parsed = parsePutItems(args, slug);
2750
3343
  if (typeof parsed === "string") return parsed;
2751
3344
  return handlePutItems(collection, parsed, deps);
2752
3345
  }
2753
- function makeManageCollectionTool(deps = {}) {
2754
- return {
2755
- definition: {
2756
- name: "manageCollection",
2757
- description: "Read and write a schema-driven collection through the host — both its records and its structure. getItems returns records WITH computed values (derived formulas, toggles, embeds) the stored JSON files don't contain; putItems validates each row against the schema before writing. getOntology maps the whole workspace: every collection with its record count and outbound ref/embed relations — call it first for cross-collection questions. schemaDocs returns the collection-authoring reference; getSchema/putSchema read and validate-then-write the collection's schema.json. Prefer it over raw file I/O on collections.",
2758
- inputSchema: {
3346
+ var MANAGE_COLLECTION_DEFINITION = {
3347
+ name: "manageCollection",
3348
+ description: "Read and write a schema-driven collection through the host — both its records and its structure. getItems returns records WITH computed values (derived formulas, toggles, embeds) the stored JSON files don't contain; putItems validates each row against the schema before writing. getOntology maps the whole workspace: every collection with its record count and outbound ref/embed relations — call it first for cross-collection questions. schemaDocs returns the collection-authoring reference; getSchema/putSchema read and validate-then-write the collection's schema.json. Prefer it over raw file I/O on collections.",
3349
+ inputSchema: {
3350
+ type: "object",
3351
+ properties: {
3352
+ action: {
3353
+ type: "string",
3354
+ enum: [
3355
+ "getItems",
3356
+ "putItems",
3357
+ "queryItems",
3358
+ "getOntology",
3359
+ "schemaDocs",
3360
+ "getSchema",
3361
+ "putSchema"
3362
+ ],
3363
+ description: "What to do."
3364
+ },
3365
+ slug: {
3366
+ type: "string",
3367
+ description: "The collection's slug (its directory name, e.g. `stock-quotes`). Required for everything except schemaDocs and getOntology."
3368
+ },
3369
+ ids: {
3370
+ type: "array",
3371
+ items: { type: "string" },
3372
+ description: "getItems: only these record ids (primary-key values). Omit for all records."
3373
+ },
3374
+ fields: {
3375
+ type: "array",
3376
+ items: { type: "string" },
3377
+ description: "getItems: only these fields per record (the primary key is always included). Omit for all fields. Use on large collections."
3378
+ },
3379
+ items: {
3380
+ type: "array",
3381
+ items: { type: "object" },
3382
+ description: "putItems: the record objects to store. Each must carry the schema's primaryKey value (it doubles as the filename)."
3383
+ },
3384
+ mode: {
3385
+ type: "string",
3386
+ enum: [
3387
+ "upsert",
3388
+ "create",
3389
+ "merge"
3390
+ ],
3391
+ description: "putItems: \"upsert\" (default) replaces existing records WHOLE; \"create\" rejects rows whose id already exists; \"merge\" updates only the fields a row carries, keeping the rest of the existing record (rejects unknown ids). Use \"merge\" when changing a few fields."
3392
+ },
3393
+ schema: {
3394
+ type: "object",
3395
+ description: "putSchema: the full collection schema object (same shape as schema.json — title, icon, dataPath, primaryKey, fields, …). Call getSchema first for the current one, and schemaDocs for the field DSL."
3396
+ },
3397
+ query: {
2759
3398
  type: "object",
2760
- properties: {
2761
- action: {
2762
- type: "string",
2763
- enum: [
2764
- "getItems",
2765
- "putItems",
2766
- "getOntology",
2767
- "schemaDocs",
2768
- "getSchema",
2769
- "putSchema"
2770
- ],
2771
- description: "What to do."
2772
- },
2773
- slug: {
2774
- type: "string",
2775
- description: "The collection's slug (its directory name, e.g. `stock-quotes`). Required for everything except schemaDocs and getOntology."
2776
- },
2777
- ids: {
2778
- type: "array",
2779
- items: { type: "string" },
2780
- description: "getItems: only these record ids (primary-key values). Omit for all records."
2781
- },
2782
- fields: {
2783
- type: "array",
2784
- items: { type: "string" },
2785
- description: "getItems: only these fields per record (the primary key is always included). Omit for all fields. Use on large collections."
2786
- },
2787
- items: {
2788
- type: "array",
2789
- items: { type: "object" },
2790
- description: "putItems: the record objects to store. Each must carry the schema's primaryKey value (it doubles as the filename)."
2791
- },
2792
- mode: {
2793
- type: "string",
2794
- enum: [
2795
- "upsert",
2796
- "create",
2797
- "merge"
2798
- ],
2799
- description: "putItems: \"upsert\" (default) replaces existing records WHOLE; \"create\" rejects rows whose id already exists; \"merge\" updates only the fields a row carries, keeping the rest of the existing record (rejects unknown ids). Use \"merge\" when changing a few fields."
2800
- },
2801
- schema: {
2802
- type: "object",
2803
- description: "putSchema: the full collection schema object (same shape as schema.json — title, icon, dataPath, primaryKey, fields, …). Call getSchema first for the current one, and schemaDocs for the field DSL."
2804
- }
2805
- },
2806
- required: ["action"]
3399
+ description: "queryItems (dataSource/CSV collections only): a structured aggregation query over the WHOLE file — `{ groupBy?: [\"col\"], aggregates?: { alias: { op: \"count\"|\"sum\"|\"avg\"|\"min\"|\"max\", column? } }, where?: [{ field, op, value }], orderBy?: [{ field, dir? }], limit? }`. At least one of groupBy/aggregates. Runs uncapped over the full file (unlike getItems), so use it for counts / sums / group-bys on large CSVs."
2807
3400
  }
2808
3401
  },
3402
+ required: ["action"]
3403
+ }
3404
+ };
3405
+ function makeManageCollectionTool(deps = {}) {
3406
+ return {
3407
+ definition: MANAGE_COLLECTION_DEFINITION,
2809
3408
  alwaysActive: true,
2810
3409
  prompt: MANAGE_COLLECTION_PROMPT,
2811
3410
  handler: (args) => manageCollectionHandler(deps, args)
2812
3411
  };
2813
3412
  }
2814
3413
  //#endregion
2815
- export { resolveTemplatePath as $, validateCollectionRecords as A, promptPathsFor as B, discoverCollections as C, CollectionSchemaZ as D, toSummary as E, buildCollectionActionSeedPrompt as F, resolveCreateItemId as G, readCustomViewI18n as H, deleteItem as I, SCHEMA_FILE$1 as J, writeItem as K, generateItemId as L, compileRecordZ as M, recordFieldProblem as N, applyMutateAction as O, buildActionSeedPrompt as P, resolveDataDir as Q, isRegularFile as R, acceptParsedSchema as S, toDetail as T, readItem as U, readCustomViewHtml as V, readSkillTemplate as W, isContainedInWorkspace as X, isContainedInRoot as Y, itemFilePath as Z, errorMessage as _, deleteCollection as a, log as at, buildWorkspaceOntology as b, computeSuccessor as c, isTriggerDue as d, safeRecordId as et, maybeSpawnSuccessor as f, ONE_SECOND_MS as g, successorId as h, deleteCustomView as i, getWorkspaceRoot as it, validateRecordObject as j, firstMutateParamProblem as k, daysInMonth as l, resolveEvery as m, MAX_UNSELECTIVE_ITEMS as n, collectionsRegistriesConfigPath as nt, deleteCollectionRefusalMessage as o, publishCollectionChange as ot, parseCivil as p, writeFileAtomic as q, makeManageCollectionTool as r, configureCollectionHost as rt, advanceTriggerDate as s, setCollectionChangePublisher as st, MAX_SCHEMA_ISSUES as t, safeSlugName as tt, formatCivil as u, computeCollectionIcon as v, loadCollection as w, schemaRelations as x, enrichItems as y, listItems as z };
3414
+ export { promptPathsFor as $, validateCollectionRecords as A, dedupeByRecordId as B, discoverCollections as C, CollectionSchemaZ as D, toSummary as E, readOnlyRefusal as F, DEFAULT_QUERY_ROWS as G, normalizeCsvValue as H, storeFor as I, buildCollectionActionSeedPrompt as J, MAX_QUERY_ROWS as K, MAX_CSV_ROWS as L, compileRecordZ as M, recordFieldProblem as N, applyMutateAction as O, collectionWritable as P, listItems as Q, csvRowToItem as R, acceptParsedSchema as S, toDetail as T, compileCsvQuery as U, encodeCsvRecordId as V, CollectionQueryZ as W, generateItemId as X, deleteItem as Y, isRegularFile as Z, errorMessage as _, getWorkspaceRoot as _t, deleteCollection as a, writeItem as at, buildWorkspaceOntology as b, setCollectionChangePublisher as bt, computeSuccessor as c, isContainedInRoot as ct, isTriggerDue as d, resolveDataDir as dt, readCustomViewHtml as et, maybeSpawnSuccessor as f, resolveTemplatePath as ft, ONE_SECOND_MS as g, configureCollectionHost as gt, successorId as h, collectionsRegistriesConfigPath as ht, deleteCustomView as i, resolveCreateItemId as it, validateRecordObject as j, firstMutateParamProblem as k, daysInMonth as l, isContainedInWorkspace as lt, resolveEvery as m, safeSlugName as mt, MAX_UNSELECTIVE_ITEMS as n, readItem as nt, deleteCollectionRefusalMessage as o, writeFileAtomic as ot, parseCivil as p, safeRecordId as pt, buildActionSeedPrompt as q, makeManageCollectionTool as r, readSkillTemplate as rt, advanceTriggerDate as s, SCHEMA_FILE$1 as st, MAX_SCHEMA_ISSUES as t, readCustomViewI18n as tt, formatCivil as u, itemFilePath as ut, computeCollectionIcon as v, log as vt, loadCollection as w, schemaRelations as x, enrichItems as y, publishCollectionChange as yt, decodeCsvRecordId as z };
2816
3415
 
2817
- //# sourceMappingURL=server-Bd8l1bhv.js.map
3416
+ //# sourceMappingURL=server--FgDORd3.js.map