dreamteamer 0.13.3 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,9 +40,17 @@ schema:
40
40
  description: Folder holding the records, workspace-relative. `data/` for content, `state/` for operational records, `system/` for sources.
41
41
  codec:
42
42
  type: string
43
- enum: [md, yaml, json]
43
+ enum: [md, yaml, json, file]
44
44
  default: md
45
- description: File format. Use `md` whenever the record has a body a human will read.
45
+ description: 'File format. Use `md` whenever the record has a body a human will read. `file` makes the record an OPAQUE file — any extension, no frontmatter, fields DERIVED (`ext`, `bytes`), written with `add --from <path>` and never with `set`. For icons, logos and images.'
46
+ max_bytes:
47
+ type: integer
48
+ default: 204800
49
+ description: '`codec: file` only — the largest a record may be, in bytes. `check` reports anything over it. A record is a small file; a big one belongs outside the vault.'
50
+ extensions:
51
+ type: array
52
+ items: { type: string }
53
+ description: '`codec: file` only — the extensions this collection accepts, lowercase and without the dot. Omitted means any.'
46
54
  shape:
47
55
  type: string
48
56
  enum: [file, folder]
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dreamteamer",
3
- "version": "0.13.3",
3
+ "version": "0.14.0",
4
4
  "description": "A workspace compiler for coding agents — schema-validated records as plain files over git, compiled into every harness",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Gilad Khen <giladkhen@gmail.com>",
@@ -86,12 +86,16 @@ templates: [collection-templates/provenance] # merged at compile, every time
86
86
  WRITTEN, so a back-dated import files under the import month; derive from the domain's own date
87
87
  field instead). `id.pattern` must accept everything the template can produce — non-latin titles
88
88
  slug to a deterministic short hash, so `[a-z0-9-]` still holds.
89
- - **The `x-` keywords carry the domain semantics.** `x-reference` (a target collection, or `"*"` for
90
- any) is what lets `check` and `rename` follow a field. `x-body` marks the single field that becomes
91
- the md body. `x-inverse` declares a two-way link and makes `check` enforce both directions.
92
- `x-title-template` overrides how a VALUE of that field is labelled rarely needed, because a
93
- reference already inherits its TARGET collection's `title_template`; author it there instead, once,
94
- rather than on every field pointing at it.
89
+ - **The `x-` keywords carry the domain semantics.** `x-reference` (a target collection, a LIST of
90
+ them for a union, or `"*"` for any) is what lets `check` and `rename` follow a field. On input, a
91
+ single-target field also accepts a bare id (`standup`, not `meetings/standup`) it is qualified
92
+ before disk, so the file always carries the fully-qualified form; a union or `"*"` field has no
93
+ single target to infer from, so it still requires the qualified spelling. `x-body` marks the
94
+ single field that becomes the md body. `x-inverse` declares a two-way link and makes `check`
95
+ enforce both directions. `x-title-template` overrides how a VALUE of that field is labelled —
96
+ rarely needed, because a reference already inherits its TARGET collection's `title_template`;
97
+ author it there instead, once, rather than on every field pointing at it (a union field inherits a
98
+ template only when every member's target collection agrees on one).
95
99
  - **Do not enum a field after the fact.** Enumerating a vocabulary the records already violate makes
96
100
  `check` fail on every pre-existing value. `dt values <collection> <field>` derives the real
97
101
  vocabulary from the data — a filter dropdown gets it for free without locking the set.
package/src/check.js CHANGED
@@ -5,9 +5,10 @@ import fs from 'node:fs';
5
5
  import path from 'node:path';
6
6
  import Ajv from 'ajv';
7
7
  import addFormats from 'ajv-formats';
8
- import { parseRecord, patternRe, fmtAjvError, unknownFields, walk, EXT } from './records.js';
8
+ import { parseRecord, patternRe, fmtAjvError, unknownFields, walk, idFromRecordPath, MAX_RECORD_BYTES } from './records.js';
9
9
  import { NO_RUNTIME, loadDescriptors, runtimeDir, namespaces as compiledNamespaces } from './runtime.js';
10
10
  import { parseRef } from './namespace.js';
11
+ import { refTargetsOf } from './ref.js';
11
12
 
12
13
  export function check({ root }) {
13
14
  const RUNTIME = runtimeDir(root);
@@ -63,11 +64,28 @@ export function check({ root }) {
63
64
  else strays.push({ collection: name, file: rel(p), note: `missing entry file ${d.storage.entry}` });
64
65
  }
65
66
  } else {
66
- const tail = `.${d.storage.suffix}${EXT[d.storage.codec ?? 'md']}`;
67
+ const opaque = (d.storage.codec ?? 'md') === 'file';
68
+ const allowed = d.storage.extensions; // undefined = any
69
+ const max = d.storage.max_bytes ?? MAX_RECORD_BYTES;
67
70
  for (const f of walk(dir)) {
68
- const r = path.relative(dir, f);
69
- if (r.endsWith(tail)) ids.set(r.slice(0, -tail.length), f);
70
- else strays.push({ collection: name, file: rel(f) });
71
+ const id = idFromRecordPath(d, path.relative(dir, f));
72
+ if (id === null) { strays.push({ collection: name, file: rel(f) }); continue; }
73
+ // One id is one file. Under a fixed-extension codec this cannot happen; under `file` it
74
+ // can, and picking one silently is how a replaced logo keeps rendering as its predecessor.
75
+ if (ids.has(id)) {
76
+ violations.push({ file: rel(f), msg: `collection "${name}" holds the id "${id}" twice — ${rel(ids.get(id))} and ${rel(f)}. Remove one.` });
77
+ continue;
78
+ }
79
+ ids.set(id, f);
80
+ if (!opaque) continue;
81
+ const ext = path.extname(f).slice(1).toLowerCase();
82
+ if (allowed && !allowed.includes(ext)) {
83
+ violations.push({ file: rel(f), msg: `collection "${name}" does not accept .${ext} — its declared extensions are ${allowed.join(', ')}` });
84
+ }
85
+ const size = fs.statSync(f).size;
86
+ if (size > max) {
87
+ violations.push({ file: rel(f), msg: `is ${size} bytes, over collection "${name}"'s max_bytes of ${max} — a record is a small file; a big one belongs outside the vault` });
88
+ }
71
89
  }
72
90
  }
73
91
  }
@@ -77,7 +95,7 @@ export function check({ root }) {
77
95
 
78
96
  // parsed fields, kept for the symmetric-ref pass below (parse each record exactly once)
79
97
  const parsed = new Map();
80
- const inverseRules = []; // [collection, fieldPath, targetCollection, inverseField]
98
+ const inverseRules = []; // [collection, fieldPath, inverseField]
81
99
  const softRefs = new Map(); // absent-but-declared peer collection -> how many refs point at it
82
100
 
83
101
  for (const [name, d] of descriptors) {
@@ -87,7 +105,7 @@ export function check({ root }) {
87
105
  const bodyField = Object.entries(d.schema.properties ?? {}).find(([, s]) => s?.['x-body'])?.[0];
88
106
  parsed.set(name, new Map());
89
107
  for (const [fieldPath, target, inverse] of refFields) {
90
- if (inverse) inverseRules.push([name, fieldPath, target, inverse]);
108
+ if (inverse) inverseRules.push([name, fieldPath, inverse]);
91
109
  }
92
110
 
93
111
  for (const [id, file] of index.get(name)) {
@@ -121,25 +139,25 @@ export function check({ root }) {
121
139
  // only, so some predicates are only expressible from one side and both directions have to exist
122
140
  // — which makes an invariant mandatory, not optional. `x-inverse` on a ref field names the field
123
141
  // on the target that must point back; a one-sided link is a violation on the side that is missing.
124
- for (const [name, fieldPath, target, inverse] of inverseRules) {
142
+ for (const [name, fieldPath, inverse] of inverseRules) {
125
143
  for (const [id, fields] of parsed.get(name)) {
126
144
  const self = `${name}/${id}`;
127
145
  for (const value of valuesAt(fields, fieldPath)) {
128
146
  if (typeof value !== 'string' || value.startsWith('@')) continue;
129
- const targetId = parseRef(value, namespaces)?.id;
130
- if (targetId === undefined) continue; // already flagged as malformed
131
- const targetFields = parsed.get(target)?.get(targetId);
132
- if (!targetFields) continue; // already flagged as dangling
147
+ const ref = parseRef(value, namespaces);
148
+ if (!ref) continue; // already flagged as malformed
149
+ const targetFields = parsed.get(ref.collection)?.get(ref.id);
150
+ if (!targetFields) continue; // already flagged as dangling
133
151
  const back = [...valuesAt(targetFields, [inverse])];
134
152
  if (!back.includes(self)) {
135
- flag(index.get(target).get(targetId),
153
+ flag(index.get(ref.collection).get(ref.id),
136
154
  `${inverse}: must point back to "${self}" (${self} declares ${fieldPath.join('.')}: ${value})`);
137
155
  }
138
156
  }
139
157
  }
140
158
  }
141
159
 
142
- function checkRef(file, fieldPath, value, target, softTargets) {
160
+ function checkRef(file, fieldPath, value, targets, softTargets) {
143
161
  if (typeof value !== 'string') return;
144
162
  if (value.startsWith('@')) return; // runtime tokens (@me, @initiator) are legal
145
163
  // The SAME parser the store writes through (src/namespace.js) — `check` disagreeing with the
@@ -147,8 +165,9 @@ export function check({ root }) {
147
165
  const ref = parseRef(value, namespaces);
148
166
  if (!ref) return flag(file, `${fieldPath.join('.')}: reference "${value}" is not <collection>/<id>`);
149
167
  const { collection: coll, id } = ref;
150
- if (target !== '*' && coll !== target) {
151
- return flag(file, `${fieldPath.join('.')}: reference "${value}" should target collection "${target}"`);
168
+ if (targets !== '*' && !targets.includes(coll)) {
169
+ const want = targets.length === 1 ? `collection "${targets[0]}"` : `one of: ${targets.join(', ')}`;
170
+ return flag(file, `${fieldPath.join('.')}: reference "${value}" should target ${want}`);
152
171
  }
153
172
  if (!descriptors.has(coll)) {
154
173
  // A collection the owning module DECLARED as a peer and nothing installed provides is the
@@ -188,15 +207,25 @@ export function check({ root }) {
188
207
  }
189
208
 
190
209
 
191
- // collect [fieldPath, targetCollection, inverseField] for every x-reference in the schema.
210
+ // collect [fieldPath, targets, inverseField] for every x-reference in the schema, where `targets`
211
+ // is '*' or the normalized array of declared collections (see refTargetsOf).
192
212
  // `x-inverse` names the field on the TARGET collection that must point back — see checkSymmetry.
193
213
  function collectRefFields(schema, prefix = []) {
194
214
  const out = [];
195
215
  for (const [key, s] of Object.entries(schema.properties ?? {})) {
196
216
  if (!s || typeof s !== 'object') continue;
197
217
  const p = [...prefix, key];
198
- if (s['x-reference']) out.push([p, s['x-reference'], s['x-inverse']]);
199
- if (s.items?.['x-reference']) out.push([p, s.items['x-reference'], s['x-inverse'] ?? s.items['x-inverse']]);
218
+ const targets = refTargetsOf(s);
219
+ if (targets) {
220
+ // s.items ?? s, not the reverse: `s['x-reference']` treats a falsy-but-present keyword
221
+ // ('', false) as absent, which refTargetsOf does not — that mismatch left `holder`
222
+ // undefined and the next line threw. Nonsense-but-authored case this still does NOT
223
+ // cover: `x-inverse` on `items` beside a SCALAR `x-reference` on the property is not
224
+ // hoisted by compile (there is no array to hoist onto), so its symmetry rule is never
225
+ // evaluated here either — bad authoring, not a bug in this guard.
226
+ const holder = s.items ?? s;
227
+ out.push([p, targets, holder['x-inverse']]);
228
+ }
200
229
  if (s.properties) out.push(...collectRefFields(s, p));
201
230
  if (s.items?.properties) out.push(...collectRefFields(s.items, p));
202
231
  }
@@ -119,6 +119,16 @@ export function collectionCommand(ws, collection, verb, args) {
119
119
  return 0;
120
120
  }
121
121
  case 'add': {
122
+ // An opaque collection is written by IMPORTING a file: the id is positional (nothing can
123
+ // generate it from fields that do not exist) and the bytes come from --from.
124
+ if ((d.storage.codec ?? 'md') === 'file') {
125
+ const id = need(pos, 0, 'id');
126
+ if (!flags.from) throw new Error(`"${collection}" is a \`codec: file\` collection — pass --from <path> with the file to import`);
127
+ const { id: written, file } = store.addFile(collection, id, flags.from, { force: !!flags.force });
128
+ flags.json ? emit(JSON.stringify({ id: written, path: rel(ws.root, file) })) : console.log(`✔ ${rel(ws.root, file)}`);
129
+ return 0;
130
+ }
131
+ if (flags.from) throw new Error(`--from imports a file as a record, and "${collection}" is not a \`codec: file\` collection`);
122
132
  const fields = coerceArrays(d, stripMeta(flags));
123
133
  const { id, file } = store.add(collection, fields, { id: flags.id });
124
134
  flags.json ? emit(JSON.stringify({ id, path: rel(ws.root, file) })) : console.log(`✔ ${rel(ws.root, file)}`);
package/src/compile.js CHANGED
@@ -73,7 +73,8 @@ function staleDisplayKeywords(schema, prefix = '') {
73
73
  /**
74
74
  * Every `x-reference` in a schema, as `[fieldPath, target]` — the same traversal check.js uses to
75
75
  * resolve refs in records, here to verify the SHAPE against the module dependency graph. Nested
76
- * objects and array items both carry the keyword, so both are walked.
76
+ * objects and array items both carry the keyword, so both are walked. `target` is the RAW keyword
77
+ * value — a string, or a list of strings for the union form — unvalidated; the caller checks shape.
77
78
  */
78
79
  function refTargets(schema, prefix = '') {
79
80
  const out = [];
@@ -88,6 +89,32 @@ function refTargets(schema, prefix = '') {
88
89
  return out;
89
90
  }
90
91
 
92
+ /**
93
+ * Hoist per-relation keywords onto the node that CARRIES `x-reference` — `items` for array fields.
94
+ * Both places were historically tolerated and check.js read `s['x-inverse'] ?? s.items['x-inverse']`,
95
+ * a two-place read every future consumer would have had to copy. After this, every runtime consumer
96
+ * reads exactly one place. Conflicting duplicates fail loudly: silently preferring one is how a
97
+ * hand-authored value gets shadowed with no error anywhere.
98
+ */
99
+ function normalizeRelationKeywords(schema, name, prefix = '') {
100
+ for (const [key, prop] of Object.entries(schema?.properties ?? {})) {
101
+ if (!prop || typeof prop !== 'object') continue;
102
+ const at = `${prefix}${key}`;
103
+ if (prop.items && typeof prop.items === 'object' && prop.items['x-reference']) {
104
+ for (const kw of ['x-inverse', 'x-title-template']) {
105
+ if (!(kw in prop)) continue;
106
+ if (kw in prop.items && prop.items[kw] !== prop[kw]) {
107
+ fail(`collection "${name}": field "${at}" declares conflicting ${kw} on the property and its items — keep one.`);
108
+ }
109
+ prop.items[kw] = prop[kw];
110
+ delete prop[kw];
111
+ }
112
+ }
113
+ if (prop.properties) normalizeRelationKeywords(prop, name, `${at}.`);
114
+ if (prop.items?.properties) normalizeRelationKeywords(prop.items, name, `${at}[].`);
115
+ }
116
+ }
117
+
91
118
  export const KINDS = ['collections', 'skills', 'agents', 'commands', 'command-bindings', 'ui-views', 'collection-templates'];
92
119
  const FOLDER_KINDS = new Set(['skills']); // folder-shape entities: copy the whole record folder
93
120
  // DERIVED_KINDS (projected, not staged) lives in runtime.js — the boundary both halves read. Not in
@@ -433,7 +460,13 @@ export function compile({ root, pkg }) {
433
460
  // descriptors merge via 'extends' — collect per collection name
434
461
  const bytes = fs.readFileSync(srcPath);
435
462
  const doc = load(bytes.toString('utf8'));
436
- if (!doc.name || (!doc.schema && !doc.extends)) fail(`${rel(srcPath)}: descriptor needs 'name' and 'schema' (or 'extends')`);
463
+ // `codec: file` records are opaque bytes: there are no fields, so there is no schema to
464
+ // require and none to honour. Every other codec parses text into fields and must declare
465
+ // what they are.
466
+ const opaque = doc.storage?.codec === 'file';
467
+ if (!doc.name || (!doc.schema && !doc.extends && !opaque)) fail(`${rel(srcPath)}: descriptor needs 'name' and 'schema' (or 'extends')`);
468
+ if (opaque && (doc.storage.shape ?? 'file') === 'folder') fail(`${rel(srcPath)}: collection "${doc.name}" is \`codec: file\` — that is one file per record, not a folder; drop \`shape: folder\``);
469
+ if (opaque && Object.keys(doc.schema?.properties ?? {}).length) console.warn(`⚠ collection ${doc.name}: \`schema\` is ignored under \`codec: file\` — an opaque record's fields are derived (ext, bytes)`);
437
470
  if (!descriptorGroups.has(doc.name)) descriptorGroups.set(doc.name, []);
438
471
  descriptorGroups.get(doc.name).push({ src: { path: rel(srcPath), hash: sha256(bytes) }, doc, moduleName: source.name });
439
472
  contributed.add(source.name);
@@ -628,6 +661,19 @@ export function compile({ root, pkg }) {
628
661
  merged.storage.repo = '.';
629
662
  }
630
663
  storageEntries.push({ name, path: merged.storage.path, base: merged.storage.base });
664
+ // An opaque record has no AUTHORED schema, but it does have fields — derived ones. Stating them
665
+ // here rather than special-casing every reader is what keeps `codec: file` a codec instead of a
666
+ // feature: ajv, the field list, `dt values`, the form and the diagram all carry on unchanged,
667
+ // and what they read is true. Any authored schema was warned about and is replaced.
668
+ if ((merged.storage.codec ?? 'md') === 'file') {
669
+ merged.schema = {
670
+ type: 'object',
671
+ properties: {
672
+ ext: { type: 'string', description: "The file's extension, lowercase and without the dot. Derived from the file — never written." },
673
+ bytes: { type: 'integer', description: "The file's size in bytes. Derived from the file — never written." },
674
+ },
675
+ };
676
+ }
631
677
  for (const [at, tpl, target] of staleDisplayKeywords(merged.schema)) {
632
678
  const fix = target
633
679
  ? `either DELETE it (a reference to "${target}" now inherits that collection's \`title_template\`) or rename it to \`x-title-template\` if this field really needs its own`
@@ -688,8 +734,9 @@ export function compile({ root, pkg }) {
688
734
  const declaredDeps = new Set(groupModules.flatMap((m) => moduleDeps.get(m) ?? []));
689
735
  const declaredPeers = new Set(groupModules.flatMap((m) => modulePeers.get(m) ?? []));
690
736
  const owns = (t) => groupModules.includes(collOwner.get(t));
691
- for (const [at, target] of refTargets(merged.schema)) {
692
- if (target === '*') {
737
+ normalizeRelationKeywords(merged.schema, name);
738
+ for (const [at, raw] of refTargets(merged.schema)) {
739
+ if (raw === '*') {
693
740
  // The workspace module is the orchestrating parent and may reference anything —
694
741
  // including modules that do not exist yet, which is what `tasks.item` means.
695
742
  // Anywhere else a wildcard is a cross-module surface no declaration can cover.
@@ -698,14 +745,24 @@ export function compile({ root, pkg }) {
698
745
  }
699
746
  continue;
700
747
  }
701
- if (CORE_COLLECTIONS.has(target) || owns(target)) continue;
702
- const owner = collOwner.get(target);
703
- if (owner && declaredDeps.has(owner)) continue;
704
- if (declaredPeers.has(target)) continue;
705
- const fix = owner
706
- ? `add "${owner}" to dreamteamer.dependencies, or "${target}" to dreamteamer.peerDependencies if the module should work without it`
707
- : `add "${target}" to dreamteamer.peerDependenciesno installed module provides it`;
708
- fail(`collection "${name}": field "${at}" references "${target}", which ${groupModules.join('/')} neither owns nor declares.\n ${fix}.`);
748
+ // `x-reference` accepts a scalar or a LIST of targets (the union) — run the identical
749
+ // per-target contract check over every member.
750
+ const targets = Array.isArray(raw) ? raw : [raw];
751
+ // scalar-or-list: the list is the union form. '*' may not appear INSIDE a list — the
752
+ // wildcard is a scalar-only sentinel, and a union that includes "anything" is not a union.
753
+ if (targets.length === 0 || targets.some((t) => typeof t !== 'string' || t === '' || t === '*')) {
754
+ fail(`collection "${name}": field "${at}" has an invalid x-reference ${JSON.stringify(raw)}expected a collection name, a non-empty list of collection names, or '*'.`);
755
+ }
756
+ for (const target of targets) {
757
+ if (CORE_COLLECTIONS.has(target) || owns(target)) continue;
758
+ const owner = collOwner.get(target);
759
+ if (owner && declaredDeps.has(owner)) continue;
760
+ if (declaredPeers.has(target)) continue;
761
+ const fix = owner
762
+ ? `add "${owner}" to dreamteamer.dependencies, or "${target}" to dreamteamer.peerDependencies if the module should work without it`
763
+ : `add "${target}" to dreamteamer.peerDependencies — no installed module provides it`;
764
+ fail(`collection "${name}": field "${at}" references "${target}", which ${groupModules.join('/')} neither owns nor declares.\n ${fix}.`);
765
+ }
709
766
  }
710
767
  // Declared peers that nothing provides, stated as DATA on the descriptor so `check` can
711
768
  // excuse their references without learning what a module is (the `storage.base` precedent —
package/src/events.js CHANGED
@@ -3,7 +3,7 @@
3
3
  // auditable and replayable forever. history IS the queue; there is no events file.
4
4
  import path from 'node:path';
5
5
  import { execFileSync } from 'node:child_process';
6
- import { EXT } from './records.js';
6
+ import { idFromRecordPath } from './records.js';
7
7
 
8
8
  /** Record events between two points, across EVERY repo that holds records. `from` is a sha or a
9
9
  * date — a sha is meaningless in another repo, so it is resolved to its commit DATE and each
@@ -110,9 +110,8 @@ export function pathToRecord(descriptors, relPath) {
110
110
  if (!rest.endsWith('/' + entry)) return null;
111
111
  return { collection: best.name, id: rest.slice(0, -(entry.length + 1)) };
112
112
  }
113
- const tail = `.${best.storage.suffix}${EXT[best.storage.codec ?? 'md']}`;
114
- if (!rest.endsWith(tail)) return null;
115
- return { collection: best.name, id: rest.slice(0, -tail.length) };
113
+ const id = idFromRecordPath(best, rest);
114
+ return id === null ? null : { collection: best.name, id };
116
115
  }
117
116
 
118
117
  /** the commit that last touched `path` inside the range — the event's provenance sha,
@@ -6,6 +6,7 @@
6
6
  // the studio components already speak (field {field,type,meta,schema}).
7
7
 
8
8
  import { sourceHint } from './runtime.js';
9
+ import { refTargetsOf } from './ref.js';
9
10
 
10
11
  /** the projection for every collection: rows keyed by collection name + collection meta. */
11
12
  export function presentation(descriptors) {
@@ -39,9 +40,11 @@ export function presentation(descriptors) {
39
40
  for (const [name, prop] of Object.entries(d.schema?.properties ?? {})) {
40
41
  if (name === 'id') continue;
41
42
  rows.push(fieldRow(d, name, prop, new Set(d.schema?.required ?? []).has(name), descriptors));
42
- const target = referenceTargetOf(prop);
43
- if (target) {
44
- relations.push({ collection: d.name, field: name, related_collection: target, list: prop.type === 'array' });
43
+ const targets = referenceTargetsOf(prop);
44
+ if (targets) {
45
+ for (const target of targets) {
46
+ relations.push({ collection: d.name, field: name, related_collection: target, list: prop.type === 'array' });
47
+ }
45
48
  }
46
49
  }
47
50
  fields[d.name] = rows;
@@ -79,10 +82,10 @@ function collectionRow(d) {
79
82
  return { collection: d.name, meta, system };
80
83
  }
81
84
 
82
- function referenceTargetOf(prop) {
83
- const ref = prop.type === 'array' ? prop.items?.['x-reference'] : prop['x-reference'];
84
- if (typeof ref !== 'string' || ref === '' || ref === '*') return null;
85
- return ref;
85
+ /** The named target collections of a reference field — null for a non-ref and for '*'. */
86
+ function referenceTargetsOf(prop) {
87
+ const targets = refTargetsOf(prop);
88
+ return targets && targets !== '*' ? targets : null;
86
89
  }
87
90
 
88
91
  /**
@@ -92,14 +95,22 @@ function referenceTargetOf(prop) {
92
95
  * `title_template` — because "a company is labelled by its name" is a fact about companies, not
93
96
  * about each of the eleven fields that point at one. Before this, that fact was hand-copied onto
94
97
  * every referencing field as `x-display: '{{ name }}'`; 51 of the 54 sites in this workspace were
95
- * exactly what the target already implies.
98
+ * exactly what the target already implies. A UNION field inherits only when every member agrees —
99
+ * a template that renders half the values wrong is worse than the raw qualified ref, which is at
100
+ * least always correct.
96
101
  */
97
102
  function titleTemplateOf(prop, descriptors) {
98
- const own = prop.type === 'array' ? prop.items?.['x-title-template'] : prop['x-title-template'];
103
+ // Keyed off prop.items itself, not prop.type === 'array': the compile hoist moves an authored
104
+ // x-title-template onto whichever node CARRIES x-reference, which for an items-bearing property
105
+ // is `items` regardless of whether `type: array` was also spelled out explicitly. Reading by
106
+ // `type` diverged from that and silently dropped the authored template for such a field.
107
+ const own = prop.items?.['x-title-template'] ?? prop['x-title-template'];
99
108
  if (typeof own === 'string' && own.length > 0) return own;
100
- const target = referenceTargetOf(prop);
101
- const inherited = target ? descriptors.get(target)?.title_template : undefined;
102
- return typeof inherited === 'string' && inherited.length > 0 ? inherited : undefined;
109
+ const targets = referenceTargetsOf(prop);
110
+ if (!targets) return undefined;
111
+ const inherited = targets.map((t) => descriptors.get(t)?.title_template);
112
+ const first = inherited[0];
113
+ return typeof first === 'string' && first.length > 0 && inherited.every((v) => v === first) ? first : undefined;
103
114
  }
104
115
 
105
116
  function fieldRow(d, name, prop, isRequired, descriptors) {
@@ -114,15 +125,15 @@ function fieldRow(d, name, prop, isRequired, descriptors) {
114
125
  if (typeof prop.title === 'string' && prop.title.length > 0) meta.title = prop.title;
115
126
 
116
127
  let type = 'string';
117
- const target = referenceTargetOf(prop);
128
+ const targets = referenceTargetsOf(prop); // plural: x-reference may name several collections; only truthiness is used below
118
129
 
119
130
  if (prop['x-body'] === true) {
120
131
  type = 'text';
121
132
  meta.special = ['dt-body'];
122
133
  meta.edit = 'input-rich-text-md';
123
- } else if (target && prop.type !== 'array') {
134
+ } else if (targets && prop.type !== 'array') {
124
135
  meta.special = ['dt-relation-path'];
125
- } else if (target && prop.type === 'array') {
136
+ } else if (targets && prop.type === 'array') {
126
137
  type = 'json';
127
138
  meta.special = ['dt-relation-path', 'dt-relation-list'];
128
139
  } else if (prop.type === 'array' && prop.items?.type === 'object') {
package/src/records.js CHANGED
@@ -5,6 +5,11 @@ import path from 'node:path';
5
5
  import { load } from './yaml.js';
6
6
 
7
7
  export function parseRecord(file, d, bodyField) {
8
+ // An opaque record IS its bytes: there is nothing to parse, and reading a PNG as utf8 would
9
+ // corrupt it on the way back out. What a reader gets instead is derived from the file itself.
10
+ if ((d.storage.codec ?? 'md') === 'file') {
11
+ return { ext: path.extname(file).slice(1).toLowerCase(), bytes: fs.statSync(file).size };
12
+ }
8
13
  return parseRecordText(fs.readFileSync(file, 'utf8'), d, bodyField);
9
14
  }
10
15
 
@@ -47,6 +52,31 @@ export function unknownFields(schema, fields) {
47
52
 
48
53
  export const EXT = { md: '.md', yaml: '.yaml', json: '.json' };
49
54
 
55
+ /** How big a `codec: file` record may be before `check` says something, when its collection does not
56
+ * say otherwise. 200 KB fits an icon, a logo, a small illustration or a compressed photo, and does
57
+ * not fit the video someone will one day try to make a record. */
58
+ export const MAX_RECORD_BYTES = 204800;
59
+
60
+ /** The id a record file carries, or null when this path is not a record of `d`.
61
+ * `relPath` is relative to the collection's data directory.
62
+ *
63
+ * THE ONLY PLACE a filename becomes an id. store, check and events each carried their own copy of
64
+ * `endsWith('.' + suffix + EXT[codec])`, which is three places to update and two to forget.
65
+ *
66
+ * `codec: file` records are opaque bytes whose extension is whatever was imported, so their tail is
67
+ * `.<suffix>.<ONE extension segment>` — one, because `x.asset.tar.gz` in the folder is an archive
68
+ * someone dropped there, and calling it a record would hide it from `check`'s stray report. */
69
+ export function idFromRecordPath(d, relPath) {
70
+ const suffix = d.storage.suffix;
71
+ if ((d.storage.codec ?? 'md') === 'file') {
72
+ const lit = suffix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
73
+ const m = new RegExp(`^(.+)\\.${lit}\\.[A-Za-z0-9]+$`).exec(relPath);
74
+ return m ? m[1] : null;
75
+ }
76
+ const tail = `.${suffix}${EXT[d.storage.codec ?? 'md']}`;
77
+ return relPath.endsWith(tail) ? relPath.slice(0, -tail.length) : null;
78
+ }
79
+
50
80
  const JUNK_DIRS = new Set(['__pycache__', 'node_modules']);
51
81
  const JUNK_FILE = /\.(pyc|pyo)$|^\.DS_Store$/;
52
82
 
package/src/ref.js CHANGED
@@ -11,3 +11,24 @@ export function splitRef(descriptors, ref) {
11
11
  if (ref === best) throw new Error(`reference "${ref}" names a collection but no record id`);
12
12
  return { collection: best, id: ref.slice(best.length + 1) };
13
13
  }
14
+
15
+ /**
16
+ * The declared target set of a reference property: '*', an array of collection names, or null when
17
+ * the property is not a reference. `x-reference` accepts a scalar or a LIST of targets; this is the
18
+ * ONE place that widening is decoded — every consumer reads targets through here (or applies the
19
+ * identical one-liner where importing would cross a layer), so the two spellings can never mean
20
+ * different things in different subsystems.
21
+ *
22
+ * This function normalizes SPELLING only, not shape: `[]` decodes to `[]` and `''` decodes to
23
+ * `['']`, neither of which is a valid target set. Whether the array is non-empty, every member is a
24
+ * non-empty string, and `'*'` never hides inside a list is compile's job (see compile.js's
25
+ * validation of each union member) — a consumer calling this at runtime is reading an
26
+ * already-compiled, already-validated descriptor and can rely on the shape holding, but should not
27
+ * re-derive that guarantee from this decoder.
28
+ */
29
+ export function refTargetsOf(prop) {
30
+ const raw = prop?.['x-reference'] ?? prop?.items?.['x-reference'];
31
+ if (raw == null) return null;
32
+ if (raw === '*') return '*';
33
+ return Array.isArray(raw) ? raw : [raw];
34
+ }
package/src/schema-ops.js CHANGED
@@ -11,11 +11,12 @@ import { load, dump } from './yaml.js';
11
11
  import { compile, kindDir, titleCase } from './compile.js';
12
12
  import { readManifest, runtimeKindDir } from './runtime.js';
13
13
  import { normalizeNamespaces, namespaceOf, baseNameOf, qualify, defaultStoragePath } from './namespace.js';
14
+ import { refTargetsOf } from './ref.js';
14
15
 
15
16
  // Same rule as store.js: a git failure we CATCH must not also print git's own error on top of the
16
17
  // clean message we throw. stdout stays piped because some callers read it.
17
18
  const GIT_QUIET = ['ignore', 'pipe', 'ignore'];
18
- import { walk, EXT } from './records.js';
19
+ import { walk, idFromRecordPath } from './records.js';
19
20
 
20
21
  // ---- the gate -------------------------------------------------------------------
21
22
 
@@ -383,10 +384,13 @@ export function renameCollection(ws, store, oldName, newName) {
383
384
  pruneEmpty(path.dirname(oldDir), path.join(ws.root, dataPath));
384
385
  }
385
386
  if (newSuffix !== oldSuffix && fs.existsSync(newDir)) {
386
- const ext = EXT[d.storage.codec ?? 'md'];
387
+ // Match on the OLD suffix, keep whatever extension the file already had — an opaque
388
+ // record's extension is its own, and a re-suffix must not rename it into another format.
389
+ const old = { storage: { ...d.storage, suffix: oldSuffix } };
387
390
  for (const file of walk(newDir)) {
388
- if (!file.endsWith(`.${oldSuffix}${ext}`)) continue;
389
- const to = file.slice(0, -(oldSuffix.length + ext.length + 1)) + `.${newSuffix}${ext}`;
391
+ const id = idFromRecordPath(old, path.relative(newDir, file));
392
+ if (id === null) continue;
393
+ const to = path.join(newDir, `${id}.${newSuffix}${path.basename(file).slice(path.basename(id).length + oldSuffix.length + 1)}`);
390
394
  fs.renameSync(file, to);
391
395
  resuffixed.push([file, to]);
392
396
  }
@@ -409,14 +413,13 @@ export function renameCollection(ws, store, oldName, newName) {
409
413
  const before = fs.readFileSync(f, 'utf8');
410
414
  const probe = load(before);
411
415
  if (!probe || !retargetRefs(probe.schema, oldName, newName)) continue;
412
- // ⚠ the boundary must cover BOTH spellings. A descriptor may write the block form
413
- // (`x-reference: accounts` to end of line) or the inline flow form
414
- // (`{ type: string, x-reference: accounts }`), where the value ends at `,` or `}`.
415
- // Anchoring on `$` alone silently matched nothing in the flow form — and the assert
416
- // below turned that silence into a refusal, which is how it was found.
417
- const after = before.replace(
418
- new RegExp(`(x-reference:\\s*)(['"]?)${oldName.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')}\\2(?=\\s*(?:[,}]|#|$))`, 'gm'),
419
- (_m, lead) => `${lead}${newName.includes('/') ? `'${newName}'` : newName}`);
416
+ // ⚠ the boundary must cover THREE spellings: the block form (`x-reference: accounts` to
417
+ // end of line), the inline flow form (`{ type: string, x-reference: accounts }`, where
418
+ // the value ends at `,` or `}`), and a LIST flow (`x-reference: [a, accounts, b]`) or
419
+ // block (`x-reference:` + `- accounts` items). Anchoring on `$` alone silently matched
420
+ // nothing in the flow form — and the assert below turned that silence into a refusal,
421
+ // which is how it was found.
422
+ const after = retargetRefText(before, oldName, newName);
420
423
  const reparsed = load(after);
421
424
  if (!reparsed || retargetRefs(reparsed.schema, oldName, newName)) {
422
425
  throw new Error(`could not retarget x-reference "${oldName}" in ${path.relative(ws.root, f)} without reformatting it — nothing was changed.`);
@@ -487,15 +490,22 @@ function descriptorSources(ws, store) {
487
490
  return out;
488
491
  }
489
492
 
490
- /** Rewrite `x-reference: old` → new anywhere in a schema. Returns true if anything changed. */
493
+ /** Rewrite `x-reference: old` → new anywhere in a schema — scalar or list entry. Returns true if anything changed. */
491
494
  function retargetRefs(schema, oldName, newName) {
492
495
  let changed = false;
493
496
  for (const prop of Object.values(schema?.properties ?? {})) {
494
497
  if (!prop || typeof prop !== 'object') continue;
495
498
  for (const holder of [prop, prop.items]) {
496
- if (holder && typeof holder === 'object' && holder['x-reference'] === oldName) {
499
+ if (!holder || typeof holder !== 'object') continue;
500
+ if (holder['x-reference'] === oldName) {
497
501
  holder['x-reference'] = newName;
498
502
  changed = true;
503
+ } else if (Array.isArray(holder['x-reference'])) {
504
+ const i = holder['x-reference'].indexOf(oldName);
505
+ if (i !== -1) {
506
+ holder['x-reference'][i] = newName;
507
+ changed = true;
508
+ }
499
509
  }
500
510
  }
501
511
  if (prop.properties && retargetRefs(prop, oldName, newName)) changed = true;
@@ -504,6 +514,73 @@ function retargetRefs(schema, oldName, newName) {
504
514
  return changed;
505
515
  }
506
516
 
517
+ /**
518
+ * The TEXTUAL x-reference retarget — a line edit, never load→dump, so comments survive (see the
519
+ * step-4 comment in renameCollection for the 17-descriptor lesson).
520
+ *
521
+ * Handles three spellings: scalar (`x-reference: old`), flow list (`x-reference: [a, old, b]`),
522
+ * and block sequence (`x-reference:` + `- old` items tracked by indent under the key):
523
+ *
524
+ * ```
525
+ * x-reference:
526
+ * - doctors
527
+ * - nurses
528
+ * ```
529
+ *
530
+ * Two YAML styles are deliberately NOT rewritten and fall through unchanged to the caller's
531
+ * reparse-assert (which throws if the return does not compile), failing closed rather than
532
+ * half-written:
533
+ *
534
+ * - **Same-indent block sequence**: YAML allows `- items` at the PARENT's own indent
535
+ * (`x-reference:` and `- doctors` at the same indent level). The indent state machine
536
+ * requires strictly deeper dashes (`item[1].length > listIndent`), so this spelling is
537
+ * never entered and goes reparse-asserted instead.
538
+ * - **Multi-line flow list**: a flow list split across lines — `[` on one line, `]` on
539
+ * another. The flow regex requires both brackets and the body on the same line, so this
540
+ * spelling is not matched and goes reparse-asserted instead.
541
+ * - **A blank or comment line BETWEEN block-sequence items**: the item regex requires a
542
+ * leading `-`, so a blank line or a `#`-comment line inside the list resets `listIndent`
543
+ * early. Every item after the gap is then read as ordinary text rather than a list member,
544
+ * and (having no `x-reference:` key on its own line) is left untouched.
545
+ *
546
+ * All three fail closed on purpose: the engine's own `dump()` always emits deeper-indented,
547
+ * gap-free sequences and single-line flow lists, so only a hand-authored descriptor can reach
548
+ * these styles. Anything trickier than the three handled spellings falls through unchanged —
549
+ * the caller reparses and REFUSES rather than guessing.
550
+ */
551
+ function retargetRefText(text, oldName, newName) {
552
+ const esc = oldName.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
553
+ const quoted = newName.includes('/') ? `'${newName}'` : newName;
554
+ const retag = (part) => {
555
+ const m = part.match(/^(\s*)(['"]?)(.*?)\2(\s*)$/);
556
+ return m && m[3] === oldName ? `${m[1]}${quoted}${m[4]}` : part;
557
+ };
558
+ const lines = text.split('\n');
559
+ let listIndent = -1; // >= 0 while inside a block-sequence x-reference list
560
+ for (let i = 0; i < lines.length; i++) {
561
+ const line = lines[i];
562
+ if (listIndent >= 0) {
563
+ const item = line.match(/^(\s*)-\s*(['"]?)(.*?)\2\s*(#.*)?$/);
564
+ if (item && item[1].length > listIndent) {
565
+ if (item[3] === oldName) {
566
+ lines[i] = line.replace(new RegExp(`(-\\s*)(['"]?)${esc}\\2`), (_m, lead) => `${lead}${quoted}`);
567
+ }
568
+ continue;
569
+ }
570
+ listIndent = -1;
571
+ }
572
+ const key = line.match(/^(\s*)x-reference:\s*(.*)$/);
573
+ if (key && (key[2] === '' || key[2].startsWith('#'))) {
574
+ listIndent = key[1].length;
575
+ continue;
576
+ }
577
+ lines[i] = line
578
+ .replace(new RegExp(`(x-reference:\\s*)(['"]?)${esc}\\2(?=\\s*(?:[,}]|#|$))`, 'gm'), (_m, lead) => `${lead}${quoted}`)
579
+ .replace(/(x-reference:\s*\[)([^\]]*)(\])/g, (_m, open, body, close) => open + body.split(',').map(retag).join(',') + close);
580
+ }
581
+ return lines.join('\n');
582
+ }
583
+
507
584
  /** Remove now-empty parents up to (not including) the data root — a moved collection leaves its
508
585
  * namespace folder behind otherwise. */
509
586
  function pruneEmpty(dir, stopAt) {
@@ -565,9 +642,13 @@ function upsertField(ws, store, collection, fieldName, prop, required, verb) {
565
642
  // target collection's `title_template`, so a field drawer that round-trips that projection
566
643
  // writes the inherited value back onto the field — hand-recreating exactly the 49 duplicated
567
644
  // `x-display` lines the inheritance replaced. Only a template that DIFFERS from the target's
568
- // is a real authored override.
569
- const ref = prop['x-reference'] ?? prop.items?.['x-reference'];
570
- const inherited = ref && ref !== '*' ? store.descriptors.get(ref)?.title_template : undefined;
645
+ // is a real authored override. For a UNION (`x-reference` a list), presentation inherits only
646
+ // when every member's `title_template` agrees — so the cleanup here computes the same
647
+ // unanimous value, not just the first member's.
648
+ const targets = refTargetsOf(prop) ?? [];
649
+ const tpls = targets === '*' ? [] : targets.map((t) => store.descriptors.get(t)?.title_template);
650
+ const first = tpls[0];
651
+ const inherited = typeof first === 'string' && first.length > 0 && tpls.every((v) => v === first) ? first : undefined;
571
652
  if (inherited) {
572
653
  if (prop['x-title-template'] === inherited) {
573
654
  prop = { ...prop };
@@ -597,23 +678,95 @@ function upsertField(ws, store, collection, fieldName, prop, required, verb) {
597
678
  return { collection, field: fieldName, file: dest, extends: doc.extends };
598
679
  }
599
680
 
681
+ /**
682
+ * WHERE A UI-VIEW'S SOURCE ACTUALLY LIVES — asked of the manifest, exactly as `descriptorSourceDir`
683
+ * asks it for a collection, and for the same reason: the guard that matters is "will `npm install`
684
+ * erase this write", not "which module owns it".
685
+ *
686
+ * ⚠ This used to be `workspaceSystemDir` unconditionally, which silently meant a view could only be
687
+ * saved if the WORKSPACE MODULE happened to ship it. Saving one shipped by any other inline module
688
+ * wrote a SECOND file carrying the same id, and compile refuses that by name — so the whole write
689
+ * rolled back and the surface reported `name collision on ui-view "…"` instead of saving. Measured
690
+ * on gk-brain 2026-08-28: every one of its module-shipped views (`modules/family`, `modules/rnd`,
691
+ * `modules/services`) was unsaveable, and the failure said nothing about why.
692
+ *
693
+ * Returns `{ file, shipped }` — where to write, and the workspace-relative source that already
694
+ * exists (null for a new view, which lands in the workspace module as before).
695
+ */
696
+ function uiViewSourceFile(ws, id) {
697
+ const src = readManifest(ws.root)?.entries?.[`ui-views/${id}.ui-view.yaml`]?.sources?.[0];
698
+ // sources are `{path, hash}`; tolerate the pre-0.10 string form, same as compile's staleness check
699
+ const shipped = typeof src === 'string' ? src : src?.path;
700
+ if (!shipped) return { file: path.join(workspaceSystemDir(ws, 'ui-views'), `${id}.ui-view.yaml`), shipped: null };
701
+ return { file: path.join(ws.root, shipped), shipped };
702
+ }
703
+
704
+ /**
705
+ * Re-attach a rewritten YAML source's COMMENTS — the part `dump` cannot round-trip.
706
+ *
707
+ * js-yaml drops every comment on `load` → `dump`. That is fine for a generated artifact and wrong
708
+ * for a module SOURCE, which is where this project writes down why something exists (`setScalar`
709
+ * above exists for the same reason). A real round-trip needs a different YAML library and core is
710
+ * not taking one on for this, so this does the narrow thing that is actually safe: a comment block
711
+ * sitting directly above a TOP-LEVEL key is carried back above that same key, if the key survived.
712
+ * The file header comes along for free — it is the block above the first key.
713
+ *
714
+ * ⚠ Deliberately top-level only. A comment above a NESTED key cannot be re-placed without knowing
715
+ * where that key ended up, and a misplaced comment is worse than an absent one: it would attach an
716
+ * explanation to something it does not explain. Those are still lost. Measured against
717
+ * `modules/family/ui-views/health-labs-abnormal.ui-view.yaml`, whose two blocks — the file header
718
+ * and the ⚠ above `filter:` — are both top-level and both survive.
719
+ */
720
+ function reattachComments(oldText, newText) {
721
+ const TOP_KEY = /^([A-Za-z_][\w-]*):/;
722
+ const blocks = new Map(); // surviving key -> the comment lines that sat above it
723
+ let pending = [];
724
+ for (const line of oldText.split('\n')) {
725
+ if (line.startsWith('#') || line.trim() === '') { pending.push(line); continue; }
726
+ const key = TOP_KEY.exec(line)?.[1];
727
+ if (key && pending.some((l) => l.startsWith('#'))) {
728
+ while (pending.length && pending[pending.length - 1].trim() === '') pending.pop();
729
+ blocks.set(key, pending);
730
+ }
731
+ pending = [];
732
+ }
733
+ if (!blocks.size) return newText;
734
+
735
+ const out = [];
736
+ for (const line of newText.split('\n')) {
737
+ const block = blocks.get(TOP_KEY.exec(line)?.[1]);
738
+ if (block) out.push(...block);
739
+ out.push(line);
740
+ }
741
+ return out.join('\n');
742
+ }
743
+
600
744
  // saved views (M3): a studio-saved view IS a ui-view record — but ui-views are
601
745
  // system-stored (sources + compile), so the write goes through the same gate as any
602
746
  // other schema op. the studio "save view" button lands here.
603
747
  export function saveUiView(ws, store, { id, view }) {
604
748
  if (!id || !/^[a-z0-9][a-z0-9-/]*$/.test(id)) throw new Error(`invalid ui-view id "${id}" — lowercase slug required`);
605
- const dest = path.join(workspaceSystemDir(ws, 'ui-views'), `${id}.ui-view.yaml`);
749
+ const { file: dest, shipped } = uiViewSourceFile(ws, id);
750
+ if (shipped && /(^|\/)node_modules\//.test(shipped))
751
+ throw new Error(`ui-view "${id}" is shipped by an installed package (${shipped}) — a write there is erased by the next npm install.\n save it under a different name, or disable it (dreamteamer.disable) and re-create it.`);
606
752
  const existed = fs.existsSync(dest);
753
+ // A module source is where this project writes down WHY a view exists; `dump` cannot keep that.
754
+ const previous = existed ? fs.readFileSync(dest, 'utf8') : null;
607
755
  writeGated(ws, store, [dest], `dreamteamer: ui-views ${existed ? 'update' : 'add'} ${id}`, () => {
608
756
  fs.mkdirSync(path.dirname(dest), { recursive: true });
609
- fs.writeFileSync(dest, dump(view));
757
+ fs.writeFileSync(dest, previous === null ? dump(view) : reattachComments(previous, dump(view)));
610
758
  });
611
759
  return { id, file: dest, updated: existed };
612
760
  }
613
761
 
614
762
  export function removeUiView(ws, store, id) {
615
- const dest = path.join(workspaceSystemDir(ws, 'ui-views'), `${id}.ui-view.yaml`);
616
- if (!fs.existsSync(dest)) throw new Error(`ui-view "${id}" is not workspace-owned (module-shipped views are removed via dreamteamer.disable)`);
763
+ // Same source resolution as the save above — an inline module's view is under this repo's git
764
+ // history like everything else, so deleting it is one revertable commit. Refusing it while
765
+ // ALLOWING a save to the same file would be an asymmetry with nothing behind it.
766
+ const { file: dest, shipped } = uiViewSourceFile(ws, id);
767
+ if (shipped && /(^|\/)node_modules\//.test(shipped))
768
+ throw new Error(`ui-view "${id}" is shipped by an installed package (${shipped}) — removing the file would be undone by the next npm install.\n disable it instead: add "<module>/${id}" to dreamteamer.disable in package.json.`);
769
+ if (!fs.existsSync(dest)) throw new Error(`ui-view "${id}" does not exist`);
617
770
  writeGated(ws, store, [dest], `dreamteamer: ui-views rm ${id}`, () => fs.rmSync(dest));
618
771
  return { removed: id };
619
772
  }
package/src/store.js CHANGED
@@ -9,10 +9,11 @@ import Ajv from 'ajv';
9
9
  import addFormats from 'ajv-formats';
10
10
  import { dump } from './yaml.js';
11
11
  import { generateId } from './template.js';
12
- import { parseRecord, parseRecordText, patternRe, fmtAjvError, unknownFields, walk, EXT, assertSafeId } from './records.js';
12
+ import { parseRecord, parseRecordText, patternRe, fmtAjvError, unknownFields, walk, EXT, assertSafeId, idFromRecordPath, MAX_RECORD_BYTES } from './records.js';
13
13
  import { normalizeRecord } from './temporal.js';
14
14
  import { NO_RUNTIME, sourceHint, loadDescriptors, runtimeDir, namespaces as compiledNamespaces, sourceRoots as compiledSourceRoots } from './runtime.js';
15
15
  import { parseRef } from './namespace.js';
16
+ import { refTargetsOf } from './ref.js';
16
17
 
17
18
  // git calls whose failure we CATCH must not print git's own error: execFileSync forwards the
18
19
  // child's stderr to ours unless told otherwise, so a handled "not a git repository" still
@@ -65,19 +66,33 @@ export class Store {
65
66
  return path.join(d.storage.base === 'runtime' ? this.runtime : this.root, d.storage.path);
66
67
  }
67
68
 
68
- filePath(d, id) {
69
+ filePath(d, id, ext) {
69
70
  assertSafeId(id); // never fs-join an id that can climb out of the collection
70
71
  if (d.storage.shape === 'folder') {
71
72
  if (!d.storage.entry) throw new Error(`collection "${d.name}" is folder-shape but declares no storage.entry`);
72
73
  return path.join(this.dir(d), id, d.storage.entry);
73
74
  }
75
+ if ((d.storage.codec ?? 'md') === 'file') {
76
+ // An opaque record's extension is not derivable from its id. A caller that WRITES says what
77
+ // it is; a caller that reads goes through the id index instead (recordRoot, below).
78
+ if (!ext) throw new Error(`collection "${d.name}" is \`codec: file\` — its path needs the file's extension`);
79
+ return path.join(this.dir(d), `${id}.${d.storage.suffix}.${ext}`);
80
+ }
74
81
  return path.join(this.dir(d), `${id}.${d.storage.suffix}${EXT[d.storage.codec ?? 'md']}`);
75
82
  }
76
83
 
77
84
  // the on-disk unit of a record: its folder for folder shapes, its file otherwise
78
85
  recordRoot(d, id) {
79
86
  assertSafeId(id);
80
- return d.storage.shape === 'folder' ? path.join(this.dir(d), id) : this.filePath(d, id);
87
+ if (d.storage.shape === 'folder') return path.join(this.dir(d), id);
88
+ // Only the index knows an opaque record's extension, so the on-disk unit is looked up rather
89
+ // than derived. An unknown id is the caller's error either way — `read` says so first.
90
+ if ((d.storage.codec ?? 'md') === 'file') {
91
+ const file = this.ids(d.name).get(id);
92
+ if (!file) throw new Error(`${d.name}/${id}: no such record`);
93
+ return file;
94
+ }
95
+ return this.filePath(d, id);
81
96
  }
82
97
 
83
98
  // current HEAD — one cheap rev-parse per cache check vs a multi-thousand-file walk
@@ -112,10 +127,9 @@ export class Store {
112
127
  }
113
128
  return ids;
114
129
  }
115
- const tail = `.${d.storage.suffix}${EXT[d.storage.codec ?? 'md']}`;
116
130
  for (const f of walk(dir)) {
117
- const r = path.relative(dir, f);
118
- if (r.endsWith(tail)) ids.set(r.slice(0, -tail.length), f);
131
+ const id = idFromRecordPath(d, path.relative(dir, f));
132
+ if (id !== null) ids.set(id, f);
119
133
  }
120
134
  return ids;
121
135
  }
@@ -173,6 +187,13 @@ export class Store {
173
187
  // canonical, offset-carrying value. ajv's `date-time` accepts exactly one spelling; without
174
188
  // this every human-shaped input is a validation error (see src/temporal.js).
175
189
  normalizeRecord(d.schema, fields);
190
+ // qualifyBareRefs must ALSO run before ajv.compile(d.schema) below, for the same "one choke
191
+ // point" reason but a different consequence: `validate(fields)` is what triggers useDefaults,
192
+ // materializing any schema `default:` onto `fields` for the first time — a bare value sitting
193
+ // in a single-target ref field's `default:` is never seen by qualifyBareRefs and would reach
194
+ // checkRefs unqualified, failing as malformed rather than as the dangling reference it should
195
+ // read as. In practice no shipped descriptor defaults a ref field, so this is latent, not hit.
196
+ this.qualifyBareRefs(d, fields);
176
197
  const validate = this.ajv.compile(d.schema); // useDefaults mutates: defaults materialize
177
198
  if (!validate(fields)) {
178
199
  const msgs = validate.errors.map((e) => ' ' + fmtAjvError(e, fields));
@@ -182,10 +203,37 @@ export class Store {
182
203
  return fields;
183
204
  }
184
205
 
206
+ // Bare ids are accepted on INPUT for a field whose target set has exactly one member, and
207
+ // qualified HERE — the same choke point that canonicalizes datetimes (normalizeRecord), for the
208
+ // same reason: add/set reach disk through validate(), so the file always carries the one
209
+ // canonical spelling. The deliberate exception is `revert`: it restores committed historical
210
+ // BYTES verbatim (that is the whole point of a revert), calling validate() only to prove the
211
+ // historical content still parses — never on the text that actually reaches atomicWrite. So a
212
+ // bare ref that was committed past this choke point (hand-edited, or written by an older engine)
213
+ // stays bare when reverted TO; `check` is what flags it, not this method. Union and '*' fields
214
+ // never qualify: the prefix is the only type information those values carry. A value that
215
+ // already parses as a ref is never rewritten — so a qualified-but-wrong id fails downstream as a
216
+ // precise dangling reference, not as malformed syntax. Known limit: a slash-carrying bare id
217
+ // (path-shaped ids) parses as a ref and is not qualified; the checkRefs error then names the
218
+ // misread collection.
219
+ qualifyBareRefs(d, fields) {
220
+ for (const [key, s] of Object.entries(d.schema.properties ?? {})) {
221
+ const targets = refTargetsOf(s);
222
+ if (!targets || targets === '*' || targets.length !== 1) continue;
223
+ const raw = fields[key];
224
+ if (raw == null) continue;
225
+ const qualify = (v) =>
226
+ typeof v === 'string' && v !== '' && !v.startsWith('@') && !parseRef(v, this.namespaces)
227
+ ? `${targets[0]}/${v}`
228
+ : v;
229
+ fields[key] = Array.isArray(raw) ? raw.map(qualify) : qualify(raw);
230
+ }
231
+ }
232
+
185
233
  checkRefs(d, fields, prefix = []) {
186
234
  for (const [key, s] of Object.entries(d.schema.properties ?? {})) {
187
- const target = s?.['x-reference'] ?? s?.items?.['x-reference'];
188
- if (!target) continue;
235
+ const targets = refTargetsOf(s);
236
+ if (!targets) continue;
189
237
  const raw = fields[key];
190
238
  if (raw == null) continue;
191
239
  for (const value of Array.isArray(raw) ? raw : [raw]) {
@@ -196,7 +244,10 @@ export class Store {
196
244
  const parsed = parseRef(value, this.namespaces);
197
245
  if (!parsed) throw new Error(`${key}: reference "${value}" is not <collection>/<id> — nothing was written.`);
198
246
  const { collection: coll, id } = parsed;
199
- if (target !== '*' && coll !== target) throw new Error(`${key}: reference "${value}" must target collection "${target}" — nothing was written.`);
247
+ if (targets !== '*' && !targets.includes(coll)) {
248
+ const want = targets.length === 1 ? `collection "${targets[0]}"` : `one of: ${targets.join(', ')}`;
249
+ throw new Error(`${key}: reference "${value}" must target ${want} — nothing was written.`);
250
+ }
200
251
  if (!this.descriptors.has(coll)) throw new Error(`${key}: reference "${value}" targets unknown collection "${coll}" — nothing was written.`);
201
252
  if (!this.ids(coll).has(id)) throw new Error(`${key}: dangling reference "${value}" — no such record. nothing was written.`);
202
253
  }
@@ -226,8 +277,45 @@ export class Store {
226
277
  });
227
278
  }
228
279
 
280
+ /** Import a file AS a record. There are no fields to validate and nothing to serialize, which is
281
+ * why this is a sibling of add() rather than a branch inside it — the two share their last three
282
+ * lines and nothing else. */
283
+ addFile(collection, id, srcPath, { force = false } = {}) {
284
+ const d = this.writableDescriptor(collection);
285
+ if ((d.storage.codec ?? 'md') !== 'file') throw new Error(`"${collection}" is not a \`codec: file\` collection — add its records with --<field> values, not --from`);
286
+ assertSafeId(id);
287
+ if (d.id?.pattern && !patternRe(d.id.pattern).test(id)) {
288
+ throw new Error(`id "${id}" does not match pattern ${d.id.pattern} — nothing was written.`);
289
+ }
290
+ const ext = path.extname(srcPath).slice(1).toLowerCase();
291
+ if (!ext) throw new Error(`${srcPath} has no extension — a file record is named by one. Nothing was written.`);
292
+ const allowed = d.storage.extensions;
293
+ if (allowed && !allowed.includes(ext)) throw new Error(`"${collection}" does not accept .${ext} — its declared extensions are ${allowed.join(', ')}. Nothing was written.`);
294
+ const size = fs.statSync(srcPath).size;
295
+ const max = d.storage.max_bytes ?? MAX_RECORD_BYTES;
296
+ if (size > max) throw new Error(`${srcPath} is ${size} bytes, over "${collection}"'s max_bytes of ${max} — a record is a small file. Nothing was written.`);
297
+ const existing = this.ids(collection).get(id);
298
+ if (existing && !force) throw new Error(`${collection}/${id} already exists — pass --force to replace it. Nothing was written.`);
299
+ const file = this.filePath(d, id, ext);
300
+ return this.withWriteLock(() => {
301
+ this._idsCache.delete(collection);
302
+ fs.mkdirSync(path.dirname(file), { recursive: true });
303
+ // A replacement whose extension changed would otherwise leave its predecessor behind, and
304
+ // two files under one id is the ambiguity `check` reports. One id is one file.
305
+ const stale = existing && existing !== file ? existing : null;
306
+ const restore = snapshot([file, ...(stale ? [stale] : [])]);
307
+ if (stale) fs.rmSync(stale, { force: true });
308
+ fs.copyFileSync(srcPath, file);
309
+ this.commit([file, ...(stale ? [stale] : [])], `dreamteamer: ${collection} add ${id}`, restore, d.storage.repo ?? '.');
310
+ return { id, file };
311
+ });
312
+ }
313
+
229
314
  set(collection, id, changes) {
230
315
  const d = this.writableDescriptor(collection);
316
+ if ((d.storage.codec ?? 'md') === 'file') {
317
+ throw new Error(`${collection}/${id} is a file record — its fields are derived from the file, so there is nothing to set. Replace it with \`dreamteamer add ${collection} ${id} --from <path> --force\`.`);
318
+ }
231
319
  const { fields, file } = this.read(collection, id);
232
320
  const previous = fs.readFileSync(file, 'utf8');
233
321
  const next = { ...fields, ...changes };