dreamteamer 0.8.0 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "dreamteamer",
-	"version": "0.8.0",
+	"version": "0.9.1",
 	"description": "A workspace compiler for coding agents \u2014 schema-validated records as plain files over git, compiled into every harness",
 	"license": "Apache-2.0",
 	"author": "Gilad Khen <giladkhen@gmail.com>",

package/src/compile.js

@@ -840,7 +840,13 @@ export function compile({ root, pkg }) {
 	// clear each kind's folder, plus `system/` — a runtime compiled by a pre-flatten engine has the
 	// whole tree under there, and leaving it would keep stale descriptors on disk beside the fresh
 	// ones. Never `rm -rf` the runtime root itself: it also holds the write lock.
-	for (const kind of KINDS) fs.rmSync(path.join(RUNTIME, kind), { recursive: true, force: true });
+	// ⚠ DERIVED_KINDS too, not just KINDS. `modules/` is projected rather than staged, so it was not
+	// in this loop and never got cleared — a module that was RENAMED or REMOVED left its old record
+	// behind forever, listing collections that no longer exist. `check` reads those records like any
+	// other, so it surfaced as a dangling reference in a file nobody had touched, twice in one day
+	// (`hq3-workspace` after the workspace-module rename, and again after `crm` was folded in). The
+	// runtime is build output; stale build output is the compiler's problem, not the reader's.
+	for (const kind of [...KINDS, ...DERIVED_KINDS]) fs.rmSync(path.join(RUNTIME, kind), { recursive: true, force: true });
 	fs.rmSync(path.join(RUNTIME, 'system'), { recursive: true, force: true });
 	fs.rmSync(path.join(RUNTIME, 'ui'), { recursive: true, force: true });
 	for (const [rt, e] of entries) {

package/src/schema-ops.js

@@ -64,6 +64,81 @@ export function workspaceSystemDir(ws, kind) {
 	return kindDir(wm ? path.join(ws.root, 'modules', wm) : ws.root, kind);
 }
 
+/**
+ * WHERE A COLLECTION'S DESCRIPTOR ACTUALLY LIVES — asked of the manifest, not assumed.
+ *
+ * `renameCollection` used to derive this from `workspaceSystemDir`, which silently meant "only the
+ * workspace module's own collections can be renamed". That is the wrong line. The guard exists to
+ * stop a write that will be ERASED, and the thing that erases writes is `npm install` — so the test
+ * is `node_modules/`, not "which module". A module whose sources are inline in the workspace repo is
+ * under the same git history as everything else and is perfectly safe to rewrite; refusing it made
+ * `collections rename` unusable for exactly the migration it was built for, because a workspace's
+ * domain collections almost always live in a module.
+ *
+ * Returns `{ dir, sources }` — the kind dir to write into (the SAME module the descriptor came from,
+ * so a rename never teleports a collection into the workspace module), and every descriptor source
+ * that contributed, so the caller can refuse the cases this cannot honestly do.
+ */
+function descriptorSourceDir(ws, name) {
+	const entry = readManifest(ws.root)?.entries?.[`collections/${name}.collection.yaml`];
+	// `sources` mixes the descriptor with any collection-templates it merged, so match on the shape
+	// of a descriptor path for THIS collection. A namespaced name is nested, hence the full suffix.
+	const suffix = `collections/${name}.collection.yaml`;
+	const sources = (entry?.sources ?? [])
+		.map((s) => s.path)
+		.filter((p) => p.endsWith(suffix));
+	if (!sources.length) return { dir: null, sources };
+	// The BASE descriptor is the one to move. With an overlay present there are two, and the overlay's
+	// `extends` names the base by its old qualified id — rewriting that is a second, different
+	// migration, so the caller refuses rather than half-doing it.
+	const moduleRoot = path.join(ws.root, sources[0].slice(0, sources[0].length - suffix.length));
+	return { dir: kindDir(moduleRoot, 'collections'), sources };
+}
+
+/**
+ * Set one scalar in a YAML document TEXTUALLY, so comments and key order survive.
+ *
+ * This exists because `load` → mutate → `dump` is lossy in the one way that matters here: it drops
+ * every comment. That is fine for a generated artifact and wrong for a module SOURCE, which is where
+ * this project writes down why a collection exists. Only `renameCollection` uses it, and only for the
+ * three scalars a rename changes; anything more ambitious belongs in a real round-trip YAML library,
+ * not in a regex.
+ *
+ * Handles both spellings the descriptors actually use — a top-level key, a nested block mapping, and
+ * the inline `storage: { path: x, suffix: y }` flow form. Callers MUST re-parse and assert, because a
+ * shape not covered here fails by changing nothing rather than by throwing.
+ */
+function setScalar(text, keyPath, value) {
+	const [head, child] = keyPath;
+	if (!child) return text.replace(new RegExp(`^${head}:.*$`, 'm'), `${head}: ${value}`);
+
+	// inline flow mapping: `storage: { path: data/x, suffix: y }`
+	const flow = new RegExp(`^${head}:\\s*\\{([^}]*)\\}\\s*$`, 'm').exec(text);
+	if (flow) {
+		let body = flow[1];
+		body = new RegExp(`\\b${child}:\\s*[^,}]+`).test(body)
+			? body.replace(new RegExp(`(\\b${child}:\\s*)[^,}]+`), `$1${value}`)
+			: `${body.trimEnd()}, ${child}: ${value}`;
+		return text.slice(0, flow.index) + `${head}: {${body}}` + text.slice(flow.index + flow[0].length);
+	}
+
+	// block mapping: `storage:\n  path: data/x`
+	const block = new RegExp(`^${head}:\\n(?:[ \\t]+.*\\n)*?[ \\t]+${child}:.*$`, 'm').exec(text);
+	if (block) {
+		return text.slice(0, block.index)
+			+ block[0].replace(new RegExp(`([ \\t]+${child}:).*$`, 'm'), `$1 ${value}`)
+			+ text.slice(block.index + block[0].length);
+	}
+
+	// the key is absent under an existing block — insert it directly after the parent
+	const parent = new RegExp(`^${head}:\\s*$`, 'm').exec(text);
+	if (parent) {
+		const at = parent.index + parent[0].length + 1;
+		return text.slice(0, at) + `  ${child}: ${value}\n` + text.slice(at);
+	}
+	return text;
+}
+
 // ---- ops ------------------------------------------------------------------------
 
 export function createCollection(ws, store, { name, template, namespace }) {
@@ -143,6 +218,15 @@ export function removeCollection(ws, store, name, { force = false } = {}) {
  * already scopes prose to `[[wikilinks]]` (decision 7) — a fresh `oldName/` pattern would have to
  * relearn both, and would corrupt `data/tasks/` in a path or a URL on its first outing. N passes over
  * the record files is the price, and at human scale it is worth paying for reusing the correct code.
+ *
+ * ⚠ MEASURED 2026-08-17, so the cost is a number rather than a hope: a 2,291-record collection in a
+ * 3,391-file workspace — gk-brain's `finance-transactions` — takes **3 minutes**, of which 142s is
+ * system time. That is 7.7M file reads to rewrite ZERO references, because the pass runs per id
+ * whether or not anything points at the collection. Tolerable for a one-time migration and left
+ * alone on that basis; it is O(records x files), so a workspace 3x larger pays 27 minutes. The fix
+ * when it is needed is a batch entry point on the store that reads each file ONCE and loops the ref
+ * set in memory, with `text.includes(oldName + '/')` as a cheap NEGATIVE filter only — never as the
+ * matcher, for the reason above.
  */
 export function renameCollection(ws, store, oldName, newName) {
 	const d = store.descriptor(oldName); // throws with the known-collection list if absent
@@ -156,10 +240,21 @@ export function renameCollection(ws, store, oldName, newName) {
 	if (store.descriptors.has(newName)) throw new Error(`collection "${newName}" already exists`);
 	if (d.storage.base === 'runtime') throw new Error(`"${oldName}" is a compiled source, not a data collection — it cannot be renamed`);
 
-	const src = path.join(workspaceSystemDir(ws, 'collections'), `${oldName}.collection.yaml`);
-	const dest = path.join(workspaceSystemDir(ws, 'collections'), `${newName}.collection.yaml`);
+	// The descriptor is renamed IN THE MODULE THAT SHIPS IT — see `descriptorSourceDir`. Two cases
+	// this refuses, both because doing them halfway is worse than not doing them:
+	const { dir: sourceDir, sources } = descriptorSourceDir(ws, oldName);
+	if (sources.length > 1) {
+		throw new Error(`"${oldName}" is overlaid — ${sources.length} modules contribute a descriptor (${sources.join(', ')}).\n  the overlay's \`extends\` names the base by its current id, so renaming the base alone would break it. merge or remove the overlay first.`);
+	}
+	if (sources.some((p) => p.split(path.sep).includes('node_modules'))) {
+		throw new Error(`"${oldName}" ships from node_modules (${sources[0]}) — a write there is erased by the next \`npm install\`. rename it in its own repo and release, or overlay it with \`extends\`.`);
+	}
+	const src = sourceDir
+		? path.join(sourceDir, `${oldName}.collection.yaml`)
+		: path.join(workspaceSystemDir(ws, 'collections'), `${oldName}.collection.yaml`);
+	const dest = path.join(sourceDir ?? workspaceSystemDir(ws, 'collections'), `${newName}.collection.yaml`);
 	if (!fs.existsSync(src)) {
-		throw new Error(`"${oldName}" is not workspace-owned — it ships with a module, so rename it there (or overlay it with \`extends\`)`);
+		throw new Error(`"${oldName}" has no writable descriptor source — the manifest names none under a module in this workspace. it may be contributed by the engine itself; overlay it with \`extends\` instead.`);
 	}
 
 	const doc = load(fs.readFileSync(src, 'utf8'));
@@ -212,21 +307,66 @@ export function renameCollection(ws, store, oldName, newName) {
 		};
 		for (const id of ids) captureRefs(`${oldName}/${id}`);
 		captureRefs(`collections/${oldName}`);
-		const restoreRefs = () => { for (const [f, bytes] of refFiles) fs.writeFileSync(f, bytes); };
+		const restoreRefs = () => {
+			for (const [f, bytes] of refFiles) {
+				fs.mkdirSync(path.dirname(f), { recursive: true }); // pruneEmpty may have taken the parent
+				fs.writeFileSync(f, bytes);
+			}
+		};
 
 		const touched = new Set();
 		let rewrites = 0;
 		try {
-			// 1. the descriptor source, at its new path
+			// 1. the descriptor source, at its new path — EDITED TEXTUALLY, never re-dumped.
+			//
+			// ⚠ `fs.writeFileSync(dest, dump(doc))` destroyed every comment in the descriptor, and a
+			// descriptor's comments are where this project keeps its reasoning: 194 lines across 24
+			// files in one real migration, including 22-line headers stating what belongs in a
+			// collection and which failure mode it guards against. The record survived; the thinking
+			// did not, and nothing said so.
+			//
+			// A rename changes exactly three scalars. Rewriting those three in place keeps the
+			// comments, the key order and the author's formatting — and the parse afterwards proves
+			// the edit landed rather than trusting the regex.
+			const edited = setScalar(setScalar(setScalar(srcBytes.toString('utf8'),
+				['name'], newName),
+				['storage', 'path'], newPath),
+				['storage', 'suffix'], newSuffix);
+			const parsed = load(edited);
+			if (parsed?.name !== newName || parsed?.storage?.path !== newPath || parsed?.storage?.suffix !== newSuffix) {
+				throw new Error(`could not rewrite ${path.relative(ws.root, src)} in place — name/storage.path/storage.suffix did not take. nothing was changed.`);
+			}
 			doc.name = newName;
 			doc.storage = { ...doc.storage, path: newPath, suffix: newSuffix };
 			fs.mkdirSync(path.dirname(dest), { recursive: true });
-			fs.writeFileSync(dest, dump(doc));
+			fs.writeFileSync(dest, edited);
 			if (dest !== src) fs.rmSync(src);
 			touched.add(src);
 			touched.add(dest);
 
-			// 2. the record folder, then the per-file suffix if it was derived
+			// 2. INBOUND REFERENCES FIRST, while the records are still where the store thinks they are.
+			//
+			// ⚠ This used to run AFTER the folder move and it silently missed every SELF-reference.
+			// `store.rewriteRefs` walks `recordFiles()`, which resolves each collection's directory
+			// from the descriptor loaded when the Store was built — i.e. the OLD `storage.path`. Move
+			// the records first and that walk finds an empty directory, so a record pointing at its
+			// own collection is never rewritten and dangles the moment compile catches up.
+			//
+			// It is not a corner case: it hit `finance/accounts`, where every card and loan carries
+			// `settled_by: <the account that settles it>` — 5 dangling refs out of 11 records, found
+			// only because `check` ran afterwards. Doing the rewrite first needs no descriptor reload
+			// and no second code path: the files are still at the old path, which is exactly what the
+			// old refs say.
+			for (const id of ids) {
+				const out = store.rewriteRefs(`${oldName}/${id}`, `${newName}/${id}`);
+				rewrites += out.rewrites;
+				for (const f of out.touched) touched.add(f);
+			}
+			const collOut = store.rewriteRefs(`collections/${oldName}`, `collections/${newName}`);
+			rewrites += collOut.rewrites;
+			for (const f of collOut.touched) touched.add(f);
+
+			// 3. the record folder, then the per-file suffix if it was derived
 			if (newDir !== oldDir && fs.existsSync(oldDir)) {
 				fs.mkdirSync(path.dirname(newDir), { recursive: true });
 				fs.renameSync(oldDir, newDir);
@@ -244,33 +384,48 @@ export function renameCollection(ws, store, oldName, newName) {
 			}
 			if (movedData) { touched.add(oldDir); touched.add(newDir); }
 
-			// 3. inbound references: per record id, plus the collection's own id in `collections`
-			//    (which is what ui-views and command-bindings point at).
-			for (const id of ids) {
-				const out = store.rewriteRefs(`${oldName}/${id}`, `${newName}/${id}`);
-				rewrites += out.rewrites;
-				for (const f of out.touched) touched.add(f);
-			}
-			const collOut = store.rewriteRefs(`collections/${oldName}`, `collections/${newName}`);
-			rewrites += collOut.rewrites;
-			for (const f of collOut.touched) touched.add(f);
-
 			// 4. bare `x-reference: <oldName>` in every descriptor SOURCE. Not a `<collection>/<id>`
-			//    ref, so step 3 cannot see it — and leaving it makes compile fail on an unknown target.
+			//    ref, so step 2 cannot see it — and leaving it makes compile fail on an unknown target.
+			//
+			// ⚠ TEXTUAL, for the same reason step 1 is. This used to `load` → mutate → `dump`, which
+			// meant that ANY descriptor needing a retarget lost every comment in it — including the
+			// renamed one itself when it self-references, which is how step 1's careful preservation
+			// was undone one step later. 17 of the 24 descriptors stripped in the migration that
+			// found this were stripped HERE, not there.
+			//
+			// `retargetRefs` still decides WHETHER a file is affected — it walks the parsed schema and
+			// knows about nested properties and `items` — but the write is a line edit, and the parse
+			// afterwards proves it landed.
 			for (const f of descriptorSources(ws, store)) {
 				const before = fs.readFileSync(f, 'utf8');
-				const doc2 = load(before);
-				if (!doc2 || !retargetRefs(doc2.schema, oldName, newName)) continue;
+				const probe = load(before);
+				if (!probe || !retargetRefs(probe.schema, oldName, newName)) continue;
+				// ⚠ the boundary must cover BOTH spellings. A descriptor may write the block form
+				// (`x-reference: accounts` to end of line) or the inline flow form
+				// (`{ type: string, x-reference: accounts }`), where the value ends at `,` or `}`.
+				// Anchoring on `$` alone silently matched nothing in the flow form — and the assert
+				// below turned that silence into a refusal, which is how it was found.
+				const after = before.replace(
+					new RegExp(`(x-reference:\\s*)(['"]?)${oldName.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')}\\2(?=\\s*(?:[,}]|#|$))`, 'gm'),
+					(_m, lead) => `${lead}${newName.includes('/') ? `'${newName}'` : newName}`);
+				const reparsed = load(after);
+				if (!reparsed || retargetRefs(reparsed.schema, oldName, newName)) {
+					throw new Error(`could not retarget x-reference "${oldName}" in ${path.relative(ws.root, f)} without reformatting it — nothing was changed.`);
+				}
 				if (!refFiles.has(f)) refFiles.set(f, Buffer.from(before));
-				fs.writeFileSync(f, dump(doc2));
+				fs.writeFileSync(f, after);
 				touched.add(f);
 				rewrites++;
 			}
 
 			compile(ws); // the gate: an uncompilable rename never reaches history
 		} catch (e) {
-			restoreRefs();
+			// ⚠ undo() FIRST. A captured file can be a SELF-reference — a record of the collection being
+			// renamed — so its path only exists again once undo() has moved the folder back. Restoring
+			// before that wrote into a directory that was no longer there, and the ENOENT masked the
+			// error actually being rolled back from.
 			undo();
+			restoreRefs();
 			try { compile(ws); } catch { /* pre-rename sources were compilable */ }
 			throw e;
 		}
@@ -287,8 +442,8 @@ export function renameCollection(ws, store, oldName, newName) {
 			execFileSync('git', ['commit', '--quiet', '-m', `dreamteamer: collections rename ${oldName} → ${newName}`, '--', ...rels], { cwd: ws.root, stdio: GIT_QUIET });
 		} catch (e) {
 			try { execFileSync('git', ['reset', '--quiet', '--', ...rels], { cwd: ws.root, stdio: GIT_QUIET }); } catch { /* nothing staged */ }
-			restoreRefs();
 			undo();
+			restoreRefs();
 			try { compile(ws); } catch { /* pre-rename sources were compilable */ }
 			throw new Error(`git commit failed — the rename was rolled back, nothing was changed. (${e.message.split('\n')[0]})`);
 		}

package/src/store.js

@@ -299,13 +299,31 @@ export class Store {
 		return new RegExp(`${ref.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w/-])`, 'g');
 	}
 
+	// ⚠ EACH FILE EXACTLY ONCE. The `modules` collection's storage.path is `modules` and
+	// `sourceRoots()` includes the workspace root, so walking it RE-YIELDS every module source that
+	// its own kind's walk already produced — 173 files in one real vault, every module source
+	// among them.
+	//
+	// That was harmless while every rewrite was idempotent, and stopped being harmless the day
+	// namespaces arrived: replacing `draft-docs/x` with `rnd/draft-docs/x` is NOT idempotent,
+	// because the result still contains the pattern. A second pass wrote
+	// `data/rnd/rnd/draft-docs/x` into module-source comments during a real migration. Dedupe HERE
+	// rather than making each caller idempotent — `findInboundRefs` is a caller too, and its counts
+	// were quietly doubled by the same walk.
 	*recordFiles() {
+		const seen = new Set();
 		for (const d of this.descriptors.values()) {
 			// for runtime-based collections, inbound-ref surgery targets SOURCES, not the runtime
 			const roots = d.storage.base === 'runtime' ? this.sourceRoots() : [this.root];
 			for (const srcRoot of roots) {
 				const dir = path.join(srcRoot, d.storage.path);
-				if (fs.existsSync(dir)) yield* walk(dir);
+				if (!fs.existsSync(dir)) continue;
+				for (const f of walk(dir)) {
+					const key = path.resolve(f);
+					if (seen.has(key)) continue;
+					seen.add(key);
+					yield f;
+				}
 			}
 		}
 	}