dreamteamer 0.6.0

package/src/runtime.js

@@ -0,0 +1,96 @@
+// THE BOUNDARY — `.dreamteamer/` is the one artifact the two halves of this engine share.
+//
+// The workspace compiler WRITES it (compile.js: modules × sources → merged descriptors + manifest).
+// The record layer READS it, and reads nothing else: store.js has never needed to know that
+// modules, channels, `extends` or `templates` exist. That seam was already real — it just wasn't
+// enforceable, because reaching the compiled output meant importing compile.js, which put a
+// record-layer → compiler edge in the graph for what is really a file-format dependency.
+//
+// Everything the record layer actually wanted is already IN the compiled output: the merged
+// descriptors, and (in the manifest) which directories hold the sources behind runtime-based
+// records. So this module owns the runtime's shape, both halves import it, and `npm run layers`
+// fails if the old edge comes back.
+import fs from 'node:fs';
+import path from 'node:path';
+import { load } from './yaml.js';
+
+export const RUNTIME_DIR = '.dreamteamer';
+
+/** One message, two callers with different manners: the store throws it, `check` prints it. */
+export const NO_RUNTIME = 'no compiled runtime — run `dreamteamer compile` first';
+
+export function runtimeDir(root) {
+	return path.join(root, RUNTIME_DIR);
+}
+
+export function readManifest(root) {
+	try { return load(fs.readFileSync(path.join(runtimeDir(root), 'manifest.yaml'), 'utf8')); } catch { return null; }
+}
+
+/**
+ * A kind's folder inside the compiled runtime. Flat (`.dreamteamer/<kind>`) is what compile writes;
+ * `.dreamteamer/system/<kind>` is probed because the runtime on disk may have been compiled by an
+ * engine from before the flatten — a stale runtime is the normal state between a `git pull` and the
+ * next `dt compile`, and answering "no compiled runtime" for one would be a lie.
+ */
+export function runtimeKindDir(root, kind) {
+	const flat = path.join(runtimeDir(root), kind);
+	if (fs.existsSync(flat)) return flat;
+	const nested = path.join(runtimeDir(root), 'system', kind);
+	return fs.existsSync(nested) ? nested : flat;
+}
+
+/**
+ * The merged collection descriptors, keyed by name — or `null` when nothing has been compiled.
+ *
+ * The ONE place descriptors are read, so the one place `storage.base` is guaranteed present. Both
+ * readers (store.js, check.js) had their own copy of this loop; the store's copy threw and check's
+ * printed-and-returned-2, so the difference is kept at the call site rather than in the loader.
+ */
+export function loadDescriptors(root) {
+	const dir = runtimeKindDir(root, 'collections');
+	if (!fs.existsSync(dir)) return null;
+	const out = new Map();
+	for (const f of fs.readdirSync(dir).sort()) {
+		if (!f.endsWith('.collection.yaml')) continue;
+		const d = load(fs.readFileSync(path.join(dir, f), 'utf8'));
+		d.storage ??= {};
+		d.storage.base ??= derivedBase(d);
+		out.set(d.name, d);
+	}
+	return out;
+}
+
+/**
+ * Which root `storage.path` is relative to — the whole of what the record layer needs to know about
+ * the system/data distinction, as DATA rather than as a string test it has to perform. `runtime` =
+ * compiled sources (skills, agents, ui-views, the descriptors themselves): generated, gitignored,
+ * and therefore not writable through the store. `workspace` = data/ and state/ records.
+ *
+ * compile.js writes the field. This derivation is compat for a runtime compiled by an OLDER engine,
+ * and is not optional: without it those collections resolve under the workspace root, where — in the
+ * `workspace-module` layout — they do not exist. That reads as zero records (a silent success) and
+ * lets `writableDescriptor` treat a compiled artifact as writable. Wrong in the expensive direction.
+ *
+ * ⚠ It tests for `system/`, which the flatten removed — that is correct and not a leftover. The only
+ * descriptors reaching it are ones compiled BEFORE `base` existed, and those necessarily still spell
+ * their runtime paths `system/<kind>`. Every descriptor this engine writes carries `base` explicitly.
+ */
+function derivedBase(d) {
+	return String(d.storage?.path ?? '').startsWith('system/') ? 'runtime' : 'workspace';
+}
+
+/**
+ * Absolute directories that may hold the SOURCES behind runtime-based records — every compiled
+ * module except npm copies (foreign installed artifacts, never rewrite targets), plus the workspace
+ * root. Used by ref surgery: renaming `collections/x` has to reach the descriptor in whichever
+ * module ships it, not the merged copy under `.dreamteamer/`.
+ *
+ * Read from the manifest rather than by re-running module discovery, which is both cheaper and more
+ * honest: the manifest records what was actually compiled, so a shadowed copy is already excluded.
+ */
+export function sourceRoots(root) {
+	const modules = readManifest(root)?.modules ?? [];
+	const roots = [root, ...modules.filter((m) => m.channel !== 'npm').map((m) => path.resolve(root, m.root))];
+	return [...new Set(roots)];
+}

package/src/schema-ops.js

@@ -0,0 +1,263 @@
+// schema operations — source-writing mutations shared by the CLI meta verbs and the
+// server's schema endpoints. the contract (audit finding 11, clean-room bug 2): an op
+// writes sources, proves them with a REAL compile, and only then commits — an
+// uncompilable source can never land in history. the successful gate compile also
+// leaves the runtime fresh, which kills the add-field-right-after-collections-add
+// papercut (review finding 7): schema ops ARE explicit compiles.
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { load, dump } from './yaml.js';
+import { compile, kindDir, titleCase } from './compile.js';
+import { readManifest, runtimeKindDir } from './runtime.js';
+
+// ---- the gate -------------------------------------------------------------------
+
+function writeGated(ws, store, files, subject, mutate) {
+	// same guarantees as record writes (docs-audit catch): the STORE's cross-process lock
+	// serializes schema ops too, and a failed git commit rolls the source back — a schema
+	// op fails closed exactly like a record mutation.
+	return store.withWriteLock(() => {
+		const snapshots = files.map((f) => ({ f, prev: fs.existsSync(f) ? fs.readFileSync(f) : null }));
+		const restore = () => {
+			for (const { f, prev } of snapshots) {
+				if (prev === null) fs.rmSync(f, { force: true });
+				else fs.writeFileSync(f, prev);
+			}
+		};
+		mutate();
+		try {
+			compile(ws); // dry-run that doubles as the materialization — throws CompileError on bad sources
+		} catch (e) {
+			restore();
+			try { compile(ws); } catch { /* runtime was already broken before this op */ }
+			throw e;
+		}
+		const rels = files.map((f) => path.relative(ws.root, f));
+		// Schema ops commit UNCONDITIONALLY — `auto-commit` governs RECORD writes only. A source
+		// change is inseparable from the compile that validated it, and `dt commit` scopes itself
+		// to record directories, so a deferred source edit would be publishable by nothing.
+		// Extending `dt commit` to module sources is the natural follow-on; it is not this wave.
+		try {
+			execFileSync('git', ['add', '--', ...rels], { cwd: ws.root });
+			execFileSync('git', ['commit', '--quiet', '-m', subject, '--', ...rels], { cwd: ws.root });
+		} catch (e) {
+			try { execFileSync('git', ['reset', '--quiet', '--', ...rels], { cwd: ws.root }); } catch { /* nothing staged */ }
+			restore();
+			try { compile(ws); } catch { /* pre-op sources were compilable */ }
+			throw new Error(`git commit failed — the schema change was rolled back, nothing was changed. (${e.message.split('\n')[0]})`);
+		}
+	});
+}
+
+/** The workspace's writable source dir for a kind (workspace-module aware). `kindDir` picks the
+ *  layout that module already uses and falls back to flat, so a `collections add` never splits a
+ *  half-moved module across both. */
+export function workspaceSystemDir(ws, kind) {
+	const wm = ws.pkg.dreamteamer?.['workspace-module'];
+	return kindDir(wm ? path.join(ws.root, 'modules', wm) : ws.root, kind);
+}
+
+// ---- ops ------------------------------------------------------------------------
+
+export function createCollection(ws, store, { name, template }) {
+	if (!name) throw new Error('missing collection name');
+	if (store.descriptors.has(name)) throw new Error(`collection "${name}" already exists`);
+	const dest = path.join(workspaceSystemDir(ws, 'collections'), `${name}.collection.yaml`);
+	if (fs.existsSync(dest)) throw new Error(`${path.relative(ws.root, dest)} already exists`);
+
+	let descriptor = { name };
+	if (template) {
+		const tplFile = path.join(runtimeKindDir(ws.root, 'collection-templates'), `${template}.collection-template.yaml`);
+		if (!fs.existsSync(tplFile)) throw new Error(`unknown collection-template "${template}"`);
+		descriptor = { name, ...structuredClone(load(fs.readFileSync(tplFile, 'utf8')).template) };
+	} else {
+		// templateless: MINIMAL but compilable — grow it with add-field
+		descriptor.id = { generate: '{{ name | slug }}' };
+		descriptor.schema = { type: 'object', required: ['name'], properties: { name: { type: 'string' } } };
+	}
+	descriptor.storage = {
+		path: `${ws.pkg.dreamteamer?.['data-path'] ?? 'data'}/${name}`,
+		codec: 'md', shape: 'file',
+		...descriptor.storage,
+		suffix: descriptor.storage?.suffix ?? singular(name),
+	};
+	writeGated(ws, store, [dest], `dreamteamer: collections add ${name}`, () => {
+		fs.mkdirSync(path.dirname(dest), { recursive: true });
+		fs.writeFileSync(dest, dump(descriptor));
+	});
+	return { file: dest, descriptor };
+}
+
+export function removeCollection(ws, store, name, { force = false } = {}) {
+	const d = store.descriptor(name);
+	const dest = path.join(workspaceSystemDir(ws, 'collections'), `${name}.collection.yaml`);
+	if (!fs.existsSync(dest)) throw new Error(`"${name}" is not workspace-owned — it ships with a module; add "<module>/${name}" to dreamteamer.disable instead`);
+	const dataDir = path.join(ws.root, d.storage.path);
+	const hasRecords = fs.existsSync(dataDir) && fs.readdirSync(dataDir).some((e) => !e.startsWith('.'));
+	if (hasRecords && !force) throw new Error(`collection "${name}" still has records under ${d.storage.path} — remove them first or pass force`);
+	writeGated(ws, store, [dest], `dreamteamer: collections rm ${name}`, () => fs.rmSync(dest));
+	return { removed: name };
+}
+
+export function addField(ws, store, collection, { name: fieldName, prop, required }) {
+	store.descriptor(collection); // must exist in the compiled runtime
+	if (!fieldName) throw new Error('missing field name');
+	if (store.descriptor(collection).schema?.properties?.[fieldName]) throw new Error(`field "${fieldName}" already exists on ${collection}`);
+	return upsertField(ws, store, collection, fieldName, prop, required, `add-field ${fieldName}`);
+}
+
+export function updateField(ws, store, collection, fieldName, { prop, required }) {
+	const d = store.descriptor(collection);
+	if (!d.schema?.properties?.[fieldName]) throw new Error(`no field "${fieldName}" on ${collection}`);
+	// upsertField REPLACES the prop, so retyping a field would silently drop its hand-authored
+	// `description`. Changing a field's type is not a decision to undocument it. Same for an
+	// authored `title` — but ONLY an authored one: a derived title is compile's output, not a
+	// human's choice, and `titleCase` is how the two are told apart.
+	const previous = d.schema.properties[fieldName];
+	if (prop.description === undefined && typeof previous.description === 'string') prop = { ...prop, description: previous.description };
+	if (prop.title === undefined && typeof previous.title === 'string' && previous.title !== titleCase(fieldName)) prop = { ...prop, title: previous.title };
+	return upsertField(ws, store, collection, fieldName, prop, required, `update-field ${fieldName}`);
+}
+
+export function removeField(ws, store, collection, fieldName) {
+	const d = store.descriptor(collection);
+	if (!d.schema?.properties?.[fieldName]) throw new Error(`no field "${fieldName}" on ${collection}`);
+	const dest = path.join(workspaceSystemDir(ws, 'collections'), `${collection}.collection.yaml`);
+	if (!fs.existsSync(dest)) throw new Error(`"${collection}" is module-shipped; the workspace can only OVERRIDE fields (extends), not remove them`);
+	const doc = load(fs.readFileSync(dest, 'utf8'));
+	if (!doc.schema?.properties?.[fieldName]) throw new Error(`field "${fieldName}" is inherited from the base module — the workspace descriptor doesn't declare it`);
+	writeGated(ws, store, [dest], `dreamteamer: ${collection} remove-field ${fieldName}`, () => {
+		delete doc.schema.properties[fieldName];
+		if (Array.isArray(doc.schema.required)) doc.schema.required = doc.schema.required.filter((r) => r !== fieldName);
+		fs.writeFileSync(dest, dump(doc));
+	});
+	return { collection, removed: fieldName };
+}
+
+function upsertField(ws, store, collection, fieldName, prop, required, verb) {
+	if (prop == null || typeof prop !== 'object' || Array.isArray(prop)) {
+		throw new Error(`field "${fieldName}": prop must be a JSON-Schema object (got ${Array.isArray(prop) ? 'array' : typeof prop}) — nothing was written.`);
+	}
+	// compile resolves `prop.title` into the COMPILED descriptor, and both writers rebuild a prop
+	// from that projection — this one and the studio's field drawer. Without this, retyping any
+	// field through the UI writes the DERIVED label back into the source as though a human chose
+	// it, and 51 collections fill with `title: Due Date` noise no longer distinguishable from a
+	// real override. The webview applies the identical rule in `lib/field-prop.ts`.
+	if (prop.title === titleCase(fieldName)) {
+		prop = { ...prop };
+		delete prop.title;
+	}
+	// Same rule for the value template. presentation INHERITS a reference's template from its
+	// target collection's `title_template`, so a field drawer that round-trips that projection
+	// writes the inherited value back onto the field — hand-recreating exactly the 49 duplicated
+	// `x-display` lines the inheritance replaced. Only a template that DIFFERS from the target's
+	// is a real authored override.
+	const ref = prop['x-reference'] ?? prop.items?.['x-reference'];
+	const inherited = ref && ref !== '*' ? store.descriptors.get(ref)?.title_template : undefined;
+	if (inherited) {
+		if (prop['x-title-template'] === inherited) {
+			prop = { ...prop };
+			delete prop['x-title-template'];
+		}
+		if (prop.items?.['x-title-template'] === inherited) {
+			prop = { ...prop, items: { ...prop.items } };
+			delete prop.items['x-title-template'];
+		}
+	}
+	const dest = path.join(workspaceSystemDir(ws, 'collections'), `${collection}.collection.yaml`);
+	let doc;
+	if (fs.existsSync(dest)) {
+		doc = load(fs.readFileSync(dest, 'utf8'));
+	} else {
+		doc = { name: collection, extends: baseModuleRef(ws.root, collection), schema: { properties: {} } };
+	}
+	writeGated(ws, store, [dest], `dreamteamer: ${collection} ${verb}`, () => {
+		doc.schema ??= { properties: {} };
+		doc.schema.properties ??= {};
+		doc.schema.properties[fieldName] = prop;
+		if (required === true) doc.schema.required = [...new Set([...(doc.schema.required ?? []), fieldName])];
+		if (required === false && Array.isArray(doc.schema.required)) doc.schema.required = doc.schema.required.filter((r) => r !== fieldName);
+		fs.mkdirSync(path.dirname(dest), { recursive: true });
+		fs.writeFileSync(dest, dump(doc));
+	});
+	return { collection, field: fieldName, file: dest, extends: doc.extends };
+}
+
+// saved views (M3): a studio-saved view IS a ui-view record — but ui-views are
+// system-stored (sources + compile), so the write goes through the same gate as any
+// other schema op. the studio "save view" button lands here.
+export function saveUiView(ws, store, { id, view }) {
+	if (!id || !/^[a-z0-9][a-z0-9-/]*$/.test(id)) throw new Error(`invalid ui-view id "${id}" — lowercase slug required`);
+	const dest = path.join(workspaceSystemDir(ws, 'ui-views'), `${id}.ui-view.yaml`);
+	const existed = fs.existsSync(dest);
+	writeGated(ws, store, [dest], `dreamteamer: ui-views ${existed ? 'update' : 'add'} ${id}`, () => {
+		fs.mkdirSync(path.dirname(dest), { recursive: true });
+		fs.writeFileSync(dest, dump(view));
+	});
+	return { id, file: dest, updated: existed };
+}
+
+export function removeUiView(ws, store, id) {
+	const dest = path.join(workspaceSystemDir(ws, 'ui-views'), `${id}.ui-view.yaml`);
+	if (!fs.existsSync(dest)) throw new Error(`ui-view "${id}" is not workspace-owned (module-shipped views are removed via dreamteamer.disable)`);
+	writeGated(ws, store, [dest], `dreamteamer: ui-views rm ${id}`, () => fs.rmSync(dest));
+	return { removed: id };
+}
+
+// base module for an extends pointer — resolved via manifest.modules across ALL channels
+// (audit open finding 1: the old regex only understood inline modules/… paths)
+function baseModuleRef(root, collection) {
+	const manifest = readManifest(root) ?? {};
+	// entry keys are runtime-relative and lost their `system/` prefix in the flatten; a manifest
+	// written by an older engine still carries it, and this reads whatever is on disk
+	const entry = manifest.entries?.[`collections/${collection}.collection.yaml`]
+		?? manifest.entries?.[`system/collections/${collection}.collection.yaml`];
+	const src = entry?.sources?.[0];
+	const srcPath = typeof src === 'string' ? src : src?.path;
+	if (!srcPath) throw new Error(`cannot determine the base module for "${collection}"`);
+	for (const m of manifest.modules ?? []) {
+		const modRoot = m.root === '.' ? '' : `${m.root}/`;
+		if (modRoot && srcPath.startsWith(modRoot)) return `${m.name}/${collection}`;
+	}
+	throw new Error(`cannot determine the base module for "${collection}" — its source is ${srcPath}; edit that descriptor directly`);
+}
+
+// CLI/API type sugar → JSON Schema property
+export function fieldDef(store, flags) {
+	const t = flags.type ?? 'string';
+	const def = flags['default-value'] ?? flags.default;
+	const p = (() => {
+		switch (t) {
+			case 'string': case 'text': return { type: 'string' };
+			case 'markdown': return { type: 'string', format: 'markdown' };
+			case 'boolean': return { type: 'boolean' };
+			case 'number': return { type: 'number' };
+			case 'integer': return { type: 'integer' };
+			case 'date': return { type: 'string', format: 'date' };
+			// `timestamp` is the WIRE type presentation.js projects `date-time` to, and therefore what
+			// the studio's field drawer round-trips. Accepting it here means the vocabulary you read
+			// out of `presentation` is the vocabulary you can type back into the CLI.
+			case 'datetime': case 'timestamp': return { type: 'string', format: 'date-time' };
+			case 'enum': {
+				if (!flags.options) throw new Error('enum needs options "a,b,c"');
+				const opts = Array.isArray(flags.options) ? flags.options : flags.options.split(',').map((s) => s.trim());
+				return { type: 'string', enum: opts };
+			}
+			case 'tags': return { type: 'array', items: { type: 'string' } };
+			default:
+				if (store.descriptors.has(t)) return { type: 'string', 'x-reference': t };
+				if (t === 'reference') return { type: 'string', 'x-reference': flags.target ?? '*' };
+				throw new Error(`unknown field type "${t}"`);
+		}
+	})();
+	if (def !== undefined) p.default = p.type === 'boolean' ? def === 'true' || def === true : p.type === 'number' || p.type === 'integer' ? Number(def) : def;
+	// what the field MEANS, in one line — JSON Schema's own keyword, projected to every surface by
+	// presentation.js. A field whose name doesn't say enough is documented here, not in a comment.
+	if (typeof flags.description === 'string' && flags.description.length > 0) p.description = flags.description;
+	return p;
+}
+
+function singular(name) {
+	return name.endsWith('ies') ? name.slice(0, -3) + 'y' : name.endsWith('s') ? name.slice(0, -1) : name;
+}

package/src/semver.js

@@ -0,0 +1,32 @@
+// tiny semver-range checker for engine-pin checks — deliberately NOT npm's semver.
+// supported ranges: exact "1.2.3", "^1.2.3", "~1.2.3", ">=1.2.3", "*".
+// NOT supported (returns null, "can't tell"): "||" alternatives, hyphen ranges,
+// x-ranges (1.2.x), combined comparators (">=1.0.0 <2.0.0"), <, <=, >,
+// prerelease/build tags. callers treat null as "warn, don't guess".
+
+const parse = (v) => {
+	const m = /^(\d+)\.(\d+)\.(\d+)$/.exec(String(v).trim());
+	return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;
+};
+
+const cmp = (a, b) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
+
+// true = in range, false = out of range, null = version or range not understood
+export function satisfies(version, range) {
+	const v = parse(version);
+	if (!v) return null;
+	const r = String(range).trim();
+	if (r === '*') return true;
+	const m = /^(>=|\^|~)?(\d+\.\d+\.\d+)$/.exec(r);
+	if (!m) return null;
+	const base = parse(m[2]);
+	switch (m[1]) {
+		case '>=': return cmp(v, base) >= 0;
+		case '^': // same major, >= base; zero-major pins the minor too (npm's ^0.x rule)
+			if (cmp(v, base) < 0) return false;
+			return base[0] === 0 ? v[0] === 0 && v[1] === base[1] : v[0] === base[0];
+		case '~': // same major.minor, >= base
+			return cmp(v, base) >= 0 && v[0] === base[0] && v[1] === base[1];
+		default: return cmp(v, base) === 0; // exact
+	}
+}