dreamteamer 0.30.0 → 0.32.0

package/skills/using-dreamteamer/references/sessions.md

@@ -6,7 +6,7 @@ refusals that keep that from looping, lying, or carrying private data somewhere
 
 The whole design turns on one asymmetry. Two sessions in the same tree are conflict-BLIND, and so are
 two sessions in one conversation: **the second write wins and nobody is told.** Sessions are
-`worktrees`' twin — observed, never stored, and the registry is the authority rather than anyone's
+git worktrees' twin — observed, never stored, and the registry is the authority rather than anyone's
 memory of it.
 
 | the question | read |
@@ -175,10 +175,9 @@ A coordinator spans repos by construction, and one of them may publish.
    records it may not message a publishing session at all, whatever it means to say. This is what
    keeps rule 1 safe after compaction, when it can no longer recall precisely what it read.
 6. ⚠ **`cwd` is not the repo, and one repo is not one tree.** Harnesses put worktrees *inside* the
-   repo or *under the home directory* depending on the harness (`references/worktrees.md`). A `cwd`
-   under a harness's own worktree root is still that repo and carries its full boundary — so a
-   path-prefix test against the primary root gets it wrong in the dangerous direction. `dt list
-   worktrees` is the instrument.
+   repo or *under the home directory* depending on the harness. A `cwd` under a harness's own
+   worktree root is still that repo and carries its full boundary — so a path-prefix test against
+   the primary root gets it wrong in the dangerous direction. `git worktree list` is the instrument.
 
 ## not stepping on your own toes
 

package/skills/using-dreamteamer/references/skills.md

@@ -160,12 +160,10 @@ Skills ship with modules and are read by any operator on any machine:
   renamed verb or a changed limit in a skill sends every future session down the old path
   confidently. When you catch a skill lying, fixing it is part of the task you are on, not a
   follow-up.
-- **Give it a proof, and know what a proof cannot cover.** A `proofs` record (`proofs.md`) pins the
+- **Give it a mechanical check, and know what one cannot cover.** Something must pin the
   mechanical half: the script the skill names runs, the record it promises appears, the path it
-  files to exists. Write it in the same commit — `dt add skills` nudges you with the path the moment
-  it writes the skill (compile's own nudge covers new commands and scripts, never skills), and
-  `dt list proofs --missing` names every artifact nobody claimed anything about. ⚠ **A
-  green proof is not evidence the skill TEACHES.** Whether a fresh session finds it, loads it and
+  files to exists — in a workspace with a proofs extension, that is a `proofs` record, written
+  in the same commit. ⚠ **A green check is not evidence the skill TEACHES.** Whether a fresh session finds it, loads it and
   does the job right is the eval layer: real tasks, blind sessions, a scoring sheet — a procedure,
   never something the engine runs.
 - **Retire what nothing loads.** A skill nobody uses still costs its line in every session's index.

package/src/api.d.ts

@@ -0,0 +1,109 @@
+// The typed contract of `import … from 'dreamteamer'` (src/api.js). It re-exports the record half
+// (`dreamteamer/records`, src/records-api.d.ts) and adds the workspace half. A test pins each runtime
+// export list to its declaration, so a name added to one and not the other fails the suite.
+
+export * from './records-api.js';
+import type { Fields, Descriptor, Descriptors, Manifest, Store } from './records-api.js';
+
+// ---- the workspace and extensions ------------------------------------------------------------------
+
+export interface Workspace {
+	/** absolute path of the workspace root */
+	root: string;
+	/** the workspace package.json */
+	pkg: { name?: string; dependencies?: Record<string, string>; devDependencies?: Record<string, string>; dreamteamer?: Record<string, any>; [k: string]: unknown };
+	/** the ACTIVATED extensions, present on a handle from `openWorkspace` */
+	extensions?: LoadedExtension[];
+}
+
+/** What an extension's `activate(dt)` returns. Every key is optional. */
+export interface Contribution {
+	commands?: Record<string, { usage?: string; run(ws: Workspace, argv: string[]): number | void | Promise<number | void> }>;
+	sourceKinds?: (string | { kind: string; exclude?: string[] })[];
+	analyze?(draft: CompileDraft): { errors?: string[]; warnings?: string[]; notes?: string[] } | void;
+	harnesses?: Record<string, (ctx: HarnessContext) => { blocks?: Record<string, string | null>; summary?: string } | void>;
+	orientation?: string | ((ctx: { entries: Map<string, Entry> }) => string);
+	hooks?: Record<string, string>;
+}
+export type Activate = (dt: typeof import('./api.js')) => Contribution | Promise<Contribution>;
+export interface LoadedExtension {
+	name: string;
+	version: string;
+	commands: NonNullable<Contribution['commands']>;
+	sourceKinds: { kind: string; exclude: string[]; extension: string }[];
+	analyze: Contribution['analyze'] | null;
+	harnesses: NonNullable<Contribution['harnesses']>;
+	orientation: Contribution['orientation'] | null;
+	hooks: Record<string, string>;
+}
+export type Entry = { sources: { path: string; hash: string }[]; bytes: Buffer };
+export interface CompileDraft {
+	/** runtime-relative path → staged entry; read-only */
+	readonly entries: Map<string, Entry>;
+	/** the FINAL merged descriptors */
+	readonly descriptors: Descriptors;
+	readonly modules: { id: string; name: string; root: string; channel: 'inline' | 'git' | 'npm' }[];
+	/** names only — never values */
+	readonly declaredVars: string[];
+	readonly declaredEnv: string[];
+	readonly previousManifest: Manifest | null;
+	/** parse one staged YAML entry, with its source path in any error */
+	parse(runtimePath: string): any;
+}
+export interface HarnessContext {
+	entries: Map<string, Entry>;
+	version: string;
+	collections: { name: string; generated: boolean; systemGroup: boolean; description: string; module: string; sensitive: boolean; sensitiveFields: string[] }[];
+	modules: { id: string; title: string; description: string; path: string }[];
+}
+
+export const EXTENSION_API: 1;
+export function openWorkspace(start?: string): Promise<Workspace & { extensions: LoadedExtension[] }>;
+export function findWorkspace(start?: string): Workspace;
+export function declaredExtensions(ws: Workspace): { name: string; version: string; dir: string; entry: string }[];
+export const engineBin: string;
+export const engineRoot: string;
+
+export function envContext(ws: Workspace): unknown;
+export function renderTemplate(template: string, ctx: unknown): string;
+export function parseEnvValues(text: string): Map<string, string>;
+export function satisfies(version: string, range: string): boolean | null;
+
+// ---- the compiler, schema and module operations --------------------------------------------------
+
+/** throws CompileError on a bad source; prints its summary; returns 0 */
+export function compile(ws: Workspace): 0;
+export function staleness(root: string): { compiled: boolean; stale: string[]; manifest?: Manifest; message?: string };
+export function warnIfStale(root: string): ReturnType<typeof staleness>;
+export function discoverModules(root: string, pkg: Workspace['pkg']): { modules: { name: string; root: string; channel: string }[]; shadows: unknown[]; disabledModules: string[] };
+export class CompileError extends Error {}
+export const KINDS: readonly string[];
+export const MANAGED_BLOCKS: readonly { id: 'orientation' | 'instructions'; begin: string; end: string }[];
+type SchemaOp = (ws: Workspace, store: Store, ...args: any[]) => any;
+export const createCollection: SchemaOp, removeCollection: SchemaOp, renameCollection: SchemaOp, moveCollection: SchemaOp, setCollectionScalars: SchemaOp;
+export const addField: SchemaOp, updateField: SchemaOp, removeField: SchemaOp, removeFieldPlan: SchemaOp, renameField: SchemaOp, renameFieldPlan: SchemaOp;
+export function fieldDef(store: Store, flags: Record<string, unknown>, collection: string): any;
+export function statedKeywords(flags: Record<string, unknown>): any;
+export const saveUiView: SchemaOp, removeUiView: SchemaOp;
+export const createModule: SchemaOp, setModule: SchemaOp, renameModule: SchemaOp, removeModule: SchemaOp;
+export const createSkill: SchemaOp, refuseHandAuthored: SchemaOp, removeEntity: SchemaOp, renameEntity: SchemaOp, setEntityFrontmatter: SchemaOp;
+export function init(opts?: { flags?: Record<string, string> }): 0;
+export function ensureRepo(ws: Workspace, id: string): { path: string; cloned: boolean };
+export function ensureAllRepos(ws: Workspace): { path: string; cloned: boolean }[];
+export function listRepos(ws: Workspace): { id: string; path: string; present: boolean; unresolved?: string }[];
+export function installClone(ws: Workspace, url: string, name?: string): number;
+
+// ---- the checkout -------------------------------------------------------------------------------------
+
+export function describeCheckout(root: string, git?: (args: string[], cwd: string) => string): { root: string; kind: 'primary' | 'linked'; gitDir: string; commonDir: string; primary: string; insideRoot: boolean };
+export function installCommand(ws: Workspace, argv: string[], opts?: { open?: (at: string) => Promise<Workspace> }): Promise<number>;
+export function resolveNpm(execPath?: string, env?: NodeJS.ProcessEnv): string | null;
+export function childEnv(): NodeJS.ProcessEnv;
+export function readHookInput(stdinText: string): { cwd: string | null; name: string | null; raw: Record<string, unknown> };
+export function readStdin(isTTY?: boolean): string;
+
+// ---- CLI helpers ------------------------------------------------------------------------------------------
+
+/** write `text` plus a trailing newline, synchronously, looping on short writes — safe before process.exit */
+export function emit(text: string, fd?: number): void;
+export function parseArgs(argv: string[]): { flags: Record<string, string | boolean | string[]>; pos: string[] };

package/src/api.js

@@ -0,0 +1,70 @@
+// The PUBLIC API — `import … from 'dreamteamer'`. Everything a surface (the VS Code extension, the
+// mobile app) or an extension (http, workflows, notebooklm) may call, and nothing else: package.json
+// `exports` hides `src/*`, so an internal file can be split, renamed or deleted without a cross-repo
+// break. That break used to be the rule — the extension imported fifteen internal files by path, and
+// deleting one took `activate()` down before the tree view existed.
+//
+// Grouped by task. Each name is the ONE implementation of its operation; nothing here wraps or
+// re-implements. `api.d.ts` beside this file is the typed contract, and a test pins the two together.
+//
+// Importing this module prints nothing, writes nothing, binds no port and touches no network.
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as self from './api.js';
+import { findWorkspace } from './workspace.js';
+import { loadExtensions } from './extensions.js';
+import { KINDS } from './compile.js';
+import { DERIVED_KINDS } from './runtime.js';
+import { KNOWN_HARNESSES } from './harnesses.js';
+import { CORE_VERBS } from './cli.js';
+import { installCommand as installCheckout } from './checkout.js';
+
+// Everything the record half exports (the browser-safe entry, `dreamteamer/records`), then the rest.
+export * from './records-api.js';
+
+// ---- the workspace ------------------------------------------------------------------------------
+
+/**
+ * The workspace at (or above) `start`, with its declared extensions ACTIVATED against this engine.
+ * The handle every other call takes: `{ root, pkg, extensions }`. Performs no compile, install,
+ * network call or write, and never changes the process cwd.
+ */
+export async function openWorkspace(start = process.cwd()) {
+	const ws = findWorkspace(start);
+	ws.extensions = await loadExtensions(ws, self, { verbs: CORE_VERBS, kinds: [...KINDS, ...DERIVED_KINDS], harnesses: KNOWN_HARNESSES });
+	return ws;
+}
+export { findWorkspace };
+export { EXTENSION_API, declaredExtensions } from './extensions.js';
+
+/** The engine's own CLI entry — what a tool spawns to run `dt` in another checkout. */
+export const engineBin = fileURLToPath(new URL('../bin/dreamteamer.js', import.meta.url));
+export const engineRoot = path.dirname(path.dirname(engineBin));
+
+// ---- values the workspace half adds ---------------------------------------------------------------
+export { envContext, renderTemplate, parseEnvValues } from './env-vars.js';
+export { satisfies } from './semver.js';
+
+// ---- the compiler, schema and module operations ---------------------------------------------------
+export { compile, staleness, warnIfStale, discoverModules, CompileError, KINDS } from './compile.js';
+export { MANAGED_BLOCKS } from './harnesses.js';
+export {
+	createCollection, removeCollection, renameCollection, moveCollection, setCollectionScalars,
+	addField, updateField, removeField, removeFieldPlan, renameField, renameFieldPlan, fieldDef, statedKeywords,
+	saveUiView, removeUiView,
+	createModule, setModule, renameModule, removeModule,
+	createSkill, refuseHandAuthored, removeEntity, renameEntity, setEntityFrontmatter,
+} from './schema-ops.js';
+export { init, ensureRepo, ensureAllRepos, listRepos, installClone } from './init.js';
+
+// ---- the checkout: which one this is, and how it becomes ready ---------------------------------
+export { describeCheckout, resolveNpm, childEnv, readHookInput, readStdin } from './checkout.js';
+
+/** `dt install` on the checkout `ws`, as the CLI runs it. The compile step reopens the workspace with
+ *  `opts.open` — `openWorkspace` unless a caller injects another — so the extensions npm just
+ *  installed take part in that compile. The one name here that supplies a default rather than
+ *  re-exporting: the checkout layer cannot import the opener that activates extensions. */
+export const installCommand = (ws, argv, opts = {}) => installCheckout(ws, argv, { open: openWorkspace, ...opts });
+
+// ---- CLI helpers an extension command reuses -----------------------------------------------------
+export { emit, parseArgs } from './collections-cli.js';

package/src/check.js

@@ -10,6 +10,7 @@ import { NO_RUNTIME, loadDescriptors, runtimeDir, namespaces as compiledNamespac
 import { parseRef } from './namespace.js';
 import { refTargetsOf, refIsSoft } from './ref.js';
 import { relationsOf, expectedMirrors } from './relations.js';
+import { placementOf, placedRecords, ownerIdOf, symlinkedChildRoots } from './placement.js';
 
 export function check({ root }) {
 	const RUNTIME = runtimeDir(root);
@@ -34,16 +35,19 @@ export function check({ root }) {
 
 	// ---- index all records: collection -> Map<id, filePath> ------------------------
 	const index = new Map();
+	// placed collections only: id -> the parent id its file was FOUND under (null = the fallback root)
+	const observed = new Map();
 	const strays = [];
 	// declared here rather than beside the validation pass: indexing can itself produce a
 	// finding (an unreachable data root, below) before a single record is read.
 	const violations = [];
+	const dirOf = (d) => path.join(d.storage.base === 'runtime' ? RUNTIME : root, d.storage.path);
 	for (const [name, d] of descriptors) {
 		const ids = new Map();
 		index.set(name, ids);
 		// runtime-based (knowhow/meta) collections are read from the COMPILED runtime —
 		// their sources may live in any module; .dreamteamer is the merged read surface
-		const dir = path.join(d.storage.base === 'runtime' ? RUNTIME : root, d.storage.path);
+		const dir = dirOf(d);
 		// An unreachable data ROOT is a finding, not a skip: a collection whose module clone is
 		// missing otherwise reports zero records and a clean check — a silent success. An EMPTY
 		// directory stays fine (a module with no records yet is normal); only a missing owning
@@ -53,13 +57,46 @@ export function check({ root }) {
 			violations.push({ file: d.storage.path, msg: `collection "${name}" is owned by ${d.storage.repo}, which is not present — every record in it is unreadable` });
 			continue;
 		}
+		const under = placementOf(d);
+		if (under) {
+			// ONE logical collection across the fallback root and every parent folder — the same walk
+			// the store indexes with (src/placement.js), so the two cannot disagree about which files
+			// are records. The first file to claim an id keeps it; every later one is a violation,
+			// because a `get` that silently answered from whichever folder sorted later is the
+			// failure this report exists to make visible.
+			const seen = new Map();
+			observed.set(name, seen);
+			const parentDir = dirOf(descriptors.get(under.collection));
+			// a child root that is (or sits behind) a symlink is not read — whatever it points at is not
+			// this parent's folder — and it is named here rather than silently skipped
+			for (const link of symlinkedChildRoots(under, parentDir)) {
+				violations.push({ file: rel(link), msg: `is a symlink — ${name} records are read only from real folders inside ${under.collection} records; whatever this points at is not indexed. Replace it with a real folder.` });
+			}
+			const onLink = (p) => violations.push({ file: rel(p), msg: `is a symlink inside a ${under.collection} record's folder — nothing behind it is read as a ${name} record, and nothing is written through it. Replace it with a real folder or file.` });
+			for (const r of placedRecords(d, dir, parentDir, onLink)) {
+				if (ids.has(r.id)) {
+					violations.push({ file: rel(r.file), msg: `collection "${name}" holds the id "${r.id}" twice — ${rel(ids.get(r.id))} and ${rel(r.file)}. Remove one.` });
+					continue;
+				}
+				ids.set(r.id, r.file);
+				seen.set(r.id, r.parentId);
+			}
+			continue;
+		}
 		if (!fs.existsSync(dir)) continue;
 		const shape = d.storage.shape ?? 'file';
 		if (shape === 'folder') {
+			// a FILE record at the root of a folder-shape collection is the state a collection is in
+			// right after its shape changed — named as such, with the verb that finishes the change
+			const asFile = { ...d, storage: { ...d.storage, shape: 'file' } };
 			for (const entry of fs.readdirSync(dir).sort()) {
 				if (entry.startsWith('.')) continue;
 				const p = path.join(dir, entry);
-				if (!fs.statSync(p).isDirectory()) { strays.push({ collection: name, file: rel(p) }); continue; }
+				if (!fs.statSync(p).isDirectory()) {
+					const legacy = idFromRecordPath(asFile, entry) !== null;
+					strays.push({ collection: name, file: rel(p), note: legacy ? `a file-shape record in a folder-shape collection — dreamteamer relocate ${name} moves it to ${entry.split('.')[0]}/${d.storage.entry}` : undefined });
+					continue;
+				}
 				const main = path.join(p, d.storage.entry ?? 'SKILL.md');
 				if (fs.existsSync(main)) ids.set(entry, main);
 				else strays.push({ collection: name, file: rel(p), note: `missing entry file ${d.storage.entry}` });
@@ -127,6 +164,24 @@ export function check({ root }) {
 					checkRef(file, fieldPath, value, target, softTargets, soft);
 				}
 			}
+			// ---- placement: the FIELD is the intended owner, the FOLDER is observed placement ---
+			// Reported, never repaired: a record found under the wrong company is either a hand move
+			// (the field is right, run relocate) or a hand edit of the field (the folder is right, set
+			// it back) and only a person knows which. A malformed owner value is skipped here — the
+			// reference check above has already named it, and "placed under X but owner is empty"
+			// on top of that would be a second report of one typo.
+			const under = placementOf(d);
+			if (under) {
+				const raw = fields[under.field];
+				const wellFormed = raw == null || raw === '' || parseRef(raw, namespaces);
+				const want = ownerIdOf(fields, under, (v) => parseRef(v, namespaces));
+				const got = observed.get(name).get(id);
+				if (wellFormed && want !== got) {
+					const where = got ? `under ${under.collection}/${got}` : `in its own root (${d.storage.path})`;
+					const should = want ? `${under.field} is ${under.collection}/${want}` : `${under.field} is empty`;
+					flag(file, `placed ${where} but ${should} — the file is not where its owner puts it. Run: dreamteamer relocate ${name}/${id}`);
+				}
+			}
 			parsed.get(name).set(id, fields);
 		}
 	}