@human-synthesis/norns 0.1.0 → 0.2.1

package/src/kernel/meta.js

@@ -151,6 +151,128 @@ const Component = v.strictObject({
 	slots: v.optional(v.array(ident))
 });
 
+// Snippet (U-07) — a typed render fragment for a palette slot (cell
+// renderers, empty states). Declared args are the slot's calling
+// convention; the body lives in `src/<m>/snippets/<name>.n` and the page
+// emitter wraps it in a `+snippet` forwarding the args as props.
+const Snippet = v.strictObject({
+	uid,
+	args: v.optional(v.array(ident)),
+	description: v.optional(v.string())
+});
+
+// Service (D15) — an external system as a typed operation manifest.
+// Credentials never appear here: `auth.binding` is an env binding *name*
+// (UPPER_SNAKE); a literal secret anywhere in a service is refused at
+// generate time (SECRET_IN_SPEC, generate.js).
+const bindingName = v.pipe(
+	v.string(),
+	v.regex(/^[A-Z][A-Z0-9_]*$/, 'must be an UPPER_SNAKE env binding name, never a secret value')
+);
+
+const ServiceAuth = v.pipe(
+	v.strictObject({
+		mode: v.picklist(['none', 'bearer', 'basic', 'hmac', 'header']),
+		binding: v.optional(bindingName),
+		header: v.optional(v.pipe(v.string(), v.regex(/^[A-Za-z][A-Za-z0-9-]*$/, 'must be a header name')))
+	}),
+	v.check(
+		(a) => (a.mode === 'none' ? a.binding === undefined : a.binding !== undefined),
+		"auth modes other than 'none' require a `binding` name; 'none' must not have one"
+	),
+	v.check((a) => a.mode === 'header' || a.header === undefined, "`header` is only valid with mode 'header'"),
+	v.check((a) => a.mode !== 'header' || a.header !== undefined, "auth mode 'header' requires `header`")
+);
+
+const ServiceOperation = v.strictObject({
+	method: v.optional(v.picklist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])),
+	path: v.optional(v.pipe(v.string(), v.regex(/^\//, 'path must start with "/"'))),
+	input: v.optional(v.record(ident, v.union([v.string(), v.record(v.string(), v.unknown())]))),
+	output: v.optional(v.unknown())
+});
+
+const Service = v.strictObject({
+	uid,
+	base: v.pipe(v.string(), v.url('base must be an absolute URL')),
+	auth: ServiceAuth,
+	operations: v.pipe(
+		v.record(ident, ServiceOperation),
+		v.check((ops) => Object.keys(ops).length > 0, 'services require at least one operation')
+	)
+});
+
+// Job (D14/K-22) — durable work. Retry policy is required by guardrail:
+// a job without declared failure behavior is refused at the shape level.
+// Jobs run from `enqueue` steps via the events bus (`job:<address>`
+// messages) — Cloudflare Queues in production, inline in dev.
+const queueName = v.pipe(
+	v.string(),
+	v.regex(/^[a-z][a-z0-9-]*$/, 'must be a queue name (lowercase, digits, dashes)')
+);
+
+const Job = v.pipe(
+	v.strictObject({
+		uid,
+		input: v.optional(v.record(ident, v.union([v.string(), v.record(v.string(), v.unknown())]))),
+		retry: v.strictObject({
+			attempts: v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(20)),
+			backoff: v.picklist(['none', 'fixed', 'exponential']),
+			baseMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1)))
+		}),
+		dlq: v.optional(queueName),
+		concurrency: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(100))),
+		steps: v.optional(v.array(v.record(v.string(), v.unknown()))),
+		emits: v.optional(v.array(v.string())),
+		examples: v.optional(v.array(example)),
+		impl: v.optional(v.picklist(['generated', 'custom']))
+	}),
+	v.check(requiresExamplesWhenCustom, CUSTOM_NEEDS_EXAMPLES),
+	v.check(
+		(j) => j.impl === 'custom' || (Array.isArray(j.steps) && j.steps.length > 0),
+		'generated jobs need at least one step (or `impl: custom`)'
+	)
+);
+
+// Endpoint (D14/K-23) — a Route grown up: declared route/method/auth and
+// IO contract in spec, body in `src/<m>/endpoints/<name>.c`. `stream`
+// declares an SSE output mode with typed frames (the chat/AI-token path).
+// `auth` is required — public endpoints declare `{ mode: 'none' }` explicitly.
+const Endpoint = v.pipe(
+	v.strictObject({
+		uid,
+		route: v.pipe(v.string(), v.regex(/^\//, 'route must start with "/"')),
+		method: v.optional(v.picklist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])),
+		auth: ServiceAuth,
+		input: v.optional(v.record(ident, v.union([v.string(), v.record(v.string(), v.unknown())]))),
+		output: v.optional(v.record(ident, v.unknown())),
+		stream: v.optional(
+			v.strictObject({
+				frame: v.pipe(
+					v.record(ident, v.unknown()),
+					v.check((f) => Object.keys(f).length > 0, 'stream frames need at least one field')
+				)
+			})
+		),
+		impl: v.optional(v.literal('custom')),
+		examples: v.optional(v.array(example))
+	}),
+	v.check((e) => !(e.output && e.stream), 'declare either `output` or `stream`, not both'),
+	v.check(requiresExamplesWhenCustom, CUSTOM_NEEDS_EXAMPLES)
+);
+
+// Room contract (D14) — Workers with `room: true` may declare state and
+// message schemas plus script examples: message sequences driven against
+// the Room class headless, checked as expected state + broadcasts (K-25).
+const roomScriptStep = v.strictObject({
+	send: ident,
+	with: v.optional(v.record(v.string(), v.unknown()))
+});
+
+const roomExample = v.strictObject({
+	script: v.pipe(v.array(roomScriptStep), v.minLength(1, 'room examples need at least one script step')),
+	expect: v.optional(v.unknown())
+});
+
 // L3 kinds: whole Civet files with declared auth + capabilities.
 // `validate` refuses them without an auth declaration (PLAN §6).
 const level3 = (extra = {}) =>
@@ -179,8 +301,26 @@ export const UNIT_SCHEMAS = {
 	Trigger,
 	Function,
 	Component,
+	Snippet,
+	Service,
+	Job,
+	Endpoint,
 	Route: level3(),
-	Worker: level3({ room: v.optional(v.boolean()) }),
+	Worker: level3({
+		room: v.optional(v.boolean()),
+		state: v.optional(v.record(ident, v.string())),
+		messages: v.optional(
+			v.record(
+				ident,
+				v.strictObject({
+					in: v.optional(v.record(v.string(), v.unknown())),
+					out: v.optional(v.record(v.string(), v.unknown()))
+				})
+			)
+		),
+		tickMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0))),
+		examples: v.optional(v.array(roomExample))
+	}),
 	Adapter: level3(),
 	Middleware: level3(),
 	Plugin
@@ -201,6 +341,10 @@ export const MODULE_SCHEMA = v.strictObject({
 	triggers: collection,
 	functions: collection,
 	components: collection,
+	snippets: collection,
+	services: collection,
+	jobs: collection,
+	endpoints: collection,
 	routes: collection,
 	workers: collection,
 	adapters: collection,

package/src/kernel/refine.js

@@ -69,6 +69,54 @@ export function refineSpecs(specs) {
 		}
 	}
 
+	/**
+	 * `call` steps may target a service operation
+	 * (`<module>.Service.<name>.<op>`) — the service and operation must both
+	 * exist. Other call targets are container tokens, resolved at runtime.
+	 */
+	function checkServiceCall(at, fromModule, call) {
+		const m = /^([a-z][a-z0-9_]*)\.Service\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)$/.exec(
+			call
+		);
+		if (!m) return;
+		const [, module, service, op] = m;
+		if (!(module in modules)) {
+			if (dependsOf(fromModule).includes(module)) return;
+			issues.push({
+				level: 'error',
+				address: at,
+				message: `call "${call}" points into unknown module "${module}" (not loaded, not in depends)`
+			});
+			return;
+		}
+		const svc = modules[module]?.services?.[service];
+		if (!svc) {
+			issues.push({
+				level: 'error',
+				address: at,
+				message: `call "${call}" does not resolve (no ${module}.Service.${service})`
+			});
+			return;
+		}
+		if (!svc.operations?.[op]) {
+			issues.push({
+				level: 'error',
+				address: at,
+				message: `call "${call}": service "${service}" has no operation "${op}"`
+			});
+		}
+	}
+
+	/** Shared step checks for Actions and Jobs: service calls and job enqueues resolve. */
+	function checkFlowSteps(at, fromModule, unitValue) {
+		for (const step of Array.isArray(unitValue?.steps) ? unitValue.steps : []) {
+			if (typeof step?.call === 'string') checkServiceCall(at, fromModule, step.call);
+			if (typeof step?.enqueue === 'string') {
+				checkRef(at, fromModule, step.enqueue, 'Job', 'enqueue');
+			}
+		}
+	}
+
 	// depends: modules exist (or are external-by-convention? no — depends
 	// names must be loaded or the well-known platform module "core") and form a DAG.
 	for (const [name, spec] of Object.entries(modules)) {
@@ -144,6 +192,7 @@ export function refineSpecs(specs) {
 				for (const ref of Array.isArray(value?.refresh) ? value.refresh : []) {
 					checkRef(at, mod, ref, 'Query', 'refresh');
 				}
+				checkFlowSteps(at, mod, value);
 				if (value?.transport === 'remote' && !remoteEnabled) {
 					issues.push({
 						level: 'error',
@@ -186,6 +235,35 @@ export function refineSpecs(specs) {
 				checkRef(at, mod, action, 'Action', 'trigger action');
 				break;
 			}
+			case 'Job': {
+				checkFlowSteps(at, mod, value);
+				break;
+			}
+			case 'Worker': {
+				if (value?.messages) {
+					for (const [i, ex] of (value.examples ?? []).entries()) {
+						for (const step of ex.script ?? []) {
+							if (!(step.send in value.messages)) {
+								issues.push({
+									level: 'error',
+									address: at,
+									message: `example ${i}: script sends undeclared message "${step.send}"`
+								});
+							}
+						}
+					}
+				}
+				break;
+			}
+			case 'Route': {
+				issues.push({
+					level: 'warning',
+					address: at,
+					message:
+						'schema-less Route is deprecated — declare an Endpoint (route/method/auth/input/output) instead; v3.1 refuses bare Routes'
+				});
+				break;
+			}
 			case 'Component': {
 				for (const [event, target] of Object.entries(value?.events ?? {})) {
 					checkRef(at, mod, target, 'Action', `events.${event}`);

package/src/kernel/trace.js

@@ -23,6 +23,7 @@ import { dirname, join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
 import { actionEntity } from './emit-units.js';
+import { shapeIssues } from '../server/service.js';
 import { normalizeField } from './emit-schema.js';
 import { generateApp } from './generate.js';
 import { loadSpecs } from './validate.js';
@@ -53,29 +54,34 @@ function rewriteImports(js, { scratch, appRoot, compileCivet }) {
 			return `${from}${q}${pathToFileURL(target).href}${q}`;
 		}
 		if (spec.startsWith('$custom/')) {
-			const rel = spec.slice('$custom/'.length);
-			const target = join(scratch, 'custom', rel.replace(/\.c$/, '.js'));
-			if (!existsSync(target)) {
-				const src = join(appRoot, 'src', rel);
-				mkdirSync(dirname(target), { recursive: true });
-				if (existsSync(src)) {
-					writeFileSync(
-						target,
-						rewriteImports(compileCivet(readFileSync(src, 'utf-8')), { scratch, appRoot, compileCivet })
-					);
-				} else {
-					writeFileSync(
-						target,
-						`export default () => { throw new Error(${JSON.stringify(`missing custom body: src/${rel}`)}) }\n`
-					);
-				}
-			}
+			const target = ensureCustom(spec.slice('$custom/'.length), { scratch, appRoot, compileCivet });
 			return `${from}${q}${pathToFileURL(target).href}${q}`;
 		}
 		return `${from}${q}${resolveBare(spec)}${q}`;
 	});
 }
 
+/** Compile a custom body `src/<rel>` into the scratch tree; missing bodies throw when invoked. */
+function ensureCustom(rel, { scratch, appRoot, compileCivet }) {
+	const target = join(scratch, 'custom', rel.replace(/\.c$/, '.js'));
+	if (!existsSync(target)) {
+		const src = join(appRoot, 'src', rel);
+		mkdirSync(dirname(target), { recursive: true });
+		if (existsSync(src)) {
+			writeFileSync(
+				target,
+				rewriteImports(compileCivet(readFileSync(src, 'utf-8')), { scratch, appRoot, compileCivet })
+			);
+		} else {
+			writeFileSync(
+				target,
+				`export default () => { throw new Error(${JSON.stringify(`missing custom body: src/${rel}`)}) }\n`
+			);
+		}
+	}
+	return target;
+}
+
 /** Compile `lib/**` of the generated tree into `<scratch>/lib` as JS. */
 function buildScratch(genRoot, scratch, appRoot) {
 	const { compile } = require('@danielx/civet');
@@ -166,7 +172,7 @@ function seedRow(id, entitySpec) {
 /* Runner                                                              */
 /* ------------------------------------------------------------------ */
 
-function recordingContainer(db, { fixtures = {}, events, calls }) {
+function recordingContainer(db, { fixtures = {}, auto = {}, events, calls }) {
 	return {
 		resolve(name) {
 			if (name === 'db') return db;
@@ -177,18 +183,84 @@ function recordingContainer(db, { fixtures = {}, events, calls }) {
 					}
 				};
 			}
+			if (name === 'jobs') {
+				return {
+					enqueue: async (address, input) => {
+						calls.push({ name: 'jobs.enqueue', args: [address, input] });
+					}
+				};
+			}
 			if (name in fixtures) return fixtures[name];
+			if (name in auto) {
+				return async (...args) => {
+					calls.push({ name, args });
+					return auto[name];
+				};
+			}
 			return async (...args) => {
 				calls.push({ name, args });
 			};
-		}
+		},
+		// `has` makes serviceClient's op-level fixture hook fire for every
+		// declared service operation — traced flows never touch the network.
+		has: (name) => name in fixtures || name in auto
 	};
 }
 
+/** Service op address → response fabricated from the op's declared output shape. */
+function serviceAutoFixtures(specs) {
+	const auto = {};
+	for (const [moduleName, moduleSpec] of Object.entries(specs.modules)) {
+		for (const [serviceName, svc] of Object.entries(moduleSpec.services ?? {})) {
+			for (const [opName, op] of Object.entries(svc.operations ?? {})) {
+				const out = {};
+				for (const [key, t] of Object.entries(op.output && typeof op.output === 'object' ? op.output : {})) {
+					const type = (typeof t === 'string' ? t : (t?.type ?? 'json')).replace(/\?$/, '');
+					out[key] = SEED_VALUES[type] ?? 'trace';
+				}
+				auto[`${moduleName}.Service.${serviceName}.${opName}`] = out;
+			}
+		}
+	}
+	return auto;
+}
+
 const looseEq = (a, b) => JSON.stringify(a) === JSON.stringify(b);
 
+function matchCalls(calls, want) {
+	return (Array.isArray(want) ? want : []).every((w) => {
+		const name = typeof w === 'string' ? w : w?.name;
+		return (calls ?? []).some(
+			(c) => c.name === name && (typeof w === 'string' || w.with === undefined || looseEq(c.args[0], w.with))
+		);
+	});
+}
+
+/**
+ * `expect` semantics shared by every traced kind: plain keys check the
+ * entity row then the return value; `calls` checks recorded external
+ * calls (by name, optionally `with` args); `frames` the collected SSE
+ * frames; `state`/`broadcasts` the Room script outcome.
+ */
+function checkExpect(record) {
+	record.pass = Object.entries(record.expect).every(([key, want]) => {
+		if (key === 'calls') return matchCalls(record.calls, want);
+		if (key === 'frames') return looseEq(record.frames, want);
+		if (key === 'state') {
+			return Object.entries(want ?? {}).every(([field, v2]) => looseEq(record.state?.[field], v2));
+		}
+		if (key === 'broadcasts') {
+			return (Array.isArray(want) ? want : []).every((n) => (record.broadcasts ?? []).some((b) => b?.type === n));
+		}
+		return looseEq(record.row?.[key], want) || looseEq(record.result?.[key], want);
+	});
+}
+
 /**
- * Run every Action example in the app's specs.
+ * Run every example in the app's specs: Actions against the sandbox DB,
+ * plus L3 units (K-25) — Jobs and custom Functions/Endpoints run with the
+ * same recording container (Service calls auto-fixtured from their output
+ * schemas), Room workers run their script examples headless.
  *
  * @param {string} [dir] specs directory, defaults to `<cwd>/specs`
  * @param {{ out?: string, fixtures?: Record<string, *>, user?: * }} [opts]
@@ -210,6 +282,27 @@ export async function traceApp(dir, opts = {}) {
 		const ddl = await tableDDL(Object.values(tables));
 		const { betterSqlite } = await import('../server/db.js');
 		const { sql, eq } = await import('drizzle-orm');
+		const { compile } = require('@danielx/civet');
+		const ctxc = { scratch, appRoot, compileCivet: (src) => compile(src, { sync: true, js: true }) };
+		const auto = serviceAutoFixtures(specs);
+
+		const runL3 = async (address, index, example, invoke) => {
+			const events = [];
+			const calls = [];
+			const record = { address, index, input: example.input ?? {}, expect: example.expect ?? {}, events, calls };
+			try {
+				const db = await betterSqlite(':memory:');
+				for (const stmt of ddl) await db.run(sql.raw(stmt));
+				const container = recordingContainer(db, { fixtures: opts.fixtures, auto, events, calls });
+				record.result = await invoke({ input: record.input, container, user, event: null });
+				checkExpect(record);
+			} catch (e) {
+				record.pass = false;
+				record.error = e?.body?.message ?? e?.message ?? String(e);
+				record.status = e?.status;
+			}
+			return record;
+		};
 
 		for (const [moduleName, moduleSpec] of Object.entries(specs.modules)) {
 			const actionsFile = join(scratch, 'lib', moduleName, 'actions.js');
@@ -217,10 +310,10 @@ export async function traceApp(dir, opts = {}) {
 				.filter(([, a]) => (a.examples ?? []).length > 0)
 				.map(([n]) => n)
 				.sort();
-			if (actionNames.length === 0 || !existsSync(actionsFile)) continue;
-			const mod = await import(pathToFileURL(actionsFile).href);
+			const mod =
+				actionNames.length > 0 && existsSync(actionsFile) ? await import(pathToFileURL(actionsFile).href) : null;
 
-			for (const name of actionNames) {
+			for (const name of mod ? actionNames : []) {
 				const actionSpec = moduleSpec.actions[name];
 				const address = `${moduleName}.Action.${name}`;
 				const target = actionEntity(moduleName, actionSpec, specs);
@@ -248,7 +341,7 @@ export async function traceApp(dir, opts = {}) {
 							if (ref?.endsWith('.id')) seededId = { table, value };
 						}
 
-						const container = recordingContainer(db, { fixtures: opts.fixtures, events, calls });
+						const container = recordingContainer(db, { fixtures: opts.fixtures, auto, events, calls });
 						record.result = await mod[name].run({ input: example.input ?? {}, container, user });
 
 						if (seededId && target) {
@@ -257,9 +350,7 @@ export async function traceApp(dir, opts = {}) {
 							)[0];
 						}
 
-						record.pass = Object.entries(record.expect).every(
-							([k, want]) => looseEq(record.row?.[k], want) || looseEq(record.result?.[k], want)
-						);
+						checkExpect(record);
 					} catch (e) {
 						record.pass = false;
 						record.error = e?.body?.message ?? e?.message ?? String(e);
@@ -267,6 +358,103 @@ export async function traceApp(dir, opts = {}) {
 					}
 				}
 			}
+
+			// Jobs — run the generated (or custom) `run` inline (K-25)
+			const jobsFile = join(scratch, 'lib', moduleName, 'jobs.js');
+			const jobNames = Object.entries(moduleSpec.jobs ?? {})
+				.filter(([, j]) => (j.examples ?? []).length > 0)
+				.map(([n]) => n)
+				.sort();
+			if (jobNames.length > 0 && existsSync(jobsFile)) {
+				const jobsMod = await import(pathToFileURL(jobsFile).href);
+				for (const name of jobNames) {
+					for (const [index, example] of moduleSpec.jobs[name].examples.entries()) {
+						cases.push(
+							await runL3(`${moduleName}.Job.${name}`, index, example, (ctx) => jobsMod[name].run(ctx))
+						);
+					}
+				}
+			}
+
+			// Functions — custom bodies with contract examples
+			for (const name of Object.keys(moduleSpec.functions ?? {}).sort()) {
+				const fnSpec = moduleSpec.functions[name];
+				if ((fnSpec.examples ?? []).length === 0) continue;
+				const target = ensureCustom(`${moduleName}/functions/${name}.c`, ctxc);
+				const body = (await import(pathToFileURL(target).href)).default;
+				for (const [index, example] of fnSpec.examples.entries()) {
+					cases.push(await runL3(`${moduleName}.Function.${name}`, index, example, (ctx) => body(ctx)));
+				}
+			}
+
+			// Endpoints — body-level trace with the declared IO contract enforced
+			for (const name of Object.keys(moduleSpec.endpoints ?? {}).sort()) {
+				const ep = moduleSpec.endpoints[name];
+				if ((ep.examples ?? []).length === 0) continue;
+				const target = ensureCustom(`${moduleName}/endpoints/${name}.c`, ctxc);
+				const body = (await import(pathToFileURL(target).href)).default;
+				for (const [index, example] of ep.examples.entries()) {
+					const record = await runL3(`${moduleName}.Endpoint.${name}`, index, example, async (ctx) => {
+						let result = body(ctx);
+						if (ep.stream) {
+							if (!result?.[Symbol.asyncIterator]) result = await result;
+							const frames = [];
+							for await (const frame of result) {
+								const issues = shapeIssues(ep.stream.frame ?? {}, frame, 'frame');
+								if (issues.length > 0) throw new Error(issues.join('; '));
+								frames.push(frame);
+							}
+							return frames;
+						}
+						result = await result;
+						if (ep.output) {
+							const issues = shapeIssues(ep.output, result, 'output');
+							if (issues.length > 0) throw new Error(`output contract: ${issues.join('; ')}`);
+						}
+						return result;
+					});
+					if (ep.stream && Array.isArray(record.result)) {
+						record.frames = record.result;
+						checkExpect(record);
+					}
+					cases.push(record);
+				}
+			}
+
+			// Room workers — script examples driven headless against the class
+			for (const name of Object.keys(moduleSpec.workers ?? {}).sort()) {
+				const w = moduleSpec.workers[name];
+				if (w?.room !== true || (w.examples ?? []).length === 0) continue;
+				const rel = w.source.startsWith('src/') ? w.source.slice(4) : w.source;
+				const target = ensureCustom(rel, ctxc);
+				const RoomClass = (await import(pathToFileURL(target).href)).default;
+				for (const [index, example] of w.examples.entries()) {
+					const record = {
+						address: `${moduleName}.Worker.${name}`,
+						index,
+						script: example.script ?? [],
+						expect: example.expect ?? {},
+						broadcasts: []
+					};
+					cases.push(record);
+					try {
+						const instance = new RoomClass({}, {});
+						instance.tickMs = 0;
+						instance.broadcast = (message) => {
+							record.broadcasts.push(typeof message === 'string' ? JSON.parse(message) : message);
+							return 0;
+						};
+						for (const step of example.script ?? []) {
+							await instance.onMessage(JSON.stringify({ type: step.send, ...(step.with ?? {}) }), null);
+						}
+						record.state = Object.fromEntries(Object.keys(w.state ?? {}).map((f) => [f, instance[f]]));
+						checkExpect(record);
+					} catch (e) {
+						record.pass = false;
+						record.error = e?.message ?? String(e);
+					}
+				}
+			}
 		}
 	} finally {
 		rmSync(scratch, { recursive: true, force: true });