@volter/twin-standard 1.0.0

package/src/derive.ts

@@ -0,0 +1,316 @@
+#!/usr/bin/env bun
+// derive — the derived-pack generator's build step (docs/contributing/architecture.md, "The derived pack"), the
+// standard's own: the grade's `derived-reproducible` runs it, so every grader derives the same way. Reads a pack's
+// vendored spec (`<pack>/spec/openapi.json[.gz]`), builds the spec IR (./spec-ir.ts) and writes the pack's
+// `src/generated/`. Deterministic: a second run over the same spec writes the same bytes. Never hand-edit what it writes.
+//
+// Usage: twin-standard derive <pack dir> [--out <dir>]   (twin-world: bun scripts/derive-pack.ts <vendor>)
+
+import { parse as parseYaml } from 'yaml';
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { gunzipSync } from 'node:zlib';
+import { CRUD_CLASSES, fromOpenAPI, fromSmithy, fromSwagger2, slug, type SpecIR } from './spec-ir.ts';
+import { fromCommandReplies, type LineSpec } from './spec-ir-lines.ts';
+import { fromProto, protoSchema, readProtoFiles } from './spec-ir-proto.ts';
+import { fromRedisCommands, type CommandTableSpec } from './spec-ir-commands.ts';
+import { fromClientCalls, type ClientOpsSpec } from './spec-ir-client.ts';
+import { discoveryMethods, fromDiscovery, isDiscovery } from './spec-ir-discovery.ts';
+import { handlerCitations, quotedPage, type Patch, type SourceRecord, urlsIn } from './check-sources.ts';
+import { applySpecPatch } from './spec-patches.ts';
+import { unitAt } from './unit.ts';
+import { mergeDocuments, readSpecDocuments } from './spec-documents.ts';
+import { p3RuleHits } from './p3-rules.ts';
+
+const vendor = process.argv[2];
+if (!vendor) {
+  console.error('usage: twin-standard derive <pack dir> [--out <dir>]');
+  process.exit(1);
+}
+const PKG = unitAt(vendor).dir;
+// an OpenAPI document, an AWS Smithy JSON model (smithy.json), the RFC a line protocol is (rfcNNNN.txt), or the
+// Protocol Buffers files an RPC service is published as (spec/proto/, every file the service imports), or the calls a
+// vendor's own client makes where the vendor publishes no spec (client-ops.json, spec-ir-client.ts), or one API's several
+// OpenAPI documents (spec/openapi/, read as their union: spec-documents.ts), or a Google API Discovery document
+// (discovery.json, spec-ir-discovery.ts)
+const rfc = existsSync(join(PKG, 'spec')) ? readdirSync(join(PKG, 'spec')).find((f) => /^rfc\d+\.txt$/.test(f)) : undefined;
+const specPath = ['openapi.json.gz', 'openapi.json', 'openapi.yaml.gz', 'openapi.yaml', 'smithy.json.gz', 'smithy.json', 'discovery.json.gz', 'discovery.json', ...(rfc ? [rfc] : []), 'openapi', 'proto', 'commands', 'client-ops.json'].map((f) => join(PKG, 'spec', f)).find((p) => existsSync(p));
+if (!specPath) {
+  console.error(`derive-pack: no spec at ${join(PKG, 'spec')}/openapi.{json,yaml}[.gz], smithy.json[.gz], rfcNNNN.txt, proto/, commands/ or client-ops.json`);
+  process.exit(1);
+}
+const rpcService = specPath.endsWith('/proto');
+const severalDocuments = specPath.endsWith('/openapi');
+// a command language's table, one vendored file per command (Redis's src/commands/*.json), with its version in SOURCE.md
+const commandTable = specPath.endsWith('/commands');
+const protoTree = (dir: string, at = ''): Record<string, string> => Object.fromEntries(readdirSync(join(dir, at), { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? Object.entries(protoTree(dir, join(at, e.name))) : e.name.endsWith('.proto') ? [[join(at, e.name), readFileSync(join(dir, at, e.name), 'utf8')]] : [])));
+const bytes = rpcService || commandTable || severalDocuments ? Buffer.from('') : readFileSync(specPath);
+const text = (specPath.endsWith('.gz') ? gunzipSync(bytes) : bytes).toString('utf8');
+// an RFC's command-reply table and a service's .proto files are read into data first, so a patch corrects them as it
+// corrects a JSON spec
+const lineProtocol = specPath.endsWith('.txt');
+const commandVersion = commandTable ? (/\*\*Version:\*\* `([^`]+)`/.exec(existsSync(join(PKG, 'spec', 'SOURCE.md')) ? readFileSync(join(PKG, 'spec', 'SOURCE.md'), 'utf8') : '')?.[1] ?? '') : '';
+let doc: any = severalDocuments ? readSpecDocuments(join(PKG, 'spec')) : commandTable ? fromRedisCommands(Object.fromEntries(readdirSync(specPath).filter((f) => f.endsWith('.json')).sort().map((f) => [f, JSON.parse(readFileSync(join(specPath, f), 'utf8'))])), commandVersion) : rpcService ? readProtoFiles(protoTree(specPath)) : lineProtocol ? fromCommandReplies(text) : /\.ya?ml(\.gz)?$/.test(specPath) ? parseYaml(text) : JSON.parse(text);
+
+// ── the evidence the pack cites ──────────────────────────────────────────────────────────────
+// A source is a page check-sources (./check-sources.ts) fetched and recorded found (spec/sources.json), a quote
+// that page holds, or `spec:<operationId or /json/pointer> "quote"`: words of the vendored spec itself
+// (read before any patch, so a patch never vouches for itself).
+const vendored = lineProtocol || commandTable ? structuredClone(doc) : JSON.parse(text.startsWith('{') ? text : JSON.stringify(doc));
+/** An RFC's section by its number (`4.1.4`): its heading's line to the next heading's. */
+const sectionOf = (n: string): string | undefined => {
+  const at = text.search(new RegExp(`^${n.replace(/\./g, '\\.')}\\.  `, 'm'));
+  if (at < 0) return undefined;
+  const next = text.slice(at + 1).search(/^\d+(\.\d+)*\. {2}/m);
+  return next < 0 ? text.slice(at) : text.slice(at, at + 1 + next);
+};
+const sourcesPath = join(PKG, 'spec', 'sources.json');
+const recorded: Record<string, SourceRecord> = existsSync(sourcesPath) ? JSON.parse(readFileSync(sourcesPath, 'utf8')) : {};
+const HTTP_METHODS = new Set(['get', 'put', 'post', 'delete', 'patch', 'head', 'options', 'trace']);
+const specOps = new Map<string, unknown>();
+// an operation by its id: the spec's operationId, or the `method_path` slug the IR mints where the spec names none (QStash)
+for (const d of severalDocuments ? Object.values(vendored.documents) as any[] : [vendored]) {
+  for (const [path, item] of Object.entries(d.paths ?? {})) {
+    for (const [method, o] of Object.entries(item as Record<string, unknown>)) {
+      if (!o || typeof o !== 'object' || !HTTP_METHODS.has(method)) continue;
+      const id = (o as { operationId?: unknown }).operationId;
+      specOps.set(typeof id === 'string' && id ? id : slug(method, path), o);
+    }
+  }
+}
+// a proto spec's rpc, message or enum is its declaring file's text, comments and all (Google documents each rpc and field
+// in the comments above it): `spec:SearchUris "…"` checks the words there
+// (the names are read from the file with its comments taken out: a comment's "the message of" names nothing)
+if (rpcService) for (const text of Object.values(protoTree(specPath))) for (const m of text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '').matchAll(/\b(?:rpc|message|enum)\s+(\w+)/g)) if (!specOps.has(m[1]!)) specOps.set(m[1]!, text);
+// a JSON pointer into several documents reads their union (a schema only one of them has)
+const pointed = severalDocuments ? mergeDocuments(vendored).doc : vendored;
+// a Discovery document's methods are its operations, by their ids
+if (isDiscovery(vendored)) for (const m of discoveryMethods(vendored.resources)) specOps.set(String(m.id), m);
+// a spec node's words: its strings and its keys (a schema's property names are what it says)
+const wordsIn = (node: unknown): string => (typeof node === 'string' ? node : node && typeof node === 'object' ? Object.entries(node).map(([k, v]) => `${Array.isArray(node) ? '' : k} ${wordsIn(v)}`).join(' ') : '');
+const bare = (s: string): string => s.replace(/\s+/g, '');
+// a JSON pointer into the spec, an operation by id, or (an RFC's) a section by number
+const specNode = (ref: string): unknown => (ref.startsWith('/') ? ref.split('/').slice(1).map((k) => k.replace(/~1/g, '/').replace(/~0/g, '~')).reduce((n: any, k) => n?.[k], severalDocuments && !ref.startsWith('/documents/') ? pointed : vendored) : lineProtocol && /^\d+(\.\d+)*$/.test(ref) ? sectionOf(ref) : specOps.get(ref));
+/** Why a citation does not hold, or undefined when it does. */
+function unheld(source: string): string | undefined {
+  const spec = /^spec:(\S+)\s+"([\s\S]*)"$/.exec(source.trim());
+  if (spec && !spec[2]!.trim()) return `spec:${spec[1]} quotes nothing`;
+  if (spec) {
+    const node = specNode(spec[1]!);
+    if (node === undefined) return `the spec has nothing at ${spec[1]}`;
+    return bare(wordsIn(node)).includes(bare(spec[2]!.replace(/\\"/g, '"'))) ? undefined : `the spec at ${spec[1]} does not say "${spec[2]}"`;
+  }
+  const urls = urlsIn(source);
+  if (!urls.length) return `"${source.slice(0, 60)}" names no page and no spec words`;
+  const dead = urls.filter((u) => recorded[u]?.status !== 200);
+  return dead.length ? dead.map((u) => `${u} is ${recorded[u] ? `answering ${recorded[u]!.status || 'nothing'}` : 'not checked'} (twin-standard check-sources ${vendor})`).join('; ') : undefined;
+}
+const evidence: string[] = [];
+// every rule a handler writes carries its citation beside it (`// source:`), held as a transition's is
+for (const c of handlerCitations(PKG)) {
+  const q = quotedPage(c.source);
+  // a quote is checked only in the two forms that say where it is; any other quote is refused, never passed unread
+  const trimmed = c.source.trim();
+  const loose = !q && (/^spec:/.test(trimmed) ? !/^spec:\S+\s+"[\s\S]*"$/.test(trimmed) : trimmed.includes('"'));
+  const held = loose ? `"${c.source.slice(0, 60)}" quotes words in no form a check reads (\`<url> "<quote>"\` or \`spec:<ref> "<quote>"\`)` : q ? (recorded[q.url]?.quotes?.[q.quote] === true ? undefined : `${q.url} does not hold "${q.quote}" as recorded (twin-standard check-sources ${vendor})`) : unheld(c.source);
+  if (held) evidence.push(`semantics/${c.where}: ${held}`);
+}
+// Corrections to the vendor's spec, as data: RFC 6902 operations (add, replace, remove, move, copy)
+// in spec/patches.json, applied before the IR so the vendored file stays the vendor's own bytes. Each says
+// `why`, citing what shows the vendor differs from its spec (its docs, the spec's own example, a recording).
+const patchPath = join(PKG, 'spec', 'patches.json');
+if (existsSync(patchPath)) {
+  for (const p of JSON.parse(readFileSync(patchPath, 'utf8')).patches as Array<{ op: string; path: string; from?: string; value?: unknown; why: string }>) {
+    if (!p.why?.trim()) { console.error(`${vendor}: spec patch ${p.op} ${p.path} gives no why`); process.exit(1); }
+    {
+      const s = (p as Patch).source;
+      const held = s?.spec ? unheld(`spec:${s.spec} "${s.quote}"`)
+        : s?.url ? (recorded[s.url]?.quotes?.[s.quote] === true ? undefined : `${s.url} does not hold "${s.quote}" as recorded`)
+        : urlsIn(p.why).length ? unheld(p.why) : 'it cites no page and quotes nothing';
+      if (held) evidence.push(`spec patch ${p.op} ${p.path}: ${held}`);
+    }
+    applySpecPatch(doc, p);
+  }
+}
+// several documents of one API are read as their union once the patches have ruled every contradiction
+if (severalDocuments) {
+  const { doc: merged, contradictions } = mergeDocuments(doc);
+  if (contradictions.length) {
+    console.error(`${vendor}: the documents contradict each other; a patch (citing what shows which holds) rules each:\n  ${contradictions.join('\n  ')}`);
+    process.exit(1);
+  }
+  doc = merged;
+}
+// --out <dir> writes the core elsewhere, touching nothing in the pack (the grader's reproducibility check)
+const outAt = process.argv.indexOf('--out');
+const out = outAt >= 0 && process.argv[outAt + 1] ? process.argv[outAt + 1]! : join(PKG, 'src', 'generated');
+mkdirSync(out, { recursive: true });
+// A line protocol's surface is its commands and the replies each may answer; its manifest's transitions
+// name commands and cite the RFC.
+if (lineProtocol || commandTable) {
+  const spec = doc as LineSpec | CommandTableSpec;
+  writeFileSync(join(out, 'surface.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', spec: specPath.split('/').pop(), ...spec })}\n`);
+  const manifestPath = join(PKG, 'src', 'manifest.ts');
+  if (existsSync(manifestPath)) {
+    const { manifest } = (await import(manifestPath)) as { manifest: { resources: Record<string, { state?: Record<string, { transitions: Array<{ operation?: string; actor?: string; to?: string; source?: string }> }> }> } };
+    for (const [name, decl] of Object.entries(manifest.resources)) {
+      for (const [field, m] of Object.entries(decl.state ?? {})) {
+        for (const tr of m.transitions) {
+          const held = unheld(String(tr.source ?? ''));
+          if (held) evidence.push(`${name}.${field} ${tr.operation ?? tr.actor ?? ''} → ${tr.to ?? '(stays)'}: ${held}`);
+          if (tr.operation && !spec.commands.some((c) => c.id === tr.operation)) evidence.push(`${name}.${field}: a transition names ${tr.operation}, which the spec does not have`);
+        }
+      }
+    }
+  }
+  if (evidence.length) {
+    console.error(`${vendor}: ${evidence.length} citation(s) do not hold:\n  ${[...new Set(evidence)].join('\n  ')}`);
+    process.exit(1);
+  }
+  console.log(`${vendor}: spec ${spec.version}, ${spec.commands.length} commands${'anyCommand' in spec ? `; any command may answer ${spec.anyCommand.join(', ')}` : ''}`);
+  process.exit(0);
+}
+const ir: SpecIR = isDiscovery(doc) ? fromDiscovery(doc) : doc?.format === 'client' ? fromClientCalls(doc as ClientOpsSpec) : rpcService ? fromProto(doc) : /smithy\.json(\.gz)?$/.test(specPath) ? fromSmithy(doc) : doc?.swagger === '2.0' ? fromSwagger2(doc) : fromOpenAPI(doc);
+// A spec that groups its operations by tag (Cloudflare's: nearly every path begins /accounts/{account_id}/, so the
+// path's first segment names one family for thousands of operations) says so in spec/grouping.json, `{ "by": "tag" }`:
+// each operation's family is then its first tag, as a file name (`Worker Script` is `worker-script`).
+const grouping = existsSync(join(PKG, 'spec', 'grouping.json')) ? (JSON.parse(readFileSync(join(PKG, 'spec', 'grouping.json'), 'utf8')) as { by?: string }) : {};
+if (grouping.by === 'tag') {
+  // an operation the spec names no operationId for is found by its method and path, in the spec as corrected (an
+  // operation a patch adds carries its own tags: PostHog's person delete)
+  const byRoute = new Map<string, { tags?: string[] }>();
+  for (const [path, item] of Object.entries((doc as { paths?: Record<string, Record<string, unknown>> }).paths ?? {})) {
+    for (const [method, op] of Object.entries(item ?? {})) if (op && typeof op === 'object') byRoute.set(`${method.toLowerCase()} ${path}`, op as { tags?: string[] });
+  }
+  for (const o of ir.operations) {
+    const op = (specOps.get(o.id) ?? byRoute.get(`${o.method.toLowerCase()} ${o.path}`)) as { tags?: string[] } | undefined;
+    const tag = op?.tags?.[0];
+    if (tag) o.family = tag.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+  }
+}
+
+// A GraphQL API beside the REST one (spec/schema.graphql[.gz]) is a second wire over the same state.
+const sdlPath = ['schema.graphql.gz', 'schema.graphql'].map((f) => join(PKG, 'spec', f)).find((p) => existsSync(p));
+// the GraphQL wire's root fields (`mutation.issueArchive`), which a transition may name as a REST operation is named
+const graphqlOperations = new Set<string>();
+if (sdlPath) {
+  const { fromGraphQL } = await import('./spec-ir-graphql.ts');
+  const sdl = (sdlPath.endsWith('.gz') ? gunzipSync(readFileSync(sdlPath)) : readFileSync(sdlPath)).toString('utf8');
+  const gql = fromGraphQL(sdl);
+  for (const o of gql.operations) graphqlOperations.add(o.id);
+  mkdirSync(out, { recursive: true });
+  writeFileSync(join(out, 'graphql.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', spec: sdlPath.split('/').pop(), ...gql })}\n`);
+  // the schema itself, for the pack's GraphQL wire to validate and execute against (imported lazily)
+  writeFileSync(join(out, 'graphql-sdl.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', sdl })}\n`);
+  const g = new Map<string, number>();
+  for (const o of gql.operations) g.set(o.class, (g.get(o.class) ?? 0) + 1);
+  console.log(`  graphql: ${gql.operations.length} root fields, ${gql.resources.length} resources; ` + [...g].sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join(' '));
+}
+
+// a proto unit's schema, for the kernel's protobuf codec and gRPC wire: fields by number, enums by number, rpcs by path
+if (rpcService) writeFileSync(join(out, 'proto.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', ...protoSchema(doc) })}\n`);
+// Data, not code: the surface is JSON so it adds nothing to a typecheck and ships as an asset.
+writeFileSync(join(out, 'surface.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', spec: specPath.split('/').pop(), ...ir })}\n`);
+
+// The event types a vendor delivers, when its spec enumerates them where a webhook subscription names the events it
+// wants (a request body's `enabled_events`): the pack records an event only under a type the vendor sends.
+const eventTypes = new Set<string>();
+for (const item of Object.values((doc?.paths ?? {}) as Record<string, Record<string, any>>)) {
+  for (const op of Object.values(item ?? {})) {
+    for (const media of Object.values((op?.requestBody?.content ?? {}) as Record<string, any>)) {
+      for (const type of (media?.schema?.properties?.enabled_events?.items?.enum ?? []) as unknown[]) if (typeof type === 'string' && type !== '*') eventTypes.add(type);
+    }
+  }
+}
+if (eventTypes.size) writeFileSync(join(out, 'events.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', spec: specPath.split('/').pop(), types: [...eventTypes].sort() })}\n`);
+
+// What the pack's UI needs to know about each resource its manifest declares: where it is listed,
+// read, created and moved, which values its state fields take and which moves are legal from which,
+// and which fields name another resource. Data the browser bundle reads; it never imports the manifest.
+const manifestPath = join(PKG, 'src', 'manifest.ts');
+if (existsSync(manifestPath)) {
+  const { manifest } = (await import(manifestPath)) as { manifest: { resources: Record<string, { state?: Record<string, { initial: unknown; transitions: Array<{ operation?: string; actor?: string; from: string[] | '*'; to?: string; source?: string }> }>; embeds?: Record<string, string> }>; unmodeled?: string[] } };
+  const base = ir.basePath ?? '';
+  const ops = ir.operations.filter((o) => !(manifest.unmodeled ?? []).includes(o.id));
+  const ui: Record<string, unknown> = {};
+  for (const [name, decl] of Object.entries(manifest.resources)) {
+    const mine = ops.filter((o) => o.resource === name);
+    const pick = (cls: string) => mine.filter((o) => o.class === cls).sort((a, b) => a.path.length - b.path.length)[0];
+    const [list, retrieve, create] = [pick('list'), pick('retrieve'), pick('create')];
+    const moves = Object.entries(decl.state ?? {}).flatMap(([field, m]) =>
+      m.transitions.filter((t) => t.operation).map((t) => {
+        const o = mine.find((x) => x.id === t.operation);
+        return o ? { operation: o.id, method: o.method.toUpperCase(), path: (o.basePath ?? base) + o.path, field, from: t.from, ...(t.to !== undefined ? { to: t.to } : {}) } : undefined;
+      }).filter(Boolean),
+    );
+    ui[name] = {
+      ...(list ? { list: (list.basePath ?? base) + list.path } : {}),
+      ...(retrieve ? { retrieve: (retrieve.basePath ?? base) + retrieve.path } : {}),
+      ...(create ? { create: { path: (create.basePath ?? base) + create.path, encoding: create.bodyEncoding, body: create.body } } : {}),
+      actions: moves,
+      states: Object.fromEntries(Object.entries(decl.state ?? {}).map(([f, m]) => [f, [...new Set([String(m.initial), ...m.transitions.flatMap((t) => [...(t.from === '*' ? [] : t.from), ...(t.to !== undefined ? [t.to] : [])])])].sort()])),
+      embeds: decl.embeds ?? {},
+    };
+  }
+  writeFileSync(join(out, 'ui.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', resources: ui })}\n`);
+  // every transition cites what shows the vendor moves that way, and the citation holds
+  for (const [name, decl] of Object.entries(manifest.resources)) {
+    for (const [field, m] of Object.entries(decl.state ?? {})) {
+      for (const tr of m.transitions) {
+        const held = unheld(String(tr.source ?? ''));
+        if (held) evidence.push(`${name}.${field} ${tr.operation ?? tr.actor ?? ''} → ${tr.to ?? '(stays)'}: ${held}`);
+        if (tr.operation && !ir.operations.some((o) => o.id === tr.operation) && !graphqlOperations.has(tr.operation)) evidence.push(`${name}.${field}: a transition names ${tr.operation}, which the spec does not have`);
+      }
+    }
+  }
+  if (evidence.length) {
+    console.error(`${vendor}: ${evidence.length} citation(s) do not hold:\n  ${[...new Set(evidence)].join('\n  ')}`);
+    process.exit(1);
+  }
+  // every state candidate of a resource the pack models is ruled on: a state field, or named not one
+  const unruled = ir.resources.flatMap((r) => {
+    const decl = manifest.resources[r.name] as { state?: Record<string, unknown>; notState?: string[] } | undefined;
+    return decl ? r.stateCandidates.filter((c) => !(c.field in (decl.state ?? {})) && !(decl.notState ?? []).includes(c.field)).map((c) => `${r.name}.${c.field}`) : [];
+  });
+  if (unruled.length) {
+    console.error(`${vendor}: the manifest does not rule on ${unruled.length} state candidate(s): ${unruled.join(', ')} (declare each a state field or list it in notState)`);
+    process.exit(1);
+  }
+  // a resource's refresh scope ("The real-system adapters", What a pack declares) names the list that enumerates it or
+  // its retrieve, or says why the vendor offers no read-back; whether every stored resource declares one is
+  // pack-standing's report, not this build's
+  // (a resource the spec names is read by its own list or retrieve; one the manifest names apart from the spec's, an RPC
+  // vendor's, by the list or read the scope names, with `items` where the answer holds them when the spec does not say)
+  const specNamed = new Set(ir.resources.map((r) => r.name));
+  const scopes = Object.entries(manifest.resources as Record<string, { refresh?: { list?: string; get?: string; none?: string; items?: string } }>).flatMap(([name, decl]) => {
+    const r = decl.refresh;
+    if (!r) return [];
+    const op = (id: string | undefined) => ir.operations.find((o) => o.id === id);
+    const ours = (o: { resource?: string }): boolean => !specNamed.has(name) || o.resource === name;
+    if (r.list !== undefined) {
+      const o = op(r.list);
+      if (!(o?.class === 'list' && ours(o))) return [`${name}: refresh.list ${r.list} is ${o ? `a ${o.class} of ${o.resource ?? 'no resource'}` : 'no operation of the spec'}, not a list of ${name}`];
+      return o.answers?.list || r.items ? [] : [`${name}: refresh.list ${r.list} answers no list the spec names; say where its items are (items)`];
+    }
+    if (r.get !== undefined) { const o = op(r.get); return o?.class === 'retrieve' && ours(o) ? [] : [`${name}: refresh.get ${r.get} is ${o ? `a ${o.class} of ${o.resource ?? 'no resource'}` : 'no operation of the spec'}, not the retrieve of ${name}`]; }
+    return typeof r.none === 'string' && r.none.trim() ? [] : [`${name}: refresh names no list, no get and no reason`];
+  });
+  if (scopes.length) {
+    console.error(`${vendor}: ${scopes.length} refresh scope(s) do not hold:\n  ${scopes.join('\n  ')}`);
+    process.exit(1);
+  }
+  // the architecture's mechanical rules over the pack's own code (./p3-rules.ts): a line breaking one records why, or
+  // the build stops
+  const hits = p3RuleHits(PKG);
+  if (hits.length) {
+    console.error(`${vendor}: ${hits.length} line(s) break a Protocol 3 rule (fix it, or record why on the line: // p3: <reason>):\n  ${hits.map((h) => `${h.file}:${h.line}: ${h.rule}\n      ${h.text}`).join('\n  ')}`);
+    process.exit(1);
+  }
+}
+
+const counts = new Map<string, number>();
+for (const o of ir.operations) counts.set(o.class, (counts.get(o.class) ?? 0) + 1);
+const crud = ir.operations.filter((o) => CRUD_CLASSES.has(o.class)).length;
+const candidates = ir.resources.reduce((n, r) => n + r.stateCandidates.length, 0);
+console.log(`${vendor}: spec ${ir.version}, ${ir.operations.length} operations (${crud} crud), ${ir.resources.length} resources, ${candidates} state candidates`);
+console.log('  ' + [...counts].sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join(' '));

package/src/gate.ts

@@ -0,0 +1,95 @@
+// The gate — what may be in a Protocol 3 pack repository or catalog: every pack and lane passes the form
+// section of the grade (docs/contributing/architecture.md, "The grade"), so nothing that is not Protocol 3 enters it:
+// nothing outside the pack layout, no handler past its contract.
+// The repository was made so that this holds (the owner, 2026-09-29: "this was WHY we created p3 so this can't
+// happen"); this is what makes it hold. It runs before a commit (the pack repository's hook, on the units it touches),
+// before a release (pack-release, on each pack it releases), and on every version submitted to the catalog
+// (twin-catalog's check, on the published tarball) — each at the version of this package it pins, so all of them judge by
+// one standard.
+//
+//   twin-standard gate <pack repository> [<pack or lane dir>…]
+//
+// It blocks on every criterion of the form section; what a pack's walked life proves (no dead code among them) is the
+// proof section's, which the gate does not read.
+import { spawnSync } from 'node:child_process';
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import ts from 'typescript';
+import { grade } from './grade.ts';
+
+/** Every unit of a pack repository: each pack (a directory with a package.json) and each lane in it (a directory with
+ *  its own src/manifest.ts and no package.json). */
+export function unitsOf(repo: string): string[] {
+  const units: string[] = [];
+  for (const name of readdirSync(repo).sort()) {
+    const dir = join(repo, name);
+    if (name.startsWith('.') || name === 'node_modules' || !statSync(dir).isDirectory() || !existsSync(join(dir, 'package.json'))) continue;
+    units.push(dir);
+    for (const sub of readdirSync(dir).sort()) {
+      const lane = join(dir, sub);
+      if (statSync(lane).isDirectory() && !existsSync(join(lane, 'package.json')) && existsSync(join(lane, 'src', 'manifest.ts'))) units.push(lane);
+    }
+  }
+  return units;
+}
+
+/** What a vendor whose every API is a lane may hold at its root: the package files, the vendor's life (journeys/), its
+ *  generated facts, its lanes, and in src/ only its manifest (its lanes' routes, discovery and descriptor, as data), the
+ *  fetch the kernel serves them by (createVendorFetch, generated) and the fixed serve factory, index and bin. Nothing
+ *  routes by hand: only the fetch reaches a lane. A directory with no manifest and no lane is no Protocol 3 pack. */
+const ROOT_ENTRIES = new Set(['README.md', 'package.json', 'LICENSE', 'CHANGELOG.md', 'tsconfig.json', 'tsconfig.build.json', 'gate.ts', 'census.json', 'vendor', 'journeys', 'generated', 'src', 'node_modules']);
+const ROOT_SRC = new Set(['manifest.ts', 'fetch.ts', 'index.ts', 'cli.ts', 'server.ts']);
+/** What the repository ignores (a build's dist/, a World's state) is no file of the pack, as protocol-3's layout reads it. */
+function ignored(dir: string, names: string[]): Set<string> {
+  if (!names.length) return new Set();
+  const run = spawnSync('git', ['check-ignore', '--', ...names], { cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+  return new Set(String(run.stdout ?? '').split('\n').filter(Boolean));
+}
+export function laneRootFailures(dir: string): string[] {
+  const lanes = readdirSync(dir).filter((n) => existsSync(join(dir, n, 'src', 'manifest.ts')));
+  if (!lanes.length) return ['layout: no src/manifest.ts and no lane: not a Protocol 3 pack'];
+  const stray = readdirSync(dir).filter((n) => !n.startsWith('.') && !ROOT_ENTRIES.has(n) && !lanes.includes(n));
+  const skip = ignored(dir, stray);
+  const outside = [
+    ...stray.filter((n) => !skip.has(n)),
+    ...(existsSync(join(dir, 'src')) ? readdirSync(join(dir, 'src')).filter((n) => !ROOT_SRC.has(n) && !n.endsWith('.d.ts') && n !== 'semantics').map((n) => `src/${n}`) : []),
+    // what the lanes share, and nothing else (architecture, "Pack layout": a vendor whose every API is a lane)
+    ...(existsSync(join(dir, 'src', 'semantics')) ? readdirSync(join(dir, 'src', 'semantics')).filter((n) => n !== 'shared.ts').map((n) => `src/semantics/${n}`) : []),
+  ];
+  const failures = outside.length ? [`layout: ${outside.length} outside the vendor's root (its lanes hold the rest): ${outside.slice(0, 8).join(', ')}${outside.length > 8 ? ` and ${outside.length - 8} more` : ''}`] : [];
+  // the vendor's front is declared and the kernel serves it: a manifest, and a fetch that is createVendorFetch over it
+  const src = (n: string): string => (existsSync(join(dir, 'src', n)) ? readFileSync(join(dir, 'src', n), 'utf8') : '');
+  if (!src('manifest.ts')) failures.push("layout: no src/manifest.ts: a vendor of lanes declares its lanes' routes, discovery and descriptor");
+  if (!/\bcreateVendorFetch\(/.test(src('fetch.ts'))) failures.push('layout: src/fetch.ts builds no createVendorFetch: the kernel routes a vendor\'s lanes from its manifest, nothing by hand');
+  // the root's own files hold to the pack contract as a unit's do (the grade's kernel-contract)
+  const beyond: string[] = [];
+  for (const n of existsSync(join(dir, 'src')) ? readdirSync(join(dir, 'src')).filter((x) => /\.tsx?$/.test(x) && !x.endsWith('.d.ts')) : []) {
+    // the file's imports as TypeScript reads them, as the grade's kernel-contract does
+    for (const from of ts.preProcessFile(readFileSync(join(dir, 'src', n), 'utf8'), true, true).importedFiles.map((i) => i.fileName)) {
+      // only the fetch reaches a lane (the kernel's routing), and the index each lane's manifest and generated surface
+      // (the kernel derives the vendor-backed half from them); the manifest is data, importing the contract's types alone
+      const laneData = n === 'index.ts' && /^\.\.\/[^/]+\/src\/(manifest\.ts|generated\/surface\.gen\.json)$/.test(from);
+      if (/^\.\.\/[^/]+\/src\//.test(from) && n !== 'fetch.ts' && !laneData) { beyond.push(`src/${n} imports ${from}: only src/fetch.ts reaches a lane`); continue; }
+      if (n === 'manifest.ts' && from !== '@volter/world-core') { beyond.push(`src/manifest.ts imports ${from}: the manifest is data`); continue; }
+      if (from.startsWith('.') || from === '@volter/world-core' || from === '@volter/world-ui') continue;
+      if (n === 'cli.ts' && (from === '@volter/world-core/args' || from === '@volter/world-core/lifecycle')) continue;
+      beyond.push(`src/${n} imports ${from}`);
+    }
+  }
+  return beyond.length ? [...failures, `kernel-contract: ${beyond.join('; ')}`] : failures;
+}
+
+/** Each unit's blocking form failures (empty when it may be in the repository). */
+export async function p3Gate(units: string[]): Promise<Array<{ unit: string; failures: string[] }>> {
+  const out: Array<{ unit: string; failures: string[] }> = [];
+  for (const unit of units) {
+    // a vendor whose every API is a lane has no spec of its own: its lanes are graded, and its root is only the vendor's
+    // front (architecture, "Pack layout", "Other wires: lanes")
+    const lanesOnly = !existsSync(join(unit, 'spec')) && readdirSync(unit).some((n) => existsSync(join(unit, n, 'src', 'manifest.ts')));
+    if (lanesOnly || !existsSync(join(unit, 'src', 'manifest.ts'))) { out.push({ unit, failures: laneRootFailures(unit) }); continue; }
+    const grades = await grade(unit, { static: true, sections: ['form'] });
+    const form = Object.values(grades).flatMap((g) => g.criteria).filter((c) => !c.pass);
+    out.push({ unit, failures: form.map((c) => `${c.id}: ${c.reason}`) });
+  }
+  return out;
+}

package/src/grade.ts

@@ -0,0 +1,47 @@
+// The grader (docs/contributing/architecture.md, "The grade"): one engine that grades a unit — a pack, or a lane of one,
+// in any repository or an unpacked published tarball — against the latest version of every standard this package
+// holds. It trusts only stamped evidence; a committed score, count or index row is never an input. The proof section
+// reads the customer life's report, which the caller recomputes and hands in (twin-world walks it on an in-memory World);
+// without one, the proof criteria fail as unproven, never pass.
+import { protocol3 } from './protocol-3.ts';
+import type { GradeUnit, ScoreReport, Standard } from './types.ts';
+import { unitAt } from './unit.ts';
+
+/** The latest version of each standard. A new protocol, or a revision, replaces or joins this list. */
+export const STANDARDS: Standard[] = [protocol3];
+
+export interface CriterionGrade { id: string; section: string; asks: string; pass: boolean; reason: string }
+export interface StandardGrade { grade: number; met: number; total: number; criteria: CriterionGrade[]; planned: string[] }
+
+export type GradeOptions = {
+  /** only the criteria that read the tree (the executed ones are the pack's typecheck and tests) */
+  static?: boolean;
+  /** only these sections (the catalog grades a submission's `form`) */
+  sections?: ReadonlyArray<'form' | 'proof' | 'soundness'>;
+  /** the unit's name when its directory does not say it (an unpacked tarball's `package/`) */
+  name?: string;
+  /** the customer life's report, recomputed by the caller, and why a walk gave none */
+  report?: ScoreReport | null;
+  walkError?: string;
+};
+
+/** Each standard's grade of the unit at `dir`, by `<standard>@<version>`. */
+export async function grade(dir: string, opts: GradeOptions = {}): Promise<Record<string, StandardGrade>> {
+  const at = unitAt(dir);
+  const name = opts.name ?? at.name;
+  const unit: GradeUnit = { name, vendor: name.split('/')[0]!, dir: at.dir, report: opts.report ?? null, ...(opts.walkError ? { walkError: opts.walkError } : {}) };
+  const grades: Record<string, StandardGrade> = {};
+  for (const standard of STANDARDS) {
+    const criteria: CriterionGrade[] = [];
+    for (const c of standard.criteria) {
+      if (opts.static && c.executed) continue;
+      if (opts.sections && !opts.sections.includes(c.section)) continue;
+      let result;
+      try { result = await c.check(unit); } catch (e) { result = { pass: false, reason: `the check threw: ${(e as Error).message.slice(0, 160)}` }; }
+      criteria.push({ id: c.id, section: c.section, asks: c.asks, ...result });
+    }
+    const met = criteria.filter((c) => c.pass).length;
+    grades[`${standard.id}@${standard.version}`] = { grade: criteria.length ? Math.floor((met / criteria.length) * 100) : 0, met, total: criteria.length, criteria, planned: standard.planned };
+  }
+  return grades;
+}

package/src/index.ts

@@ -0,0 +1,12 @@
+// @volter/twin-standard — the Protocol 3 standard (docs/contributing/architecture.md, "The grade"): the criteria a twin
+// pack is judged by, the grader, the gate a pack repository and the catalog run, and the derivation the grade checks
+// against. One package, versioned, so every place a pack is judged — its repository, its release, the catalog's review
+// of a submission — judges by the same standard at the version it pins. Changing a criterion is a release of this
+// package: a criterion's id never changes meaning, a changed criterion takes a new id and a new standard version.
+export { grade, STANDARDS, type CriterionGrade, type GradeOptions, type StandardGrade } from './grade.ts';
+export { laneRootFailures, p3Gate, unitsOf } from './gate.ts';
+export { gatePublished, vendorOfPackage, type PublishedGate } from './published.ts';
+export { handlerName, protocol3, PLANNED } from './protocol-3.ts';
+export { sharingLanesIn } from './lanes.ts';
+export { unitAt } from './unit.ts';
+export type { Criterion, CriterionResult, GradeUnit, ScoreReport, Standard } from './types.ts';

package/src/lanes.ts

@@ -0,0 +1,29 @@
+// The lanes over a vendor's state (docs/contributing/architecture.md, "Pack layout"): a lane whose manifest names the
+// vendor's service and that the vendor's serve factory serves (its server.ts, or its fixed fetch.ts when its manifest routes to the lane, imports the lane's src/). Such a lane is
+// walked, measured and graded through the vendor's life; every other lane keeps its own.
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** The service a manifest names: the top-level `service:` of its exported manifest object. */
+function serviceOf(manifestPath: string): string | undefined {
+  const text = readFileSync(manifestPath, 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '');
+  const body = /export\s+const\s+manifest\b[^=]*=\s*\{([\s\S]*)$/.exec(text)?.[1] ?? '';
+  // only the object's own first level: a nested object's `service` is not the manifest's
+  let depth = 0;
+  for (const m of body.matchAll(/[{}]|\bservice:\s*['"]([^'"]+)['"]/g)) {
+    if (m[0] === '{') depth += 1;
+    else if (m[0] === '}') { if (depth === 0) break; depth -= 1; }
+    else if (depth === 0) return m[1];
+  }
+  return undefined;
+}
+
+/** The lanes over a vendor's state in the vendor's directory `pkg`, wherever its repository is. */
+export function sharingLanesIn(pkg: string, vendor: string): string[] {
+  // the vendor serves a lane from its serve factory, or from its fixed fetch when its manifest routes to it
+  const serverText = ['server.ts', 'fetch.ts'].map((f) => join(pkg, 'src', f)).filter(existsSync).map((f) => readFileSync(f, 'utf8')).join('\n');
+  return (existsSync(pkg) ? readdirSync(pkg) : []).filter((l) => {
+    const manifest = join(pkg, l, 'src', 'manifest.ts');
+    return existsSync(manifest) && serviceOf(manifest) === vendor && new RegExp(`from ['"]\\.\\./${l}/src/`).test(serverText);
+  }).sort();
+}

package/src/p3-rules.ts

@@ -0,0 +1,44 @@
+// The architecture's mechanical rules for a derived pack's own code (docs/contributing/architecture.md, "What an author
+// writes, and how"), checked by the derive over `src/` (generated files aside). Each was broken by copying: the rules the
+// tools did not check spread from one pack to thirty (the P3 drift pass, 2026-09-30). A line may record why it is the
+// exception, in a `// p3: <reason>` comment on it or on the line before, and the check passes it.
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+export type RuleHit = { file: string; line: number; rule: string; text: string };
+
+const RULES: Array<{ rule: string; test: (line: string) => boolean }> = [
+  // ids come from ctx.mint: never a row count made into an id, a number or a label's nonce
+  { rule: 'an id or number from a row count (use ctx.mint)', test: (l) => /\b(rowsRaw|rows|rowsOf)\([^;]*?\)(\s*\.filter\([^;]*?\))?\.length\s*\+\s*1\b/.test(l) || /\b1\s*\+\s*[\w.]*\b(rowsRaw|rows|rowsOf)\([^;]*?\)(\s*\.filter\([^;]*?\))?\.length\b/.test(l) },
+  // the twin's own URL is ctx.publicBase
+  { rule: "the twin's URL from the request (use ctx.publicBase)", test: (l) => /request\.url\)\.origin/.test(l) },
+  // signing, digests and randomness are the kernel's (ctx.crypto, world-core's exports)
+  { rule: 'node:crypto in a pack (use ctx.crypto or the kernel\'s digest)', test: (l) => /from ['"](node:)?crypto['"]|require\(['"](node:)?crypto['"]\)/.test(l) },
+  // a webhook is the manifest's events; ctx.deliver is for a message that is no declared event (a push notification)
+  { rule: 'ctx.deliver where the manifest\'s events belong (or record why)', test: (l) => /ctx\.deliver\(/.test(l) },
+  // an object's bytes are ctx.blobs, never a row's field
+  { rule: "bytes kept in a row (use ctx.blobs)", test: (l) => /ctx\.(write|record)\([^;]*toString\(['"]base64['"]\)/.test(l) || /\b_bytes\s*:/.test(l) },
+  // a secret the vendor issues is drawn from ctx.secret, never computed from ids
+  { rule: 'a secret computed from ids (draw it from ctx.secret)', test: (l) => /(\b(const|let)\s+[a-zA-Z]*(secret|Secret|token|Token|password|Password|hmac|Hmac)\b\s*=|[{,]\s*(secret|token|password|hmac|api_key|client_secret|access_token|refresh_token)\s*:(?!\s*(string|number|unknown)\b))[^;,]*\b(digest|uuidFrom|lettersFrom|sha256)\(/.test(l) && !/secret\(|Secret\(|keyValue\(|tokenValue\(|credential\(/.test(l) },
+];
+
+const files = (dir: string): string[] => readdirSync(dir).flatMap((n) => {
+  const p = join(dir, n);
+  if (statSync(p).isDirectory()) return n === 'generated' || n === 'node_modules' ? [] : files(p);
+  return /\.(ts|tsx)$/.test(n) && !n.endsWith('.test.ts') ? [p] : [];
+});
+
+/** Every line of a unit's `src/` that breaks a rule and does not record why. */
+export function p3RuleHits(unit: string): RuleHit[] {
+  const src = join(unit, 'src');
+  const out: RuleHit[] = [];
+  for (const f of files(src)) {
+    const lines = readFileSync(f, 'utf8').split('\n');
+    lines.forEach((text, i) => {
+      if (/^\s*(\/\/|\*|\/\*)/.test(text)) return;
+      const excused = /\/\/ p3: \S/.test(text) || /\/\/ p3: \S/.test(lines[i - 1] ?? '');
+      for (const r of RULES) if (!excused && r.test(text)) out.push({ file: relative(unit, f), line: i + 1, rule: r.rule, text: text.trim().slice(0, 140) });
+    });
+  }
+  return out;
+}