@event4u/agent-config 16.5.0 → 17.0.0

package/src/scripts/ai_council/hand_payload.ts

@@ -0,0 +1,230 @@
+/**
+ * What a hand is sent: the slice the head cut, and nothing else.
+ *
+ * A slice is the specification of the function to build plus the signatures it
+ * must call. Implementations, repo maps, architecture docs, READMEs, roadmaps
+ * and the project's name never go — no single unpaid provider should learn the
+ * whole project. The API carries no field for any of them, so there is nothing
+ * to forget to strip.
+ *
+ * Identifiers that would let fragments be joined back together (project and
+ * package names, repository paths, domains) are replaced through the run's
+ * placeholder scope before anything leaves.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import type { CouncilEgress } from './council_egress.js';
+import { handSystemPrompt } from './hand_schemas.js';
+import type { HandKind } from '../_lib/hand_kinds.js';
+
+export type LinkingType = 'PROJECT' | 'PACKAGE' | 'REPOPATH' | 'DOMAIN';
+
+export interface LinkingIdentifier {
+    readonly type: LinkingType;
+    readonly value: string;
+}
+
+export interface HandSlice {
+    readonly kind: HandKind;
+    /** The specification of the one thing to produce. */
+    readonly spec: string;
+    /** Source of each function the result must call; only its signature is sent. */
+    readonly callees?: readonly string[];
+    readonly linking?: readonly LinkingIdentifier[];
+}
+
+export interface HandPayload {
+    readonly system: string;
+    readonly user: string;
+}
+
+export class HandPayloadError extends Error {}
+
+interface Block {
+    readonly start: number;
+    readonly end: number;
+}
+
+/** Top-level `{…}` blocks and `=>` positions, outside strings and comments. */
+function scanTopLevel(src: string): { blocks: Block[]; arrows: number[] } {
+    const blocks: Block[] = [];
+    const arrows: number[] = [];
+    let depth = 0;
+    let open = -1;
+    for (let i = 0; i < src.length; i++) {
+        const c = src[i];
+        const next = src[i + 1];
+        if (c === '/' && next === '/') {
+            const nl = src.indexOf('\n', i);
+            i = nl === -1 ? src.length : nl;
+            continue;
+        }
+        if (c === '/' && next === '*') {
+            const close = src.indexOf('*/', i + 2);
+            i = close === -1 ? src.length : close + 1;
+            continue;
+        }
+        if (c === '"' || c === "'" || c === '`') {
+            for (i++; i < src.length && src[i] !== c; i++) if (src[i] === '\\') i++;
+            continue;
+        }
+        if (c === '(' || c === '[' || c === '{') {
+            if (c === '{' && depth === 0) open = i;
+            depth++;
+        } else if (c === ')' || c === ']' || c === '}') {
+            depth--;
+            if (c === '}' && depth === 0 && open !== -1) {
+                blocks.push({ start: open, end: i });
+                open = -1;
+            }
+        } else if (c === '=' && next === '>' && depth === 0) {
+            arrows.push(i);
+        }
+    }
+    return { blocks, arrows };
+}
+
+/** The colon that closes a Python `def` header, outside brackets. */
+function pythonHeaderEnd(src: string): number {
+    let depth = 0;
+    for (let i = 0; i < src.length; i++) {
+        const c = src[i];
+        if (c === '(' || c === '[') depth++;
+        else if (c === ')' || c === ']') depth--;
+        else if (c === ':' && depth === 0) return i;
+    }
+    return -1;
+}
+
+/**
+ * A function's signature without its body. Brace languages lose the final
+ * top-level block (a return-type literal before it survives); an expression
+ * arrow loses everything from its arrow; a Python `def` keeps its header line.
+ */
+export function signatureOf(source: string): string {
+    const src = source.trim();
+    if (/^(?:async\s+)?def\s/.test(src)) {
+        const end = pythonHeaderEnd(src);
+        return end === -1 ? src : src.slice(0, end + 1);
+    }
+    const { blocks, arrows } = scanTopLevel(src);
+    const tailEnd = src.replace(/[\s;]+$/u, '').length - 1;
+    const last = blocks[blocks.length - 1];
+    let head = src;
+    if (last !== undefined && last.end === tailEnd) {
+        head = src.slice(0, last.start).trimEnd();
+        if (head.endsWith('=>')) head = head.slice(0, -2).trimEnd();
+    } else if (arrows[0] !== undefined) {
+        head = src.slice(0, arrows[0]).trimEnd();
+    }
+    return `${head.replace(/[\s;]+$/u, '')};`;
+}
+
+function escapeRegExp(s: string): string {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** The body of a callee: what its signature withholds. */
+function bodyOf(source: string, signature: string): string {
+    const sigHead = signature.replace(/;$/, '');
+    const src = source.trim();
+    return src.startsWith(sigHead) ? src.slice(sigHead.length).trim() : src;
+}
+
+/**
+ * Callee signatures under the spec, then linking identifiers and detected
+ * values through the placeholder scope. Refuses a spec that already carries a
+ * callee's body, and refuses to return a payload in which a linking identifier
+ * survived in any spelling.
+ */
+export function buildHandPayload(slice: HandSlice, egress: CouncilEgress): HandPayload {
+    const callees = slice.callees ?? [];
+    const signatures = callees.map(signatureOf);
+    callees.forEach((src, i) => {
+        const body = bodyOf(src, signatures[i] as string).replace(/^[{\s]+|[}\s;]+$/g, '');
+        if (body.length >= 12 && slice.spec.includes(body)) {
+            throw new HandPayloadError('the spec carries a called function’s implementation; send its signature only');
+        }
+    });
+    let user = slice.spec.trim();
+    if (signatures.length > 0) {
+        user += `\n\nSignatures you may call (implementations withheld):\n${signatures.join('\n')}`;
+    }
+    const linking = [...(slice.linking ?? [])].filter((l) => l.value.length > 0).sort((a, b) => b.value.length - a.value.length);
+    for (const l of linking) {
+        user = user.replace(new RegExp(escapeRegExp(l.value), 'gi'), (m) => egress.scope.mint(l.type, 'identifier', m));
+    }
+    user = egress.replace(user);
+    const lower = user.toLowerCase();
+    for (const l of linking) {
+        if (lower.includes(l.value.toLowerCase())) throw new HandPayloadError(`a ${l.type} identifier survived replacement`);
+    }
+    return { system: handSystemPrompt(slice.kind), user };
+}
+
+function readJson(file: string): Record<string, unknown> | null {
+    try {
+        const v: unknown = JSON.parse(fs.readFileSync(file, 'utf8'));
+        return typeof v === 'object' && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
+    } catch {
+        return null;
+    }
+}
+
+function urlOf(v: unknown): string | null {
+    if (typeof v === 'string') return v;
+    if (typeof v === 'object' && v !== null && typeof (v as { url?: unknown }).url === 'string') return (v as { url: string }).url;
+    return null;
+}
+
+/** Hosts, plus owner and repository segments of a repository URL. */
+function urlIdentifiers(raw: string, withPath: boolean): LinkingIdentifier[] {
+    try {
+        const u = new URL(raw.replace(/^git\+/, ''));
+        const out: LinkingIdentifier[] = [{ type: 'DOMAIN', value: u.hostname }];
+        if (withPath) {
+            for (const seg of u.pathname.split('/').filter(Boolean).slice(0, 2)) {
+                out.push({ type: 'PROJECT', value: seg.replace(/\.git$/, '') });
+            }
+        }
+        return out;
+    } catch {
+        return [];
+    }
+}
+
+/** Names a slice could be re-linked by, read from the project root; short values are dropped. */
+export function projectLinkingIdentifiers(root: string): LinkingIdentifier[] {
+    const abs = path.resolve(root);
+    const found: LinkingIdentifier[] = [
+        { type: 'REPOPATH', value: abs },
+        { type: 'PROJECT', value: path.basename(abs) },
+    ];
+    const pkg = readJson(path.join(abs, 'package.json'));
+    const pkgName = pkg?.['name'];
+    if (typeof pkgName === 'string') {
+        found.push({ type: 'PACKAGE', value: pkgName });
+        const scoped = /^@([^/]+)\/(.+)$/.exec(pkgName);
+        if (scoped !== null) found.push({ type: 'PROJECT', value: scoped[1] as string }, { type: 'PACKAGE', value: scoped[2] as string });
+    }
+    for (const key of ['homepage', 'bugs']) {
+        const u = urlOf(pkg?.[key]);
+        if (u !== null) found.push(...urlIdentifiers(u, false));
+    }
+    const repo = urlOf(pkg?.['repository']);
+    if (repo !== null) found.push(...urlIdentifiers(repo, true));
+    const composerName = readJson(path.join(abs, 'composer.json'))?.['name'];
+    if (typeof composerName === 'string') {
+        found.push({ type: 'PACKAGE', value: composerName });
+        for (const part of composerName.split('/')) found.push({ type: 'PROJECT', value: part });
+    }
+    const seen = new Set<string>();
+    return found.filter((l) => {
+        const key = l.value.toLowerCase();
+        if (l.value.length < 3 || seen.has(key)) return false;
+        seen.add(key);
+        return true;
+    });
+}

package/src/scripts/ai_council/hand_schemas.ts

@@ -0,0 +1,123 @@
+/**
+ * One JSON output schema per hand kind, and the one parser that admits a hand's
+ * return. A return that does not match is a failed call: a half-parsed answer
+ * would hand the head a proposal nobody can check against a shape.
+ *
+ * The schemas are plain JSON Schema objects so they can be shown to a hand
+ * verbatim; the validator below implements only the keywords they use, and
+ * refuses any schema keyword it does not know rather than ignoring it.
+ */
+
+import type { HandKind } from '../_lib/hand_kinds.js';
+
+export interface JsonSchema {
+    readonly type: 'object' | 'array' | 'string' | 'number';
+    readonly properties?: Readonly<Record<string, JsonSchema>>;
+    readonly required?: readonly string[];
+    readonly additionalProperties?: false;
+    readonly items?: JsonSchema;
+    readonly minItems?: number;
+    readonly maxItems?: number;
+    readonly minLength?: number;
+    readonly minimum?: number;
+    readonly maximum?: number;
+    readonly enum?: readonly string[];
+}
+
+const TEXT: JsonSchema = { type: 'string', minLength: 1 };
+
+function obj(properties: Record<string, JsonSchema>): JsonSchema {
+    return { type: 'object', properties, required: Object.keys(properties), additionalProperties: false };
+}
+
+export const HAND_KIND_SCHEMAS: Readonly<Record<HandKind, JsonSchema>> = {
+    classify: obj({ label: TEXT, confidence: { type: 'number', minimum: 0, maximum: 1 } }),
+    extract: obj({ items: { type: 'array', items: TEXT, maxItems: 200 } }),
+    'summarise-given-text': obj({ summary: TEXT }),
+    'draft-candidate': obj({ candidate: TEXT }),
+    'test-candidates': obj({
+        tests: { type: 'array', items: obj({ name: TEXT, body: TEXT }), minItems: 1, maxItems: 50 },
+    }),
+    'finding-proposal': obj({
+        findings: {
+            type: 'array',
+            items: obj({ claim: TEXT, evidence: TEXT, severity: { type: 'string', enum: ['low', 'medium', 'high'] } }),
+            maxItems: 50,
+        },
+    }),
+};
+
+function validate(value: unknown, schema: JsonSchema, at: string): string | null {
+    switch (schema.type) {
+        case 'string':
+            if (typeof value !== 'string') return `${at}: expected string`;
+            if (schema.minLength !== undefined && value.trim().length < schema.minLength) return `${at}: empty string`;
+            if (schema.enum !== undefined && !schema.enum.includes(value)) return `${at}: not one of ${schema.enum.join('|')}`;
+            return null;
+        case 'number':
+            if (typeof value !== 'number' || !Number.isFinite(value)) return `${at}: expected number`;
+            if (schema.minimum !== undefined && value < schema.minimum) return `${at}: below ${schema.minimum}`;
+            if (schema.maximum !== undefined && value > schema.maximum) return `${at}: above ${schema.maximum}`;
+            return null;
+        case 'array': {
+            if (!Array.isArray(value)) return `${at}: expected array`;
+            if (schema.minItems !== undefined && value.length < schema.minItems) return `${at}: fewer than ${schema.minItems}`;
+            if (schema.maxItems !== undefined && value.length > schema.maxItems) return `${at}: more than ${schema.maxItems}`;
+            if (schema.items === undefined) return null;
+            for (let i = 0; i < value.length; i++) {
+                const e = validate(value[i], schema.items, `${at}[${i}]`);
+                if (e !== null) return e;
+            }
+            return null;
+        }
+        case 'object': {
+            if (typeof value !== 'object' || value === null || Array.isArray(value)) return `${at}: expected object`;
+            const rec = value as Record<string, unknown>;
+            const props = schema.properties ?? {};
+            for (const key of schema.required ?? []) {
+                if (!(key in rec)) return `${at}.${key}: missing`;
+            }
+            for (const key of Object.keys(rec)) {
+                const sub = props[key];
+                if (sub === undefined) {
+                    if (schema.additionalProperties === false) return `${at}.${key}: not allowed`;
+                    continue;
+                }
+                const e = validate(rec[key], sub, `${at}.${key}`);
+                if (e !== null) return e;
+            }
+            return null;
+        }
+        default:
+            return `${at}: unsupported schema type`;
+    }
+}
+
+export type HandParse = { readonly ok: true; readonly value: unknown } | { readonly ok: false; readonly reason: string };
+
+/** Accepts bare JSON or exactly one fenced JSON block; anything else fails. */
+export function parseHandReturn(kind: HandKind, text: string): HandParse {
+    let body = text.trim();
+    const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n```$/.exec(body);
+    if (fenced !== null) body = (fenced[1] ?? '').trim();
+    let value: unknown;
+    try {
+        value = JSON.parse(body);
+    } catch {
+        return { ok: false, reason: 'not JSON' };
+    }
+    const error = validate(value, HAND_KIND_SCHEMAS[kind], '$');
+    return error === null ? { ok: true, value } : { ok: false, reason: error };
+}
+
+/**
+ * The hand's whole system prompt: the kind and its schema, nothing else. No
+ * project rule, skill or instruction-file text reaches a hand.
+ */
+export function handSystemPrompt(kind: HandKind): string {
+    return (
+        `You perform one bounded task of kind "${kind}" on the text you are given. ` +
+        'Use only that text. Reply with ONE JSON value matching this JSON Schema and nothing else: ' +
+        JSON.stringify(HAND_KIND_SCHEMAS[kind])
+    );
+}

package/src/scripts/ai_council/jury_aggregate.ts

@@ -27,6 +27,7 @@
  * inferring it from a method name.
  */
 
+import { cohensKappa } from '../check_quality_regression.js';
 import {
     judgeBothOrders,
     positionConsistency,
@@ -45,7 +46,20 @@ export interface JuryScore {
 }
 
 /** Why a panel produced no aggregate. Each value is a REFUSAL, not an error. */
-export type JuryAbsentReason = 'no_scores' | 'single_family' | 'non_finite_score';
+export type JuryAbsentReason = 'no_scores' | 'single_family' | 'non_finite_score' | 'panel_over_cap';
+
+/**
+ * K10: past five judges the effective number of independent votes stops
+ * growing, so a sixth seat adds cost and correlated agreement, not evidence.
+ * Different architectures beat more seats.
+ */
+export const MAX_JURY_JUDGES = 5;
+
+function _overCap(n: number): string | null {
+    return n > MAX_JURY_JUDGES
+        ? `${String(n)} judges exceed the cap of ${String(MAX_JURY_JUDGES)}; past five, more judges add correlated agreement, not independence`
+        : null;
+}
 
 export type JuryResult =
     | {
@@ -75,6 +89,10 @@ export function juryAggregate(scores: readonly JuryScore[]): JuryResult {
     if (scores.length === 0) {
         return { present: false, absentReason: 'no_scores', reason: 'no jury scores supplied' };
     }
+    const overCap = _overCap(scores.length);
+    if (overCap !== null) {
+        return { present: false, absentReason: 'panel_over_cap', reason: overCap };
+    }
     for (const s of scores) {
         if (!Number.isFinite(s.score)) {
             return {
@@ -146,6 +164,17 @@ export function juryPairwise(
     judges: readonly JuryPairwiseJudge[],
 ): JuryPairwiseResult {
     const families = [...new Set(judges.map((j) => j.family))].sort();
+    const overCap = _overCap(judges.length);
+    if (overCap !== null) {
+        return {
+            present: false,
+            absentReason: 'panel_over_cap',
+            reason: overCap,
+            resolution: null,
+            observations: [],
+            consistency: [],
+        };
+    }
     if (judges.length === 0 || families.length < 2) {
         return {
             present: false,
@@ -176,3 +205,128 @@ export function juryPairwise(
     }
     return { present: true, absentReason: null, reason: null, resolution, observations, consistency };
 }
+
+/** One seat on a jury. `billing` is carried, never consulted: a free seat may judge. */
+export interface JurorSeat {
+    readonly judge: string;
+    readonly family: string;
+    readonly billing?: string | null;
+}
+
+/** A panel description, validated: every judge named once, at most five. */
+export function validateJuryPanel(raw: unknown): { panel: JurorSeat[] } | { error: string } {
+    const judges = (raw as { judges?: unknown } | null)?.judges;
+    if (!Array.isArray(judges) || judges.length === 0) return { error: 'a jury panel needs a non-empty `judges` list' };
+    const overCap = _overCap(judges.length);
+    if (overCap !== null) return { error: overCap };
+    const panel: JurorSeat[] = [];
+    for (const j of judges as unknown[]) {
+        const o = j as Partial<JurorSeat> | null;
+        if (typeof o?.judge !== 'string' || typeof o.family !== 'string') {
+            return { error: `every judge needs a string \`judge\` and \`family\` (got ${JSON.stringify(j)})` };
+        }
+        if (panel.some((p) => p.judge === o.judge)) return { error: `judge ${o.judge} is listed twice` };
+        panel.push({ judge: o.judge, family: o.family, billing: o.billing ?? null });
+    }
+    return { panel };
+}
+
+/**
+ * The prompt every juror receives. It takes the task and the item and nothing
+ * else, so no juror can be shown another juror's answer by construction.
+ */
+export function jurorPrompt(task: string, item: string): string {
+    return `${task}\n\n---\n\n${item}`;
+}
+
+export interface BlindJuryResult {
+    /** Why nothing was dispatched, or null. */
+    readonly refused: string | null;
+    readonly answers: ReadonlyArray<{ readonly judge: string; readonly family: string; readonly answer: string }>;
+}
+
+/**
+ * Ask every juror at once with the same prompt. All calls start before any
+ * answer is awaited, so no answer can reach another juror before synthesis.
+ */
+export async function dispatchJuryBlind(
+    task: string,
+    item: string,
+    jurors: readonly JurorSeat[],
+    ask: (juror: JurorSeat, prompt: string) => Promise<string>,
+): Promise<BlindJuryResult> {
+    const overCap = _overCap(jurors.length);
+    if (overCap !== null) return { refused: overCap, answers: [] };
+    const prompt = jurorPrompt(task, item);
+    const pending = jurors.map((j) => ask(j, prompt));
+    const answers = await Promise.all(pending);
+    return { refused: null, answers: jurors.map((j, i) => ({ judge: j.judge, family: j.family, answer: answers[i] as string })) };
+}
+
+export interface JuryIndependence {
+    readonly k: number;
+    readonly items: number;
+    /** Each judge's Cohen's κ against the ground truth — the claim's own statistic, unchanged. */
+    readonly kappa: ReadonlyArray<{ readonly judge: string; readonly kappa: number }>;
+    /** Pairwise φ over error indicators; null where a judge's errors do not vary. */
+    readonly phi: ReadonlyArray<{ readonly a: string; readonly b: string; readonly phi: number | null }>;
+    readonly undefinedPairs: number;
+    /** Mean of the defined φ values, or null when none is defined. */
+    readonly phiBar: number | null;
+    /** Kish effective n = k / (1 + (k−1)·φ̄), with φ̄ floored at 0 so n_eff never exceeds k. */
+    readonly nEff: number | null;
+    readonly nEffRatio: number | null;
+}
+
+function _phi(x: readonly boolean[], y: readonly boolean[]): number | null {
+    let n11 = 0;
+    let n10 = 0;
+    let n01 = 0;
+    let n00 = 0;
+    x.forEach((xi, i) => {
+        const yi = y[i] === true;
+        if (xi && yi) n11 += 1;
+        else if (xi) n10 += 1;
+        else if (yi) n01 += 1;
+        else n00 += 1;
+    });
+    const den = (n11 + n10) * (n01 + n00) * (n11 + n01) * (n10 + n00);
+    return den === 0 ? null : (n11 * n00 - n10 * n01) / Math.sqrt(den);
+}
+
+/**
+ * Family count is a proxy for independence; correlated errors are the thing
+ * itself. A judge's error on item i is `labels[i] !== gold[i]`; φ is the
+ * Pearson correlation of two judges' error vectors, and n_eff is reported
+ * beside κ rather than in place of it.
+ */
+export function juryIndependence(
+    gold: readonly string[],
+    judges: ReadonlyArray<{ readonly judge: string; readonly labels: readonly string[] }>,
+): JuryIndependence {
+    const errors = judges.map((j) => gold.map((g, i) => j.labels[i] !== g));
+    const phi: Array<{ a: string; b: string; phi: number | null }> = [];
+    for (let i = 0; i < judges.length; i += 1) {
+        for (let k = i + 1; k < judges.length; k += 1) {
+            phi.push({
+                a: (judges[i] as { judge: string }).judge,
+                b: (judges[k] as { judge: string }).judge,
+                phi: _phi(errors[i] as boolean[], errors[k] as boolean[]),
+            });
+        }
+    }
+    const defined = phi.map((p) => p.phi).filter((v): v is number => v !== null);
+    const phiBar = defined.length === 0 ? null : defined.reduce((a, v) => a + v, 0) / defined.length;
+    const k = judges.length;
+    const nEff = phiBar === null ? null : k / (1 + (k - 1) * Math.max(0, phiBar));
+    return {
+        k,
+        items: gold.length,
+        kappa: judges.map((j) => ({ judge: j.judge, kappa: cohensKappa(gold, j.labels) })),
+        phi,
+        undefinedPairs: phi.length - defined.length,
+        phiBar,
+        nEff,
+        nEffRatio: nEff === null || k === 0 ? null : nEff / k,
+    };
+}