@unson/brainbase-mcp 0.4.0 → 0.5.0

package/dist/judgment-dag-replay-evaluation.js

@@ -0,0 +1,424 @@
+import { createHash } from 'node:crypto';
+import { executeJudgmentDAG } from './judgment-dag-runner.js';
+import { computeJudgmentDAGRunArtifactId } from './judgment-dag-artifact-store.js';
+export const JUDGMENT_DAG_OUTCOME_ATTACHMENT_VERSION = 'judgment-dag-outcome.v1';
+export const JUDGMENT_DAG_EVALUATION_EVENT_SET_VERSION = 'judgment-dag-evaluation-event-set.v1';
+const ARTIFACT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/u;
+export class JudgmentDAGReplayEvaluationError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = 'JudgmentDAGReplayEvaluationError';
+        this.code = code;
+    }
+}
+function fail(code, message) {
+    throw new JudgmentDAGReplayEvaluationError(code, message);
+}
+function isPlainObject(value) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function snapshotJSON(value, label, seen = new WeakSet()) {
+    if (value === null || typeof value === 'boolean' || typeof value === 'string')
+        return value;
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value))
+            fail('invalid_request', `${label} must contain only finite numbers`);
+        return Object.is(value, -0) ? 0 : value;
+    }
+    if (typeof value !== 'object')
+        fail('invalid_request', `${label} must be JSON-compatible`);
+    if (seen.has(value))
+        fail('invalid_request', `${label} must not contain cycles`);
+    seen.add(value);
+    try {
+        if (Array.isArray(value)) {
+            const result = [];
+            for (let index = 0; index < value.length; index += 1) {
+                if (!Object.hasOwn(value, index))
+                    fail('invalid_request', `${label} must not contain sparse arrays`);
+                result.push(snapshotJSON(value[index], `${label}[${index}]`, seen));
+            }
+            return result;
+        }
+        if (!isPlainObject(value))
+            fail('invalid_request', `${label} must contain only plain objects`);
+        const result = {};
+        for (const key of Object.keys(value).sort()) {
+            const descriptor = Object.getOwnPropertyDescriptor(value, key);
+            if (descriptor === undefined || !('value' in descriptor)) {
+                fail('invalid_request', `${label}.${key} must be a data property`);
+            }
+            result[key] = snapshotJSON(descriptor.value, `${label}.${key}`, seen);
+        }
+        return result;
+    }
+    finally {
+        seen.delete(value);
+    }
+}
+function canonicalJSON(value, label) {
+    return JSON.stringify(snapshotJSON(value, label));
+}
+function detached(value, label) {
+    return JSON.parse(canonicalJSON(value, label));
+}
+function deepFreeze(value) {
+    if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
+        Object.freeze(value);
+        for (const child of Object.values(value))
+            deepFreeze(child);
+    }
+    return value;
+}
+function requireNonEmptyString(value, label) {
+    if (typeof value !== 'string' || value.trim().length === 0) {
+        fail('invalid_request', `${label} must be a non-empty string`);
+    }
+    return value;
+}
+function requireArtifactId(value, label) {
+    if (typeof value !== 'string' || !ARTIFACT_ID_PATTERN.test(value)) {
+        fail('invalid_request', `${label} must be a sha256 artifact ID`);
+    }
+    return value;
+}
+function contentId(value, label) {
+    return `sha256:${createHash('sha256').update(canonicalJSON(value, label), 'utf8').digest('hex')}`;
+}
+function detachedRecord(record, label) {
+    const snapshot = detached(record, label);
+    requireNonEmptyString(snapshot.run_id, `${label}.run_id`);
+    return deepFreeze(snapshot);
+}
+function verifiedArtifactRecord(artifactIdValue, recordValue, label) {
+    const artifactId = requireArtifactId(artifactIdValue, `${label}.artifact_id`);
+    const record = detachedRecord(recordValue, `${label}.record`);
+    let expectedArtifactId;
+    try {
+        expectedArtifactId = computeJudgmentDAGRunArtifactId(record);
+    }
+    catch {
+        fail('invalid_request', `${label}.record is not a valid run artifact record`);
+    }
+    if (artifactId !== expectedArtifactId) {
+        fail('artifact_mismatch', `${label}.artifact_id does not match the run record content`);
+    }
+    return { artifactId, record };
+}
+function requiredHistoricalVersions(record) {
+    const versions = new Map();
+    for (const entry of record.runner_versions) {
+        if (versions.has(entry.runner_type))
+            fail('invalid_request', 'historical runner versions must be unique');
+        versions.set(entry.runner_type, requireNonEmptyString(entry.version, 'historical runner version'));
+    }
+    return versions;
+}
+function captureRunnerRegistrations(request, dag, expectedVersions) {
+    try {
+        const runnersDescriptor = Object.getOwnPropertyDescriptor(request, 'runners');
+        if (runnersDescriptor === undefined || !('value' in runnersDescriptor) ||
+            !isPlainObject(runnersDescriptor.value)) {
+            fail('invalid_request', 'runners must be an own data property containing a plain object');
+        }
+        const runners = runnersDescriptor.value;
+        const captured = {};
+        const runnerTypes = [...new Set(dag.nodes.map((node) => node.runner_type))]
+            .sort((left, right) => left.localeCompare(right));
+        for (const runnerType of runnerTypes) {
+            const registrationDescriptor = Object.getOwnPropertyDescriptor(runners, runnerType);
+            if (registrationDescriptor === undefined || !('value' in registrationDescriptor) ||
+                !isPlainObject(registrationDescriptor.value)) {
+                fail('invalid_request', `runner ${runnerType} must be an own data property`);
+            }
+            const versionDescriptor = Object.getOwnPropertyDescriptor(registrationDescriptor.value, 'version');
+            const runDescriptor = Object.getOwnPropertyDescriptor(registrationDescriptor.value, 'run');
+            if (versionDescriptor === undefined || !('value' in versionDescriptor) ||
+                runDescriptor === undefined || !('value' in runDescriptor) ||
+                typeof runDescriptor.value !== 'function') {
+                fail('invalid_request', `runner ${runnerType} must expose version and run as data properties`);
+            }
+            const version = requireNonEmptyString(versionDescriptor.value, `runner ${runnerType} version`);
+            const expectedVersion = expectedVersions?.get(runnerType);
+            if (expectedVersion !== undefined && version !== expectedVersion) {
+                fail('runner_version_mismatch', `historical runner ${runnerType} must use recorded version ${expectedVersion}`);
+            }
+            captured[runnerType] = { version, run: runDescriptor.value };
+        }
+        return Object.freeze(captured);
+    }
+    catch (error) {
+        if (error instanceof JudgmentDAGReplayEvaluationError)
+            throw error;
+        fail('invalid_request', 'runner registrations could not be captured safely');
+    }
+}
+export async function replayJudgmentDAGRun(request) {
+    if (!isPlainObject(request) || !isPlainObject(request.source)) {
+        fail('invalid_request', 'replay request and source must be plain objects');
+    }
+    const { artifactId, record: sourceRecord } = verifiedArtifactRecord(request.source.artifact_id, request.source.record, 'source');
+    const replayRunId = requireNonEmptyString(request.replay_run_id, 'replay_run_id');
+    if (replayRunId === sourceRecord.run_id)
+        fail('invalid_request', 'replay_run_id must differ from source run_id');
+    if (request.mode !== 'historical' && request.mode !== 'candidate')
+        fail('invalid_request', 'mode is invalid');
+    let replayDAG;
+    let expectedVersions;
+    if (request.mode === 'historical') {
+        if (request.candidate_dag !== undefined)
+            fail('invalid_request', 'historical replay cannot provide candidate_dag');
+        expectedVersions = requiredHistoricalVersions(sourceRecord);
+        replayDAG = sourceRecord.dag;
+    }
+    else {
+        if (request.candidate_dag === undefined)
+            fail('invalid_request', 'candidate replay requires candidate_dag');
+        replayDAG = detached(request.candidate_dag, 'candidate_dag');
+    }
+    const capturedRunners = captureRunnerRegistrations(request, replayDAG, expectedVersions);
+    const record = await executeJudgmentDAG({
+        run_id: replayRunId,
+        dag: replayDAG,
+        input: sourceRecord.input,
+        runners: capturedRunners
+    });
+    return deepFreeze({
+        source_artifact_id: artifactId,
+        source_run_id: sourceRecord.run_id,
+        mode: request.mode,
+        record
+    });
+}
+function normalizedObservations(record, observations) {
+    if (!Array.isArray(observations) || observations.length === 0) {
+        fail('invalid_request', 'observations must be a non-empty array');
+    }
+    const nodeIds = new Set(record.nodes.map((node) => node.node_id));
+    const identities = new Set();
+    return observations.map((raw, index) => {
+        if (!isPlainObject(raw))
+            fail('invalid_request', `observations[${index}] must be a plain object`);
+        const metricId = requireNonEmptyString(raw.metric_id, `observations[${index}].metric_id`);
+        if (raw.scope !== 'run' && raw.scope !== 'node')
+            fail('invalid_request', 'observation scope is invalid');
+        if (typeof raw.value !== 'boolean' && (typeof raw.value !== 'number' || !Number.isFinite(raw.value))) {
+            fail('invalid_request', 'observation value must be boolean or a finite number');
+        }
+        let nodeId;
+        if (raw.scope === 'node') {
+            nodeId = requireNonEmptyString(raw.node_id, `observations[${index}].node_id`);
+            if (!nodeIds.has(nodeId))
+                fail('invalid_request', `observation references unknown node ${nodeId}`);
+        }
+        else if (raw.node_id !== undefined) {
+            fail('invalid_request', 'run observation cannot have node_id');
+        }
+        const identity = `${metricId}\u0000${raw.scope}\u0000${nodeId ?? ''}`;
+        if (identities.has(identity))
+            fail('invalid_request', 'observations must be unique by metric, scope, and node');
+        identities.add(identity);
+        return nodeId === undefined
+            ? { metric_id: metricId, scope: raw.scope, value: raw.value }
+            : { metric_id: metricId, scope: raw.scope, node_id: nodeId, value: raw.value };
+    });
+}
+export function createJudgmentDAGOutcomeAttachment(request) {
+    if (!isPlainObject(request))
+        fail('invalid_request', 'outcome request must be a plain object');
+    const { artifactId, record } = verifiedArtifactRecord(request.run_artifact_id, request.record, 'run_artifact');
+    const observations = normalizedObservations(record, request.observations);
+    const payload = {
+        attachment_version: JUDGMENT_DAG_OUTCOME_ATTACHMENT_VERSION,
+        run_artifact_id: artifactId,
+        run_id: record.run_id,
+        observations
+    };
+    return deepFreeze({ ...payload, attachment_id: contentId(payload, 'outcome attachment') });
+}
+function validateOutcome(outcome, artifactId, record, label) {
+    const snapshot = detached(outcome, label);
+    if (snapshot.attachment_version !== JUDGMENT_DAG_OUTCOME_ATTACHMENT_VERSION ||
+        snapshot.run_artifact_id !== artifactId || snapshot.run_id !== record.run_id) {
+        fail('outcome_mismatch', `${label} does not bind to its run artifact`);
+    }
+    const normalized = createJudgmentDAGOutcomeAttachment({
+        run_artifact_id: artifactId,
+        record,
+        observations: snapshot.observations
+    });
+    if (normalized.attachment_id !== snapshot.attachment_id) {
+        fail('outcome_mismatch', `${label} content identity is invalid`);
+    }
+    return normalized;
+}
+function normalizedEvaluationRun(value, label) {
+    if (!isPlainObject(value))
+        fail('invalid_request', `${label} must be a plain object`);
+    const { artifactId, record } = verifiedArtifactRecord(value.artifact_id, value.record, label);
+    const outcome = validateOutcome(value.outcome, artifactId, record, `${label}.outcome`);
+    return { artifact_id: artifactId, record, outcome };
+}
+export function createJudgmentDAGEvaluationEventSet(request) {
+    if (!isPlainObject(request) || !Array.isArray(request.events) || request.events.length === 0) {
+        fail('invalid_request', 'evaluation event set requires a non-empty events array');
+    }
+    const eventIds = new Set();
+    const events = request.events.map((event, index) => {
+        if (!isPlainObject(event))
+            fail('invalid_request', `events[${index}] must be a plain object`);
+        const typedEvent = event;
+        const eventId = requireNonEmptyString(typedEvent.event_id, `events[${index}].event_id`);
+        if (eventIds.has(eventId))
+            fail('invalid_request', `duplicate event_id ${eventId}`);
+        eventIds.add(eventId);
+        const baseline = normalizedEvaluationRun(typedEvent.baseline, `events[${index}].baseline`);
+        const candidate = normalizedEvaluationRun(typedEvent.candidate, `events[${index}].candidate`);
+        if (canonicalJSON(baseline.record.input, 'baseline input') !==
+            canonicalJSON(candidate.record.input, 'candidate input')) {
+            fail('context_mismatch', `event ${eventId} does not compare the same recorded context`);
+        }
+        return { event_id: eventId, baseline, candidate };
+    });
+    const payload = {
+        event_set_version: JUDGMENT_DAG_EVALUATION_EVENT_SET_VERSION,
+        events
+    };
+    return deepFreeze({
+        ...payload,
+        event_set_id: contentId(payload, 'evaluation event set'),
+        event_ids: events.map((event) => event.event_id)
+    });
+}
+function validateEventSet(eventSet) {
+    const rebuilt = createJudgmentDAGEvaluationEventSet({ events: eventSet.events });
+    if (eventSet.event_set_version !== JUDGMENT_DAG_EVALUATION_EVENT_SET_VERSION ||
+        eventSet.event_set_id !== rebuilt.event_set_id ||
+        canonicalJSON(eventSet.event_ids, 'event_ids') !== canonicalJSON(rebuilt.event_ids, 'event_ids')) {
+        fail('invalid_request', 'evaluation event set identity is invalid');
+    }
+    return rebuilt;
+}
+function validateCriterion(value) {
+    if (!isPlainObject(value) || !isPlainObject(value.scoring)) {
+        fail('invalid_request', 'criterion and scoring must be plain objects');
+    }
+    const criterionId = requireNonEmptyString(value.criterion_id, 'criterion_id');
+    const goal = requireNonEmptyString(value.goal, 'goal');
+    const metricId = requireNonEmptyString(value.metric_id, 'metric_id');
+    let scoring;
+    if (value.scoring.kind === 'numeric') {
+        if (value.scoring.direction !== 'higher_is_better' && value.scoring.direction !== 'lower_is_better') {
+            fail('invalid_request', 'numeric direction is invalid');
+        }
+        scoring = { kind: 'numeric', direction: value.scoring.direction };
+    }
+    else if (value.scoring.kind === 'pass_fail') {
+        if (!['eq', 'gte', 'lte'].includes(value.scoring.operator))
+            fail('invalid_request', 'pass_fail operator is invalid');
+        if (typeof value.scoring.target !== 'boolean' &&
+            (typeof value.scoring.target !== 'number' || !Number.isFinite(value.scoring.target))) {
+            fail('invalid_request', 'pass_fail target must be boolean or a finite number');
+        }
+        if (typeof value.scoring.target === 'boolean' && value.scoring.operator !== 'eq') {
+            fail('invalid_request', 'boolean pass_fail target only supports eq');
+        }
+        scoring = { kind: 'pass_fail', operator: value.scoring.operator, target: value.scoring.target };
+    }
+    else {
+        fail('invalid_request', 'scoring kind is invalid');
+    }
+    return deepFreeze({ criterion_id: criterionId, goal, metric_id: metricId, scoring });
+}
+function scoreValue(value, scoring) {
+    if (scoring.kind === 'numeric') {
+        if (typeof value !== 'number' || !Number.isFinite(value)) {
+            fail('invalid_score', 'numeric scoring requires finite number observations');
+        }
+        return value;
+    }
+    if (scoring.operator === 'eq')
+        return value === scoring.target ? 1 : 0;
+    if (typeof value !== 'number' || typeof scoring.target !== 'number') {
+        fail('invalid_score', 'ordered pass_fail scoring requires number observations and target');
+    }
+    return scoring.operator === 'gte'
+        ? (value >= scoring.target ? 1 : 0)
+        : (value <= scoring.target ? 1 : 0);
+}
+function observation(attachment, metricId, scope, nodeId) {
+    return attachment.observations.find((item) => item.metric_id === metricId && item.scope === scope && item.node_id === nodeId);
+}
+function average(values) {
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+export function evaluateJudgmentDAGVersions(request) {
+    if (!isPlainObject(request))
+        fail('invalid_request', 'evaluation request must be a plain object');
+    const eventSet = validateEventSet(request.event_set);
+    const criterion = validateCriterion(request.criterion);
+    const baselineOverall = [];
+    const candidateOverall = [];
+    const allNodeIds = new Set();
+    for (const event of eventSet.events) {
+        const baselineRun = observation(event.baseline.outcome, criterion.metric_id, 'run');
+        const candidateRun = observation(event.candidate.outcome, criterion.metric_id, 'run');
+        if (baselineRun === undefined || candidateRun === undefined) {
+            fail('outcome_mismatch', `event ${event.event_id} is missing the run observation for ${criterion.metric_id}`);
+        }
+        baselineOverall.push(scoreValue(baselineRun.value, criterion.scoring));
+        candidateOverall.push(scoreValue(candidateRun.value, criterion.scoring));
+        for (const node of event.baseline.record.nodes)
+            allNodeIds.add(node.node_id);
+        for (const node of event.candidate.record.nodes)
+            allNodeIds.add(node.node_id);
+    }
+    const nodeCalibration = [...allNodeIds].sort().map((nodeId) => {
+        const baselineScores = [];
+        const candidateScores = [];
+        let baselinePresent = false;
+        let candidatePresent = false;
+        for (const event of eventSet.events) {
+            baselinePresent ||= event.baseline.record.nodes.some((node) => node.node_id === nodeId);
+            candidatePresent ||= event.candidate.record.nodes.some((node) => node.node_id === nodeId);
+            const baselineNode = observation(event.baseline.outcome, criterion.metric_id, 'node', nodeId);
+            const candidateNode = observation(event.candidate.outcome, criterion.metric_id, 'node', nodeId);
+            if (baselineNode !== undefined && candidateNode !== undefined) {
+                baselineScores.push(scoreValue(baselineNode.value, criterion.scoring));
+                candidateScores.push(scoreValue(candidateNode.value, criterion.scoring));
+            }
+        }
+        if (!baselinePresent) {
+            return { node_id: nodeId, status: 'candidate_only', baseline: null, candidate: null, delta: null, event_count: 0 };
+        }
+        if (!candidatePresent) {
+            return { node_id: nodeId, status: 'baseline_only', baseline: null, candidate: null, delta: null, event_count: 0 };
+        }
+        if (baselineScores.length === 0) {
+            return { node_id: nodeId, status: 'no_observation', baseline: null, candidate: null, delta: null, event_count: 0 };
+        }
+        const baseline = average(baselineScores);
+        const candidate = average(candidateScores);
+        return {
+            node_id: nodeId,
+            status: 'comparable',
+            baseline,
+            candidate,
+            delta: candidate - baseline,
+            event_count: baselineScores.length
+        };
+    });
+    const baseline = average(baselineOverall);
+    const candidate = average(candidateOverall);
+    return deepFreeze({
+        event_set_id: eventSet.event_set_id,
+        event_ids: eventSet.event_ids.slice(),
+        criterion,
+        overall: { baseline, candidate, delta: candidate - baseline, event_count: eventSet.events.length },
+        node_calibration: nodeCalibration
+    });
+}

package/dist/judgment-dag-runner.d.ts

@@ -0,0 +1,70 @@
+import { type JudgmentDAG, type JudgmentDAGMetadata, type JudgmentDAGNode, type JudgmentDAGRunnerType } from './judgment-dag-core.js';
+/** A JSON-compatible value accepted at every local runner boundary. */
+export type JudgmentDAGJSONValue = JudgmentDAGMetadata;
+export interface JudgmentDAGRunnerInput {
+    readonly run_id: string;
+    readonly dag: JudgmentDAG;
+    readonly node: JudgmentDAGNode;
+    readonly input: JudgmentDAGJSONValue;
+    readonly dependency_outputs: readonly JudgmentDAGDependencyOutput[];
+}
+export interface JudgmentDAGDependencyOutput {
+    readonly node_id: string;
+    readonly output: JudgmentDAGJSONValue;
+}
+export interface JudgmentDAGRunnerRegistration {
+    readonly version: string;
+    readonly run: (input: JudgmentDAGRunnerInput) => JudgmentDAGJSONValue | Promise<JudgmentDAGJSONValue>;
+}
+export interface JudgmentDAGRunRequest {
+    readonly run_id: string;
+    readonly dag: JudgmentDAG;
+    readonly input: JudgmentDAGJSONValue;
+    readonly runners: Partial<Record<JudgmentDAGRunnerType, JudgmentDAGRunnerRegistration>>;
+}
+export interface JudgmentDAGRunnerVersion {
+    readonly runner_type: JudgmentDAGRunnerType;
+    readonly version: string;
+}
+export interface JudgmentDAGNodeRunRecord {
+    readonly node_id: string;
+    readonly runner_type: JudgmentDAGRunnerType;
+    readonly runner_version: string;
+    readonly input_contract: string;
+    readonly output_contract: string;
+    readonly input: JudgmentDAGJSONValue;
+    readonly dependency_outputs: readonly JudgmentDAGDependencyOutput[];
+    readonly output: JudgmentDAGJSONValue;
+}
+export interface JudgmentDAGRunRecord {
+    readonly run_id: string;
+    readonly dag: JudgmentDAG;
+    readonly input: JudgmentDAGJSONValue;
+    readonly execution_order: readonly string[];
+    readonly runner_versions: readonly JudgmentDAGRunnerVersion[];
+    readonly nodes: readonly JudgmentDAGNodeRunRecord[];
+}
+export type JudgmentDAGExecutionCode = 'invalid_request' | 'missing_runner' | 'invalid_runner' | 'invalid_json' | 'runner_failed';
+export type JudgmentDAGRunnerFailureKind = 'sync_throw' | 'async_reject';
+export interface JudgmentDAGExecutionErrorDetails {
+    readonly node_id?: string;
+    readonly node_type?: JudgmentDAGNode['node_type'];
+    readonly runner_type?: JudgmentDAGRunnerType;
+    readonly failure_kind?: JudgmentDAGRunnerFailureKind;
+    readonly cause?: unknown;
+}
+export declare class JudgmentDAGExecutionError extends Error {
+    readonly code: JudgmentDAGExecutionCode;
+    readonly node_id?: string;
+    readonly node_type?: JudgmentDAGNode['node_type'];
+    readonly runner_type?: JudgmentDAGRunnerType;
+    readonly failure_kind?: JudgmentDAGRunnerFailureKind;
+    readonly cause?: unknown;
+    constructor(code: JudgmentDAGExecutionCode, message: string, details?: JudgmentDAGExecutionErrorDetails);
+}
+/**
+ * Execute a validated local DAG sequentially using only explicit runner
+ * registrations. No clock, randomness, filesystem, network, or persistence
+ * is consulted by this function.
+ */
+export declare function executeJudgmentDAG(request: JudgmentDAGRunRequest): Promise<JudgmentDAGRunRecord>;