@openpond/harness 0.1.0 → 0.2.0

package/dist/index.js

@@ -1,6 +1,10 @@
 export * from "./common.js";
+export * from "./evaluation-review.js";
 export * from "./harness.js";
 export * from "./harness-improvements.js";
 export * from "./harness-workspaces.js";
 export * from "./models.js";
+export * from "./refiner.js";
+export * from "./refiner-detection.js";
+export * from "./refiner-support.js";
 export * from "./tools.js";

package/dist/refiner-detection.js

@@ -0,0 +1,482 @@
+import { createImprovementObservation, createRefinementTriggerDecision, } from "./harness-improvements.js";
+import { contentHash } from "./common.js";
+export const DEFAULT_REFINEMENT_TRIGGER_POLICY = {
+    schemaVersion: "openpond.refinementTriggerPolicy.v1",
+    maxEstimatedCostUsd: 0.05,
+    cooldownMs: 0,
+    maxPendingPlans: 2,
+    maxEvidenceEvents: 20,
+    maxProposalEdits: 4,
+    maxProposalBytes: 20_000,
+};
+export function detectHarnessImprovementAtBoundary(input) {
+    const policy = input.policy ?? DEFAULT_REFINEMENT_TRIGGER_POLICY;
+    const outcomes = normalizeToolOutcomes(input.events);
+    const observations = collectObservations({ ...input, outcomes });
+    const actionable = observations.filter(isActionableObservation);
+    const deduplicationKey = contentHash({
+        runRef: input.runRef,
+        boundary: input.boundary.kind,
+        observations: actionable.map((observation) => ({
+            kind: observation.kind,
+            deterministicClass: observation.deterministicClass,
+            tool: observation.tool,
+            state: observation.state,
+            eventRefs: observation.eventRefs,
+        })),
+        turnId: input.turnId,
+    });
+    const base = {
+        schemaVersion: "openpond.refinementTriggerDecision.v1",
+        id: stableId("refinement-trigger", {
+            turnId: input.turnId,
+            boundary: input.boundary,
+            deduplicationKey,
+            priorDeduplicationKeys: [...(input.priorDeduplicationKeys ?? [])].sort(),
+            pendingPlanCount: input.pendingPlanCount ?? 0,
+            cooldownUntil: input.cooldownUntil ?? null,
+        }),
+        runRef: input.runRef,
+        turnId: input.turnId,
+        harnessRelease: input.harnessRelease,
+        overlay: input.overlay,
+        observations: actionable
+            .slice(0, policy.maxEvidenceEvents)
+            .map((observation) => ({
+            id: observation.id,
+            contentHash: observation.contentHash,
+        })),
+        deduplicationKey,
+        policy,
+        pendingPlanCount: input.pendingPlanCount ?? 0,
+        boundary: input.boundary,
+        cooldownUntil: input.cooldownUntil ?? null,
+        createdAt: input.boundary.occurredAt,
+        metadata: {
+            loadedSkillNames: [...new Set(input.loadedSkillNames ?? [])].sort(),
+        },
+    };
+    if (actionable.length === 0) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                observations: [],
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: observations.some((observation) => observation.state === "open")
+                    ? "A tool failure is still open; wait for recovery or a terminal turn boundary."
+                    : "The completed boundary contains no new reviewable turn or reusable detour.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    if (input.turnReviewAlreadyQueued) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: "This turn already has a queued background Harness review.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    if (input.priorDeduplicationKeys?.has(deduplicationKey)) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: "Equivalent improvement evidence was already routed for this run.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    if (isCoolingDown(input.boundary.occurredAt, input.cooldownUntil)) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: "The Refiner is in its configured cooldown window.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    if ((input.pendingPlanCount ?? 0) >= policy.maxPendingPlans) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: "The run already has the maximum number of pending improvement plans.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    const estimatedMaxCostUsd = input.estimatedRefinerCostUsd ?? 0.01;
+    if (estimatedMaxCostUsd > policy.maxEstimatedCostUsd) {
+        return {
+            observations,
+            trigger: createRefinementTriggerDecision({
+                ...base,
+                decision: "no_action",
+                deterministicRoute: null,
+                suggestedRoutes: [],
+                reason: "The bounded Refiner estimate exceeds the configured run budget.",
+                estimatedMaxCostUsd: 0,
+            }),
+        };
+    }
+    return {
+        observations,
+        trigger: createRefinementTriggerDecision({
+            ...base,
+            decision: "queue_refiner",
+            deterministicRoute: null,
+            suggestedRoutes: [],
+            reason: actionable.some((observation) => observation.kind === "user_turn")
+                ? "A completed user turn is ready for bounded background Harness review."
+                : "A recovered detour may contain a reusable Harness improvement.",
+            estimatedMaxCostUsd,
+        }),
+    };
+}
+function collectObservations(input) {
+    const observations = [];
+    const openFailures = [];
+    const seenFailureKeys = new Set();
+    const recoveredFailureKeys = new Set();
+    const userTurnEvent = input.boundary.kind === "turn_completed"
+        ? latestUserTurnEvent(input.events)
+        : null;
+    if (userTurnEvent) {
+        observations.push(observationForEvents({
+            input,
+            kind: "user_turn",
+            state: "terminal",
+            events: [userTurnEvent],
+            deterministicClass: null,
+            summary: "The completed user turn is available for bounded background review.",
+        }));
+    }
+    for (const outcome of input.outcomes) {
+        if (outcome.action === "refine_request" && !outcome.failed) {
+            const requestedSummary = typeof outcome.args.summary === "string"
+                ? outcome.args.summary.trim()
+                : "The agent explicitly requested bounded refinement.";
+            observations.push(observationFor({
+                input,
+                kind: "reusable_success",
+                state: "terminal",
+                outcomes: [outcome],
+                deterministicClass: "refine_requested",
+                summary: requestedSummary.slice(0, 100_000),
+            }));
+            continue;
+        }
+        if (outcome.failed) {
+            const failureKey = contentHash({
+                action: outcome.action,
+                deterministicClass: outcome.deterministicClass,
+                invocationKey: outcome.invocationKey,
+            });
+            if (!seenFailureKeys.has(failureKey)) {
+                observations.push(observationFor({
+                    input,
+                    kind: "tool_failure",
+                    state: input.boundary.kind === "turn_completed" ? "terminal" : "open",
+                    outcomes: [outcome],
+                    deterministicClass: outcome.deterministicClass,
+                    summary: `Tool ${outcome.action} failed${outcome.deterministicClass ? ` (${outcome.deterministicClass})` : ""}.`,
+                }));
+                seenFailureKeys.add(failureKey);
+            }
+            openFailures.push(outcome);
+            continue;
+        }
+        const sameActionIndex = findLastIndex(openFailures, (candidate) => candidate.action === outcome.action);
+        const priorFailureIndex = sameActionIndex >= 0
+            ? sameActionIndex
+            : openFailures.length - 1;
+        const priorFailure = openFailures[priorFailureIndex];
+        if (!priorFailure)
+            continue;
+        const failureKey = contentHash({
+            action: priorFailure.action,
+            deterministicClass: priorFailure.deterministicClass,
+            invocationKey: priorFailure.invocationKey,
+        });
+        if (recoveredFailureKeys.has(failureKey))
+            continue;
+        const failureObservationIndex = observations.findIndex((observation) => observation.kind === "tool_failure" &&
+            observation.eventRefs.some((reference) => reference.id === priorFailure.event.id));
+        if (failureObservationIndex >= 0) {
+            observations[failureObservationIndex] = observationFor({
+                input,
+                kind: "tool_failure",
+                state: "recovered",
+                outcomes: [priorFailure],
+                deterministicClass: priorFailure.deterministicClass,
+                summary: `Tool ${priorFailure.action} failed and recovered within the same run${priorFailure.deterministicClass
+                    ? ` (${priorFailure.deterministicClass})`
+                    : ""}.`,
+            });
+        }
+        if (priorFailure.action === outcome.action) {
+            observations.push(observationFor({
+                input,
+                kind: "retry",
+                state: "recovered",
+                outcomes: [priorFailure, outcome],
+                deterministicClass: priorFailure.deterministicClass,
+                summary: `Tool ${outcome.action} was retried after a failure.`,
+            }));
+        }
+        observations.push(observationFor({
+            input,
+            kind: "recovery",
+            state: "recovered",
+            outcomes: [priorFailure, outcome],
+            deterministicClass: recoveredClass(priorFailure.deterministicClass),
+            summary: priorFailure.action === outcome.action
+                ? `Tool ${outcome.action} recovered within the same run.`
+                : `Tool ${outcome.action} recovered after ${priorFailure.action} failed.`,
+        }));
+        recoveredFailureKeys.add(failureKey);
+        openFailures.splice(priorFailureIndex, 1);
+    }
+    const recovered = observations.filter((observation) => observation.kind === "recovery");
+    if (input.boundary.kind === "turn_completed" && recovered.length > 0) {
+        const relevantOutcomes = input.outcomes.filter((outcome) => recovered.some((observation) => observation.eventRefs.some((reference) => reference.id === outcome.event.id)));
+        observations.push(observationFor({
+            input,
+            kind: "completion_detour",
+            state: "recovered",
+            outcomes: relevantOutcomes,
+            deterministicClass: "completed_after_recovery",
+            summary: "The turn completed after one or more recoverable tool detours.",
+        }));
+    }
+    return observations;
+}
+function observationForEvents(input) {
+    const eventRefs = input.events.map((event) => ({
+        id: event.id,
+        sequence: event.sequence ?? null,
+        contentHash: contentHash(event),
+    }));
+    return createImprovementObservation({
+        schemaVersion: "openpond.improvementObservation.v1",
+        id: stableId("improvement-observation", {
+            runRef: input.input.runRef,
+            kind: input.kind,
+            state: input.state,
+            deterministicClass: input.deterministicClass,
+            summary: input.summary,
+            eventRefs,
+            boundary: input.input.boundary,
+        }),
+        runRef: input.input.runRef,
+        turnId: input.input.turnId,
+        harnessRelease: input.input.harnessRelease,
+        overlay: input.input.overlay,
+        eventRefs,
+        kind: input.kind,
+        state: input.state,
+        tool: null,
+        deterministicClass: input.deterministicClass,
+        summary: input.summary,
+        createdAt: input.input.boundary.occurredAt,
+        metadata: {},
+    });
+}
+function latestUserTurnEvent(events) {
+    return [...events]
+        .sort(compareEvents)
+        .reverse()
+        .find((event) => event.name === "turn.started") ?? null;
+}
+function findLastIndex(values, predicate) {
+    for (let index = values.length - 1; index >= 0; index -= 1) {
+        if (predicate(values[index]))
+            return index;
+    }
+    return -1;
+}
+function observationFor(input) {
+    const eventRefs = input.outcomes.map(({ event }) => ({
+        id: event.id,
+        sequence: event.sequence ?? null,
+        contentHash: contentHash(event),
+    }));
+    const first = input.outcomes[0] ?? null;
+    return createImprovementObservation({
+        schemaVersion: "openpond.improvementObservation.v1",
+        id: stableId("improvement-observation", {
+            runRef: input.input.runRef,
+            kind: input.kind,
+            state: input.state,
+            deterministicClass: input.deterministicClass,
+            summary: input.summary,
+            eventRefs,
+            boundary: input.input.boundary,
+        }),
+        runRef: input.input.runRef,
+        turnId: input.input.turnId,
+        harnessRelease: input.input.harnessRelease,
+        overlay: input.input.overlay,
+        eventRefs,
+        kind: input.kind,
+        state: input.state,
+        tool: first
+            ? { name: first.action, invocationKey: first.invocationKey }
+            : null,
+        deterministicClass: input.deterministicClass,
+        summary: input.summary,
+        createdAt: input.input.boundary.occurredAt,
+        metadata: {},
+    });
+}
+function normalizeToolOutcomes(events) {
+    const startedByCallId = new Map();
+    const outcomes = [];
+    for (const event of [...events].sort(compareEvents)) {
+        const callId = toolCallId(event);
+        if (event.name === "tool.started") {
+            if (callId)
+                startedByCallId.set(callId, event);
+            continue;
+        }
+        // Workspace action results are implementation details nested beneath a
+        // provider tool call. Treating cleanup/status actions as independent tool
+        // successes can falsely mark the parent tool failure as recovered.
+        if (event.name !== "tool.completed")
+            continue;
+        const action = toolAction(event);
+        const started = callId ? startedByCallId.get(callId) : undefined;
+        const failed = toolEventFailed(event);
+        outcomes.push({
+            event,
+            action,
+            args: asRecord(started?.args ?? event.args),
+            invocationKey: contentHash({
+                action,
+                args: started?.args ?? event.args ?? {},
+            }),
+            failed,
+            deterministicClass: failed ? classifyToolFailure(event) : null,
+        });
+    }
+    return outcomes;
+}
+function toolEventFailed(event) {
+    if (event.status === "failed")
+        return true;
+    const data = asRecord(event.data);
+    const result = asRecord(data.result);
+    if (result.ok === false)
+        return true;
+    const status = String(data.status ?? result.status ?? "").toLowerCase();
+    return ["failed", "error", "errored", "blocked", "timed_out"].includes(status);
+}
+function classifyToolFailure(event) {
+    const data = asRecord(event.data);
+    const result = asRecord(data.result);
+    if (result.timedOut === true)
+        return "timeout";
+    if (typeof result.exitCode === "number" && result.exitCode !== 0) {
+        return "command_exit_nonzero";
+    }
+    const structuredStatus = String(data.status ?? result.status ?? "").toLowerCase();
+    if (["timed_out", "timeout"].includes(structuredStatus))
+        return "timeout";
+    const text = [
+        event.error,
+        event.output,
+        result.error,
+        result.stderr,
+        result.stdout,
+    ].filter((value) => typeof value === "string").join("\n").toLowerCase();
+    if (text.includes("modulenotfounderror") ||
+        text.includes("module_not_found") ||
+        text.includes("cannot find module") ||
+        text.includes("no module named")) {
+        return "dependency_missing";
+    }
+    if (text.includes("research limit") || text.includes("rate limit") || text.includes("quota")) {
+        return "tool_budget_exhausted";
+    }
+    if (text.includes("permission denied") || text.includes("forbidden") || text.includes("unauthorized")) {
+        return "permission_denied";
+    }
+    if (/\btimed out\b|\btimeout\b/.test(text))
+        return "timeout";
+    if (text.includes("invalid argument") ||
+        text.includes("invalid_request") ||
+        text.includes("validation")) {
+        return "invalid_tool_arguments";
+    }
+    if (text.includes("enoent") || text.includes("no such file") || text.includes("not found")) {
+        return "missing_file_or_resource";
+    }
+    if (text.includes("exit code") || text.includes("non-zero") || text.includes("nonzero")) {
+        return "command_exit_nonzero";
+    }
+    return "unclassified_tool_failure";
+}
+function recoveredClass(deterministicClass) {
+    return deterministicClass ? `recovered_${deterministicClass}` : "recovered_tool_failure";
+}
+function isActionableObservation(observation) {
+    return (observation.kind === "recovery" ||
+        observation.kind === "completion_detour" ||
+        observation.kind === "user_turn" ||
+        observation.kind === "reusable_success" ||
+        (observation.kind === "validation" && observation.state === "terminal") ||
+        (observation.kind === "tool_failure" && observation.state === "terminal"));
+}
+function toolAction(event) {
+    if (event.action?.trim())
+        return event.action.trim();
+    const data = asRecord(event.data);
+    return typeof data.tool === "string" && data.tool.trim() ? data.tool.trim() : "unknown_tool";
+}
+function toolCallId(event) {
+    const data = asRecord(event.data);
+    for (const value of [data.toolCallId, data.id, data.callId]) {
+        if (typeof value === "string" && value.trim())
+            return value.trim();
+    }
+    return null;
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+}
+function compareEvents(left, right) {
+    if (left.sequence !== undefined || right.sequence !== undefined) {
+        return (left.sequence ?? Number.MAX_SAFE_INTEGER) -
+            (right.sequence ?? Number.MAX_SAFE_INTEGER);
+    }
+    return left.timestamp.localeCompare(right.timestamp);
+}
+function isCoolingDown(now, cooldownUntil) {
+    if (!cooldownUntil)
+        return false;
+    return Date.parse(now) < Date.parse(cooldownUntil);
+}
+function stableId(prefix, value) {
+    return `${prefix}-${contentHash(value).slice(0, 24)}`;
+}

package/dist/refiner-support.js

@@ -0,0 +1,105 @@
+import { contentHash } from "./common.js";
+export function memoryKeyFromTarget(target) {
+    const match = /^memory\/([a-z0-9][a-z0-9-]{0,119})$/.exec(target.replaceAll("\\", "/"));
+    if (!match)
+        throw new Error(`Invalid Harness memory target: ${target}.`);
+    return match[1];
+}
+export function expectedMemoryRevision(proposal, target) {
+    const expected = proposal.metadata.expectedMemory;
+    if (!expected || typeof expected !== "object" || Array.isArray(expected)) {
+        throw new Error("Memory proposal is missing its expected revision snapshot.");
+    }
+    const record = expected;
+    if (record.key !== memoryKeyFromTarget(target)) {
+        throw new Error("Memory proposal expected revision targets a different key.");
+    }
+    if (record.revision !== null && (!Number.isInteger(record.revision) || Number(record.revision) < 1)) {
+        throw new Error("Memory proposal expected revision is invalid.");
+    }
+    if (record.contentHash !== null && typeof record.contentHash !== "string") {
+        throw new Error("Memory proposal expected content hash is invalid.");
+    }
+    if (record.status !== null && record.status !== "active" && record.status !== "deleted") {
+        throw new Error("Memory proposal expected status is invalid.");
+    }
+    return {
+        revision: record.revision,
+        contentHash: record.contentHash,
+        status: record.status,
+    };
+}
+export function boundedTriggerEvidence(trigger) {
+    return {
+        id: trigger.id,
+        runRef: trigger.runRef,
+        turnId: trigger.turnId,
+        reason: trigger.reason,
+        suggestedRoutes: trigger.suggestedRoutes,
+        boundary: trigger.boundary,
+    };
+}
+export function boundedObservationEvidence(observation) {
+    return {
+        id: observation.id,
+        kind: observation.kind,
+        state: observation.state,
+        tool: observation.tool?.name ?? null,
+        deterministicClass: observation.deterministicClass,
+        summary: observation.summary,
+    };
+}
+export function proposalEvidence(observations) {
+    const byId = new Map();
+    for (const observation of observations) {
+        for (const event of observation.eventRefs) {
+            const evidence = observationEvidence(observation, event);
+            byId.set(`${evidence.kind}:${evidence.id}`, evidence);
+        }
+    }
+    return [...byId.values()];
+}
+function observationEvidence(observation, event) {
+    const kind = observation.kind === "recovery"
+        ? "recovery"
+        : observation.kind === "validation"
+            ? "validation"
+            : observation.kind === "user_turn"
+                ? "user_turn"
+                : "tool_event";
+    return {
+        kind: kind,
+        id: event.id,
+        contentHash: event.contentHash,
+    };
+}
+export function uniqueEventRefs(observations) {
+    const byId = new Map();
+    for (const observation of observations) {
+        for (const event of observation.eventRefs)
+            byId.set(event.id, event);
+    }
+    return [...byId.values()];
+}
+export function sameOverlayRef(overlay, reference) {
+    return overlay.id === reference.id &&
+        overlay.revision === reference.revision &&
+        overlay.contentHash === reference.contentHash;
+}
+export function overlayRef(overlay) {
+    return {
+        id: overlay.id,
+        revision: overlay.revision,
+        contentHash: overlay.contentHash,
+    };
+}
+export function sameWorkspaceRevision(workspace, overlay) {
+    return workspace.revision === overlay.workspace.revision &&
+        workspace.sourceRevision === overlay.workspace.sourceRevision &&
+        workspace.currentChannel.revision === overlay.workspace.channelRevision &&
+        workspace.currentChannel.release?.id === overlay.baseHarnessRelease.id &&
+        workspace.currentChannel.release.contentHash === overlay.baseHarnessRelease.contentHash;
+}
+export function stableId(prefix, value) {
+    return `${prefix}-${contentHash(value).slice(0, 24)}`;
+}