@jinn-network/client 0.1.2 → 0.1.3-canary.24b69ea4

package/dist/harnesses/impls/prediction-v1-evaluator/index.js

@@ -0,0 +1,219 @@
+/**
+ * prediction-v1-evaluator — deterministic Brier scorer for Polymarket Tasks.
+ */
+import { createHash } from 'node:crypto';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { REQUIRES_LIVE_DAEMON_READINESS } from '../../types.js';
+import { SignedEnvelopeSchema } from '../../../types/envelope.js';
+import { PredictionV1TaskSchema } from '../../../types/prediction-v1.js';
+import { PredictionV1RestorationPayloadSchema, } from '../../../types/payloads/prediction-v1.js';
+import { canonicalJson } from '../../engine/canonical-json.js';
+import { RESTORATION_ENVELOPE_CID_CONTEXT_KEY, RESTORATION_TASK_CID_CONTEXT_KEY, } from '../evaluation-context.js';
+import { checkManifestSignature, checkTaskRef, checkTaskRefMissingExpected, recomputeTopLevelSignatureHash, } from '../prediction-v0-evaluator/checks/integrity.js';
+import { getResolution, } from '../../../venues/polymarket/client.js';
+export class PredictionV1Evaluator {
+    config;
+    name = 'prediction-v1-evaluator';
+    version = '1.0.0';
+    constructor(config = {}) {
+        this.config = config;
+    }
+    supports(ctx) {
+        return ctx.solverType === 'prediction.v1' && ctx.role === 'evaluation';
+    }
+    async isReady() {
+        if (this.config.stub)
+            return { ...REQUIRES_LIVE_DAEMON_READINESS };
+        return { ready: true };
+    }
+    async canAttempt(task) {
+        if (task.solverType !== 'prediction.v1')
+            return { ok: false, reason: 'solverType is not prediction.v1' };
+        if (task.role !== 'evaluation')
+            return { ok: false, reason: 'role is not evaluation' };
+        if (typeof task.context?.['restorationResult'] !== 'string') {
+            return { ok: false, reason: 'context.restorationResult required' };
+        }
+        if (typeof task.context?.[RESTORATION_TASK_CID_CONTEXT_KEY] !== 'string') {
+            return { ok: false, reason: `context.${RESTORATION_TASK_CID_CONTEXT_KEY} required` };
+        }
+        return { ok: true };
+    }
+    async run(ctx) {
+        if (this.config.stub) {
+            throw new Error('prediction-v1-evaluator: stub registry cannot run evaluation (requires live daemon)');
+        }
+        const task = PredictionV1TaskSchema.parse(ctx.task);
+        const ids = task.spec.source.identifiers;
+        const checks = [];
+        const manifestJson = ctx.task.context['restorationResult'];
+        const rawEnvelope = JSON.parse(manifestJson);
+        const solutionEnvelopeSha256 = createHash('sha256').update(canonicalJson(rawEnvelope)).digest('hex');
+        const solutionEnvelopeCid = ctx.task.context?.[RESTORATION_ENVELOPE_CID_CONTEXT_KEY]
+            ?? ctx.task.restorationRequestId
+            ?? '';
+        let solution = null;
+        try {
+            const envelope = SignedEnvelopeSchema.parse(rawEnvelope);
+            if (envelope.solverType !== 'prediction.v1' || envelope.role !== 'restoration') {
+                checks.push({
+                    name: 'solution.envelope',
+                    status: 'FAIL',
+                    detail: `expected prediction.v1/restoration, got ${envelope.solverType}/${envelope.role}`,
+                });
+            }
+            else {
+                checks.push({ name: 'solution.envelope', status: 'PASS' });
+            }
+            checks.push(await checkManifestSignature(recomputeTopLevelSignatureHash(rawEnvelope), envelope.signature));
+            const expectedTaskCid = ctx.task.context?.[RESTORATION_TASK_CID_CONTEXT_KEY];
+            if (expectedTaskCid) {
+                checks.push(checkTaskRef(envelope.task.cid, expectedTaskCid));
+            }
+            else {
+                checks.push(checkTaskRefMissingExpected());
+            }
+            solution = PredictionV1RestorationPayloadSchema.parse(envelope.payload);
+            checks.push({ name: 'solution.schema', status: 'PASS' });
+        }
+        catch (err) {
+            checks.push({
+                name: 'solution.schema',
+                status: 'FAIL',
+                detail: err instanceof Error ? err.message : String(err),
+            });
+        }
+        if (solution) {
+            const submittedAt = Date.parse(solution.submittedAt);
+            if (Number.isFinite(submittedAt) && submittedAt >= task.window.startTs && submittedAt <= task.window.endTs) {
+                checks.push({ name: 'solution.window', status: 'PASS' });
+            }
+            else {
+                checks.push({
+                    name: 'solution.window',
+                    status: 'FAIL',
+                    detail: { submittedAt: solution.submittedAt, window: task.window },
+                });
+            }
+        }
+        const resolution = await this.readResolution(ids.marketId, ids.conditionId, task.spec.source.url);
+        checks.push(checkResolutionIdentity(resolution, ids));
+        if (resolution.status === 'resolved') {
+            checks.push({ name: 'market.resolution', status: 'PASS' });
+        }
+        else if (resolution.status === 'unresolved') {
+            checks.push({ name: 'market.resolution', status: 'INDETERMINATE' });
+        }
+        else {
+            checks.push({ name: 'market.resolution', status: 'FAIL', detail: resolution.status });
+        }
+        const verdict = deriveVerdict(checks, resolution);
+        const scores = verdict === 'SCORED' && solution && resolution.outcome
+            ? scoreBrier(solution.probabilityYes, task.spec.consensusSnapshot.probabilityYes, resolution.outcome)
+            : undefined;
+        const payload = {
+            verdict,
+            ...(resolution.outcome ? { outcome: resolution.outcome } : verdict === 'INVALID' ? { outcome: 'INVALID' } : {}),
+            ...(resolution.resolvedAt ? { resolvedAt: resolution.resolvedAt } : {}),
+            resolutionSource: {
+                venue: 'polymarket',
+                url: resolution.sourceUrl,
+                marketId: ids.marketId,
+                conditionId: ids.conditionId,
+            },
+            task: {
+                cid: ctx.taskCid ?? '',
+                id: task.id,
+            },
+            solutionEnvelope: {
+                cid: solutionEnvelopeCid,
+                sha256: solutionEnvelopeSha256,
+            },
+            ...(solution
+                ? {
+                    claimed: {
+                        probabilityYes: solution.probabilityYes,
+                        submittedAt: solution.submittedAt,
+                        modelId: solution.modelId,
+                    },
+                }
+                : {}),
+            benchmark: {
+                probabilityYes: task.spec.consensusSnapshot.probabilityYes,
+                sampledAt: task.spec.consensusSnapshot.sampledAt,
+                method: task.spec.consensusSnapshot.method,
+            },
+            ...(scores ? { scores } : {}),
+            checks,
+        };
+        writeFileSync(join(ctx.workingDir, 'prediction-v1-verdict.json'), JSON.stringify(payload, null, 2));
+        return {
+            venueRef: { name: 'polymarket' },
+            gating: {
+                verdict,
+                outcome: resolution.outcome ?? resolution.status,
+                ...(scores ?? {}),
+            },
+            informational: {
+                resolution,
+            },
+            verdictPayload: payload,
+            artifacts: [
+                {
+                    path: 'prediction-v1-verdict.json',
+                    artifactType: 'prediction_v1_verdict',
+                    metadata: { verdict, ...(scores ?? {}) },
+                    access: { priceUsdc: '0' },
+                },
+            ],
+        };
+    }
+    async readResolution(marketId, conditionId, sourceUrl) {
+        const read = this.config._testDeps?.getResolution ?? getResolution;
+        return read({ marketId, conditionId, sourceUrl });
+    }
+}
+function deriveVerdict(checks, resolution) {
+    if (checks.some((check) => check.status === 'FAIL' && check.name !== 'market.resolution'))
+        return 'REJECTED';
+    if (checks.some((check) => check.status === 'INDETERMINATE'))
+        return 'INDETERMINATE';
+    if (resolution.status === 'resolved' && resolution.outcome)
+        return 'SCORED';
+    if (resolution.status === 'unresolved')
+        return 'INDETERMINATE';
+    return 'INVALID';
+}
+function checkResolutionIdentity(resolution, ids) {
+    const marketMatches = resolution.marketId === ids.marketId;
+    const conditionMatches = resolution.conditionId.toLowerCase() === ids.conditionId.toLowerCase();
+    if (marketMatches && conditionMatches) {
+        return { name: 'market.identity', status: 'PASS' };
+    }
+    return {
+        name: 'market.identity',
+        status: 'FAIL',
+        detail: {
+            expectedMarketId: ids.marketId,
+            actualMarketId: resolution.marketId,
+            expectedConditionId: ids.conditionId,
+            actualConditionId: resolution.conditionId,
+        },
+    };
+}
+function scoreBrier(probabilityYes, consensusProbabilityYes, outcome) {
+    const target = outcome === 'YES' ? 1 : 0;
+    const solver = Number(probabilityYes);
+    const consensus = Number(consensusProbabilityYes);
+    const solverBrier = (solver - target) ** 2;
+    const consensusBrier = (consensus - target) ** 2;
+    return {
+        scoreBasis: 'brier-loss.v1',
+        solverBrier: solverBrier.toFixed(6),
+        consensusBrier: consensusBrier.toFixed(6),
+        brierSpread: (solverBrier - consensusBrier).toFixed(6),
+    };
+}
+export default PredictionV1Evaluator;
+//# sourceMappingURL=index.js.map

package/dist/harnesses/impls/prediction-v1-evaluator/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/prediction-v1-evaluator/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,8BAA8B,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EACL,oCAAoC,GAErC,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EACL,oCAAoC,EACpC,gCAAgC,GACjC,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,sBAAsB,EACtB,YAAY,EACZ,2BAA2B,EAC3B,8BAA8B,GAC/B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EACL,aAAa,GAEd,MAAM,sCAAsC,CAAC;AAe9C,MAAM,OAAO,qBAAqB;IAIH;IAHpB,IAAI,GAAG,yBAAyB,CAAC;IACjC,OAAO,GAAG,OAAO,CAAC;IAE3B,YAA6B,SAAsC,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;IAAG,CAAC;IAEzE,QAAQ,CAAC,GAAgE;QACvE,OAAO,GAAG,CAAC,UAAU,KAAK,eAAe,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,8BAA8B,EAAE,CAAC;QACnE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAU;QACzB,IAAI,IAAI,CAAC,UAAU,KAAK,eAAe;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAiC,EAAE,CAAC;QACzG,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC;QACvF,IAAI,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC,mBAAmB,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC5D,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAC;QACrE,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC,gCAAgC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,gCAAgC,WAAW,EAAE,CAAC;QACvF,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAmB;QAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,qFAAqF,CAAC,CAAC;QACzG,CAAC;QACD,MAAM,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;QACzC,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,MAAM,YAAY,GAAG,GAAG,CAAC,IAAI,CAAC,OAAQ,CAAC,mBAAmB,CAAW,CAAC;QACtE,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAA4B,CAAC;QACxE,MAAM,sBAAsB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrG,MAAM,mBAAmB,GACtB,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,oCAAoC,CAAwB;eAC7E,GAAG,CAAC,IAAI,CAAC,oBAAoB;eAC7B,EAAE,CAAC;QAER,IAAI,QAAQ,GAA0C,IAAI,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACzD,IAAI,QAAQ,CAAC,UAAU,KAAK,eAAe,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAC/E,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,mBAAmB;oBACzB,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,2CAA2C,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE;iBAC1F,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,MAAM,sBAAsB,CAAC,8BAA8B,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;YAC3G,MAAM,eAAe,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,gCAAgC,CAAuB,CAAC;YACnG,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC;YACjE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,QAAQ,GAAG,oCAAoC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACzD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC3G,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,iBAAiB;oBACvB,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;iBACnE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClG,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,UAAU,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,OAAO,KAAK,QAAQ,IAAI,QAAQ,IAAI,UAAU,CAAC,OAAO;YACnE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,cAAc,EAAE,UAAU,CAAC,OAAO,CAAC;YACrG,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,OAAO,GAA4B;YACvC,OAAO;YACP,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/G,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,gBAAgB,EAAE;gBAChB,KAAK,EAAE,YAAY;gBACnB,GAAG,EAAE,UAAU,CAAC,SAAS;gBACzB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;aAC7B;YACD,IAAI,EAAE;gBACJ,GAAG,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;gBACtB,EAAE,EAAE,IAAI,CAAC,EAAE;aACZ;YACD,gBAAgB,EAAE;gBAChB,GAAG,EAAE,mBAAmB;gBACxB,MAAM,EAAE,sBAAsB;aAC/B;YACD,GAAG,CAAC,QAAQ;gBACV,CAAC,CAAC;oBACE,OAAO,EAAE;wBACP,cAAc,EAAE,QAAQ,CAAC,cAAc;wBACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;wBACjC,OAAO,EAAE,QAAQ,CAAC,OAAO;qBAC1B;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE;gBACT,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,cAAc;gBAC1D,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS;gBAChD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM;aAC3C;YACD,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,MAAM;SACP,CAAC;QAEF,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,4BAA4B,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEpG,OAAO;YACL,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;YAChC,MAAM,EAAE;gBACN,OAAO;gBACP,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM;gBAChD,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;aAClB;YACD,aAAa,EAAE;gBACb,UAAU;aACX;YACD,cAAc,EAAE,OAAO;YACvB,SAAS,EAAE;gBACT;oBACE,IAAI,EAAE,4BAA4B;oBAClC,YAAY,EAAE,uBAAuB;oBACrC,QAAQ,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE;oBACxC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE;iBAC3B;aACF;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAgB,EAChB,WAAmB,EACnB,SAAiB;QAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,IAAI,aAAa,CAAC;QACnE,OAAO,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC;IACpD,CAAC;CACF;AAED,SAAS,aAAa,CAAC,MAAe,EAAE,UAA8B;IACpE,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,CAAC;QAAE,OAAO,UAAU,CAAC;IAC7G,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,eAAe,CAAC;QAAE,OAAO,eAAe,CAAC;IACrF,IAAI,UAAU,CAAC,MAAM,KAAK,UAAU,IAAI,UAAU,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAC;IAC5E,IAAI,UAAU,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,eAAe,CAAC;IAC/D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,uBAAuB,CAC9B,UAA8B,EAC9B,GAA8C;IAE9C,MAAM,aAAa,GAAG,UAAU,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC;IAC3D,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;IAChG,IAAI,aAAa,IAAI,gBAAgB,EAAE,CAAC;QACtC,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IACrD,CAAC;IACD,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE;YACN,gBAAgB,EAAE,GAAG,CAAC,QAAQ;YAC9B,cAAc,EAAE,UAAU,CAAC,QAAQ;YACnC,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,iBAAiB,EAAE,UAAU,CAAC,WAAW;SAC1C;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,cAAsB,EACtB,uBAA+B,EAC/B,OAAqB;IAOrB,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACjD,OAAO;QACL,UAAU,EAAE,eAAe;QAC3B,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACnC,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QACzC,WAAW,EAAE,CAAC,WAAW,GAAG,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,eAAe,qBAAqB,CAAC"}

package/dist/harnesses/impls/session-derived-evaluator/index.d.ts

@@ -0,0 +1,4 @@
+export * from './signals.js';
+export * from './structural-similarity.js';
+export * from './test-suite-rerun.js';
+export * from './llm-judge.js';

package/dist/harnesses/impls/session-derived-evaluator/index.js

@@ -0,0 +1,5 @@
+export * from './signals.js';
+export * from './structural-similarity.js';
+export * from './test-suite-rerun.js';
+export * from './llm-judge.js';
+//# sourceMappingURL=index.js.map

package/dist/harnesses/impls/session-derived-evaluator/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/session-derived-evaluator/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,uBAAuB,CAAC;AACtC,cAAc,gBAAgB,CAAC"}

package/dist/harnesses/impls/session-derived-evaluator/llm-judge.d.ts

@@ -0,0 +1,16 @@
+import { type EvaluationSignal } from './signals.js';
+export interface LlmJudgeInput {
+    problemStatement: string;
+    candidatePatch?: string | null;
+    referencePatch?: string | null;
+    judge: (input: {
+        problemStatement: string;
+        candidatePatch?: string | null;
+        referencePatch?: string | null;
+    }) => Promise<{
+        score: number;
+        reasoning: string;
+    }>;
+    weight?: number;
+}
+export declare function llmJudgeSignal(input: LlmJudgeInput): Promise<EvaluationSignal>;

package/dist/harnesses/impls/session-derived-evaluator/llm-judge.js

@@ -0,0 +1,16 @@
+import { clampScore } from './signals.js';
+export async function llmJudgeSignal(input) {
+    const weight = input.weight ?? 0.2;
+    const judged = await input.judge({
+        problemStatement: input.problemStatement,
+        candidatePatch: input.candidatePatch,
+        referencePatch: input.referencePatch,
+    });
+    return {
+        present: true,
+        score: clampScore(judged.score),
+        weight,
+        reasoning: judged.reasoning,
+    };
+}
+//# sourceMappingURL=llm-judge.js.map

package/dist/harnesses/impls/session-derived-evaluator/llm-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/session-derived-evaluator/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAyB,MAAM,cAAc,CAAC;AAcjE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAoB;IACvD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC;QAC/B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,cAAc,EAAE,KAAK,CAAC,cAAc;KACrC,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;QAC/B,MAAM;QACN,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC"}

package/dist/harnesses/impls/session-derived-evaluator/signals.d.ts

@@ -0,0 +1,13 @@
+export interface EvaluationSignal {
+    present: boolean;
+    score: number;
+    weight: number;
+    reasoning?: string;
+}
+export interface CompositeEvaluation {
+    compositeScore: number;
+    presentWeight: number;
+    signalsPresent: number;
+}
+export declare function clampScore(score: number): number;
+export declare function composite(signals: readonly EvaluationSignal[]): CompositeEvaluation;

package/dist/harnesses/impls/session-derived-evaluator/signals.js

@@ -0,0 +1,15 @@
+export function clampScore(score) {
+    if (!Number.isFinite(score))
+        return 0;
+    return Math.max(0, Math.min(1, score));
+}
+export function composite(signals) {
+    const present = signals.filter((signal) => signal.present);
+    const presentWeight = present.reduce((sum, signal) => sum + signal.weight, 0);
+    if (present.length === 0 || presentWeight <= 0) {
+        return { compositeScore: 0, presentWeight: 0, signalsPresent: 0 };
+    }
+    const compositeScore = present.reduce((sum, signal) => sum + (clampScore(signal.score) * signal.weight) / presentWeight, 0);
+    return { compositeScore, presentWeight, signalsPresent: present.length };
+}
+//# sourceMappingURL=signals.js.map

package/dist/harnesses/impls/session-derived-evaluator/signals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"signals.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/session-derived-evaluator/signals.ts"],"names":[],"mappings":"AAaA,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAoC;IAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,aAAa,EACjF,CAAC,CACF,CAAC;IACF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC"}

package/dist/harnesses/impls/session-derived-evaluator/structural-similarity.d.ts

@@ -0,0 +1,7 @@
+import { type EvaluationSignal } from './signals.js';
+export interface StructuralSimilarityInput {
+    candidatePatch?: string | null;
+    referencePatch?: string | null;
+    weight?: number;
+}
+export declare function structuralSimilaritySignal(input: StructuralSimilarityInput): EvaluationSignal;

package/dist/harnesses/impls/session-derived-evaluator/structural-similarity.js

@@ -0,0 +1,28 @@
+import { clampScore } from './signals.js';
+export function structuralSimilaritySignal(input) {
+    const weight = input.weight ?? 0.3;
+    if (!input.candidatePatch || !input.referencePatch) {
+        return { present: false, score: 0, weight, reasoning: 'candidate or reference patch missing' };
+    }
+    const candidate = normalizedPatchLines(input.candidatePatch);
+    const reference = normalizedPatchLines(input.referencePatch);
+    if (candidate.length === 0 || reference.length === 0) {
+        return { present: false, score: 0, weight, reasoning: 'patch has no comparable changed lines' };
+    }
+    const candidateSet = new Set(candidate);
+    const referenceSet = new Set(reference);
+    let overlap = 0;
+    for (const line of candidateSet) {
+        if (referenceSet.has(line))
+            overlap += 1;
+    }
+    const dice = (2 * overlap) / (candidateSet.size + referenceSet.size);
+    return { present: true, score: clampScore(dice), weight };
+}
+function normalizedPatchLines(patch) {
+    return patch
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => (line.startsWith('+') || line.startsWith('-')) && !line.startsWith('+++') && !line.startsWith('---'));
+}
+//# sourceMappingURL=structural-similarity.js.map

package/dist/harnesses/impls/session-derived-evaluator/structural-similarity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structural-similarity.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/session-derived-evaluator/structural-similarity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAyB,MAAM,cAAc,CAAC;AAQjE,MAAM,UAAU,0BAA0B,CAAC,KAAgC;IACzE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC;IACnC,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QACnD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,sCAAsC,EAAE,CAAC;IACjG,CAAC;IACD,MAAM,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,uCAAuC,EAAE,CAAC;IAClG,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,KAAK;SACT,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5H,CAAC"}

package/dist/harnesses/impls/session-derived-evaluator/test-suite-rerun.d.ts

@@ -0,0 +1,15 @@
+import type { EvaluationSignal } from './signals.js';
+export interface TestSuiteRunResult {
+    passed: boolean;
+    log?: string;
+}
+export interface TestSuiteRerunInput {
+    testSuiteRef?: string | null;
+    candidatePatch?: string | null;
+    runTests: (input: {
+        testSuiteRef: string;
+        candidatePatch: string;
+    }) => Promise<TestSuiteRunResult>;
+    weight?: number;
+}
+export declare function testSuiteRerunSignal(input: TestSuiteRerunInput): Promise<EvaluationSignal>;

package/dist/harnesses/impls/session-derived-evaluator/test-suite-rerun.js

@@ -0,0 +1,17 @@
+export async function testSuiteRerunSignal(input) {
+    const weight = input.weight ?? 0.5;
+    if (!input.testSuiteRef || !input.candidatePatch) {
+        return { present: false, score: 0, weight, reasoning: 'test suite or candidate patch missing' };
+    }
+    const result = await input.runTests({
+        testSuiteRef: input.testSuiteRef,
+        candidatePatch: input.candidatePatch,
+    });
+    return {
+        present: true,
+        score: result.passed ? 1 : 0,
+        weight,
+        ...(result.log ? { reasoning: result.log } : {}),
+    };
+}
+//# sourceMappingURL=test-suite-rerun.js.map

package/dist/harnesses/impls/session-derived-evaluator/test-suite-rerun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-suite-rerun.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/session-derived-evaluator/test-suite-rerun.ts"],"names":[],"mappings":"AAcA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAA0B;IACnE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC;IACnC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,uCAAuC,EAAE,CAAC;IAClG,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC;QAClC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,cAAc;KACrC,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM;QACN,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACjD,CAAC;AACJ,CAAC"}

package/dist/harnesses/impls/swe-rebench-v2-evaluator/eval-runner.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Thin Python-subprocess wrapper around `scripts/eval.py` from the upstream
+ * SWE-rebench/SWE-rebench-V2 repo (MIT). Operators install the upstream
+ * harness as a Python dependency; this runner shells out and parses the
+ * structured JSON report.
+ */
+import type { EvalRunner } from './index.js';
+export interface PythonEvalRunnerOptions {
+    /** Path to the cloned SWE-rebench-V2 repo (cached locally). */
+    upstreamRepoDir: string;
+    /** Override Python executable. Defaults to `python3`. */
+    pythonBin?: string;
+    /** Workers for parallel eval (defaults to 1; we run one task at a time). */
+    maxWorkers?: number;
+}
+export declare class PythonEvalRunner implements EvalRunner {
+    private readonly opts;
+    constructor(opts: PythonEvalRunnerOptions);
+    runEval(args: Parameters<EvalRunner['runEval']>[0]): ReturnType<EvalRunner['runEval']>;
+}

package/dist/harnesses/impls/swe-rebench-v2-evaluator/eval-runner.js

@@ -0,0 +1,107 @@
+/**
+ * Thin Python-subprocess wrapper around `scripts/eval.py` from the upstream
+ * SWE-rebench/SWE-rebench-V2 repo (MIT). Operators install the upstream
+ * harness as a Python dependency; this runner shells out and parses the
+ * structured JSON report.
+ */
+import { spawn } from 'node:child_process';
+import { mkdtemp, writeFile, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+export class PythonEvalRunner {
+    opts;
+    constructor(opts) {
+        this.opts = opts;
+    }
+    async runEval(args) {
+        const tmp = await mkdtemp(join(tmpdir(), 'swerebench-eval-'));
+        // Single-task runner: eval.py matches the patch override by instance_id.
+        const INSTANCE_ID = args.instance_id;
+        const taskJson = [{
+                instance_id: INSTANCE_ID,
+                // SWE-rebench Docker images place the checked-out repository at
+                // /testbed. The upstream eval.py derives its docker workdir from the
+                // repo slug, so use a synthetic slug that resolves to /testbed while
+                // preserving the real repo separately in the Jinn task/HF row.
+                repo: 'jinn/testbed',
+                image_name: args.image,
+                FAIL_TO_PASS: args.fail_to_pass,
+                PASS_TO_PASS: args.pass_to_pass,
+                test_patch: args.test_patch,
+                install_config: {
+                    test_cmd: [
+                        ...normalizeCommands(args.install),
+                        ...normalizeCommands(args.test_cmd),
+                    ],
+                    log_parser: args.log_parser,
+                },
+            }];
+        // Upstream eval.py expects --patches to be a JSON list of
+        // `{instance_id, patch, test_patch?}` overrides keyed by instance_id.
+        const patchesJson = [{ instance_id: INSTANCE_ID, patch: args.patch }];
+        const taskJsonPath = join(tmp, 'task.json');
+        const patchesJsonPath = join(tmp, 'patches.json');
+        const reportPath = join(tmp, 'report.json');
+        await writeFile(taskJsonPath, JSON.stringify(taskJson));
+        await writeFile(patchesJsonPath, JSON.stringify(patchesJson));
+        const pyArgs = [
+            '-m', 'scripts.eval',
+            '--json', taskJsonPath,
+            '--patches', patchesJsonPath,
+            '--max-workers', String(this.opts.maxWorkers ?? 1),
+            '--report-json', reportPath,
+        ];
+        const child = spawn(this.opts.pythonBin ?? 'python3', pyArgs, {
+            cwd: this.opts.upstreamRepoDir,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let stderr = '';
+        child.stderr.on('data', (d) => { stderr += d.toString(); });
+        let stdout = '';
+        child.stdout.on('data', (d) => { stdout += d.toString(); });
+        const exitCode = await new Promise((resolve, reject) => {
+            child.on('close', (code) => resolve(code ?? 1));
+            child.on('error', reject);
+        });
+        let report;
+        try {
+            report = JSON.parse(await readFile(reportPath, 'utf8'));
+        }
+        catch {
+            await rm(tmp, { recursive: true, force: true });
+            throw new Error(`Eval runner failed: exitCode=${exitCode}, stderr=${stderr.slice(-500)}`);
+        }
+        // Upstream report shape: { total, passed, items: [{instance_id, passed_match,
+        // from_fail_to_pass, failed_from_pass_to_pass, exit_code, log_path, error}] }.
+        const items = Array.isArray(report.items) ? report.items : [];
+        const item = items.find((i) => i['instance_id'] === INSTANCE_ID) ?? items[0] ?? {};
+        let logBody = '';
+        const logPath = item['log_path'];
+        if (typeof logPath === 'string' && logPath.length > 0) {
+            try {
+                logBody = await readFile(logPath, 'utf8');
+            }
+            catch {
+                logBody = '';
+            }
+        }
+        await rm(tmp, { recursive: true, force: true });
+        return {
+            passed_match: item['passed_match'] === true,
+            passed: Array.isArray(item['from_fail_to_pass']) ? item['from_fail_to_pass'] : [],
+            failed: Array.isArray(item['failed_from_pass_to_pass']) ? item['failed_from_pass_to_pass'] : [],
+            log: stdout + logBody,
+            exitCode,
+        };
+    }
+}
+function normalizeCommands(value) {
+    if (typeof value === 'string') {
+        return value.trim() ? [value] : [];
+    }
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value.filter((item) => typeof item === 'string' && item.trim().length > 0);
+}
+//# sourceMappingURL=eval-runner.js.map

package/dist/harnesses/impls/swe-rebench-v2-evaluator/eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../../../../src/harnesses/impls/swe-rebench-v2-evaluator/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAYjC,MAAM,OAAO,gBAAgB;IACE;IAA7B,YAA6B,IAA6B;QAA7B,SAAI,GAAJ,IAAI,CAAyB;IAAG,CAAC;IAE9D,KAAK,CAAC,OAAO,CAAC,IAA0C;QACtD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAC9D,yEAAyE;QACzE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,MAAM,QAAQ,GAAG,CAAC;gBAChB,WAAW,EAAE,WAAW;gBACxB,gEAAgE;gBAChE,qEAAqE;gBACrE,qEAAqE;gBACrE,+DAA+D;gBAC/D,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,IAAI,CAAC,KAAK;gBACtB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,cAAc,EAAE;oBACd,QAAQ,EAAE;wBACR,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC;wBAClC,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC;qBACpC;oBACD,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B;aACF,CAAC,CAAC;QACH,0DAA0D;QAC1D,sEAAsE;QACtE,MAAM,WAAW,GAAG,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAC5C,MAAM,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxD,MAAM,SAAS,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,cAAc;YACpB,QAAQ,EAAE,YAAY;YACtB,WAAW,EAAE,eAAe;YAC5B,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YAClD,eAAe,EAAE,UAAU;SAC5B,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,MAAM,EAAE;YAC5D,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe;YAC9B,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,IAAI,MAAkD,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAkB,CAAC;QAC3E,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,YAAY,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;QAED,8EAA8E;QAC9E,+EAA+E;QAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEnF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAEhD,OAAO;YACL,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI;YAC3C,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,mBAAmB,CAAc,CAAC,CAAC,CAAC,EAAE;YAC/F,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,0BAA0B,CAAc,CAAC,CAAC,CAAC,EAAE;YAC7G,GAAG,EAAE,MAAM,GAAG,OAAO;YACrB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,KAAoC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpG,CAAC"}

package/dist/harnesses/impls/swe-rebench-v2-evaluator/harness.d.ts

@@ -0,0 +1,74 @@
+/**
+ * SWE-rebench v2 Evaluator Harness — wraps the {@link SweRebenchV2Evaluator}
+ * grading library as a first-class {@link Harness} so the daemon dispatches
+ * `swe-rebench-v2.v1` evaluation tasks to it.
+ *
+ * Operator setup is automated via {@link SweRebenchV2EvaluatorHarness.onEnable}:
+ *   `jinn harnesses enable swe-rebench-v2-evaluator`
+ *   - Validates Docker + Python availability.
+ *   - Clones the upstream `SWE-rebench/SWE-rebench-V2` repo into
+ *     `<implStateDir>/upstream` (one-time; idempotent on rerun).
+ *   - Writes a `state.json` marker with `enabled: true`.
+ *
+ * `isReady()` reports `ready: false` (with a `nextStep` pointing at the
+ * enable command) until the marker file is written.
+ *
+ * Spec: docs/superpowers/specs/2026-05-06-agent-harness-solvernet-design.md §3.4
+ */
+import { type SpawnOptions } from 'node:child_process';
+import type { EnableResult, Harness, HarnessContext, HarnessEnableContext, HarnessEnableMetadata, ReadyStatus, Solution } from '../../types.js';
+import type { Task } from '../../../types/task.js';
+import { uploadToIpfs } from '../../../adapters/mech/ipfs.js';
+import { type EvalRunner, type HfFetcher } from './index.js';
+export interface SweRebenchV2EvaluatorHarnessOptions {
+    /** Marks a stub registry — `isReady()` reports requires-live-daemon. */
+    stub?: boolean;
+    /**
+     * Per-impl state directory (e.g. `~/.jinn-client/engine/impl-state/swe-rebench-v2-evaluator`).
+     * The upstream repo is cloned to `<implStateDir>/upstream` and the enabled
+     * marker lives at `<implStateDir>/state.json`.
+     */
+    implStateDir?: string;
+    /** IPFS registry URL used to pin test logs. Defaults to Autonolas gateway. */
+    ipfsRegistryUrl?: string;
+    /**
+     * Test-only injection points. Production runs use Node's child_process
+     * + node:fs + the bundled HFFetcher / PythonEvalRunner.
+     */
+    _testDeps?: {
+        runCommand?: typeof runCommand;
+        fetcher?: HfFetcher;
+        runner?: EvalRunner;
+        uploadToIpfs?: typeof uploadToIpfs;
+    };
+}
+export declare class SweRebenchV2EvaluatorHarness implements Harness {
+    readonly name = "swe-rebench-v2-evaluator";
+    readonly version = "1.0.0";
+    private readonly stub;
+    private readonly implStateDir;
+    private readonly ipfsRegistryUrl;
+    private readonly deps;
+    constructor(opts?: SweRebenchV2EvaluatorHarnessOptions);
+    supports(ctx: {
+        solverType: string;
+        role?: 'restoration' | 'evaluation';
+    }): boolean;
+    canAttempt(task: Task): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
+    isReady(): Promise<ReadyStatus>;
+    enableMetadata(): HarnessEnableMetadata;
+    onEnable(_ctx: HarnessEnableContext): Promise<EnableResult>;
+    run(ctx: HarnessContext): Promise<Solution>;
+}
+interface CommandResult {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}
+declare function runCommand(bin: string, args: string[], opts?: SpawnOptions): Promise<CommandResult>;
+export { runCommand };