@sema-agent/server 7.125.1 โ 7.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +229 -2
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/USAGE.md +346 -145
- package/deploy/sema-up/README.md +1 -1
- package/deploy/sema-up/chart/templates/networkpolicy-sandbox.yaml +57 -0
- package/deploy/sema-up/chart/templates/sandbox-rbac.yaml +32 -1
- package/deploy/sema-up/chart/templates/server-v4pro.yaml +3 -2
- package/deploy/sema-up/chart/templates/server.yaml +2 -1
- package/deploy/sema-up/chart/values.yaml +22 -0
- package/deploy/sema-up/cluster-up.sh +10 -10
- package/deploy/sema-up/kube-up.sh +20 -13
- package/deploy/sema-up/lib/dataroot.sh +1 -1
- package/deploy/sema-up/lib/envf.sh +5 -5
- package/deploy/sema-up/lib/flagshape.sh +5 -5
- package/deploy/sema-up/lib/lanes.py +1 -1
- package/deploy/sema-up/lib/node-addrs.sh +42 -0
- package/deploy/sema-up/lib/parse-machines.py +2 -2
- package/deploy/sema-up/lib/shell-access.sh +1 -1
- package/deploy/sema-up/preflight.sh +4 -4
- package/deploy/sema-up/sema-up.sh +33 -33
- package/deploy/sema-up/smoke.sh +3 -3
- package/deploy/sema-up/stack/compose.yaml +1 -1
- package/deploy/sema-up/stack/sema-stack-cluster.yaml +26 -3
- package/dist/approval-card.d.ts +20 -4
- package/dist/approval-content-kind.d.ts +38 -22
- package/dist/approval-content-kind.js +20 -4
- package/dist/audit.js +3 -0
- package/dist/bench/s1/arms.d.ts +9 -37
- package/dist/bench/s1/arms.js +4 -61
- package/dist/bench/s1/live-deps.d.ts +4 -10
- package/dist/bench/s1/live-deps.js +1 -222
- package/dist/bench/s1/oracle.d.ts +1 -1
- package/dist/bench/s1/reviewer.d.ts +1 -1
- package/dist/bench/s1/row.d.ts +5 -7
- package/dist/bench/s1/row.js +0 -2
- package/dist/bench/s1/run-firm.d.ts +2 -2
- package/dist/bench/s1/run-firm.js +1 -4
- package/dist/bench/s1/runner-ctx.d.ts +17 -29
- package/dist/bench/s1/runner-ctx.js +3 -7
- package/dist/bench/s1/tasks.d.ts +9 -46
- package/dist/bench/s1/tasks.js +4 -63
- package/dist/boot/execution-env.d.ts +38 -26
- package/dist/boot/execution-env.js +87 -41
- package/dist/boot/leader.d.ts +2 -1
- package/dist/boot/memory-boundary.d.ts +1 -1
- package/dist/boot/memory-consolidation.d.ts +37 -3
- package/dist/boot/memory-consolidation.js +18 -2
- package/dist/boot/parked-revive-gate.d.ts +0 -3
- package/dist/boot/parked-revive-gate.js +2 -2
- package/dist/boot/resolve-spec.d.ts +0 -2
- package/dist/boot/resolve-spec.js +12 -17
- package/dist/boot/retention-lane.d.ts +12 -1
- package/dist/boot/retention-lane.js +2 -0
- package/dist/boot/runner-deps.d.ts +7 -6
- package/dist/boot/runner-deps.js +5 -16
- package/dist/boot/session-faces.d.ts +7 -2
- package/dist/boot/session-faces.js +37 -1
- package/dist/boot/stage-03-store-backend.d.ts +1 -0
- package/dist/boot/stage-03-store-backend.js +2 -2
- package/dist/boot/stage-04-stores.d.ts +1 -0
- package/dist/boot/stage-05-execution-env.d.ts +2 -2
- package/dist/boot/stage-05-execution-env.js +7 -4
- package/dist/boot/stage-06-runners.d.ts +2 -3
- package/dist/boot/stage-06-runners.js +13 -9
- package/dist/boot/stage-07-capability-layer.d.ts +2 -3
- package/dist/boot/stage-07-capability-layer.js +3 -4
- package/dist/boot/stage-08-reapers.d.ts +2 -3
- package/dist/boot/stage-08-reapers.js +2 -2
- package/dist/boot/stage-09-leader.d.ts +2 -3
- package/dist/boot/stage-09-leader.js +2 -1
- package/dist/boot/stage-10-http-server.d.ts +2 -3
- package/dist/boot/stage-10-http-server.js +2 -2
- package/dist/boot/stores.d.ts +1 -0
- package/dist/boot/stores.js +4 -3
- package/dist/boot/web-admin.d.ts +5 -0
- package/dist/boot/web-admin.js +28 -14
- package/dist/brain.d.ts +2 -2
- package/dist/brain.js +1 -1
- package/dist/capabilities/builtin-tools.d.ts +2 -1
- package/dist/capabilities/composite-members.d.ts +1 -1
- package/dist/capabilities/composite-members.js +1 -1
- package/dist/capabilities/hands-lane.d.ts +9 -6
- package/dist/capabilities/hands-lane.js +2 -2
- package/dist/capabilities/memory-notice.d.ts +27 -22
- package/dist/capabilities/memory-notice.js +15 -17
- package/dist/capabilities/scenarios.d.ts +9 -14
- package/dist/capabilities/scenarios.js +5 -50
- package/dist/capability-facts.d.ts +22 -11
- package/dist/capability-facts.js +13 -6
- package/dist/config-catalog.js +14 -5
- package/dist/config-provider.d.ts +5 -1
- package/dist/config-provider.js +23 -4
- package/dist/config-types.d.ts +19 -2
- package/dist/config.js +41 -6
- package/dist/deployment-governance.d.ts +19 -21
- package/dist/deployment-governance.js +7 -8
- package/dist/env-name-allowlist-knobs.d.ts +5 -5
- package/dist/env-name-allowlist-knobs.js +1 -0
- package/dist/execution-lane-caps.d.ts +12 -0
- package/dist/execution-lane-caps.js +18 -4
- package/dist/fleet/subagent-tail-bus.js +15 -5
- package/dist/hooks/hook-runner.d.ts +84 -30
- package/dist/hooks/hook-runner.js +74 -41
- package/dist/http/admission.js +17 -10
- package/dist/http/durable-run-leg.d.ts +5 -4
- package/dist/http/durable-run-leg.js +2 -4
- package/dist/http/idempotency.d.ts +28 -6
- package/dist/http/idempotency.js +10 -2
- package/dist/http/principal-gate.d.ts +14 -0
- package/dist/http/principal-gate.js +7 -0
- package/dist/http/request-guards.d.ts +2 -1
- package/dist/http/request-guards.js +14 -14
- package/dist/http/resume-legs.d.ts +2 -2
- package/dist/http/resume-legs.js +20 -10
- package/dist/http/route-ctx.d.ts +15 -7
- package/dist/http/routes/a2a-serve.js +11 -7
- package/dist/http/routes/approvals-assistant.js +18 -3
- package/dist/http/routes/background.d.ts +1 -1
- package/dist/http/routes/background.js +1 -1
- package/dist/http/routes/capabilities.js +10 -2
- package/dist/http/routes/mcp-reconnect.js +1 -1
- package/dist/http/routes/runs.d.ts +4 -3
- package/dist/http/routes/runs.js +35 -14
- package/dist/http/routes/session-sync.js +14 -11
- package/dist/http/routes/sessions.js +9 -9
- package/dist/http/routes/side-query.js +6 -19
- package/dist/http/routes/tasks.js +29 -19
- package/dist/http/routes/workflows.js +8 -0
- package/dist/http/server.d.ts +4 -0
- package/dist/http/server.js +3 -5
- package/dist/http/submit-replay.d.ts +15 -8
- package/dist/http/submit-replay.js +6 -0
- package/dist/http/wire-types.d.ts +7 -9
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/internal-run-env.d.ts +30 -0
- package/dist/internal-run-env.js +11 -0
- package/dist/leader/wire.js +3 -2
- package/dist/lsp/manager.js +1 -1
- package/dist/memory-scope.d.ts +19 -7
- package/dist/memory-scope.js +25 -8
- package/dist/observability/fail-open.d.ts +31 -3
- package/dist/observability/fail-open.js +31 -3
- package/dist/observability/metrics.d.ts +0 -1
- package/dist/observability/metrics.js +1 -4
- package/dist/observability/run-terminal.js +72 -4
- package/dist/permission-rule-vocab.d.ts +3 -2
- package/dist/permission-rule-vocab.js +1 -1
- package/dist/plan-cache-probe.d.ts +3 -2
- package/dist/plugins/approval-ask-store-sql.d.ts +1 -1
- package/dist/plugins/caching-session-store.d.ts +42 -22
- package/dist/plugins/caching-session-store.js +19 -9
- package/dist/plugins/checkpoint-store-sql.d.ts +42 -28
- package/dist/plugins/checkpoint-store-sql.js +57 -11
- package/dist/plugins/file-run-store.d.ts +25 -21
- package/dist/plugins/file-run-store.js +70 -34
- package/dist/plugins/k8s-exec-protocol.js +3 -0
- package/dist/plugins/local-checkpoint-store.d.ts +8 -5
- package/dist/plugins/local-checkpoint-store.js +5 -15
- package/dist/plugins/local-session-store.d.ts +66 -7
- package/dist/plugins/local-session-store.js +168 -101
- package/dist/plugins/mailbox-recipient-lifecycle.d.ts +1 -1
- package/dist/plugins/memory-run-store.d.ts +19 -3
- package/dist/plugins/memory-run-store.js +38 -3
- package/dist/plugins/one-time-migrations.d.ts +165 -39
- package/dist/plugins/one-time-migrations.js +51 -144
- package/dist/plugins/pg-pool.js +11 -1
- package/dist/plugins/pg-session-storage.d.ts +2 -0
- package/dist/plugins/pg-session-storage.js +9 -5
- package/dist/plugins/remote-env-adb.d.ts +3 -0
- package/dist/plugins/remote-env-adb.js +1 -0
- package/dist/plugins/remote-env-device.d.ts +3 -0
- package/dist/plugins/remote-env-device.js +1 -0
- package/dist/plugins/remote-env-e2b.d.ts +4 -0
- package/dist/plugins/remote-env-e2b.js +2 -1
- package/dist/plugins/remote-env-host.d.ts +23 -1
- package/dist/plugins/remote-env-host.js +20 -2
- package/dist/plugins/remote-env-k8s.d.ts +264 -41
- package/dist/plugins/remote-env-k8s.js +551 -82
- package/dist/plugins/remote-env-local-docker.d.ts +4 -0
- package/dist/plugins/remote-env-local-docker.js +2 -1
- package/dist/plugins/remote-env-ssh.d.ts +3 -0
- package/dist/plugins/remote-env-ssh.js +1 -0
- package/dist/plugins/retention-store-sql.d.ts +4 -1
- package/dist/plugins/retention-store-sql.js +11 -9
- package/dist/plugins/run-store-sql.d.ts +44 -6
- package/dist/plugins/run-store-sql.js +92 -27
- package/dist/plugins/session-capture-record-store-sql.d.ts +2 -3
- package/dist/plugins/session-store.d.ts +4 -3
- package/dist/plugins/session-store.js +1 -1
- package/dist/plugins/sql-driver.d.ts +2 -1
- package/dist/plugins/store-backend.d.ts +12 -5
- package/dist/plugins/store-backend.js +13 -4
- package/dist/plugins/store-contracts.d.ts +20 -3
- package/dist/plugins/store-open-wait.js +3 -0
- package/dist/plugins/tidb-pool.js +15 -3
- package/dist/plugins/tidb-session-storage.js +3 -2
- package/dist/plugins/tidb-session-store.d.ts +2 -0
- package/dist/plugins/tidb-session-store.js +6 -2
- package/dist/request-key-closure.d.ts +32 -7
- package/dist/request-key-closure.js +25 -9
- package/dist/router/route-orchestration.d.ts +3 -11
- package/dist/router/route-orchestration.js +0 -4
- package/dist/rule-entry-list.js +0 -4
- package/dist/rules-consent.js +3 -0
- package/dist/run-acceptance.d.ts +51 -0
- package/dist/run-acceptance.js +27 -0
- package/dist/run-local.d.ts +5 -4
- package/dist/run-local.js +23 -17
- package/dist/runs.d.ts +7 -11
- package/dist/runs.js +6 -16
- package/dist/sandbox-egress-policy.d.ts +61 -0
- package/dist/sandbox-egress-policy.js +235 -0
- package/dist/security.d.ts +7 -0
- package/dist/security.js +5 -0
- package/dist/server-secret-env.d.ts +10 -4
- package/dist/server-secret-env.js +4 -1
- package/dist/session-error-wire.d.ts +2 -2
- package/dist/session-incarnation.d.ts +46 -2
- package/dist/session-incarnation.js +14 -3
- package/dist/session-sync-kernel.d.ts +10 -3
- package/dist/session-titler.d.ts +5 -3
- package/dist/spec-fields.d.ts +24 -8
- package/dist/spec-fields.js +21 -2
- package/dist/task-workflow.d.ts +6 -4
- package/dist/task-workflow.js +3 -2
- package/dist/tool-approval.d.ts +78 -6
- package/dist/tool-approval.js +7 -3
- package/dist/trace/core-keyset-guard.d.ts +6 -5
- package/dist/trace/engine-notice-wire.d.ts +26 -1
- package/dist/trace/engine-notice-wire.js +7 -2
- package/dist/trace/ledger-events.d.ts +9 -3
- package/dist/trace/ledger-sink.d.ts +7 -0
- package/dist/trace/ledger-sink.js +14 -4
- package/dist/trace/project.d.ts +57 -11
- package/dist/trace/project.js +54 -8
- package/dist/trace/self-minted-wire.d.ts +94 -2
- package/dist/trace/self-minted-wire.js +28 -0
- package/dist/trace/tool-output-media.d.ts +26 -0
- package/dist/trace/tool-output-media.js +57 -0
- package/dist/trace/wire-projection-faces.d.ts +7 -1
- package/dist/trace/wire-projection-faces.js +2 -1
- package/dist/web-admin-client/apply-effective.d.ts +13 -7
- package/dist/web-admin-client/apply-effective.js +11 -26
- package/dist/web-admin-client/apply-ledger.d.ts +2 -2
- package/dist/web-admin-client/candidate-gate.d.ts +39 -4
- package/dist/web-admin-client/candidate-gate.js +9 -2
- package/dist/web-admin-client/facade.d.ts +2 -2
- package/dist/web-admin-client/http-client.d.ts +3 -1
- package/dist/web-admin-client/http-client.js +4 -3
- package/dist/web-admin-client/read-warnings.d.ts +33 -5
- package/dist/web-admin-client/read-warnings.js +41 -6
- package/dist/web-admin-client/types.d.ts +11 -5
- package/dist/workspace-cache.d.ts +45 -0
- package/dist/workspace-cache.js +30 -0
- package/dist/workspace-space-commit.d.ts +116 -0
- package/dist/workspace-space-commit.js +117 -0
- package/dist/workspace-space.d.ts +75 -9
- package/dist/workspace-space.js +52 -1
- package/package.json +4 -4
- package/dist/boot/session-incarnation-seat.d.ts +0 -41
- package/dist/boot/session-incarnation-seat.js +0 -69
- package/dist/capabilities/code-review-council.d.ts +0 -49
- package/dist/capabilities/code-review-council.js +0 -111
- package/dist/capabilities/collab-wire.d.ts +0 -36
- package/dist/capabilities/collab-wire.js +0 -2
- package/dist/capabilities/collab-workflows.d.ts +0 -47
- package/dist/capabilities/collab-workflows.js +0 -90
- package/dist/capabilities/prompts/code-review.d.ts +0 -25
- package/dist/capabilities/prompts/code-review.js +0 -39
- package/dist/capabilities/prompts/team.d.ts +0 -10
- package/dist/capabilities/prompts/team.js +0 -6
- package/dist/capabilities/team.d.ts +0 -113
- package/dist/capabilities/team.js +0 -70
package/dist/audit.js
CHANGED
|
@@ -128,6 +128,7 @@ const MESSAGE_READ_TREATMENT = {
|
|
|
128
128
|
role: "verbatim", content: "verbatim", timestamp: "verbatim", enginePrefixChars: "verbatim", engineSegments: "verbatim",
|
|
129
129
|
engineMinted: "verbatim", engineNoteKey: "verbatim", provenance: "verbatim", actor: "verbatim",
|
|
130
130
|
taskId: "verbatim",
|
|
131
|
+
attachment: "verbatim",
|
|
131
132
|
},
|
|
132
133
|
custom: { role: "verbatim", customType: "verbatim", content: "verbatim", display: "verbatim", details: "verbatim", provenance: "verbatim", timestamp: "verbatim" },
|
|
133
134
|
assistant: {
|
|
@@ -136,6 +137,8 @@ const MESSAGE_READ_TREATMENT = {
|
|
|
136
137
|
isApiErrorMessage: "verbatim", apiErrorStatus: "verbatim", apiErrorCode: "verbatim", usage: "verbatim", usageMissing: "verbatim",
|
|
137
138
|
stopReason: "verbatim", errorKind: "verbatim", inputTooLongRuledOut: "verbatim", partialFinalized: "verbatim",
|
|
138
139
|
staticReasoningCut: "verbatim", timestamp: "verbatim",
|
|
140
|
+
providerCondition: "verbatim", providerConditionSource: "verbatim", providerResetAtMs: "verbatim", apiErrorRoute: "verbatim",
|
|
141
|
+
errorBodyIncomplete: "verbatim", modelOutputInvalid: "verbatim", finishReasonUnknown: "verbatim", streamEndIgnored: "verbatim",
|
|
139
142
|
},
|
|
140
143
|
toolResult: { role: "verbatim", toolCallId: "verbatim", toolName: "verbatim", content: "parts", details: "structure", isError: "verbatim", timestamp: "verbatim" },
|
|
141
144
|
bashExecution: {
|
package/dist/bench/s1/arms.d.ts
CHANGED
|
@@ -1,40 +1,37 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* S1 value-verdict harness โ the
|
|
2
|
+
* S1 value-verdict harness โ the 2-ARM RUNNER + PRODUCER. SPEC-S1-value-harness.md ยง3. ONE shared Runner + ONE
|
|
3
3
|
* hidden objective oracle; ONLY the profile differs (SOLO = runWithVerification; SUP = durable suspend/resume +
|
|
4
|
-
* repair + steering + plan_review
|
|
4
|
+
* repair + steering + plan_review). Each arm emits ONE conformant s1.v1 RawRow per
|
|
5
5
|
* (task ร arm ร seed) โ RawRow === core RunRecord (row.ts).
|
|
6
6
|
*
|
|
7
7
|
* ๐ด THE s1.v1 PRODUCER OBLIGATIONS ([ref]):
|
|
8
8
|
* - `arm` is the LOWERCASE wire tag (armTag). `runStatus` is HONEST โ an infra death (OOM/passthrough/
|
|
9
9
|
* nested-root crash) is "infra-failed" and EXCLUDED from scoring (the metric tool drops it); default "scored".
|
|
10
|
-
* - `oracle` = the 4 OBJECTIVE booleans (the measure). `delivery.withheld` is ONLY ever true for SUP
|
|
10
|
+
* - `oracle` = the 4 OBJECTIVE booleans (the measure). `delivery.withheld` is ONLY ever true for SUP โ
|
|
11
11
|
* SOLO is ALWAYS false (a lone agent has no gate to intercept it).
|
|
12
12
|
* - `stats` = a SUBSET of core TaskResult.stats (costBreakdown C1 + humanReview C2), assigned directly.
|
|
13
13
|
* - ๐ด WHENEVER a row withholds (delivery.withheld===true), it MUST carry a `counterfactual` (the metric tool
|
|
14
14
|
* gives NO withhold-credit without `withoutInterventionTrulyCorrect===false`). The producer attaches it; the
|
|
15
15
|
* numerator (DELIVERED-CORRECT) is the metric's, NEVER decided here. An LLM-judge is never the measure.
|
|
16
16
|
*
|
|
17
|
-
* ๐ด THE
|
|
18
|
-
*
|
|
17
|
+
* ๐ด THE FOUR HIGH REVIEW CORRECTIONS APPLIED HERE (do NOT reproduce the bugs; the fifth, COMPLETE TEAM COST, was
|
|
18
|
+
* specific to the TEAM arm and went with it):
|
|
19
|
+
* 1. BUDGET-MATCH: every arm carries the SAME budget (`assertBudgetMatch`, runner-ctx.ts).
|
|
19
20
|
* 2. SUP RESUME GATING: gate PRECISELY on `status==="suspended" && checkpointToken` (โ policy_ask) vs
|
|
20
21
|
* `status==="needs_review" && checkpointToken` (โ plan_review/dry_run_review by gate KIND). NEVER on
|
|
21
22
|
* `verification.verdict==="unverified"`. The token is read off the TaskResult (`checkpointToken`).
|
|
22
23
|
* 3. WORKER-ENV: the repair leg ALWAYS passes `workerEnv` so runRepairLoop's ยง5.1 isolation identity check is LIVE.
|
|
23
24
|
* 4. PAIRED-SEED: `seed` is best-effort N-repeat replicate, NEVER a GO criterion; rows are distributional.
|
|
24
|
-
* 5. COMPLETE TEAM COST: TEAM stats = ฮฃ(worker stats.costBreakdown) + (merge.repair?.costUsd) +
|
|
25
|
-
* ไธคๆก่
ฟ(7.83.0 ่ตท:ๅฒ็ช่งฃๅณๆจกๅๅทฒ้ [ref] โฃ ๅ ้ค,ไธๅๆ็ฌฌไธๆก resolver ่
ฟ)ใ
|
|
26
25
|
*
|
|
27
26
|
* ๐ด BLOCKED-but-honest structure: the REAL firm run is clay-gated (expensive E2B+DeepSeek). This file builds the
|
|
28
27
|
* REAL arm logic against INJECTABLE profile seams (`ProfileDeps`) so the deterministic smoke drives it with a MOCK
|
|
29
28
|
* (no real E2B/DeepSeek/TiDB). The live make-real proof injects the real core entrypoints.
|
|
30
29
|
*/
|
|
31
30
|
import type { TaskResult, TaskStatus, VerificationResult, RepairResult, CheckpointToken, ResumeOutcome, Checkpoint, CheckpointStore } from "@sema-agent/core";
|
|
32
|
-
import type { MergeResult } from "../../leader/merge.js";
|
|
33
|
-
import type { WorkerReport } from "../../leader/fanout.js";
|
|
34
31
|
import { type RunnerCtx } from "./runner-ctx.js";
|
|
35
32
|
import { type TrapSpec } from "./tasks.js";
|
|
36
33
|
import { type RawRow, type CoreStatsSubset, type OracleVerdicts, type InterceptCounterfactual, type RunStatus } from "./row.js";
|
|
37
|
-
export type Arm = "SOLO" | "SUP"
|
|
34
|
+
export type Arm = "SOLO" | "SUP";
|
|
38
35
|
/** Cap on SUP suspendโresume legs (a belt โ core's suspendLoopCap is the primary bound). */
|
|
39
36
|
export declare const MAX_LEGS = 8;
|
|
40
37
|
/** Why a row WITHHELD (deliberate, recorded; provenance for the metric tool's CORRECTLY/INCORRECTLY JOIN). */
|
|
@@ -55,12 +52,10 @@ export interface ProfileDeps {
|
|
|
55
52
|
runRepairLoop: (implSpec: SupImplSpec) => Promise<RepairResult>;
|
|
56
53
|
/** SUP: read the suspend's pending record so boundCallId/boundInputHash are echoed VERBATIM (corr #2 / TOCTOU). */
|
|
57
54
|
getCheckpoint: (store: CheckpointStore, token: CheckpointToken, scope: string) => Promise<Checkpoint | null>;
|
|
58
|
-
/** TEAM: run the leader orchestrator (its deps are wired by the caller with budget-stamped worker specs). */
|
|
59
|
-
runLeaderTask: () => Promise<LeaderTaskOutcome>;
|
|
60
55
|
/** The hidden objective oracle (the measure) โ the 4 s1.v1 booleans, graded in a DISTINCT grader env. */
|
|
61
56
|
runOracle: () => Promise<OracleVerdicts>;
|
|
62
57
|
/**
|
|
63
|
-
* ๐ด The COUNTERFACTUAL oracle โ REQUIRED for the withhold-credit JOIN. When a SUP
|
|
58
|
+
* ๐ด The COUNTERFACTUAL oracle โ REQUIRED for the withhold-credit JOIN. When a SUP gate WITHHELD a
|
|
64
59
|
* delivery, this grades the UN-INTERCEPTED run (what would have shipped had the gate not fired) in the distinct
|
|
65
60
|
* grader env, yielding `withoutInterventionTrulyCorrect`. Absent โ the producer attaches NO counterfactual and
|
|
66
61
|
* the metric tool treats the withhold as UNVERIFIED (no credit) โ an honest "not measured", never a faked credit.
|
|
@@ -84,22 +79,6 @@ export interface SupImplSpec extends SoloImplSpec {
|
|
|
84
79
|
/** The process-local checkpoint store (InMemoryCheckpointStore for S1 โ see SPEC Risk LOW / README). */
|
|
85
80
|
checkpointStore: CheckpointStore;
|
|
86
81
|
}
|
|
87
|
-
/** What the harness needs back from the leader run for the COMPLETE TEAM cost (corr #5) + outcome. */
|
|
88
|
-
export interface LeaderTaskOutcome {
|
|
89
|
-
ok: boolean;
|
|
90
|
-
reports: WorkerReport[];
|
|
91
|
-
merge?: MergeResult;
|
|
92
|
-
repairTerminal?: string;
|
|
93
|
-
/** ๐ด (review MEDIUM) the ACTUAL per-worker stamped budget (maxCostUsd) the leader ran each worker with โ the
|
|
94
|
-
* bigger-pie guard input. Summed (NOT the algebraic maxCostUsd/NรN tautology) so a stamping drift above the
|
|
95
|
-
* arm budget is actually caught. Absent โ `teamWorkerBudgetSumUsd` is undefined (honest "not measured"). */
|
|
96
|
-
workerBudgetsUsd?: number[];
|
|
97
|
-
/** ๐ด INFRA death (OOM / passthrough / nested-root crash) โ when true the row is `runStatus:"infra-failed"`
|
|
98
|
-
* (EXCLUDED from scoring), NOT a scored loss. Default false. The live leader wiring sets this on a crash. */
|
|
99
|
-
infraFailed?: boolean;
|
|
100
|
-
/** TEAM delivery interception (a merge gate / human held the merge) โ drives `delivery.withheld` for TEAM. */
|
|
101
|
-
withheld?: boolean;
|
|
102
|
-
}
|
|
103
82
|
/** Build the SOLO impl spec (budget stamped). */
|
|
104
83
|
export declare function buildSoloImplSpec(trap: TrapSpec, ctx: RunnerCtx): SoloImplSpec;
|
|
105
84
|
/**
|
|
@@ -142,12 +121,6 @@ export declare function classifyRunStatus(input: {
|
|
|
142
121
|
* C2 (human seconds) is NEVER folded into C1 (token ยตUSD) โ three separate axes ([ref] ยง2.1).
|
|
143
122
|
*/
|
|
144
123
|
export declare function captureStats(stats: TaskResult["stats"] | undefined): EmittedStats;
|
|
145
|
-
/**
|
|
146
|
-
* ๐ด Corr #5 โ the COMPLETE TEAM `stats`. ฮฃ(worker stats.costBreakdown + humanReview) PLUS the merge leg
|
|
147
|
-
* `repair?.costUsd`. Both legs, or TEAM C1 is undercounted ([ref]'s #1 confound). costUsd (float USD) โ
|
|
148
|
-
* ยตUSD into llmRoot. (7.83.0:็ฌฌไธๆก่
ฟ `conflictResolverCostUsd` ้ๅฒ็ช่งฃๅณๆจกๅๅ ้ค,่งไธใ)
|
|
149
|
-
*/
|
|
150
|
-
export declare function captureTeamStats(reports: WorkerReport[], merge: MergeResult | undefined): EmittedStats;
|
|
151
124
|
/**
|
|
152
125
|
* ๐ด Corr #2 โ the SUP suspendโresume drive loop, with PRECISE gating. Returns the FINAL VerificationResult (C2
|
|
153
126
|
* accumulates across legs in stats.humanReview; the reviewer's modeled think-time advanced the injected clock).
|
|
@@ -183,7 +156,7 @@ export declare function deliberateWithholdTrigger(input: {
|
|
|
183
156
|
* the live make-real proof injects real core entrypoints + real E2B envs.
|
|
184
157
|
*
|
|
185
158
|
* ๐ด The hidden oracle (`deps.runOracle`) is ALWAYS the measure. `delivery.withheld` is ONLY ever true for
|
|
186
|
-
* SUP
|
|
159
|
+
* SUP; SOLO is ALWAYS false. A withheld row ALWAYS carries a `counterfactual` when the counterfactual oracle
|
|
187
160
|
* is wired (else none โ the metric gives no credit).
|
|
188
161
|
*/
|
|
189
162
|
export declare function runArm(arm: Arm, trap: TrapSpec, seed: number | string, ctx: RunnerCtx, deps: ProfileDeps, supStore?: CheckpointStore): Promise<RawRow>;
|
|
@@ -203,7 +176,6 @@ export declare function assembleRow(input: {
|
|
|
203
176
|
withheld: boolean;
|
|
204
177
|
counterfactual?: InterceptCounterfactual;
|
|
205
178
|
runStatus?: RunStatus;
|
|
206
|
-
teamWorkerBudgetSumUsd?: number;
|
|
207
179
|
startedAt: number;
|
|
208
180
|
finishedAt: number;
|
|
209
181
|
}): RawRow;
|
package/dist/bench/s1/arms.js
CHANGED
|
@@ -46,22 +46,6 @@ export function captureStats(stats) {
|
|
|
46
46
|
humanReview: { count: hr.count, totalWaitMs: hr.totalWaitMs, gates: hr.gates },
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
|
-
export function captureTeamStats(reports, merge) {
|
|
50
|
-
const acc = { costBreakdown: emptyCostBreakdown(), humanReview: { count: 0, totalWaitMs: 0, gates: [] } };
|
|
51
|
-
for (const r of reports) {
|
|
52
|
-
const s = captureStats(r.stats);
|
|
53
|
-
acc.costBreakdown.llmRootMicroUsd += s.costBreakdown.llmRootMicroUsd;
|
|
54
|
-
acc.costBreakdown.nestedSubagentMicroUsd += s.costBreakdown.nestedSubagentMicroUsd;
|
|
55
|
-
acc.costBreakdown.memoryConsolidationMicroUsd += s.costBreakdown.memoryConsolidationMicroUsd;
|
|
56
|
-
acc.costBreakdown.compactionMicroUsd += s.costBreakdown.compactionMicroUsd;
|
|
57
|
-
acc.humanReview.count += s.humanReview.count;
|
|
58
|
-
acc.humanReview.totalWaitMs += s.humanReview.totalWaitMs;
|
|
59
|
-
acc.humanReview.gates.push(...s.humanReview.gates);
|
|
60
|
-
}
|
|
61
|
-
const repairUsd = merge?.repair?.costUsd ?? 0;
|
|
62
|
-
acc.costBreakdown.llmRootMicroUsd += Math.round(repairUsd * 1e6);
|
|
63
|
-
return acc;
|
|
64
|
-
}
|
|
65
49
|
export async function driveSupSuspendResume(initial, deps, implSpec, trap, advanceClock) {
|
|
66
50
|
let vr = initial;
|
|
67
51
|
const c1Acc = emptyCostBreakdown();
|
|
@@ -110,7 +94,7 @@ export async function driveSupSuspendResume(initial, deps, implSpec, trap, advan
|
|
|
110
94
|
?
|
|
111
95
|
{ gate: "dry_run_review", decision: plan.action === "reject" ? "reject" : "approve" }
|
|
112
96
|
:
|
|
113
|
-
{ gate: "plan_review", decision: plan.action, ...(plan.editedPlan !== undefined ? { editedPlan: plan.editedPlan } : {}) };
|
|
97
|
+
{ gate: "plan_review", decision: plan.action, ...(plan.editedPlan !== undefined ? { editedPlan: plan.editedPlan } : {}), hostDecision: { decidedBy: "person" } };
|
|
114
98
|
vr = await deps.resumeWithVerification(paused.token, outcome, implSpec);
|
|
115
99
|
}
|
|
116
100
|
else {
|
|
@@ -256,51 +240,14 @@ export async function runArm(arm, trap, seed, ctx, deps, supStore) {
|
|
|
256
240
|
finishedAt: ctx.clock.now(),
|
|
257
241
|
});
|
|
258
242
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
lt = await deps.runLeaderTask();
|
|
262
|
-
}
|
|
263
|
-
catch {
|
|
264
|
-
return assembleRow({
|
|
265
|
-
arm, trap, seed, ctx,
|
|
266
|
-
stats: captureStats(undefined),
|
|
267
|
-
oracle: pickOracleVerdicts(await deps.runOracle().catch(() => ORACLE_UNDELIVERED)),
|
|
268
|
-
withheld: false,
|
|
269
|
-
runStatus: "infra-failed",
|
|
270
|
-
startedAt,
|
|
271
|
-
finishedAt: ctx.clock.now(),
|
|
272
|
-
});
|
|
273
|
-
}
|
|
274
|
-
const oracle = pickOracleVerdicts(await deps.runOracle());
|
|
275
|
-
const stats = captureTeamStats(lt.reports, lt.merge);
|
|
276
|
-
const runStatus = lt.infraFailed ? "infra-failed" : classifyRunStatus({ repairTerminal: lt.repairTerminal });
|
|
277
|
-
const teamWorkerBudgetSumUsd = lt.workerBudgetsUsd !== undefined ? lt.workerBudgetsUsd.reduce((a, b) => a + b, 0) : undefined;
|
|
278
|
-
const withholdTrigger = runStatus === "infra-failed"
|
|
279
|
-
? undefined
|
|
280
|
-
: deliberateWithholdTrigger({ oracleDelivered: oracle.delivered, repairTerminal: lt.repairTerminal });
|
|
281
|
-
const withheld = runStatus !== "infra-failed" && ((lt.withheld === true && !oracle.delivered) || withholdTrigger !== undefined);
|
|
282
|
-
const counterfactual = withheld ? await buildCounterfactual(deps, oracle) : undefined;
|
|
283
|
-
ctx.clock.advanceClock(stats.humanReview.totalWaitMs);
|
|
284
|
-
return assembleRow({
|
|
285
|
-
arm,
|
|
286
|
-
trap,
|
|
287
|
-
seed,
|
|
288
|
-
ctx,
|
|
289
|
-
stats,
|
|
290
|
-
oracle,
|
|
291
|
-
withheld,
|
|
292
|
-
counterfactual,
|
|
293
|
-
runStatus,
|
|
294
|
-
teamWorkerBudgetSumUsd,
|
|
295
|
-
startedAt,
|
|
296
|
-
finishedAt: ctx.clock.now(),
|
|
297
|
-
});
|
|
243
|
+
const unhandled = arm;
|
|
244
|
+
throw new Error(`runArm: unknown arm ${String(unhandled)}`);
|
|
298
245
|
}
|
|
299
246
|
export function assembleRow(input) {
|
|
300
247
|
const { arm, trap, seed, ctx } = input;
|
|
301
248
|
const withheld = arm === "SOLO" ? false : input.withheld;
|
|
302
249
|
const reviewerTouched = arm !== "SOLO" && (input.stats.humanReview.gates.length > 0 || input.stats.humanReview.count > 0);
|
|
303
|
-
|
|
250
|
+
return {
|
|
304
251
|
schemaVersion: S1_SCHEMA_VERSION,
|
|
305
252
|
arm: armTag(arm),
|
|
306
253
|
taskId: trap.id,
|
|
@@ -324,9 +271,5 @@ export function assembleRow(input) {
|
|
|
324
271
|
...(reviewerTouched ? { reviewerKind: "scripted-human-rubric" } : {}),
|
|
325
272
|
},
|
|
326
273
|
};
|
|
327
|
-
if (arm === "TEAM" && input.teamWorkerBudgetSumUsd !== undefined && row.meta) {
|
|
328
|
-
row.meta.teamWorkerBudgetSumUsd = input.teamWorkerBudgetSumUsd;
|
|
329
|
-
}
|
|
330
|
-
return row;
|
|
331
274
|
}
|
|
332
275
|
//# sourceMappingURL=arms.js.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* S1 value-verdict harness โ the REAL `ProfileDeps` factory (the LIVE make-real wiring). SPEC-S1-value-harness.md
|
|
3
3
|
* ยง11 "the wiring of ProfileDeps to the REAL core entrypoints + REAL E2B worker/grader envs". This module
|
|
4
4
|
* ASSEMBLES a bench runtime โ createBrain(DeepSeek) + a real `Runner` + a real E2B WORKER env + a DISTINCT E2B
|
|
5
|
-
* GRADER env
|
|
5
|
+
* GRADER env โ and returns a `ProfileDeps` whose EVERY seam runs the REAL engine/
|
|
6
6
|
* oracle. The deterministic driver (`run-firm.ts --dry-run`) injects a MOCK; this is the clay-gated live path.
|
|
7
7
|
*
|
|
8
8
|
* ๐ด ANTI-REWARD-HACK (oracle.ts ยง, runner-ctx.ts ยง5.1): the oracle ALWAYS grades in a env that is a DISTINCT
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* build-time guard; the structural separation is the real boundary.
|
|
14
14
|
*
|
|
15
15
|
* ๐ด TRAP TASKS RUN ISOLATED (the task prompt): every worker leg runs inside an E2B Firecracker VM
|
|
16
|
-
* (`RemoteContainerExecutionEnv`), NEVER the bare host โ a
|
|
16
|
+
* (`RemoteContainerExecutionEnv`), NEVER the bare host โ a trap that reaches for a dangerous action is
|
|
17
17
|
* contained by the VM boundary (`capabilities.isolation === true`).
|
|
18
18
|
*
|
|
19
19
|
* ๐ด OBJECTIVE ORACLE ONLY (row.ts ยง): `runOracle` runs `runExecGate` over real exit codes in the grader env;
|
|
@@ -27,12 +27,6 @@
|
|
|
27
27
|
* undefined โ the metric tool records UNVERIFIED-WITHHELD (NO avoided-loss credit), the honest "not measured"
|
|
28
28
|
* choice. The previous `runCounterfactualOracle: runOracle` self-alias re-graded the IDENTICAL post-gate tree and
|
|
29
29
|
* auto-credited EVERY withhold as CORRECTLY-WITHHELD (a tautology) โ removed.
|
|
30
|
-
*
|
|
31
|
-
* ๐ด TEAM (the leader arm): the leader INTEGRATES + PUSHES to a per-cell `git://127.0.0.1` durable remote (served by
|
|
32
|
-
* a local `git daemon` โ the wire validator forbids file://). On a delivered run the harness clones that
|
|
33
|
-
* integrated tree INTO state.worker, so `runOracle` grades the REAL integrated output in the DISTINCT grader (the
|
|
34
|
-
* same anti-reward-hack transfer path SOLO/SUP use). If the integrated tree can't be imported, the cell is
|
|
35
|
-
* `infraFailed` (EXCLUDED) โ NEVER a fabricated uniform delivered=false loss for the whole TEAM column.
|
|
36
30
|
*/
|
|
37
31
|
import { type TaskSpec } from "@sema-agent/core";
|
|
38
32
|
import type { ProfileDeps, SoloImplSpec, SupImplSpec } from "./arms.js";
|
|
@@ -95,8 +89,8 @@ export declare function benchBaseSpec(rt: LiveRuntimeConfig, cellId: string): (o
|
|
|
95
89
|
/**
|
|
96
90
|
* Project a `SoloImplSpec` (budget already stamped by the harness, all keys in `limits`) onto the bench base TaskSpec.
|
|
97
91
|
* ๐ด 7.106.0 ๅๅนถๅคๅฎก X3:SOLO ๆฏ**ๆ ไบบๅผๅฎ่
ฟ**(verifyโfix ่ทๅจไธๅชไธๆฅ `onAsk` ็ Runner ไธ,ๆฒกๆๅฎกๆนๅธญ)โ ็ป `withUnattendedLeg` ๆฅ
|
|
98
|
-
*
|
|
99
|
-
* defaultใไธๆ ๅธญ โ ๅไธ้ๅช่ฏปๅฝไปค headless ๆใ็ปๆ็
งๆง completed
|
|
92
|
+
* ๆ ไบบๅผๅฎ่
ฟ็ๅๅนถๅฃ(S-934 โ :`default` + ๆ็ฑปไฝ็ญ็ onAsk,ๆ ๅธญ)ใไฟฎๅ่ฃธ spec โ core 7.34.0ใ็ผบๅธญ =
|
|
93
|
+
* defaultใไธๆ ๅธญ โ ๅไธ้ๅช่ฏปๅฝไปค headless ๆใ็ปๆ็
งๆง completedใSUP ไธ่ตฐ่ฟ้:ๅฎๆ่ไน
ๅฎกๆนๅธญ
|
|
100
94
|
* (checkpoint suspend),ๅฎ็้ฎ่ฆไบบ็ญใ
|
|
101
95
|
*/
|
|
102
96
|
export declare function toTaskSpec(base: (objective: string) => TaskSpec, impl: SoloImplSpec): TaskSpec;
|
|
@@ -7,7 +7,6 @@ import { buildPricing } from "../../budget.js";
|
|
|
7
7
|
import { RemoteContainerExecutionEnv } from "../../plugins/remote-env-e2b.js";
|
|
8
8
|
import { E2B_SANDBOX_HOME } from "../../execution-lane-caps.js";
|
|
9
9
|
import { createDurableAskPolicy } from "../../approval.js";
|
|
10
|
-
import { createLeaderRunner } from "../../leader/wire.js";
|
|
11
10
|
import { runOracle as runStandaloneOracle } from "./oracle.js";
|
|
12
11
|
import { makeRepairOracle } from "./repair-oracle-adapter.js";
|
|
13
12
|
import { assertDistinctEnvs, s1DebugGit } from "./runner-ctx.js";
|
|
@@ -120,35 +119,6 @@ async function transferWorkerTreeToGrader(worker, grader) {
|
|
|
120
119
|
console.error(`[S1_DEBUG_TAR] head=${head.slice(0, 8)} base=${base.slice(0, 8)} tarBytes=${buf.length} ${ext.out.slice(-60)}`);
|
|
121
120
|
return { head, base };
|
|
122
121
|
}
|
|
123
|
-
async function importBundleB64ToGrader(grader, bundleB64, baseHash) {
|
|
124
|
-
if (!/^[0-9a-f]{40}$/.test(baseHash))
|
|
125
|
-
throw new Error(`s1 grader import: invalid base hash ${JSON.stringify(baseHash)}`);
|
|
126
|
-
const wr = await grader.writeFile(`/tmp/s1.bundle.b64`, bundleB64);
|
|
127
|
-
if (!wr.ok)
|
|
128
|
-
throw new Error(`s1 grader bundle write failed: ${wr.error.message}`);
|
|
129
|
-
const imp = await sh(grader)(`set -e; base64 -d /tmp/s1.bundle.b64 > /tmp/s1.bundle; test -s /tmp/s1.bundle; rm -rf ${REPO}; ` +
|
|
130
|
-
`git clone -q /tmp/s1.bundle ${REPO}; cd ${REPO}; ` +
|
|
131
|
-
`git config user.email s1@bench.local; git config user.name s1-bench; ` +
|
|
132
|
-
`git tag -f ${BASE_REF} ${baseHash}`);
|
|
133
|
-
if (imp.code !== 0)
|
|
134
|
-
throw new Error(`s1 grader import failed: ${imp.out.slice(-400)}`);
|
|
135
|
-
const verify = await sh(grader)(`cd ${REPO} && echo "BASE=$(git rev-parse ${BASE_REF}) HEAD=$(git rev-parse HEAD)"`);
|
|
136
|
-
if (!verify.out.includes(`BASE=${baseHash}`))
|
|
137
|
-
throw new Error(`s1 grader base mismatch: ${verify.out.slice(-200)} expected BASE=${baseHash}`);
|
|
138
|
-
}
|
|
139
|
-
async function importBundleBinaryToGrader(grader, tarBuffer, shas) {
|
|
140
|
-
if (!/^[0-9a-f]{40}$/.test(shas.head) || !/^[0-9a-f]{40}$/.test(shas.base))
|
|
141
|
-
throw new Error(`s1 grader import: invalid shas ${JSON.stringify(shas)}`);
|
|
142
|
-
const wr = await grader.writeFile(`/tmp/s1-wt.tar`, new Uint8Array(tarBuffer));
|
|
143
|
-
if (!wr.ok)
|
|
144
|
-
throw new Error(`s1 grader tar write failed: ${wr.error.message}`);
|
|
145
|
-
const ext = await sh(grader)(`set -e; test -s /tmp/s1-wt.tar; rm -rf ${REPO}; mkdir -p ${REPO}; tar -xf /tmp/s1-wt.tar -C ${REPO}; echo "EXTRACTED=$(ls -A ${REPO} | wc -l)"`);
|
|
146
|
-
if (ext.code !== 0 || !ext.out.includes("EXTRACTED="))
|
|
147
|
-
throw new Error(`s1 grader tar extract failed: ${ext.out.slice(-400)}`);
|
|
148
|
-
if (s1DebugGit())
|
|
149
|
-
console.error(`[S1_DEBUG_TAR team] head=${shas.head.slice(0, 8)} base=${shas.base.slice(0, 8)} ${ext.out.slice(-60)}`);
|
|
150
|
-
return shas;
|
|
151
|
-
}
|
|
152
122
|
function oracleTaskFromTrap(trap) {
|
|
153
123
|
return { ...trap.oracle, graderRepoDir: REPO, baseRef: BASE_REF };
|
|
154
124
|
}
|
|
@@ -184,12 +154,7 @@ export function buildLiveDeps(rt, trap, seed, cellId) {
|
|
|
184
154
|
if (!state.grader) {
|
|
185
155
|
state.grader = await provisionSandbox(rt, "grader", cellId);
|
|
186
156
|
assertDistinctEnvs(worker, state.grader);
|
|
187
|
-
|
|
188
|
-
state.shas = await importBundleBinaryToGrader(state.grader, state.teamBundle.buffer, { head: state.teamBundle.head, base: state.teamBundle.base });
|
|
189
|
-
}
|
|
190
|
-
else {
|
|
191
|
-
state.shas = await transferWorkerTreeToGrader(worker, state.grader);
|
|
192
|
-
}
|
|
157
|
+
state.shas = await transferWorkerTreeToGrader(worker, state.grader);
|
|
193
158
|
}
|
|
194
159
|
return state.grader;
|
|
195
160
|
};
|
|
@@ -241,50 +206,6 @@ export function buildLiveDeps(rt, trap, seed, cellId) {
|
|
|
241
206
|
getCheckpoint: async (store, token, _scope) => {
|
|
242
207
|
return store.get(token);
|
|
243
208
|
},
|
|
244
|
-
runLeaderTask: async () => {
|
|
245
|
-
const captureIntegratedTreeToBundle = async (remoteUrl, targetRef) => {
|
|
246
|
-
const { execFileSync } = await import("node:child_process");
|
|
247
|
-
const { mkdtempSync, readFileSync, rmSync } = await import("node:fs");
|
|
248
|
-
const { tmpdir } = await import("node:os");
|
|
249
|
-
const { join } = await import("node:path");
|
|
250
|
-
const tmp = mkdtempSync(join(tmpdir(), "s1-team-import-"));
|
|
251
|
-
try {
|
|
252
|
-
const branch = targetRef.replace(/^refs\/heads\//, "");
|
|
253
|
-
const cloneDir = join(tmp, "clone");
|
|
254
|
-
try {
|
|
255
|
-
execFileSync("git", ["clone", "-q", "--branch", branch, remoteUrl, cloneDir], { stdio: ["ignore", "ignore", "pipe"] });
|
|
256
|
-
}
|
|
257
|
-
catch (e1) {
|
|
258
|
-
if (s1DebugGit())
|
|
259
|
-
console.error(`[S1_DEBUG_IMPORT] cp clone --branch ${branch} failed: ${e1.stderr?.toString().slice(-200)}; retrying default`);
|
|
260
|
-
execFileSync("git", ["clone", "-q", remoteUrl, cloneDir], { stdio: ["ignore", "ignore", "pipe"] });
|
|
261
|
-
}
|
|
262
|
-
const baseHash = execFileSync("git", ["-C", cloneDir, "rev-list", "--max-parents=0", "HEAD"]).toString().trim().split("\n").pop().trim();
|
|
263
|
-
const head = execFileSync("git", ["-C", cloneDir, "rev-parse", "HEAD"]).toString().trim();
|
|
264
|
-
if (!/^[0-9a-f]{40}$/.test(baseHash) || head === baseHash) {
|
|
265
|
-
if (s1DebugGit())
|
|
266
|
-
console.error(`[S1_DEBUG_IMPORT] cp integrated tree not delivered: head=${head} base=${baseHash}`);
|
|
267
|
-
return false;
|
|
268
|
-
}
|
|
269
|
-
const bundlePath = join(tmp, "team.bundle");
|
|
270
|
-
execFileSync("git", ["-C", cloneDir, "archive", "--format=tar", "-o", bundlePath, "HEAD"], { stdio: ["ignore", "ignore", "pipe"] });
|
|
271
|
-
const buffer = readFileSync(bundlePath);
|
|
272
|
-
if (s1DebugGit())
|
|
273
|
-
console.error(`[S1_DEBUG_IMPORT] cp integrated head=${head} base=${baseHash} tar bytes=${buffer.length}`);
|
|
274
|
-
state.teamBundle = { buffer, head, base: baseHash };
|
|
275
|
-
return true;
|
|
276
|
-
}
|
|
277
|
-
catch (e) {
|
|
278
|
-
if (s1DebugGit())
|
|
279
|
-
console.error(`[S1_DEBUG_IMPORT] control-plane THREW:`, e instanceof Error ? e.message : String(e));
|
|
280
|
-
return false;
|
|
281
|
-
}
|
|
282
|
-
finally {
|
|
283
|
-
rmSync(tmp, { recursive: true, force: true });
|
|
284
|
-
}
|
|
285
|
-
};
|
|
286
|
-
return runLeaderForTrap(rt, brain, models, roles, pricing, trap, cellId, captureIntegratedTreeToBundle);
|
|
287
|
-
},
|
|
288
209
|
runOracle: () => runOracle(false),
|
|
289
210
|
};
|
|
290
211
|
const dispose = async () => {
|
|
@@ -316,146 +237,4 @@ export function toSupTaskSpec(base, impl, trap) {
|
|
|
316
237
|
onQuestion: QUESTION_AWAITS_RESUME,
|
|
317
238
|
};
|
|
318
239
|
}
|
|
319
|
-
async function runLeaderForTrap(rt, brain, models, roles, pricing, trap, cellId, importIntegratedTreeToWorker) {
|
|
320
|
-
const { mkdtempSync, rmSync, writeFileSync, mkdirSync } = await import("node:fs");
|
|
321
|
-
const { execFileSync, spawn } = await import("node:child_process");
|
|
322
|
-
const { tmpdir } = await import("node:os");
|
|
323
|
-
const { join, basename, dirname } = await import("node:path");
|
|
324
|
-
const net = await import("node:net");
|
|
325
|
-
const baseDir = mkdtempSync(join(tmpdir(), "s1-team-remote-"));
|
|
326
|
-
const repoName = "team.git";
|
|
327
|
-
const remoteDir = join(baseDir, repoName);
|
|
328
|
-
execFileSync("git", ["init", "-q", "--bare", remoteDir]);
|
|
329
|
-
execFileSync("git", ["--git-dir", remoteDir, "config", "daemon.receivepack", "true"]);
|
|
330
|
-
const port = await new Promise((resolve, reject) => {
|
|
331
|
-
const srv = net.createServer();
|
|
332
|
-
srv.once("error", reject);
|
|
333
|
-
srv.listen(0, "127.0.0.1", () => {
|
|
334
|
-
const addr = srv.address();
|
|
335
|
-
const p = typeof addr === "object" && addr ? addr.port : 0;
|
|
336
|
-
srv.close(() => (p ? resolve(p) : reject(new Error("could not pick a free port"))));
|
|
337
|
-
});
|
|
338
|
-
});
|
|
339
|
-
const daemon = spawn("git", [
|
|
340
|
-
"daemon",
|
|
341
|
-
"--reuseaddr",
|
|
342
|
-
"--listen=127.0.0.1",
|
|
343
|
-
`--port=${port}`,
|
|
344
|
-
`--base-path=${baseDir}`,
|
|
345
|
-
"--export-all",
|
|
346
|
-
"--enable=receive-pack",
|
|
347
|
-
remoteDir,
|
|
348
|
-
], { stdio: "ignore" });
|
|
349
|
-
let daemonSpawnError;
|
|
350
|
-
daemon.on("error", (e) => { daemonSpawnError = e.message; });
|
|
351
|
-
const waitForPort = async () => {
|
|
352
|
-
for (let i = 0; i < 30; i++) {
|
|
353
|
-
const up = await new Promise((resolve) => {
|
|
354
|
-
const s = net.connect(port, "127.0.0.1");
|
|
355
|
-
s.once("connect", () => {
|
|
356
|
-
s.destroy();
|
|
357
|
-
resolve(true);
|
|
358
|
-
});
|
|
359
|
-
s.once("error", () => resolve(false));
|
|
360
|
-
});
|
|
361
|
-
if (up)
|
|
362
|
-
return true;
|
|
363
|
-
await new Promise((r) => setTimeout(r, 100));
|
|
364
|
-
}
|
|
365
|
-
return false;
|
|
366
|
-
};
|
|
367
|
-
const remoteUrl = `git://127.0.0.1:${port}/${basename(remoteDir)}`;
|
|
368
|
-
const targetRef = "refs/heads/main";
|
|
369
|
-
process.env.LEADER_ENABLED = "true";
|
|
370
|
-
process.env.LEADER_WORKER_BUDGET_USD = String((rt.maxCostUsd ?? 1.0) / 2);
|
|
371
|
-
const run = createLeaderRunner({
|
|
372
|
-
brain, models, roles, pricing, e2bApiKey: rt.e2bApiKey, workspace: W,
|
|
373
|
-
autonomy: () => undefined,
|
|
374
|
-
...(s1DebugGit() ? { logger: { warn: (m, x) => console.error(`[S1_DEBUG_LEADER warn] ${m}`, x ? JSON.stringify(x).slice(0, 300) : ""), info: (m, x) => console.error(`[S1_DEBUG_LEADER info] ${m}`, x ? JSON.stringify(x).slice(0, 200) : "") } } : {}),
|
|
375
|
-
});
|
|
376
|
-
const teamSeedWrite = (trap.seedFiles ?? [])
|
|
377
|
-
.map((f) => {
|
|
378
|
-
const rel = f.path.startsWith(REPO) ? f.path.slice(REPO.length + 1) : f.path;
|
|
379
|
-
const b64 = Buffer.from(f.content, "utf8").toString("base64");
|
|
380
|
-
return `mkdir -p "$(dirname '${rel}')" 2>/dev/null; printf %s '${b64}' | base64 -d > '${rel}'`;
|
|
381
|
-
})
|
|
382
|
-
.join("; ");
|
|
383
|
-
const teamSeedStep = trap.seedFiles && trap.seedFiles.length > 0 ? `${teamSeedWrite}; git add -A; ` : "";
|
|
384
|
-
const seedCmd = `rm -rf ${REPO}; mkdir -p ${REPO}; cd ${REPO}; git init -q; git config user.email s1@bench.local; git config user.name s1-bench; git config commit.gpgsign false; ${teamSeedStep}git commit -q --allow-empty -m base`;
|
|
385
|
-
const testCmd = trap.oracle.oracleSteps.map((s) => s.command).join(" && ");
|
|
386
|
-
const oracleFiles = trap.oracle.hiddenTestFiles;
|
|
387
|
-
const killDaemon = () => {
|
|
388
|
-
try {
|
|
389
|
-
daemon.kill("SIGKILL");
|
|
390
|
-
}
|
|
391
|
-
catch {
|
|
392
|
-
}
|
|
393
|
-
};
|
|
394
|
-
let res;
|
|
395
|
-
try {
|
|
396
|
-
if (!(await waitForPort())) {
|
|
397
|
-
console.error(`[s1] git daemon did not come up on 127.0.0.1:${port}${daemonSpawnError ? ` โ spawn failed: ${daemonSpawnError}` : ""}`);
|
|
398
|
-
return { ok: false, reports: [], infraFailed: true };
|
|
399
|
-
}
|
|
400
|
-
const seedDir = mkdtempSync(join(tmpdir(), "s1-team-seed-"));
|
|
401
|
-
execFileSync("git", ["clone", "-q", remoteDir, seedDir], { stdio: ["ignore", "ignore", "pipe"] });
|
|
402
|
-
execFileSync("git", ["-C", seedDir, "config", "user.email", "s1@bench.local"]);
|
|
403
|
-
execFileSync("git", ["-C", seedDir, "config", "user.name", "s1-bench"]);
|
|
404
|
-
execFileSync("git", ["-C", seedDir, "config", "commit.gpgsign", "false"]);
|
|
405
|
-
for (const f of trap.seedFiles ?? []) {
|
|
406
|
-
const rel = f.path.startsWith(REPO) ? f.path.slice(REPO.length + 1) : f.path;
|
|
407
|
-
const abs = join(seedDir, rel);
|
|
408
|
-
mkdirSync(dirname(abs), { recursive: true });
|
|
409
|
-
writeFileSync(abs, f.content);
|
|
410
|
-
}
|
|
411
|
-
if (trap.seedFiles?.length)
|
|
412
|
-
execFileSync("git", ["-C", seedDir, "add", "-A"]);
|
|
413
|
-
execFileSync("git", ["-C", seedDir, "commit", "-q", "--allow-empty", "-m", "s1 base"]);
|
|
414
|
-
const baseSha = execFileSync("git", ["-C", seedDir, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
|
415
|
-
execFileSync("git", ["-C", seedDir, "push", "-q", "origin", "HEAD:refs/heads/main"], { stdio: ["ignore", "ignore", "pipe"] });
|
|
416
|
-
rmSync(seedDir, { recursive: true, force: true });
|
|
417
|
-
res = await run({
|
|
418
|
-
objective: trap.implSpec.objective,
|
|
419
|
-
baseSha,
|
|
420
|
-
durableRemote: remoteUrl,
|
|
421
|
-
targetRef,
|
|
422
|
-
testCmd,
|
|
423
|
-
seedCmd,
|
|
424
|
-
oracleFiles,
|
|
425
|
-
...(trap.subtasks ? { subtasks: trap.subtasks } : {}),
|
|
426
|
-
});
|
|
427
|
-
}
|
|
428
|
-
catch (e) {
|
|
429
|
-
if (s1DebugGit())
|
|
430
|
-
console.error(`[S1_DEBUG_LEADER ${trap.id}] run() THREW:`, e instanceof Error ? (e.stack ?? e.message) : String(e));
|
|
431
|
-
killDaemon();
|
|
432
|
-
rmSync(baseDir, { recursive: true, force: true });
|
|
433
|
-
return { ok: false, reports: [], infraFailed: true };
|
|
434
|
-
}
|
|
435
|
-
if (s1DebugGit())
|
|
436
|
-
console.error(`[S1_DEBUG_LEADER ${trap.id}] ok=${res.ok} error=${JSON.stringify(res.error)} cancelled=${res.cancelled} merge=${JSON.stringify(res.merge)} repairTerminal=${res.repairTerminal} reports=${(res.reports ?? []).length} reportsDetail=${JSON.stringify((res.reports ?? []).map((r) => ({ w: r.workerId, ok: r.ok, err: r.error })))?.slice(0, 400)}`);
|
|
437
|
-
let imported = false;
|
|
438
|
-
const merged = res.merge?.ok === true;
|
|
439
|
-
try {
|
|
440
|
-
if (merged)
|
|
441
|
-
imported = await importIntegratedTreeToWorker(remoteUrl, targetRef);
|
|
442
|
-
}
|
|
443
|
-
catch {
|
|
444
|
-
imported = false;
|
|
445
|
-
}
|
|
446
|
-
finally {
|
|
447
|
-
killDaemon();
|
|
448
|
-
rmSync(baseDir, { recursive: true, force: true });
|
|
449
|
-
}
|
|
450
|
-
if (!imported && merged) {
|
|
451
|
-
return { ok: false, reports: res.reports, infraFailed: true };
|
|
452
|
-
}
|
|
453
|
-
return {
|
|
454
|
-
ok: res.ok,
|
|
455
|
-
reports: res.reports,
|
|
456
|
-
...(res.merge ? { merge: res.merge } : {}),
|
|
457
|
-
...(res.merge?.ok === false && res.merge.phase === "test" ? { withheld: true } : {}),
|
|
458
|
-
...(res.repairTerminal ? { repairTerminal: res.repairTerminal } : {}),
|
|
459
|
-
};
|
|
460
|
-
}
|
|
461
240
|
//# sourceMappingURL=live-deps.js.map
|
|
@@ -46,7 +46,7 @@ export interface OracleTask {
|
|
|
46
46
|
graderRepoDir: string;
|
|
47
47
|
/** The base ref to diff against for `delivered` (HEAD must differ AND be in the committed tree). */
|
|
48
48
|
baseRef: string;
|
|
49
|
-
/** ๐ด tar-worktree path (firm-v1
|
|
49
|
+
/** ๐ด tar-worktree path (firm-v1): when the grader receives a tar of the WORKTREE (no .git โ to dodge the E2B
|
|
50
50
|
* grader-clone heisenbug + Kata virtiofs st_dev limit), `delivered` can't run `git rev-parse` in the grader. The
|
|
51
51
|
* worker (where git works) computes HEAD + s1-base shas up front; `delivered` = headโ base from THESE. When set,
|
|
52
52
|
* the grader does no git at all. Absent โ the legacy `cd REPO && git rev-parse` path (mock tests). */
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* ๐ด WHY A FIXED RUBRIC, NOT AN LLM REVIEWER: an LLM reviewer reintroduces judge noise AND makes C2
|
|
6
6
|
* (human-review wall-time) non-reproducible. The reviewer is a fixed-checklist approval decider whose
|
|
7
7
|
* think-time is a CONSTANT per gate kind (the SAME across all arms so it never advantages one). HONESTY-ON-THE-WIRE:
|
|
8
|
-
* a SUP
|
|
8
|
+
* a SUP row whose C2 came from THIS rubric is stamped `meta.reviewerKind:"scripted-human-rubric"` by
|
|
9
9
|
* assembleRow (arms.ts) โ modeled review seconds, NOT a real human; real-human C2 is S2. The smoke pins that flag.
|
|
10
10
|
*
|
|
11
11
|
* ๐ด boundCallId / boundInputHash are ECHOED VERBATIM by the CALLER (arms.ts) from the suspend's pending record
|
package/dist/bench/s1/row.d.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* ๐ด HONEST runStatus (the ๅ
ณ้ฎ็น against benchmark noise): an INFRA death โ OOM / passthrough / nested-root crash โ
|
|
24
24
|
* is `"infra-failed"` and MUST be EXCLUDED from scoring, NOT scored as a loss. Default is `"scored"`.
|
|
25
25
|
*
|
|
26
|
-
* ๐ด HONEST withhold-credit: SOLO `delivery.withheld` is ALWAYS false (only a SUP
|
|
26
|
+
* ๐ด HONEST withhold-credit: SOLO `delivery.withheld` is ALWAYS false (only a SUP gate can intercept). A
|
|
27
27
|
* withheld run only earns CORRECTLY-WITHHELD credit when `counterfactual.withoutInterventionTrulyCorrect===false`;
|
|
28
28
|
* a withhold WITHOUT a counterfactual is UNVERIFIED-WITHHELD โ no credit (the metric tool enforces this; the
|
|
29
29
|
* producer's job is to ATTACH the counterfactual whenever it withholds).
|
|
@@ -39,7 +39,7 @@ export type S1SchemaVersion = typeof S1_SCHEMA_VERSION;
|
|
|
39
39
|
export declare const S1_CORE_VERSION: string;
|
|
40
40
|
/** The arm tag as emitted on the wire (lowercase โ the s1.v1 contract value; the internal `Arm` is uppercase). The
|
|
41
41
|
* harness emits a SUBSET of core's `RunRecord["arm"]` (which also carries "sup-vf", an S1b ablation S1 does not run). */
|
|
42
|
-
export type ArmTag = "solo" | "sup"
|
|
42
|
+
export type ArmTag = "solo" | "sup";
|
|
43
43
|
/** Map the internal uppercase `Arm` to its lowercase s1.v1 wire tag. */
|
|
44
44
|
export declare function armTag(arm: Arm): ArmTag;
|
|
45
45
|
/**
|
|
@@ -75,14 +75,12 @@ export interface RawRow extends RunRecord {
|
|
|
75
75
|
coreVersion: typeof S1_CORE_VERSION;
|
|
76
76
|
gitSha: string;
|
|
77
77
|
runId: string;
|
|
78
|
-
/** ๐ด HONESTY-ON-THE-WIRE: stamped on a SUP
|
|
78
|
+
/** ๐ด HONESTY-ON-THE-WIRE: stamped on a SUP row whose C2 (stats.humanReview) came from the DETERMINISTIC
|
|
79
79
|
* rubric reviewer (reviewer.ts THINK_MS), NOT a real human (real-human C2 is S2). Absent โ no modeled review
|
|
80
|
-
* touched the row (e.g. SOLO, or a SUP
|
|
80
|
+
* touched the row (e.g. SOLO, or a SUP run with no humanReview gate). A non-scoring provenance flag.
|
|
81
81
|
* (core's RunRecord carries `judgeMode:"scripted-human-initjudge"` for the same honesty at the record level;
|
|
82
82
|
* arms.ts also stamps that field. This provenance copy stays on `meta` for the search-side readers.) */
|
|
83
83
|
reviewerKind?: "scripted-human-rubric";
|
|
84
|
-
/** TEAM bigger-pie guard input (ฮฃ the leader's ACTUAL stamped per-worker budgets); read by assertBudgetMatch. */
|
|
85
|
-
teamWorkerBudgetSumUsd?: number;
|
|
86
84
|
};
|
|
87
85
|
}
|
|
88
86
|
/** Serialize a RawRow to one JSONL line (no trailing newline โ the writer joins with "\n"). */
|
|
@@ -97,7 +95,7 @@ export declare function rowsToJsonl(rows: RawRow[]): string;
|
|
|
97
95
|
* `RunRecord[]` input (RawRow extends RunRecord), so the harness produces the verdict from its OWN rows with no
|
|
98
96
|
* re-mapping โ the metric definition lives in core, the producer only emits honest inputs.
|
|
99
97
|
*
|
|
100
|
-
* The live SOLO/SUP
|
|
98
|
+
* The live SOLO/SUP run that GENERATES rows is clay-gated (expensive E2B+DeepSeek); this is purely the
|
|
101
99
|
* CONSUME path (test-covered on synthetic conformant rows in test/s1-report.test.ts).
|
|
102
100
|
*/
|
|
103
101
|
export declare function buildS1Report(rows: RawRow[], opts?: {
|
package/dist/bench/s1/row.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* S1 value-verdict harness โ the RESUMABLE FIRM-RUN DRIVER. SPEC-S1-value-harness.md ยง11 ("the human's post-step
|
|
3
|
-
* that wires this scaffold into a runnable batch CLI"). Enumerates the firm cells {C1,C2
|
|
3
|
+
* that wires this scaffold into a runnable batch CLI"). Enumerates the firm cells {C1,C2} ร the arms each trap
|
|
4
4
|
* declares ร N seeds, runs each MISSING cell via `runArm` against a REAL `ProfileDeps` (live-deps.ts), records an
|
|
5
5
|
* append-only JSONL LEDGER keyed by (taskId, arm, seed), and at the end builds the `ValueJudgmentReport` via
|
|
6
6
|
* `buildS1Report`. The report IS the output โ this driver never asserts which arm wins.
|
|
@@ -46,7 +46,7 @@ export interface RunFirmOptions {
|
|
|
46
46
|
/** Seeds per (trap, arm). Default 5 (clay: comprehensive โ CLI `--seeds N`). */
|
|
47
47
|
seeds?: number;
|
|
48
48
|
/** Optional cell filters for a controlled live run (single-cell smoke / re-run one trap-arm). `only` = a trapId
|
|
49
|
-
* (e.g. "C1"), `arm` = a lowercase arm tag ("solo"|"sup"
|
|
49
|
+
* (e.g. "C1"), `arm` = a lowercase arm tag ("solo"|"sup"). The ledger keys are unchanged โ a filtered run
|
|
50
50
|
* still resumes/merges into the same run. */
|
|
51
51
|
only?: string;
|
|
52
52
|
arm?: string;
|
|
@@ -65,10 +65,8 @@ function armUpper(tag) {
|
|
|
65
65
|
return "SOLO";
|
|
66
66
|
case "sup":
|
|
67
67
|
return "SUP";
|
|
68
|
-
case "team":
|
|
69
|
-
return "TEAM";
|
|
70
68
|
default:
|
|
71
|
-
throw new Error(`s1 run-firm: unknown arm tag ${JSON.stringify(tag)} in ledger row (expected solo|sup
|
|
69
|
+
throw new Error(`s1 run-firm: unknown arm tag ${JSON.stringify(tag)} in ledger row (expected solo|sup)`);
|
|
72
70
|
}
|
|
73
71
|
}
|
|
74
72
|
function createLedgerWriter(ledgerPath) {
|
|
@@ -242,7 +240,6 @@ function dryRunMockFactory() {
|
|
|
242
240
|
resumeWithVerification: async () => vr,
|
|
243
241
|
runRepairLoop: async () => ({ ...vr, repairTerminal: "candidate_only", oracleCostMicroUsd: 0, bundle: { failureTrace: "", diagnostics: [], rejectedHypotheses: [], attemptCount: 1, oracleTier: "trusted_hidden" } }),
|
|
244
242
|
getCheckpoint: async () => null,
|
|
245
|
-
runLeaderTask: async () => ({ ok: true, reports: [], merge: undefined, workerBudgetsUsd: [0.5, 0.5] }),
|
|
246
243
|
runOracle: async () => oracle,
|
|
247
244
|
runCounterfactualOracle: async () => oracle,
|
|
248
245
|
};
|