@pome-sh/cli 0.23.3 → 0.23.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/build-info.json +3 -3
- package/dist/{checks-MSUPRIKQ.js → checks-HIGL2LWC.js} +2 -2
- package/dist/{chunk-PQYIAA6K.js → chunk-7AMIVWUB.js} +4 -3
- package/dist/{chunk-LI7RLYVA.js → chunk-GFTFMA3T.js} +1 -1
- package/dist/{chunk-ZJU4JMHY.js → chunk-RSQY6UIL.js} +31 -8
- package/dist/{chunk-KUVTL4NZ.js → chunk-XOWIA7NR.js} +1 -1
- package/dist/{chunk-M4P2435M.js → chunk-YUKMLGYF.js} +1 -1
- package/dist/{runDemo-LLR4FAS4.js → runDemo-4J5TDA5N.js} +3 -3
- package/dist/{runTrialGroup-G4OOARX2.js → runTrialGroup-NIBA5KRF.js} +4 -4
- package/dist/src/cli/main.js +69 -20
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -99,6 +99,16 @@ Three rules CI must honor:
|
|
|
99
99
|
excluded from the verdict fraction (`3 of 4 passed · 1 incomplete`) so neither
|
|
100
100
|
is counted as a pass nor charged to the agent as a loss — but a group holding
|
|
101
101
|
one cannot exit `0`.
|
|
102
|
+
- **`pome fix-prompt` uses the same codes, and its `1` is only ever
|
|
103
|
+
INCOMPLETE.** Building a prompt for a failed run set exits `0` (the prompt is
|
|
104
|
+
on stdout, and stdout being non-empty is the signal that there was something
|
|
105
|
+
to fix); an all-green root exits `0` with nothing on stdout; a bad argument or
|
|
106
|
+
a root with no readable run sets exits `5`. `1` is reserved for the one case
|
|
107
|
+
where the newest non-passing set was never fully graded: no prompt is built,
|
|
108
|
+
because a run whose checks never ran is not evidence of an agent defect. This
|
|
109
|
+
matches `pome run`, where `1` also covers INCOMPLETE — the two commands do not
|
|
110
|
+
disagree about what an ungraded run exits, and `verdict.json`'s `state` stays
|
|
111
|
+
the field to read when a script needs the reason rather than the code.
|
|
102
112
|
|
|
103
113
|
## Development
|
|
104
114
|
|
package/dist/build-info.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"package": "pome-sh",
|
|
3
|
-
"version": "0.23.
|
|
4
|
-
"git_sha": "
|
|
5
|
-
"build_time": "2026-08-
|
|
3
|
+
"version": "0.23.5",
|
|
4
|
+
"git_sha": "4f55e6deab92cf0ff34dd769c73763fc9b480375",
|
|
5
|
+
"build_time": "2026-08-10T15:36:42.602Z"
|
|
6
6
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { findManifestPath, readManifest } from './chunk-
|
|
1
|
+
import { findManifestPath, readManifest } from './chunk-XOWIA7NR.js';
|
|
2
2
|
import { getAvailablePort } from './chunk-XDU6TD4O.js';
|
|
3
3
|
import { buildEgressAllowlist } from './chunk-CBFKZZBR.js';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-7AMIVWUB.js';
|
|
5
5
|
import './chunk-VBATFCWR.js';
|
|
6
6
|
import './chunk-SG6ZTIMT.js';
|
|
7
7
|
import { serve } from '@hono/node-server';
|
|
@@ -948,9 +948,10 @@ var agentResponseSchema = z.object({
|
|
|
948
948
|
display_name: z.string(),
|
|
949
949
|
judge_model: z.string(),
|
|
950
950
|
// Manifest identity (F-818): registered agent.framework / agent.description /
|
|
951
|
-
// agent.version, nullable where the server has nothing stored
|
|
952
|
-
// the pre-F-820 cloud, which
|
|
953
|
-
|
|
951
|
+
// agent.version, nullable where the server has nothing stored (F-1213:
|
|
952
|
+
// unset, never a guessed default). Optional for the pre-F-820 cloud, which
|
|
953
|
+
// omits them.
|
|
954
|
+
framework: z.string().nullable().optional(),
|
|
954
955
|
description: z.string().nullable().optional(),
|
|
955
956
|
version: z.string().nullable().optional(),
|
|
956
957
|
// F-818 resolver semantics: true when POST /v1/agents auto-registered a new
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { criterionSchema, finalizeResponseSchema, HostedDiscardRefusedError, HostedOrchError, HostedAuthError, HostedQuotaError, submitResultResponseSchema, createEvalSessionResponseSchema, createSessionResponseSchema, sessionPublicSchema } from './chunk-
|
|
1
|
+
import { criterionSchema, finalizeResponseSchema, HostedDiscardRefusedError, HostedOrchError, HostedAuthError, HostedQuotaError, submitResultResponseSchema, createEvalSessionResponseSchema, createSessionResponseSchema, sessionPublicSchema } from './chunk-7AMIVWUB.js';
|
|
2
2
|
import { seedSchema, parseSeed, defaultSeedState as defaultSeedState$2 } from './chunk-SGDUD7KK.js';
|
|
3
3
|
import { gmailSeedSchema, defaultSeedState } from './chunk-NJ246QPJ.js';
|
|
4
4
|
import { linearSeedSchema, defaultSeedState as defaultSeedState$1 } from './chunk-ZKID2HS3.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readManifest, normalizeManifestTwins } from './chunk-
|
|
2
|
-
import { createHostedClient, perTwinReturnedByCloud, parseTaskFile, runAgentCommand, writeRunArtifactsCore, toTwinHttpEvent, redactJsonl, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, evaluationCounts } from './chunk-
|
|
3
|
-
import { MOUNTED_TWINS, HostedAuthError, HostedDiscardRefusedError, HostedQuotaError, HostedOrchError, HostedTrialError, agentResponseSchema } from './chunk-
|
|
1
|
+
import { readManifest, normalizeManifestTwins } from './chunk-XOWIA7NR.js';
|
|
2
|
+
import { createHostedClient, perTwinReturnedByCloud, parseTaskFile, runAgentCommand, writeRunArtifactsCore, toTwinHttpEvent, redactJsonl, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, evaluationCounts } from './chunk-GFTFMA3T.js';
|
|
3
|
+
import { MOUNTED_TWINS, HostedAuthError, HostedDiscardRefusedError, HostedQuotaError, HostedOrchError, HostedTrialError, agentResponseSchema } from './chunk-7AMIVWUB.js';
|
|
4
4
|
import { redactSecrets, redactEvent } from './chunk-SG6ZTIMT.js';
|
|
5
5
|
import { existsSync } from 'node:fs';
|
|
6
6
|
import { readFile, rm, stat, mkdir, writeFile, chmod, mkdtemp, readdir } from 'node:fs/promises';
|
|
@@ -117,13 +117,15 @@ function groupRunSets(trials) {
|
|
|
117
117
|
(a, b) => a.verdict.finalized_at.localeCompare(b.verdict.finalized_at)
|
|
118
118
|
);
|
|
119
119
|
const last = bucket[bucket.length - 1];
|
|
120
|
+
const hasFailed = bucket.some((t) => t.verdict.state === "fail");
|
|
121
|
+
const allPassed = bucket.every((t) => t.verdict.state === "pass");
|
|
120
122
|
sets.push({
|
|
121
123
|
groupId: bucket[0].verdict.group_id,
|
|
122
124
|
taskName: bucket[0].verdict.task_name,
|
|
123
125
|
taskPath: bucket[0].verdict.task_path,
|
|
124
126
|
trials: bucket,
|
|
125
127
|
latestFinalizedAt: last.verdict.finalized_at,
|
|
126
|
-
|
|
128
|
+
outcome: hasFailed ? "fail" : allPassed ? "pass" : "incomplete"
|
|
127
129
|
});
|
|
128
130
|
}
|
|
129
131
|
sets.sort((a, b) => a.latestFinalizedAt.localeCompare(b.latestFinalizedAt));
|
|
@@ -131,14 +133,26 @@ function groupRunSets(trials) {
|
|
|
131
133
|
}
|
|
132
134
|
function latestFailedRunSet(sets) {
|
|
133
135
|
for (let i = sets.length - 1; i >= 0; i -= 1) {
|
|
134
|
-
if (sets[i].
|
|
136
|
+
if (sets[i].outcome === "fail") return sets[i];
|
|
137
|
+
}
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
function latestIncompleteRunSet(sets) {
|
|
141
|
+
for (let i = sets.length - 1; i >= 0; i -= 1) {
|
|
142
|
+
if (sets[i].outcome === "incomplete") return sets[i];
|
|
135
143
|
}
|
|
136
144
|
return null;
|
|
137
145
|
}
|
|
138
146
|
async function discoverRunSet(target) {
|
|
139
147
|
const anchorResult = await readVerdictArtifactDetailed(target);
|
|
140
148
|
if (anchorResult.status === "stale-version") {
|
|
141
|
-
return {
|
|
149
|
+
return {
|
|
150
|
+
kind: "trial-dir",
|
|
151
|
+
set: null,
|
|
152
|
+
incompleteSet: null,
|
|
153
|
+
totalSets: 0,
|
|
154
|
+
staleVersionCount: 1
|
|
155
|
+
};
|
|
142
156
|
}
|
|
143
157
|
if (anchorResult.status === "ok") {
|
|
144
158
|
const anchor = anchorResult.trial;
|
|
@@ -151,18 +165,27 @@ async function discoverRunSet(target) {
|
|
|
151
165
|
return {
|
|
152
166
|
kind: "trial-dir",
|
|
153
167
|
set: own,
|
|
168
|
+
incompleteSet: null,
|
|
154
169
|
totalSets: Math.max(sets2.length, 1),
|
|
155
170
|
staleVersionCount: staleVersionDirs2.length
|
|
156
171
|
};
|
|
157
172
|
}
|
|
158
173
|
if (!existsSync(target)) {
|
|
159
|
-
return {
|
|
174
|
+
return {
|
|
175
|
+
kind: "root",
|
|
176
|
+
set: null,
|
|
177
|
+
incompleteSet: null,
|
|
178
|
+
totalSets: 0,
|
|
179
|
+
staleVersionCount: 0
|
|
180
|
+
};
|
|
160
181
|
}
|
|
161
182
|
const { trials, staleVersionDirs } = await scanVerdictArtifactsDetailed(target);
|
|
162
183
|
const sets = groupRunSets(trials);
|
|
184
|
+
const failedSet = latestFailedRunSet(sets);
|
|
163
185
|
return {
|
|
164
186
|
kind: "root",
|
|
165
|
-
set:
|
|
187
|
+
set: failedSet,
|
|
188
|
+
incompleteSet: failedSet ? null : latestIncompleteRunSet(sets),
|
|
166
189
|
totalSets: sets.length,
|
|
167
190
|
staleVersionCount: staleVersionDirs.length
|
|
168
191
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { HostedOrchError, manifestSchema, deriveAgentSlug, SLUG_RE } from './chunk-
|
|
1
|
+
import { HostedOrchError, manifestSchema, deriveAgentSlug, SLUG_RE } from './chunk-7AMIVWUB.js';
|
|
2
2
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { resolve, join, dirname } from 'node:path';
|
|
4
4
|
import { stringify, parse } from 'yaml';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { getAvailablePort } from './chunk-XDU6TD4O.js';
|
|
2
2
|
import { buildEgressAllowlist, readBlockedEgress } from './chunk-CBFKZZBR.js';
|
|
3
|
-
import { parseTaskFile, seedStateForTwin, runAgentCommand, writeRunArtifactsCore } from './chunk-
|
|
3
|
+
import { parseTaskFile, seedStateForTwin, runAgentCommand, writeRunArtifactsCore } from './chunk-GFTFMA3T.js';
|
|
4
4
|
import { createRecorder, bootTwin } from './chunk-2R46XQAL.js';
|
|
5
5
|
import { eventSchema } from './chunk-VBATFCWR.js';
|
|
6
6
|
import { redactSecrets, redactEvent } from './chunk-SG6ZTIMT.js';
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { newGroupId, reassuranceBox, twinReadyLine, trialsHeaderLine, trialLine, summaryLines, evaluatingLine, criterionPhrase } from './chunk-RGZBC7NF.js';
|
|
2
2
|
import { DemoCapacityError, capacityLabel, parseCapacityMarker, capacityKindFrom } from './chunk-ZX4WNSZ5.js';
|
|
3
|
-
import { runTask, demoTaskPath, DEMO_TASK_NAME, DEMO_REPO } from './chunk-
|
|
3
|
+
import { runTask, demoTaskPath, DEMO_TASK_NAME, DEMO_REPO } from './chunk-YUKMLGYF.js';
|
|
4
4
|
import { getAvailablePort } from './chunk-XDU6TD4O.js';
|
|
5
5
|
import './chunk-CBFKZZBR.js';
|
|
6
|
-
import { createHostedClient, parseTaskFile, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, outcomeOf } from './chunk-
|
|
6
|
+
import { createHostedClient, parseTaskFile, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, outcomeOf } from './chunk-GFTFMA3T.js';
|
|
7
7
|
import './chunk-NW7HGA2K.js';
|
|
8
|
-
import { HostedQuotaError, HostedOrchError } from './chunk-
|
|
8
|
+
import { HostedQuotaError, HostedOrchError } from './chunk-7AMIVWUB.js';
|
|
9
9
|
import './chunk-SGDUD7KK.js';
|
|
10
10
|
import './chunk-NJ246QPJ.js';
|
|
11
11
|
import './chunk-ZKID2HS3.js';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { newGroupId, criterionPhrase } from './chunk-RGZBC7NF.js';
|
|
2
|
-
import { runTaskHosted, resolveRunAgentIdentity } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { createHostedClient, parseTaskFile, outcomeOf } from './chunk-
|
|
2
|
+
import { runTaskHosted, resolveRunAgentIdentity } from './chunk-RSQY6UIL.js';
|
|
3
|
+
import './chunk-XOWIA7NR.js';
|
|
4
|
+
import { createHostedClient, parseTaskFile, outcomeOf } from './chunk-GFTFMA3T.js';
|
|
5
5
|
import './chunk-NW7HGA2K.js';
|
|
6
|
-
import { HostedQuotaError, HostedTrialError } from './chunk-
|
|
6
|
+
import { HostedQuotaError, HostedTrialError } from './chunk-7AMIVWUB.js';
|
|
7
7
|
import './chunk-SGDUD7KK.js';
|
|
8
8
|
import './chunk-NJ246QPJ.js';
|
|
9
9
|
import './chunk-ZKID2HS3.js';
|
package/dist/src/cli/main.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, VERDICT_ARTIFACT_VERSION, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-
|
|
3
|
-
import { readManifest, writeManifest, MANIFEST_JSON, readRequiredManifest, normalizeManifestTwins } from '../../chunk-
|
|
4
|
-
import { assetPath, runTask, resolvePackageRoot, DEMO_TASK_NAME, demoTaskPath } from '../../chunk-
|
|
2
|
+
import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, VERDICT_ARTIFACT_VERSION, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-RSQY6UIL.js';
|
|
3
|
+
import { readManifest, writeManifest, MANIFEST_JSON, readRequiredManifest, normalizeManifestTwins } from '../../chunk-XOWIA7NR.js';
|
|
4
|
+
import { assetPath, runTask, resolvePackageRoot, DEMO_TASK_NAME, demoTaskPath } from '../../chunk-YUKMLGYF.js';
|
|
5
5
|
import '../../chunk-XDU6TD4O.js';
|
|
6
6
|
import '../../chunk-CBFKZZBR.js';
|
|
7
|
-
import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, readConfigTwins, scoreCountsSummary, markerFor,
|
|
7
|
+
import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, outcomeOf, readConfigTwins, scoreCountsSummary, markerFor, criterionMarkerLabel, twinSkipSuffix, readCodeCriteria, createHostedClient, toTwinHttpEvent, redactJsonl, scoreFromFinalizeResponse, parseGitHubSeedState, uploadRunBlobs, isPreSatisfied } from '../../chunk-GFTFMA3T.js';
|
|
8
8
|
import '../../chunk-NW7HGA2K.js';
|
|
9
|
-
import { MOUNTED_TWINS, deriveAgentSlug, exitCodeFor, HostedUsageError, HostedOrchError, HostedAuthError, HostedQuotaError } from '../../chunk-
|
|
9
|
+
import { MOUNTED_TWINS, deriveAgentSlug, exitCodeFor, HostedUsageError, HostedOrchError, HostedAuthError, HostedQuotaError } from '../../chunk-7AMIVWUB.js';
|
|
10
10
|
import { TAPE_ASSERTABLE_TOOLS } from '../../chunk-FKZZWWYC.js';
|
|
11
11
|
import { seedSchema } from '../../chunk-SGDUD7KK.js';
|
|
12
12
|
import { SLACK_CHECKS } from '../../chunk-6OT4IW25.js';
|
|
@@ -4086,8 +4086,6 @@ async function createAndPersistAgent(input) {
|
|
|
4086
4086
|
if (typeof description === "string") nextAgent.description = description;
|
|
4087
4087
|
const version = agent.version ?? existingAgent.version;
|
|
4088
4088
|
if (typeof version === "string") nextAgent.version = version;
|
|
4089
|
-
const framework = agent.framework ?? existingAgent.framework;
|
|
4090
|
-
if (typeof framework === "string") nextAgent.framework = framework;
|
|
4091
4089
|
const nextRaw = {
|
|
4092
4090
|
...input.manifestRead.raw,
|
|
4093
4091
|
agent: nextAgent
|
|
@@ -4251,7 +4249,7 @@ async function writeClaudeSdkScaffold() {
|
|
|
4251
4249
|
await mkdir(dirname(path), { recursive: true });
|
|
4252
4250
|
await writeFile(path, CLAUDE_SDK_AGENT_SOURCE, "utf8");
|
|
4253
4251
|
return {
|
|
4254
|
-
agentSdkValue: "claude",
|
|
4252
|
+
agentSdkValue: "claude-agent-sdk",
|
|
4255
4253
|
agentCommand: `npx tsx ${CLAUDE_SDK_AGENT_RELATIVE}`,
|
|
4256
4254
|
exampleAgentRelativePath: CLAUDE_SDK_AGENT_RELATIVE,
|
|
4257
4255
|
postInstallHint: "Next steps:\n 1. npm install @pome-sh/adapter-claude-sdk @anthropic-ai/claude-agent-sdk zod\n 2. pome login\n 3. pome register agent <name>\n 4. pome run tasks/01-bug-happy-path.md\n\nSee `pome docs getting-started` for a narrative walkthrough."
|
|
@@ -4350,6 +4348,14 @@ function criterionMarker(c) {
|
|
|
4350
4348
|
function failedResults(verdict) {
|
|
4351
4349
|
return verdict.criteria_results.filter((r) => outcomeOf(r) === "failed");
|
|
4352
4350
|
}
|
|
4351
|
+
function isGraded(t) {
|
|
4352
|
+
return t.verdict.state !== "incomplete";
|
|
4353
|
+
}
|
|
4354
|
+
function ungradedCount(verdict) {
|
|
4355
|
+
return verdict.criteria_results.filter(
|
|
4356
|
+
(r) => !isPreSatisfied(r) && outcomeOf(r) !== "passed" && outcomeOf(r) !== "failed"
|
|
4357
|
+
).length;
|
|
4358
|
+
}
|
|
4353
4359
|
function flattenLine(text, max = 300) {
|
|
4354
4360
|
const flat = text.replace(/\s+/g, " ").trim();
|
|
4355
4361
|
return flat.length > max ? `${flat.slice(0, max - 1)}\u2026` : flat;
|
|
@@ -4357,10 +4363,14 @@ function flattenLine(text, max = 300) {
|
|
|
4357
4363
|
function renderGroupedSignatures(trials) {
|
|
4358
4364
|
const byCriterion = /* @__PURE__ */ new Map();
|
|
4359
4365
|
const outcomesSeen = /* @__PURE__ */ new Map();
|
|
4366
|
+
const gradedFor = /* @__PURE__ */ new Map();
|
|
4360
4367
|
for (const trial of trials) {
|
|
4361
4368
|
for (const result of trial.verdict.criteria_results) {
|
|
4362
4369
|
const key = result.criterion.text;
|
|
4363
4370
|
const outcome = isPreSatisfied(result) ? "excluded" : outcomeOf(result);
|
|
4371
|
+
if (outcome === "passed" || outcome === "failed") {
|
|
4372
|
+
gradedFor.set(key, (gradedFor.get(key) ?? 0) + 1);
|
|
4373
|
+
}
|
|
4364
4374
|
if (outcome === "failed") {
|
|
4365
4375
|
const entry = byCriterion.get(key) ?? {
|
|
4366
4376
|
marker: criterionMarker(result.criterion),
|
|
@@ -4383,12 +4393,12 @@ function renderGroupedSignatures(trials) {
|
|
|
4383
4393
|
else if (seen.size === 1 && seen.has("excluded")) preSatisfiedEverywhere.push(key);
|
|
4384
4394
|
else notUniformlyEvaluated.push(key);
|
|
4385
4395
|
}
|
|
4386
|
-
const completed = trials.length;
|
|
4387
4396
|
const blocks = [...byCriterion.entries()].sort((a, b) => b[1].hits.length - a[1].hits.length).map(([text, { marker, hits }], idx) => {
|
|
4388
4397
|
const lines = hits.map(
|
|
4389
4398
|
(h) => ` - ${h.label}: ${flattenLine(h.reason)}`
|
|
4390
4399
|
);
|
|
4391
|
-
|
|
4400
|
+
const graded = gradedFor.get(text) ?? hits.length;
|
|
4401
|
+
return `${idx + 1}. ${marker} ${flattenLine(text)} \u2014 failed in ${hits.length} of ${graded} trials that graded it
|
|
4392
4402
|
${lines.join("\n")}`;
|
|
4393
4403
|
});
|
|
4394
4404
|
if (blocks.length === 0 && passedEverywhere.length === 0 && preSatisfiedEverywhere.length === 0 && notUniformlyEvaluated.length === 0) {
|
|
@@ -4416,18 +4426,19 @@ ${lines.join("\n")}`;
|
|
|
4416
4426
|
return [...blocks, ...notes].join("\n");
|
|
4417
4427
|
}
|
|
4418
4428
|
function representativeFailingTrial(trials) {
|
|
4419
|
-
const failing = trials.filter((t) =>
|
|
4429
|
+
const failing = trials.filter((t) => t.verdict.state === "fail");
|
|
4420
4430
|
if (failing.length === 0) return null;
|
|
4421
4431
|
return failing.reduce(
|
|
4422
4432
|
(worst, t) => failedResults(t.verdict).length > failedResults(worst.verdict).length ? t : worst
|
|
4423
4433
|
);
|
|
4424
4434
|
}
|
|
4425
4435
|
function buildGroupFixUserPrompt(ctx) {
|
|
4426
|
-
const
|
|
4427
|
-
const
|
|
4436
|
+
const incomplete = ctx.trials.filter((t) => !isGraded(t));
|
|
4437
|
+
const completed = ctx.trials.length - incomplete.length;
|
|
4438
|
+
const passed = ctx.trials.filter((t) => t.verdict.state === "pass").length;
|
|
4428
4439
|
const representative = representativeFailingTrial(ctx.trials);
|
|
4429
4440
|
const otherFailing = ctx.trials.filter(
|
|
4430
|
-
(t) =>
|
|
4441
|
+
(t) => t.verdict.state === "fail" && t !== representative
|
|
4431
4442
|
);
|
|
4432
4443
|
const signatures = redactSecrets(
|
|
4433
4444
|
renderGroupedSignatures(ctx.trials)
|
|
@@ -4441,9 +4452,11 @@ function buildGroupFixUserPrompt(ctx) {
|
|
|
4441
4452
|
)
|
|
4442
4453
|
);
|
|
4443
4454
|
const promptBlock = ctx.task ? redactSecrets(ctx.task.prompt) : `(task file not found at ${ctx.trials[0]?.verdict.task_path ?? "?"} \u2014 criteria above come from the cloud verdicts)`;
|
|
4455
|
+
const tally = completed === 0 ? "no trial in this set was graded end to end" : `${passed} of ${completed} completed trials passed`;
|
|
4456
|
+
const gapNote = incomplete.length > 0 ? ` \xB7 ${incomplete.length} INCOMPLETE (counted in nothing below \u2014 see the last section)` : "";
|
|
4444
4457
|
const sections = [];
|
|
4445
4458
|
sections.push(`## Run set (cloud-judged)
|
|
4446
|
-
task ${redactSecrets(ctx.taskName)} \xB7 ${ctx.groupId ? `group ${ctx.groupId}` : "single run"} \xB7 ${
|
|
4459
|
+
task ${redactSecrets(ctx.taskName)} \xB7 ${ctx.groupId ? `group ${ctx.groupId}` : "single run"} \xB7 ${tally}${gapNote}`);
|
|
4447
4460
|
sections.push(`## Grouped failure signatures (from the cloud judge)
|
|
4448
4461
|
${escapeTagContent(signatures)}`);
|
|
4449
4462
|
sections.push(`## Task prompt (what the agent was told to do)
|
|
@@ -4465,6 +4478,17 @@ ${escapeTagContent(trace)}
|
|
|
4465
4478
|
return `- ${t.label} \u2014 failed: ${failed || "(see verdict)"} \u2014 trace at ${join(t.runDir, "events.jsonl")}`;
|
|
4466
4479
|
});
|
|
4467
4480
|
sections.push(`## Other failing trials (traces on disk)
|
|
4481
|
+
${escapeTagContent(redactSecrets(lines.join("\n")))}`);
|
|
4482
|
+
}
|
|
4483
|
+
if (incomplete.length > 0) {
|
|
4484
|
+
const lines = incomplete.map(
|
|
4485
|
+
(t) => `- ${t.label} \u2014 ${ungradedCount(t.verdict)} criterion(s) never graded \u2014 trace at ${join(t.runDir, "events.jsonl")}`
|
|
4486
|
+
);
|
|
4487
|
+
sections.push(`## Trials the grader never finished (INCOMPLETE)
|
|
4488
|
+
The grader never reached every criterion in these trials, so they are neither
|
|
4489
|
+
passes nor failures and are counted in no fraction above. Do NOT treat them as
|
|
4490
|
+
evidence for or against any fix: a criterion that never ran is a grader or seed
|
|
4491
|
+
gap, not something the agent did wrong.
|
|
4468
4492
|
${escapeTagContent(redactSecrets(lines.join("\n")))}`);
|
|
4469
4493
|
}
|
|
4470
4494
|
if (passed > 0 && passed < completed) {
|
|
@@ -4497,7 +4521,7 @@ var DEFAULT_AGENT_FILE = "examples/agents/scripted-triage-agent.ts";
|
|
|
4497
4521
|
var DEFAULT_AGENT_COMMAND = `node ${DEFAULT_AGENT_FILE}`;
|
|
4498
4522
|
var MANIFEST_SCHEMA_URL = "https://pome.sh/schemas/v1/pome.json";
|
|
4499
4523
|
function readPackageVersion() {
|
|
4500
|
-
if ("0.23.
|
|
4524
|
+
if ("0.23.5".length > 0) return "0.23.5";
|
|
4501
4525
|
try {
|
|
4502
4526
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
4503
4527
|
const candidates = [
|
|
@@ -4952,7 +4976,7 @@ function createProgram() {
|
|
|
4952
4976
|
return;
|
|
4953
4977
|
}
|
|
4954
4978
|
{
|
|
4955
|
-
const { runDoctorChecks } = await import('../../checks-
|
|
4979
|
+
const { runDoctorChecks } = await import('../../checks-HIGL2LWC.js');
|
|
4956
4980
|
const { renderDoctorReport } = await import('../../render-ZQQ4UMNO.js');
|
|
4957
4981
|
const doctorReport = await runDoctorChecks({ mode: useLocal ? "full" : "hosted" });
|
|
4958
4982
|
if (!doctorReport.ok) {
|
|
@@ -4990,7 +5014,7 @@ function createProgram() {
|
|
|
4990
5014
|
taskForRuns.config.runs
|
|
4991
5015
|
);
|
|
4992
5016
|
if (k > 1) {
|
|
4993
|
-
const { runTrialGroup } = await import('../../runTrialGroup-
|
|
5017
|
+
const { runTrialGroup } = await import('../../runTrialGroup-NIBA5KRF.js');
|
|
4994
5018
|
const fileForRerun = relative(process.cwd(), file);
|
|
4995
5019
|
const rerunCommand = defaultTask ? options.trials !== void 0 ? `pome run -n ${k}` : "pome run" : `pome run ${fileForRerun && !fileForRerun.startsWith("..") ? fileForRerun : file} -n ${k}`;
|
|
4996
5020
|
const groupResult = await runTrialGroup({
|
|
@@ -5084,7 +5108,7 @@ function createProgram() {
|
|
|
5084
5108
|
process.exitCode = 5;
|
|
5085
5109
|
return;
|
|
5086
5110
|
}
|
|
5087
|
-
const { runDemo } = await import('../../runDemo-
|
|
5111
|
+
const { runDemo } = await import('../../runDemo-4J5TDA5N.js');
|
|
5088
5112
|
const result = await runDemo({
|
|
5089
5113
|
apiBase: opts.apiUrl.replace(/\/$/, ""),
|
|
5090
5114
|
dashboardBase: process.env.POME_DASHBOARD_URL ?? DEFAULT_DASHBOARD_URL,
|
|
@@ -5102,7 +5126,7 @@ function createProgram() {
|
|
|
5102
5126
|
program.command("doctor").description(
|
|
5103
5127
|
"Check the agent\u2194twin wiring: pome.json (or pome.yaml) present + valid, the local twin boots + serves, requests routed to the twin (not a hardcoded production host), egress floor active. On failure prints one named cause (file:line where knowable) + one concrete fix and exits non-zero."
|
|
5104
5128
|
).action(async () => {
|
|
5105
|
-
const { runDoctorChecks } = await import('../../checks-
|
|
5129
|
+
const { runDoctorChecks } = await import('../../checks-HIGL2LWC.js');
|
|
5106
5130
|
const { renderDoctorReport } = await import('../../render-ZQQ4UMNO.js');
|
|
5107
5131
|
const report = await runDoctorChecks();
|
|
5108
5132
|
for (const line of renderDoctorReport(report, { passNote: true })) console.error(line);
|
|
@@ -5204,6 +5228,31 @@ function createProgram() {
|
|
|
5204
5228
|
return;
|
|
5205
5229
|
}
|
|
5206
5230
|
if (!discovery.set) {
|
|
5231
|
+
const incomplete = discovery.incompleteSet;
|
|
5232
|
+
if (incomplete) {
|
|
5233
|
+
const ungradedTrials = incomplete.trials.filter(
|
|
5234
|
+
(t) => t.verdict.state === "incomplete"
|
|
5235
|
+
).length;
|
|
5236
|
+
const gradedFailures = incomplete.trials.reduce(
|
|
5237
|
+
(n, t) => n + t.verdict.criteria_results.filter((r) => outcomeOf(r) === "failed").length,
|
|
5238
|
+
0
|
|
5239
|
+
);
|
|
5240
|
+
const which = `task ${incomplete.taskName}${incomplete.groupId ? ` \xB7 group ${incomplete.groupId}` : ""}`;
|
|
5241
|
+
console.error(
|
|
5242
|
+
`Not routed to fix-prompt: no run set under ${root} failed outright. The most recent non-passing one (${which}) is INCOMPLETE \u2014 ${ungradedTrials} of ${incomplete.trials.length} trial(s) have criteria the grader never graded.`
|
|
5243
|
+
);
|
|
5244
|
+
if (gradedFailures > 0) {
|
|
5245
|
+
console.error(
|
|
5246
|
+
`${gradedFailures} criterion result(s) in that set WERE graded and did fail, so this is not only a grading gap \u2014 but no trial in it was graded end to end, and a fix prompt built from a partial grading would claim more than was checked. Re-run \`pome run ${incomplete.taskPath}\` to grade the rest, or point fix-prompt straight at one trial (\`pome fix-prompt ${incomplete.trials[0].runDir}\`) to build one from the partial grading anyway.`
|
|
5247
|
+
);
|
|
5248
|
+
} else {
|
|
5249
|
+
console.error(
|
|
5250
|
+
`Nothing in that set was graded and failed, so it is a grader/seed gap, not an agent defect, and fix-prompt will not hand it to your coding agent. Re-run \`pome run ${incomplete.taskPath}\` to grade those criteria; if they come back ungraded, the gap is in the task's checks or its seed, not in your prompt.`
|
|
5251
|
+
);
|
|
5252
|
+
}
|
|
5253
|
+
process.exitCode = 1;
|
|
5254
|
+
return;
|
|
5255
|
+
}
|
|
5207
5256
|
console.error(
|
|
5208
5257
|
`Nothing to fix: the latest run sets under ${root} all passed.`
|
|
5209
5258
|
);
|
package/package.json
CHANGED