@bli-cockpit/cli 0.2.250 → 0.2.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-prepare-memo.js +24 -3
- package/dist/adapters/raw-evidence-transcript-file.js +37 -16
- package/dist/adapters/raw-evidence.js +4 -2
- package/dist/commands/backfill-live-window.js +41 -0
- package/dist/commands/backfill-report.js +11 -1
- package/dist/commands/backfill-scan.js +8 -4
- package/dist/commands/public-root.js +1 -1
- package/dist/image-description-session.js +20 -9
- package/package.json +1 -1
|
@@ -61,19 +61,23 @@ export function prepareRecipe(options) {
|
|
|
61
61
|
export function prepareMemoPath(stateDir) {
|
|
62
62
|
return path.join(stateDir, PREPARE_MEMO_FILENAME);
|
|
63
63
|
}
|
|
64
|
-
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0 };
|
|
64
|
+
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0 };
|
|
65
65
|
export async function openPrepareMemo(stateDir) {
|
|
66
66
|
const entries = await readMemoEntries(stateDir);
|
|
67
67
|
let dirty = false;
|
|
68
68
|
let counts = { ...EMPTY_COUNTS };
|
|
69
69
|
/** Sources recalled since the last `settle`: only these have their `seen_ms` refreshed. */
|
|
70
70
|
const recalled = new Set();
|
|
71
|
+
/** In-memory only, never persisted: see `rememberForThisRun`. */
|
|
72
|
+
const thisRun = new Map();
|
|
71
73
|
return {
|
|
72
74
|
recall(sourceKey, stamp, recipe) {
|
|
73
75
|
const entry = entries.get(sourceKey);
|
|
74
76
|
if (!entry)
|
|
75
|
-
return
|
|
76
|
-
if (entry.volatile
|
|
77
|
+
return recallThisRun(thisRun, sourceKey, stamp, recipe);
|
|
78
|
+
if (entry.volatile)
|
|
79
|
+
return recallThisRun(thisRun, sourceKey, stamp, recipe);
|
|
80
|
+
if (entry.recipe !== recipe)
|
|
77
81
|
return null;
|
|
78
82
|
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
79
83
|
return null;
|
|
@@ -82,9 +86,18 @@ export async function openPrepareMemo(stateDir) {
|
|
|
82
86
|
},
|
|
83
87
|
remember(sourceKey, entry, now = Date.now()) {
|
|
84
88
|
entries.set(sourceKey, { ...entry, seen_ms: now });
|
|
89
|
+
thisRun.delete(sourceKey);
|
|
85
90
|
dirty = true;
|
|
86
91
|
},
|
|
92
|
+
rememberForThisRun(sourceKey, entry) {
|
|
93
|
+
thisRun.set(sourceKey, { ...entry, volatile: true, seen_ms: Date.now() });
|
|
94
|
+
},
|
|
87
95
|
count(outcome, bytes) {
|
|
96
|
+
if (outcome === "volatile_reused") {
|
|
97
|
+
counts.volatile_reused_count += 1;
|
|
98
|
+
counts.volatile_reused_bytes += bytes;
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
88
101
|
if (outcome === "skipped") {
|
|
89
102
|
counts.skipped_count += 1;
|
|
90
103
|
counts.skipped_bytes += bytes;
|
|
@@ -114,6 +127,14 @@ export async function openPrepareMemo(stateDir) {
|
|
|
114
127
|
},
|
|
115
128
|
};
|
|
116
129
|
}
|
|
130
|
+
function recallThisRun(thisRun, sourceKey, stamp, recipe) {
|
|
131
|
+
const entry = thisRun.get(sourceKey);
|
|
132
|
+
if (!entry || entry.recipe !== recipe)
|
|
133
|
+
return null;
|
|
134
|
+
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
135
|
+
return null;
|
|
136
|
+
return entry;
|
|
137
|
+
}
|
|
117
138
|
/**
|
|
118
139
|
* Entries recalled this pass keep their `seen_ms` fresh, lazily, so they are not forgotten while in
|
|
119
140
|
* use. Only those: an entry nobody recalls (a deleted or rotated session) ages out after
|
|
@@ -63,9 +63,9 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
63
63
|
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
64
64
|
return skip("file_too_large");
|
|
65
65
|
}
|
|
66
|
-
const { sanitized, contentHash, stubClaim, volatile, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
66
|
+
const { sanitized, contentHash, stubClaim, volatile, deferredOnly, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
67
67
|
await rememberPreparedTranscript(collection, options, names, {
|
|
68
|
-
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, redactedEntry,
|
|
68
|
+
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, deferredOnly, redactedEntry,
|
|
69
69
|
});
|
|
70
70
|
if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
|
|
71
71
|
collection.reused.push({
|
|
@@ -121,7 +121,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
|
|
|
121
121
|
byteSize: sanitized.bytes.byteLength,
|
|
122
122
|
alreadyDurable: collection.skipContentHashes.has(contentHash),
|
|
123
123
|
});
|
|
124
|
-
return { sanitized, contentHash, stubClaim, volatile: images.volatile, redactedEntry };
|
|
124
|
+
return { sanitized, contentHash, stubClaim, volatile: images.volatile, deferredOnly: images.deferredOnly, redactedEntry };
|
|
125
125
|
}
|
|
126
126
|
/**
|
|
127
127
|
* The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
|
|
@@ -133,10 +133,10 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
133
133
|
const session = collection.budget.images;
|
|
134
134
|
const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
|
|
135
135
|
if (!session || !source)
|
|
136
|
-
return { bytes: file.bytes, volatile: false };
|
|
136
|
+
return { bytes: file.bytes, volatile: false, deferredOnly: false };
|
|
137
137
|
// Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
|
|
138
138
|
if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
|
|
139
|
-
return { bytes: file.bytes, volatile: false };
|
|
139
|
+
return { bytes: file.bytes, volatile: false, deferredOnly: false };
|
|
140
140
|
try {
|
|
141
141
|
const prepared = await session.prepareTranscript({
|
|
142
142
|
text: file.bytes.toString("utf8"),
|
|
@@ -151,13 +151,15 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
151
151
|
: prepared.imageCount > 0;
|
|
152
152
|
const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
|
|
153
153
|
const volatile = imagesPending || slimFailed;
|
|
154
|
+
// BLI-22785: volatile ONLY because the budget or a backoff held images back, and the slim step held up.
|
|
155
|
+
const deferredOnly = volatile && prepared.deferredOnly && !slimFailed;
|
|
154
156
|
if (!prepared.changed)
|
|
155
|
-
return { bytes: file.bytes, volatile };
|
|
156
|
-
return { bytes: Buffer.from(prepared.text, "utf8"), volatile };
|
|
157
|
+
return { bytes: file.bytes, volatile, deferredOnly };
|
|
158
|
+
return { bytes: Buffer.from(prepared.text, "utf8"), volatile, deferredOnly };
|
|
157
159
|
}
|
|
158
160
|
catch (error) {
|
|
159
161
|
console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
|
|
160
|
-
return { bytes: file.bytes, volatile: true };
|
|
162
|
+
return { bytes: file.bytes, volatile: true, deferredOnly: false };
|
|
161
163
|
}
|
|
162
164
|
}
|
|
163
165
|
/**
|
|
@@ -178,8 +180,13 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
178
180
|
if (options.maxFileBytes && stamp.size > options.maxFileBytes)
|
|
179
181
|
return false;
|
|
180
182
|
const known = memo.recall(names.sourceKey, stamp, recipe);
|
|
181
|
-
if (!known
|
|
183
|
+
if (!known)
|
|
182
184
|
return false;
|
|
185
|
+
if (!collection.skipContentHashes.has(known.content_hash)) {
|
|
186
|
+
if (known.volatile)
|
|
187
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "hash_not_durable_full_path");
|
|
188
|
+
return false;
|
|
189
|
+
}
|
|
183
190
|
if (known.redacted) {
|
|
184
191
|
collection.redacted.push({
|
|
185
192
|
kind: options.kind,
|
|
@@ -194,16 +201,25 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
194
201
|
content_hash_sha256: known.content_hash,
|
|
195
202
|
codex_session_id: options.sessionId,
|
|
196
203
|
});
|
|
204
|
+
if (known.volatile) {
|
|
205
|
+
// BLI-22785: images held back by this run's budget or a backoff; rebuilding gives the same bytes.
|
|
206
|
+
memo.count("volatile_reused", stamp.size);
|
|
207
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "images_deferred_budget_cannot_refill_this_run");
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
197
210
|
memo.count("skipped", stamp.size);
|
|
198
211
|
return true;
|
|
199
212
|
}
|
|
213
|
+
function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
214
|
+
console.error("[raw-evidence] volatile transcript " + (reason === "hash_not_durable_full_path" ? "rebuilt" : "reused this run"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
215
|
+
}
|
|
200
216
|
/** Remember what this transcript became, but only if the file stayed put while it was built. */
|
|
201
217
|
async function rememberPreparedTranscript(collection, options, names, built) {
|
|
202
218
|
const memo = collection.budget.prepareMemo;
|
|
203
219
|
if (!memo)
|
|
204
220
|
return;
|
|
205
221
|
memo.count(built.volatile ? "volatile" : "rebuilt", built.readBytes);
|
|
206
|
-
if (built.volatile || !built.stamp)
|
|
222
|
+
if ((built.volatile && !built.deferredOnly) || !built.stamp)
|
|
207
223
|
return;
|
|
208
224
|
const after = await statSourceForMemo(options.filePath);
|
|
209
225
|
const stable = after !== null
|
|
@@ -212,10 +228,7 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
212
228
|
&& built.readBytes === built.stamp.size;
|
|
213
229
|
if (!stable)
|
|
214
230
|
return;
|
|
215
|
-
const
|
|
216
|
-
if (known?.content_hash === built.contentHash)
|
|
217
|
-
return;
|
|
218
|
-
memo.remember(names.sourceKey, {
|
|
231
|
+
const entry = {
|
|
219
232
|
...built.stamp,
|
|
220
233
|
recipe: built.recipe,
|
|
221
234
|
content_hash: built.contentHash,
|
|
@@ -223,8 +236,16 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
223
236
|
redacted: built.redactedEntry
|
|
224
237
|
? { redaction: built.redactedEntry.redaction, completenessLabel: built.redactedEntry.completenessLabel }
|
|
225
238
|
: null,
|
|
226
|
-
|
|
227
|
-
|
|
239
|
+
};
|
|
240
|
+
if (built.volatile) {
|
|
241
|
+
// BLI-22785: in memory for this process only; the memo file keeps no volatile entry.
|
|
242
|
+
memo.rememberForThisRun(names.sourceKey, entry);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
const known = memo.recall(names.sourceKey, built.stamp, built.recipe);
|
|
246
|
+
if (known && !known.volatile && known.content_hash === built.contentHash)
|
|
247
|
+
return;
|
|
248
|
+
memo.remember(names.sourceKey, { ...entry, volatile: false });
|
|
228
249
|
}
|
|
229
250
|
function nameEvidenceFile(options) {
|
|
230
251
|
const fileName = path.basename(options.filePath);
|
|
@@ -107,15 +107,17 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
107
107
|
*/
|
|
108
108
|
async function reportPrepareMemo(collection) {
|
|
109
109
|
const counts = await collection.budget.prepareMemo?.settle();
|
|
110
|
-
if (!counts || counts.skipped_count + counts.rebuilt_count === 0)
|
|
110
|
+
if (!counts || counts.skipped_count + counts.rebuilt_count + counts.volatile_reused_count === 0)
|
|
111
111
|
return;
|
|
112
|
-
console.error(counts.skipped_count > 0 ? "[raw-evidence] slim skipped unchanged" : "[raw-evidence] no unchanged transcripts to skip", JSON.stringify({
|
|
112
|
+
console.error(counts.skipped_count + counts.volatile_reused_count > 0 ? "[raw-evidence] slim skipped unchanged" : "[raw-evidence] no unchanged transcripts to skip", JSON.stringify({
|
|
113
113
|
reason: "source_unchanged_since_last_prepare",
|
|
114
114
|
count: counts.skipped_count,
|
|
115
115
|
bytes: counts.skipped_bytes,
|
|
116
116
|
rebuilt_count: counts.rebuilt_count,
|
|
117
117
|
rebuilt_bytes: counts.rebuilt_bytes,
|
|
118
118
|
rebuilt_volatile_count: counts.volatile_count,
|
|
119
|
+
rebuilt_volatile_reused_count: counts.volatile_reused_count,
|
|
120
|
+
rebuilt_volatile_reused_bytes: counts.volatile_reused_bytes,
|
|
119
121
|
}));
|
|
120
122
|
}
|
|
121
123
|
/**
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-22790: backfill is history. A session file modified inside the live window belongs to the
|
|
3
|
+
* normal sync tick, which collects the same `DEFAULT_SINCE_MINUTES` (24 h) every ~15 minutes and
|
|
4
|
+
* already uploads these sessions (`riding_already_durable`). Re-preparing a live, still-growing
|
|
5
|
+
* session in backfill held the collection lock for 232 s per tick and kept the run "partial"
|
|
6
|
+
* forever, so doctor re-ran the backfill on every tick.
|
|
7
|
+
*/
|
|
8
|
+
import { DEFAULT_SINCE_MINUTES } from "../adapters/raw-evidence-collection-state.js";
|
|
9
|
+
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
10
|
+
export const BACKFILL_LIVE_WINDOW_MS = DEFAULT_SINCE_MINUTES * 60 * 1000;
|
|
11
|
+
/** The window this run uses: none for a stub pass (it names its sessions), else the test seam or the default. */
|
|
12
|
+
export function liveWindowMsFor(command) {
|
|
13
|
+
return command.stubPass ? 0 : (command.liveWindowMs ?? BACKFILL_LIVE_WINDOW_MS);
|
|
14
|
+
}
|
|
15
|
+
/** `onlySessions` (`codex:<id>` / `claude_code:<id>`) when set, then the live window. */
|
|
16
|
+
export function scopeBackfillCandidates(candidates, options) {
|
|
17
|
+
const key = (c) => stubSessionKey(c.source, c.session_id);
|
|
18
|
+
const scoped = options.onlySessions ? candidates.filter((c) => options.onlySessions.has(key(c))) : candidates;
|
|
19
|
+
return leaveLiveSessionsToTheTick(scoped, options.live);
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Drop the candidates the tick owns: modified within `liveWindowMs` (0 keeps all), unless the
|
|
23
|
+
* server names the session a redaction stub (BLI-11102), which is re-offered even while live.
|
|
24
|
+
* Logs the count either way.
|
|
25
|
+
*/
|
|
26
|
+
export function leaveLiveSessionsToTheTick(candidates, options) {
|
|
27
|
+
let liveExcluded = 0;
|
|
28
|
+
const kept = options.liveWindowMs <= 0 ? candidates : candidates.filter((candidate) => {
|
|
29
|
+
const live = options.now.getTime() - candidate.session_file_mtime_ms < options.liveWindowMs
|
|
30
|
+
&& !options.stubbedSessions.has(stubSessionKey(candidate.source, candidate.session_id));
|
|
31
|
+
if (live)
|
|
32
|
+
liveExcluded += 1;
|
|
33
|
+
return !live;
|
|
34
|
+
});
|
|
35
|
+
console.error("[backfill] live sessions left to the tick", JSON.stringify({
|
|
36
|
+
reason: liveExcluded > 0 ? "live_owned_by_tick" : "no_live_sessions",
|
|
37
|
+
live_excluded: liveExcluded,
|
|
38
|
+
live_window_minutes: Math.round(options.liveWindowMs / 60_000),
|
|
39
|
+
}));
|
|
40
|
+
return { kept, liveExcluded };
|
|
41
|
+
}
|
|
@@ -228,7 +228,17 @@ function countRemainingHistory(options) {
|
|
|
228
228
|
async function writeAllHistoryCompletionMarker(ctx) {
|
|
229
229
|
const { paths, cursor, sources, scopedCursor, scan, oversizedCandidateKeys, now } = ctx;
|
|
230
230
|
const cappedSidecars = cappedSidecarCount(scan.candidates);
|
|
231
|
-
|
|
231
|
+
// BLI-22790: sessions left to the tick were NOT covered here. Mark coverage only up to the
|
|
232
|
+
// start of the live window, so once they age out a later backfill still sees them.
|
|
233
|
+
const coveredThrough = scan.live_excluded_count > 0
|
|
234
|
+
? new Date(now.getTime() - scan.live_window_ms)
|
|
235
|
+
: now;
|
|
236
|
+
console.error("[backfill] history coverage recorded", JSON.stringify({
|
|
237
|
+
reason: scan.live_excluded_count > 0 ? "covered_through_live_window_start" : "covered_through_now",
|
|
238
|
+
live_excluded: scan.live_excluded_count,
|
|
239
|
+
live_window_minutes: Math.round(scan.live_window_ms / 60_000),
|
|
240
|
+
}));
|
|
241
|
+
recordBackfillScanCoverage(cursor, sources, coveredThrough, now);
|
|
232
242
|
await writeBackfillCursor(paths, cursor);
|
|
233
243
|
const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
|
|
234
244
|
await writeBackfillCompletionMarker(paths, {
|
|
@@ -14,6 +14,7 @@ import { scanAndAttributeClaudeSessions, } from "../adapters/claude-attribution.
|
|
|
14
14
|
import { defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
|
|
15
15
|
import { emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
16
16
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
17
|
+
import { liveWindowMsFor, scopeBackfillCandidates } from "./backfill-live-window.js";
|
|
17
18
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
18
19
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
19
20
|
import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
|
|
@@ -23,7 +24,6 @@ import { blockedBackfillResult } from "./backfill-result.js";
|
|
|
23
24
|
import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
|
|
24
25
|
import { loadStubCursorHashes, locateStubbedSessions } from "./backfill-stub-locate.js";
|
|
25
26
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
26
|
-
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
27
27
|
/**
|
|
28
28
|
* SCAN: is this machine paired, what roots/sources/window apply, and what did
|
|
29
29
|
* the archived Codex + Claude session stores actually contain. Returns either
|
|
@@ -149,11 +149,13 @@ async function scanBackfillSessions(options) {
|
|
|
149
149
|
cursorHashes: options.stubCursorHashes,
|
|
150
150
|
sinceMinutes: options.window.since_minutes,
|
|
151
151
|
});
|
|
152
|
+
const liveWindowMs = liveWindowMsFor(options.command);
|
|
152
153
|
const selection = selectBackfillCandidates({
|
|
153
154
|
scanned,
|
|
154
155
|
cursor: options.cursor,
|
|
155
156
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
156
157
|
maxFiles: options.command.maxFiles,
|
|
158
|
+
live: { liveWindowMs, now: options.now, stubbedSessions: options.stubReoffer.sessions },
|
|
157
159
|
onlySessions: options.command.stubPass
|
|
158
160
|
? new Set(options.command.stubPass.plan.sessions.keys())
|
|
159
161
|
: undefined,
|
|
@@ -177,6 +179,8 @@ async function scanBackfillSessions(options) {
|
|
|
177
179
|
issues,
|
|
178
180
|
retryable_candidate_keys: retryableCandidateKeys(candidates),
|
|
179
181
|
omitted_candidate_count: selection.omittedCandidateCount,
|
|
182
|
+
live_excluded_count: selection.liveExcluded,
|
|
183
|
+
live_window_ms: liveWindowMs,
|
|
180
184
|
};
|
|
181
185
|
}
|
|
182
186
|
/**
|
|
@@ -220,9 +224,8 @@ async function scanClaudeHistory(request, projectsDir, scanLimit) {
|
|
|
220
224
|
* so a `--max-files` cap spends its budget on them before newer history.
|
|
221
225
|
*/
|
|
222
226
|
function selectBackfillCandidates(options) {
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
|
|
227
|
+
const { kept, liveExcluded } = scopeBackfillCandidates(options.scanned, options);
|
|
228
|
+
const allCandidates = kept
|
|
226
229
|
.filter((candidate) => isAfterCursor(candidate, options.cursor) ||
|
|
227
230
|
isPointerlessTerminalRetry(candidate, options.pointerlessTerminalSessions))
|
|
228
231
|
.sort((a, b) => compareBackfillCandidatesForRetry(a, b, options.pointerlessTerminalSessions));
|
|
@@ -230,6 +233,7 @@ function selectBackfillCandidates(options) {
|
|
|
230
233
|
return {
|
|
231
234
|
candidates,
|
|
232
235
|
omittedCandidateCount: allCandidates.length - candidates.length,
|
|
236
|
+
liveExcluded,
|
|
233
237
|
};
|
|
234
238
|
}
|
|
235
239
|
async function loadPointerlessTerminalSessions(paths, sources) {
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.251");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -10,9 +10,13 @@ import { postImageToDescribeDoor } from "./image-describe-client.js";
|
|
|
10
10
|
import { resolveImageHandlingMode } from "./image-mode-state.js";
|
|
11
11
|
import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
|
|
12
12
|
import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
|
|
13
|
-
/** New images sent to the door in one sync (all worktrees together). */
|
|
13
|
+
/** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
|
|
14
14
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* After this long spent DESCRIBING (the clock starts at the first image sent, not when the session
|
|
17
|
+
* opens: collection can reach image-bearing transcripts minutes into a tick), remaining images
|
|
18
|
+
* wait for the next tick.
|
|
19
|
+
*/
|
|
16
20
|
export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
|
|
17
21
|
const CONCURRENCY = 3;
|
|
18
22
|
const TAG = "[image-descriptions]";
|
|
@@ -25,11 +29,10 @@ export async function openImageDescriptions(options) {
|
|
|
25
29
|
const stateDirty = false;
|
|
26
30
|
const settledThisRun = new Map();
|
|
27
31
|
const now = options.now ?? Date.now;
|
|
28
|
-
const startedAt = now();
|
|
29
32
|
const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
|
|
30
33
|
const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
|
|
31
34
|
const sentThisRun = 0;
|
|
32
|
-
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now,
|
|
35
|
+
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, firstSendAt: null, maxPerRun, maxRunMs, sentThisRun };
|
|
33
36
|
return {
|
|
34
37
|
mode,
|
|
35
38
|
async prepareTranscript(input) {
|
|
@@ -52,6 +55,7 @@ function markDescribed(run, sha256) {
|
|
|
52
55
|
async function sendOne(run, image, text, input) {
|
|
53
56
|
const { options, fetchImpl, settledThisRun, failedImages } = run;
|
|
54
57
|
run.sentThisRun += 1;
|
|
58
|
+
run.firstSendAt ??= run.now();
|
|
55
59
|
const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
|
|
56
60
|
schema_version: "image-describe.v1",
|
|
57
61
|
sha256: image.sha256,
|
|
@@ -92,12 +96,12 @@ async function describeAndReplace(run, input) {
|
|
|
92
96
|
const { mode, described, options, orderedKeys, failedImages } = run;
|
|
93
97
|
const images = findEmbeddedImages(input.text);
|
|
94
98
|
if (images.length === 0)
|
|
95
|
-
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0 };
|
|
99
|
+
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
96
100
|
const unique = new Map();
|
|
97
101
|
for (const image of images)
|
|
98
102
|
if (!unique.has(image.sha256))
|
|
99
103
|
unique.set(image.sha256, image);
|
|
100
|
-
const counts = { already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
|
|
104
|
+
const counts = { settled_earlier: 0, already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
|
|
101
105
|
const toSend = selectImagesToDescribe(run, unique, counts);
|
|
102
106
|
await sendPendingImages(run, toSend, counts, input);
|
|
103
107
|
const failedReasons = countImageOutcomes(run, toSend, counts);
|
|
@@ -105,6 +109,8 @@ async function describeAndReplace(run, input) {
|
|
|
105
109
|
await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
|
|
106
110
|
run.stateDirty = false;
|
|
107
111
|
}
|
|
112
|
+
const deferredOnly = counts.over_budget + counts.backing_off > 0
|
|
113
|
+
&& counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
|
|
108
114
|
const replaced = mode === "describe_and_drop"
|
|
109
115
|
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
110
116
|
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
@@ -114,6 +120,7 @@ async function describeAndReplace(run, input) {
|
|
|
114
120
|
images: images.length,
|
|
115
121
|
unique: unique.size,
|
|
116
122
|
...counts,
|
|
123
|
+
deferred_only: deferredOnly,
|
|
117
124
|
failed_reasons: failedReasons,
|
|
118
125
|
replaced: replaced.replacedCount,
|
|
119
126
|
chars_saved: replaced.charsSaved,
|
|
@@ -125,6 +132,7 @@ async function describeAndReplace(run, input) {
|
|
|
125
132
|
uniqueCount: unique.size,
|
|
126
133
|
replacedCount: replaced.replacedCount,
|
|
127
134
|
charsSaved: replaced.charsSaved,
|
|
135
|
+
deferredOnly,
|
|
128
136
|
};
|
|
129
137
|
}
|
|
130
138
|
function selectImagesToDescribe(run, unique, counts) {
|
|
@@ -136,8 +144,10 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
136
144
|
continue;
|
|
137
145
|
}
|
|
138
146
|
const earlier = settledThisRun.get(image.sha256);
|
|
139
|
-
if (earlier)
|
|
147
|
+
if (earlier) {
|
|
148
|
+
counts.settled_earlier += 1;
|
|
140
149
|
continue;
|
|
150
|
+
}
|
|
141
151
|
if (image.byteSize > IMAGE_DESCRIBE_MAX_BYTES) {
|
|
142
152
|
settledThisRun.set(image.sha256, "too_large");
|
|
143
153
|
counts.too_large += 1;
|
|
@@ -153,12 +163,13 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
153
163
|
return toSend;
|
|
154
164
|
}
|
|
155
165
|
async function sendPendingImages(run, toSend, counts, input) {
|
|
156
|
-
const { now, maxPerRun,
|
|
166
|
+
const { now, maxPerRun, maxRunMs } = run;
|
|
157
167
|
let next = 0;
|
|
158
168
|
const worker = async () => {
|
|
159
169
|
while (next < toSend.length) {
|
|
160
170
|
const image = toSend[next++];
|
|
161
|
-
|
|
171
|
+
const timeUp = run.firstSendAt !== null && now() - run.firstSendAt > maxRunMs;
|
|
172
|
+
if (run.sentThisRun >= maxPerRun || timeUp) {
|
|
162
173
|
counts.over_budget += 1;
|
|
163
174
|
continue;
|
|
164
175
|
}
|