@bli-cockpit/cli 0.2.93 → 0.2.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-autostart.js +99 -0
- package/dist/commands/sync-followups-memory.js +123 -0
- package/dist/commands/sync-followups-self-update.js +127 -0
- package/dist/commands/sync-followups-staging.js +202 -0
- package/dist/commands/sync-followups.js +30 -511
- package/dist/commands/sync-heartbeat.js +36 -0
- package/dist/commands/sync-receipt.js +138 -0
- package/dist/commands/sync-report.js +153 -0
- package/dist/commands/sync-roots.js +28 -0
- package/dist/commands/sync-run.js +52 -0
- package/dist/commands/sync-types.js +1 -0
- package/dist/commands/sync.js +94 -357
- package/dist/disk-usage-classify.js +109 -0
- package/dist/disk-usage-facts.js +4 -0
- package/dist/disk-usage-files.js +99 -0
- package/dist/disk-usage-footprint.js +45 -0
- package/dist/disk-usage-ledger.js +49 -0
- package/dist/disk-usage-scan.js +80 -0
- package/dist/disk-usage-totals.js +83 -0
- package/dist/disk-usage.js +33 -396
- package/package.json +4 -4
package/dist/commands/sync.js
CHANGED
|
@@ -1,20 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit sync` — the tick. Every fifteen minutes on a fleet machine, and by
|
|
3
|
+
* hand whenever somebody types it.
|
|
4
|
+
*
|
|
5
|
+
* Read `runSync` below as the table of contents. It is the tick's steps, in
|
|
6
|
+
* order, and every part of this command lives in exactly one of them:
|
|
7
|
+
*
|
|
8
|
+
* rotate the scheduler's own logs ../log-rotation.ts
|
|
9
|
+
* say the tick started reportTickStarted, below
|
|
10
|
+
* collect behind both locks sync-receipt.ts
|
|
11
|
+
* check in with the heartbeat door sync-heartbeat.ts
|
|
12
|
+
* say how the tick went reportTickOutcome / reportTickThrew
|
|
13
|
+
* converge the rest of the machine sync-followups.ts
|
|
14
|
+
*
|
|
15
|
+
* The supporting modules each own one thing those steps share:
|
|
16
|
+
*
|
|
17
|
+
* sync-types.ts the shapes each step hands the next
|
|
18
|
+
* sync-roots.ts which approved roots this tick may collect
|
|
19
|
+
* sync-heartbeat.ts the check-in itself, and the backlog it carries
|
|
20
|
+
* sync-receipt.ts the two locks, and the receipt the run earns
|
|
21
|
+
* sync-run.ts the runbook under the lock: dedup, discover, sync
|
|
22
|
+
* sync-report.ts the three shapes a finished run is reported in
|
|
23
|
+
*
|
|
24
|
+
* Both arms below do the same things in the same order, because a tick that
|
|
25
|
+
* THREW is still a tick a machine survived and the fleet has to be able to tell
|
|
26
|
+
* that from silence (BLI-3551). The one difference is the staging prune, which
|
|
27
|
+
* the throwing arm has never run.
|
|
28
|
+
*
|
|
29
|
+
* Split out of commands/local.ts (BLI-3578), then into the siblings above
|
|
30
|
+
* (BLI-3982). The scan and upload engine underneath is `./session-sync.ts`;
|
|
31
|
+
* what a run reports is `./collection-report.ts`. A new sibling must also join
|
|
32
|
+
* `scripts/build-public-cli.mjs` `runtimeFiles`, or the repo tests stay green
|
|
33
|
+
* while the packed CLI breaks.
|
|
34
|
+
*/
|
|
1
35
|
import { runMemoryExperienceAfterSync } from "./memory-log.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
6
|
-
import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
|
|
7
|
-
import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
8
|
-
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
36
|
+
import { classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
37
|
+
import { sendSyncHeartbeat } from "./sync-heartbeat.js";
|
|
38
|
+
import { runSyncWithHealthReceipt } from "./sync-receipt.js";
|
|
9
39
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
10
|
-
import { describeError } from "../health-detail.js";
|
|
11
|
-
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
12
40
|
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
13
41
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
14
|
-
import { CollectionRootRequiredError, } from "../onboarding-roots.js";
|
|
15
|
-
import { rawEvidenceDedupSummary, rawEvidenceGcSummary, runRawEvidenceLocalGc, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
16
|
-
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
17
|
-
import { acquireSyncLock } from "../sync-lock.js";
|
|
18
42
|
export async function runSync(command, io) {
|
|
19
43
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
20
44
|
// BLI-3553, first thing in the tick: cap the scheduler's own logs. Nothing
|
|
@@ -22,38 +46,16 @@ export async function runSync(command, io) {
|
|
|
22
46
|
// sync.log on the reference Mac. Best-effort by construction — a rotation
|
|
23
47
|
// problem is its own log line, never a reason collection does not run.
|
|
24
48
|
await rotateCollectorLogsBestEffort(paths);
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
const minCliVersionAtStart = await reportInstallEventsBestEffort({
|
|
28
|
-
homeDir: command.homeDir,
|
|
29
|
-
dashboardUrl,
|
|
30
|
-
command: "sync",
|
|
31
|
-
events: [{ step: "sync_started", status: "ok" }],
|
|
32
|
-
json: command.json,
|
|
33
|
-
io,
|
|
34
|
-
});
|
|
49
|
+
const dashboardUrl = await tickDashboardUrl(command, paths);
|
|
50
|
+
const minCliVersionAtStart = await reportTickStarted(command, io, dashboardUrl);
|
|
35
51
|
try {
|
|
36
52
|
const result = await runSyncWithHealthReceipt(command, io);
|
|
37
53
|
// BLI-3551: every tick checks in, including one that collected nothing.
|
|
38
54
|
// This is the only writer of `last_seen_at` that does not need an envelope,
|
|
39
55
|
// so it is what separates a quiet machine from a dead one.
|
|
40
56
|
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
41
|
-
const minCliVersion = await
|
|
42
|
-
|
|
43
|
-
dashboardUrl,
|
|
44
|
-
command: "sync",
|
|
45
|
-
events: [result.completion],
|
|
46
|
-
json: command.json,
|
|
47
|
-
io,
|
|
48
|
-
});
|
|
49
|
-
// BLI-2601: self-update runs only after collection's own outcome above is
|
|
50
|
-
// already decided and reported, win or lose. See the function doc.
|
|
51
|
-
await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
|
|
52
|
-
await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
|
|
53
|
-
// BLI-3580: BLI Memory's registration converges the same way — after
|
|
54
|
-
// collection, at most once a day, its own receipt either way.
|
|
55
|
-
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
56
|
-
await runMemoryExperienceAfterSync({ ...command, dashboardUrl }, io);
|
|
57
|
+
const minCliVersion = await reportTickOutcome(command, io, dashboardUrl, result.completion);
|
|
58
|
+
await convergeAfterTick(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
|
|
57
59
|
// BLI-3619: and the disk stops growing without bound — same daily cadence,
|
|
58
60
|
// same rule that a follow-up never blocks or fails collection.
|
|
59
61
|
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
@@ -68,341 +70,76 @@ export async function runSync(command, io) {
|
|
|
68
70
|
status: "fail",
|
|
69
71
|
reason: errorCode,
|
|
70
72
|
});
|
|
71
|
-
const minCliVersion = await
|
|
72
|
-
|
|
73
|
-
dashboardUrl,
|
|
74
|
-
command: "sync",
|
|
75
|
-
events: [
|
|
76
|
-
{
|
|
77
|
-
step: "sync_complete",
|
|
78
|
-
status: "fail",
|
|
79
|
-
error_code: errorCode,
|
|
80
|
-
error_detail: redactedSyncErrorDetail(error),
|
|
81
|
-
},
|
|
82
|
-
],
|
|
83
|
-
json: command.json,
|
|
84
|
-
io,
|
|
85
|
-
});
|
|
86
|
-
await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
|
|
87
|
-
await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
|
|
88
|
-
// BLI-3580: BLI Memory's registration converges the same way — after
|
|
89
|
-
// collection, at most once a day, its own receipt either way.
|
|
90
|
-
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
91
|
-
await runMemoryExperienceAfterSync({ ...command, dashboardUrl }, io);
|
|
73
|
+
const minCliVersion = await reportTickThrew(command, io, dashboardUrl, error, errorCode);
|
|
74
|
+
await convergeAfterTick(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
|
|
92
75
|
throw error;
|
|
93
76
|
}
|
|
94
77
|
}
|
|
78
|
+
/** Where this tick reports: what the operator typed, else what it was paired to. */
|
|
79
|
+
async function tickDashboardUrl(command, paths) {
|
|
80
|
+
const config = await readLocalCollectorConfig(paths).catch(() => null);
|
|
81
|
+
return command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
|
|
82
|
+
}
|
|
95
83
|
/**
|
|
96
|
-
*
|
|
84
|
+
* Say the tick started, and learn the floor while asking.
|
|
97
85
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
86
|
+
* The reply carries `min_cli_version`, which is what the self-update follow-up
|
|
87
|
+
* needs; a tick that never reaches its own completion receipt still has this
|
|
88
|
+
* one to fall back on (BLI-2678).
|
|
101
89
|
*/
|
|
102
|
-
async function
|
|
103
|
-
|
|
104
|
-
// BLI-3797: the backlog figure rides the same check-in as the root labels, so
|
|
105
|
-
// `cockpit ops` learns about undelivered evidence on the SAME tick that proves
|
|
106
|
-
// the machine is alive. Best-effort: nulls omit the fields rather than
|
|
107
|
-
// reporting a zero nobody measured.
|
|
108
|
-
const staging = await readHeartbeatStagingFacts({
|
|
90
|
+
async function reportTickStarted(command, io, dashboardUrl) {
|
|
91
|
+
return reportInstallEventsBestEffort({
|
|
109
92
|
homeDir: command.homeDir,
|
|
110
|
-
|
|
111
|
-
|
|
93
|
+
dashboardUrl,
|
|
94
|
+
command: "sync",
|
|
95
|
+
events: [{ step: "sync_started", status: "ok" }],
|
|
96
|
+
json: command.json,
|
|
97
|
+
io,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/** Post the receipt the run earned, whatever it says. */
|
|
101
|
+
async function reportTickOutcome(command, io, dashboardUrl, completion) {
|
|
102
|
+
return reportInstallEventsBestEffort({
|
|
112
103
|
homeDir: command.homeDir,
|
|
113
104
|
dashboardUrl,
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
stagingUncommittedBytes: staging.bytes,
|
|
118
|
-
stagingUncommittedReason: staging.reason,
|
|
119
|
-
},
|
|
105
|
+
command: "sync",
|
|
106
|
+
events: [completion],
|
|
107
|
+
json: command.json,
|
|
120
108
|
io,
|
|
121
|
-
}).catch((error) => {
|
|
122
|
-
// The sender already swallows everything it knows about; this is the net
|
|
123
|
-
// for anything it does not, because a heartbeat must never fail a sync.
|
|
124
|
-
console.error("[heartbeat] the check-in threw and was dropped", JSON.stringify({ reason: "heartbeat_threw", ...describeError(error) }));
|
|
125
|
-
return false;
|
|
126
109
|
});
|
|
127
110
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
else {
|
|
142
|
-
writeLine(io.stdout, "Pausing normal sync while it catches up on old sessions.");
|
|
143
|
-
}
|
|
144
|
-
return {
|
|
145
|
-
exitCode: 0,
|
|
146
|
-
completion: {
|
|
147
|
-
step: "sync_complete",
|
|
148
|
-
status: "skipped",
|
|
149
|
-
error_code: "live_sync_paused_during_backfill",
|
|
150
|
-
},
|
|
151
|
-
heartbeat: {
|
|
152
|
-
status: "skipped",
|
|
153
|
-
reason: "live_sync_paused_during_backfill",
|
|
154
|
-
},
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
// Single-flight: a launchd timer and a manual sync must not interleave the
|
|
158
|
-
// cursor read-modify-write. A blocked invocation exits cleanly (B.4 §7).
|
|
159
|
-
const lock = await acquireSyncLock(getCollectorRuntimePaths(command.homeDir));
|
|
160
|
-
if (!lock.acquired) {
|
|
161
|
-
if (command.json) {
|
|
162
|
-
writeLine(io.stdout, JSON.stringify({
|
|
163
|
-
status: "sync_already_running",
|
|
164
|
-
reason: "another sync owns the collection lock",
|
|
165
|
-
held_since: lock.held_since,
|
|
166
|
-
collection_complete: false,
|
|
167
|
-
upload_state: "not_uploaded",
|
|
168
|
-
retryable: true,
|
|
169
|
-
}, null, 2));
|
|
170
|
-
}
|
|
171
|
-
else {
|
|
172
|
-
writeLine(io.stdout, "Tower sync already running; skipping this run.");
|
|
173
|
-
}
|
|
174
|
-
return {
|
|
175
|
-
exitCode: 0,
|
|
176
|
-
completion: {
|
|
177
|
-
step: "sync_complete",
|
|
178
|
-
status: "skipped",
|
|
179
|
-
error_code: "sync_already_running",
|
|
180
|
-
},
|
|
181
|
-
heartbeat: { status: "skipped", reason: "sync_already_running" },
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
try {
|
|
185
|
-
const { exitCode, failureReasons, failureRecords, notice, sessionsObserved, sessionsOutsideRoot, sessionsNewThisTick, sessionsPendingUpload, } = await runSyncLocked(command, io);
|
|
186
|
-
const counts = {
|
|
187
|
-
sessionsObserved,
|
|
188
|
-
sessionsOutsideRoot,
|
|
189
|
-
sessionsNewThisTick,
|
|
190
|
-
sessionsPendingUpload,
|
|
191
|
-
};
|
|
192
|
-
if (exitCode === 0) {
|
|
193
|
-
// BLI-3551: an `ok` tick can still have something to say. `nothing_in_root`
|
|
194
|
-
// is the receipt that separates "this machine is alive and its operator
|
|
195
|
-
// works outside the approved roots" from "this machine is dead", which
|
|
196
|
-
// until now looked identical from the dashboard.
|
|
197
|
-
return {
|
|
198
|
-
exitCode,
|
|
199
|
-
completion: {
|
|
200
|
-
step: "sync_complete",
|
|
201
|
-
status: "ok",
|
|
202
|
-
...(notice ? { error_detail: notice } : {}),
|
|
203
|
-
},
|
|
204
|
-
heartbeat: { status: "ok", reason: notice, ...counts },
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
// A sync that fails by exit code says exactly as much as one that throws.
|
|
208
|
-
// It used to say `sync_failed` and nothing else, so 100% of recorded
|
|
209
|
-
// failure rows carried a null detail and the real reason was reachable only
|
|
210
|
-
// by running `cockpit status` on the machine itself (BLI-2526).
|
|
211
|
-
const reasonText = failureReasons.join("; ");
|
|
212
|
-
// The bucket comes from the records the deciding branches wrote, not from
|
|
213
|
-
// this sentence (BLI-3551). The sentence is still the detail.
|
|
214
|
-
const errorCode = classifySyncFailureRecords(failureRecords);
|
|
215
|
-
return {
|
|
216
|
-
exitCode,
|
|
217
|
-
completion: {
|
|
111
|
+
/**
|
|
112
|
+
* Post a receipt for a tick that threw before it could earn one.
|
|
113
|
+
*
|
|
114
|
+
* The bucket is the classifier's, the detail is the redacted error: a receipt
|
|
115
|
+
* that can only say "failed" is one nobody can act on (BLI-2526).
|
|
116
|
+
*/
|
|
117
|
+
async function reportTickThrew(command, io, dashboardUrl, error, errorCode) {
|
|
118
|
+
return reportInstallEventsBestEffort({
|
|
119
|
+
homeDir: command.homeDir,
|
|
120
|
+
dashboardUrl,
|
|
121
|
+
command: "sync",
|
|
122
|
+
events: [
|
|
123
|
+
{
|
|
218
124
|
step: "sync_complete",
|
|
219
125
|
status: "fail",
|
|
220
126
|
error_code: errorCode,
|
|
221
|
-
error_detail: redactedSyncErrorDetail(
|
|
127
|
+
error_detail: redactedSyncErrorDetail(error),
|
|
222
128
|
},
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
await lock.handle.release();
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
/**
|
|
231
|
-
* Turn a finished run into an exit code and the reasons behind it.
|
|
232
|
-
*
|
|
233
|
-
* One place, so a future return path cannot reintroduce a code with no reason.
|
|
234
|
-
* The reasons come from the run itself — the code that decided `ok` is false is
|
|
235
|
-
* the only code that knows why.
|
|
236
|
-
*/
|
|
237
|
-
function syncResult(run) {
|
|
238
|
-
return {
|
|
239
|
-
exitCode: run.ok ? 0 : 1,
|
|
240
|
-
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
241
|
-
failureRecords: run.ok ? [] : run.failure_records,
|
|
242
|
-
notice: run.notice,
|
|
243
|
-
sessionsObserved: run.sessions_observed,
|
|
244
|
-
sessionsOutsideRoot: run.sessions_outside_root,
|
|
245
|
-
sessionsNewThisTick: run.sessions_new_this_tick,
|
|
246
|
-
sessionsPendingUpload: run.sessions_pending_upload,
|
|
247
|
-
};
|
|
129
|
+
],
|
|
130
|
+
json: command.json,
|
|
131
|
+
io,
|
|
132
|
+
});
|
|
248
133
|
}
|
|
249
134
|
/**
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
* than one branchy body.
|
|
135
|
+
* Everything the tick converges after its own outcome is already decided and
|
|
136
|
+
* reported, win or lose (BLI-2601): the CLI floor, the scheduler, and BLI
|
|
137
|
+
* Memory's registration — each at most once a day, each with its own receipt,
|
|
138
|
+
* and none of them able to block or fail collection (BLI-3580).
|
|
255
139
|
*/
|
|
256
|
-
async function
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
// anything else (BLI-3066). Safe by construction — a duplicate is identical
|
|
262
|
-
// by content hash to the survivor.
|
|
263
|
-
const dedup = await sweepDuplicateStagedRawEvidence(getCollectorRuntimePaths(command.homeDir), io.env);
|
|
264
|
-
if (!dedup.skipped && dedup.removed_dirs > 0) {
|
|
265
|
-
writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
|
|
266
|
-
}
|
|
267
|
-
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
268
|
-
maxDepth: command.maxDepth,
|
|
269
|
-
maxRepos: command.maxRepos,
|
|
270
|
-
homeDir: command.homeDir,
|
|
271
|
-
allowEmpty: true,
|
|
272
|
-
}, io);
|
|
273
|
-
const run = await runAttributedWorktreeSync({
|
|
274
|
-
homeDir: command.homeDir,
|
|
275
|
-
dashboardUrl: command.dashboardUrl,
|
|
276
|
-
collectionRoots,
|
|
277
|
-
startContexts: false,
|
|
278
|
-
worktrees,
|
|
279
|
-
fetchImpl: io.fetch,
|
|
280
|
-
});
|
|
281
|
-
if (run.outcomes.length > 1) {
|
|
282
|
-
return reportMultiRepoSync(command, io, run, dedup);
|
|
283
|
-
}
|
|
284
|
-
// Zero worktrees is a legitimate steady state, not a failure: an approved
|
|
285
|
-
// root can hold no git repos, and sessions upload independently of
|
|
286
|
-
// worktrees (session-first, BLI-2581). This used to throw "Sync produced no
|
|
287
|
-
// result", which painted ~90 false-red sync_failed receipts per day on one
|
|
288
|
-
// fleet machine with a single empty root and taught people to ignore
|
|
289
|
-
// sync_failed (BLI-2722). A genuinely broken run still fails via run.ok.
|
|
290
|
-
if (run.outcomes.length === 0) {
|
|
291
|
-
return reportNoWorktreeSync(command, io, run, dedup);
|
|
292
|
-
}
|
|
293
|
-
return reportSingleRepoSync(command, io, run, dedup);
|
|
294
|
-
}
|
|
295
|
-
async function reportMultiRepoSync(command, io, run, dedup) {
|
|
296
|
-
const rows = run.outcomes.map((outcome) => worktreeSyncRow(outcome, run));
|
|
297
|
-
const collectionRunStatus = attributedSyncRunStatus(run);
|
|
298
|
-
const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
|
|
299
|
-
if (command.json) {
|
|
300
|
-
writeLine(io.stdout, JSON.stringify({
|
|
301
|
-
mode: "multi_repo",
|
|
302
|
-
status: collectionRunStatus,
|
|
303
|
-
collection_complete: run.ok,
|
|
304
|
-
results: run.outcomes.map((outcome) => outcome.sync),
|
|
305
|
-
repos: rows,
|
|
306
|
-
codex_sessions: run.summary,
|
|
307
|
-
raw_evidence_gc: gc,
|
|
308
|
-
raw_evidence_dedup: dedup,
|
|
309
|
-
}, null, 2));
|
|
310
|
-
return syncResult(run);
|
|
311
|
-
}
|
|
312
|
-
writeLine(run.ok ? io.stdout : io.stderr, `Tower parent sync ${collectionRunStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
|
|
313
|
-
for (const outcome of run.outcomes) {
|
|
314
|
-
const { worktree, sync } = outcome;
|
|
315
|
-
const uploaded = sync.status === "uploaded";
|
|
316
|
-
const failureSuffix = sync.status === "spooled" ? ` reason:${sync.failure_reason}` : "";
|
|
317
|
-
writeLine(uploaded ? io.stdout : io.stderr, `- ${worktree.repo_label}/${worktree.worktree_label} (${worktree.branch}) head:${shortSha(sync.head_sha ?? worktree.head_sha)} ${sync.status} objects:${sync.raw_evidence_uploaded_object_count} chunks:${sync.raw_evidence_uploaded_chunk_count} reused:${sync.raw_evidence_reused_count} failed:${sync.raw_evidence_failed_count} cursor:${sync.cursor_tracked_object_count}${failureSuffix}`);
|
|
318
|
-
}
|
|
319
|
-
writeAgentSessionSummary(io, run.summary);
|
|
320
|
-
if (gc && !gc.skipped)
|
|
321
|
-
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
322
|
-
return syncResult(run);
|
|
323
|
-
}
|
|
324
|
-
async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
325
|
-
const collectionRunStatus = attributedSyncRunStatus(run);
|
|
326
|
-
const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
|
|
327
|
-
if (command.json) {
|
|
328
|
-
writeLine(io.stdout, JSON.stringify({
|
|
329
|
-
mode: "no_worktrees",
|
|
330
|
-
status: collectionRunStatus,
|
|
331
|
-
collection_complete: run.ok,
|
|
332
|
-
...(run.notice ? { notice: run.notice } : {}),
|
|
333
|
-
codex_sessions: run.summary,
|
|
334
|
-
raw_evidence_gc: gc,
|
|
335
|
-
raw_evidence_dedup: dedup,
|
|
336
|
-
}, null, 2));
|
|
337
|
-
return syncResult(run);
|
|
338
|
-
}
|
|
339
|
-
writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
|
|
340
|
-
if (run.notice) {
|
|
341
|
-
// Says out loud what the receipt now says to the dashboard: the sessions
|
|
342
|
-
// this machine ran were all outside the folders it is allowed to look at.
|
|
343
|
-
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
|
|
344
|
-
}
|
|
345
|
-
writeAgentSessionSummary(io, run.summary);
|
|
346
|
-
if (gc && !gc.skipped)
|
|
347
|
-
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
348
|
-
return syncResult(run);
|
|
349
|
-
}
|
|
350
|
-
async function reportSingleRepoSync(command, io, run, dedup) {
|
|
351
|
-
const result = run.outcomes[0]?.sync;
|
|
352
|
-
if (!result) {
|
|
353
|
-
throw new Error("Sync produced no result for the repo worktree.");
|
|
354
|
-
}
|
|
355
|
-
const collectionRunStatus = attributedSyncRunStatus(run);
|
|
356
|
-
const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
|
|
357
|
-
if (command.json) {
|
|
358
|
-
writeLine(io.stdout, JSON.stringify({
|
|
359
|
-
...result,
|
|
360
|
-
status: collectionRunStatus,
|
|
361
|
-
collection_complete: run.ok,
|
|
362
|
-
codex_sessions: run.summary,
|
|
363
|
-
raw_evidence_gc: gc,
|
|
364
|
-
raw_evidence_dedup: dedup,
|
|
365
|
-
}, null, 2));
|
|
366
|
-
return syncResult(run);
|
|
367
|
-
}
|
|
368
|
-
if (run.ok) {
|
|
369
|
-
writeLine(io.stdout, "Tower uploaded this session.");
|
|
370
|
-
writeLine(io.stdout, `Ticket: ${displayTicketId(result.ticket_id)}`);
|
|
371
|
-
writeLine(io.stdout, `Context: ${result.work_context_id}`);
|
|
372
|
-
writeLine(io.stdout, `Head: ${shortSha(result.head_sha)}`);
|
|
373
|
-
writeLine(io.stdout, `Things recorded: ${result.event_count}`);
|
|
374
|
-
writeLine(io.stdout, `Risk flags: ${result.risk_flag_count}`);
|
|
375
|
-
writeLine(io.stdout, `Raw evidence files: ${result.raw_evidence_file_count}`);
|
|
376
|
-
writeLine(io.stdout, rawEvidenceSyncLine(result));
|
|
377
|
-
writeAgentSessionSummary(io, run.summary);
|
|
378
|
-
writeLine(io.stdout, cursorStatusLine(result));
|
|
379
|
-
if (gc && !gc.skipped)
|
|
380
|
-
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
381
|
-
return syncResult(run);
|
|
382
|
-
}
|
|
383
|
-
if (result.status === "uploaded") {
|
|
384
|
-
writeLine(io.stderr, "Tower uploaded, but some sessions did not make it. Run `cockpit sync` again.");
|
|
385
|
-
writeAgentSessionSummary(io, run.summary);
|
|
386
|
-
return syncResult(run);
|
|
387
|
-
}
|
|
388
|
-
writeLine(io.stderr, "Tower could not upload. It saved a note to retry and will try again on the next sync.");
|
|
389
|
-
writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
|
|
390
|
-
writeLine(io.stderr, `Failure: ${result.failure_reason}`);
|
|
391
|
-
writeLine(io.stderr, `Retry: ${result.retry_command}`);
|
|
392
|
-
return syncResult(run);
|
|
393
|
-
}
|
|
394
|
-
async function resolveSyncCollectionRoots(command) {
|
|
395
|
-
const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
|
|
396
|
-
if (explicitRoots.length > 0) {
|
|
397
|
-
return collectionRootConsentAliases(explicitRoots);
|
|
398
|
-
}
|
|
399
|
-
const config = await readLocalCollectorConfig(getCollectorRuntimePaths(command.homeDir)).catch(() => null);
|
|
400
|
-
const savedRoots = normalizeCollectionRoots(config?.default_repo_paths ?? []);
|
|
401
|
-
if (savedRoots.length > 0) {
|
|
402
|
-
return collectionRootConsentAliases(savedRoots);
|
|
403
|
-
}
|
|
404
|
-
throw new CollectionRootRequiredError(`no explicit or saved collection root is available.`);
|
|
405
|
-
}
|
|
406
|
-
async function runSyncRawEvidenceGc(command, io) {
|
|
407
|
-
return runRawEvidenceLocalGc(getCollectorRuntimePaths(command.homeDir), io.env);
|
|
140
|
+
async function convergeAfterTick(command, io, dashboardUrl, minCliVersion) {
|
|
141
|
+
await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion);
|
|
142
|
+
await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
|
|
143
|
+
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
144
|
+
await runMemoryExperienceAfterSync({ ...command, dashboardUrl }, io);
|
|
408
145
|
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The three-state question, asked once per staged file: committed, uncommitted,
|
|
3
|
+
* or unknown — and the reason label that says which record answered.
|
|
4
|
+
*
|
|
5
|
+
* The order of the answers is the whole safety property. This machine's commit
|
|
6
|
+
* ledger wins first; a server answer an earlier `cockpit clean --reconcile`
|
|
7
|
+
* recorded wins next, because the server was actually asked; and only when
|
|
8
|
+
* neither can speak does the object fall to `unknown` with the reason its
|
|
9
|
+
* silence has. Everything downstream — what may be deleted, what is offered
|
|
10
|
+
* again, what an operator reads — is this label.
|
|
11
|
+
*/
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { hashUnnamedFile } from "./disk-usage-files.js";
|
|
14
|
+
export async function classifyObject(dir, packId, relative, size, manifest, options) {
|
|
15
|
+
const hash = manifest?.get(relative) ??
|
|
16
|
+
(await hashUnnamedFile(path.join(dir, relative), size, options.budget));
|
|
17
|
+
const stagedAt = options.stagedAt.get(`${packId}/${relative}`) ?? null;
|
|
18
|
+
const committedAt = hash ? options.ledger.committed.get(hash) : undefined;
|
|
19
|
+
const base = {
|
|
20
|
+
pack_id: packId,
|
|
21
|
+
relative_path: relative,
|
|
22
|
+
byte_size: size,
|
|
23
|
+
staged_on_disk_at: options.stagedOnDiskAt,
|
|
24
|
+
disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
|
|
25
|
+
};
|
|
26
|
+
if (committedAt) {
|
|
27
|
+
return {
|
|
28
|
+
...base,
|
|
29
|
+
content_hash: hash ?? null,
|
|
30
|
+
state: "committed",
|
|
31
|
+
reason: "committed_in_ledger",
|
|
32
|
+
decided_at: committedAt,
|
|
33
|
+
age_ms: ageMs(committedAt, options.now),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
if (hash) {
|
|
37
|
+
const reconciled = options.reconciled.get(hash);
|
|
38
|
+
if (reconciled) {
|
|
39
|
+
return classifyFromReconcileAnswer(base, hash, reconciled, options.now);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const unknownReason = unknownStateReason(hash, manifest, stagedAt, options);
|
|
43
|
+
return {
|
|
44
|
+
...base,
|
|
45
|
+
content_hash: hash ?? null,
|
|
46
|
+
state: unknownReason ? "unknown" : "uncommitted",
|
|
47
|
+
reason: unknownReason ?? "absent_from_ledger",
|
|
48
|
+
decided_at: stagedAt,
|
|
49
|
+
age_ms: ageMs(stagedAt, options.now),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The server's own answer for this hash, already on disk from an earlier
|
|
54
|
+
* `cockpit clean --reconcile` run. This takes priority over
|
|
55
|
+
* `unknownStateReason`'s guesswork about the LOCAL ledger's memory, because
|
|
56
|
+
* the server was actually asked and actually answered.
|
|
57
|
+
*/
|
|
58
|
+
function classifyFromReconcileAnswer(base, hash, reconciled, now) {
|
|
59
|
+
if (reconciled.verdict === "committed") {
|
|
60
|
+
const decidedAt = reconciled.server_committed_at ?? reconciled.checked_at;
|
|
61
|
+
return {
|
|
62
|
+
...base,
|
|
63
|
+
content_hash: hash,
|
|
64
|
+
state: "committed",
|
|
65
|
+
reason: "reconciled_committed",
|
|
66
|
+
decided_at: decidedAt,
|
|
67
|
+
age_ms: ageMs(decidedAt, now),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
...base,
|
|
72
|
+
content_hash: hash,
|
|
73
|
+
state: "uncommitted",
|
|
74
|
+
reason: reconciled.verdict === "not_committed"
|
|
75
|
+
? "reconciled_not_committed"
|
|
76
|
+
: "reconciled_unknown_to_server",
|
|
77
|
+
decided_at: reconciled.checked_at,
|
|
78
|
+
age_ms: ageMs(reconciled.checked_at, now),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Why the ledger's silence about this object is NOT the same as a "no".
|
|
83
|
+
*
|
|
84
|
+
* The `objects` map is capped at 5,000 rows and drops the oldest, so anything
|
|
85
|
+
* staged before the oldest row it still holds is outside its memory. Saying
|
|
86
|
+
* "uncommitted" there would be a claim the ledger never made.
|
|
87
|
+
*/
|
|
88
|
+
function unknownStateReason(hash, manifest, stagedAt, options) {
|
|
89
|
+
if (!manifest)
|
|
90
|
+
return "manifest_unreadable";
|
|
91
|
+
if (!hash)
|
|
92
|
+
return "no_manifest_row";
|
|
93
|
+
const reachesBackTo = options.ledger.reachesBackTo;
|
|
94
|
+
if (!reachesBackTo)
|
|
95
|
+
return "ledger_empty";
|
|
96
|
+
if (stagedAt && stagedAt.localeCompare(reachesBackTo) < 0) {
|
|
97
|
+
return "ledger_evicted";
|
|
98
|
+
}
|
|
99
|
+
if (!stagedAt)
|
|
100
|
+
return "ledger_evicted";
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
/** How old a recorded moment is, with an unreadable one worth nothing. */
|
|
104
|
+
function ageMs(at, now) {
|
|
105
|
+
if (!at)
|
|
106
|
+
return 0;
|
|
107
|
+
const parsed = Date.parse(at);
|
|
108
|
+
return Number.isFinite(parsed) ? Math.max(0, now.getTime() - parsed) : 0;
|
|
109
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Directory name of the staged raw-evidence tree under the state directory. */
|
|
2
|
+
export const RAW_EVIDENCE_DIR = "raw-evidence";
|
|
3
|
+
/** One-off evidence vaults a person made by hand; named, never auto-deleted. */
|
|
4
|
+
export const MANUAL_VAULT_PREFIX = "manual-study-evidence-vault-";
|