omp-conductor 0.19.4 → 0.19.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/resume.ts +5 -0
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +54 -8
- package/src/dashboard/controls.ts +3 -0
- package/src/doctor.ts +36 -3
- package/src/fleet.ts +51 -3
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +21 -0
- package/src/store.ts +296 -22
- package/src/types.ts +32 -3
- package/src/upgrade-journal.ts +37 -1
- package/src/upgrade-verify.ts +272 -37
- package/src/upgrade.ts +324 -23
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +54 -20
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/src/store.ts
CHANGED
|
@@ -53,6 +53,8 @@ import type {
|
|
|
53
53
|
GroomingRecord,
|
|
54
54
|
GroomingVerdict,
|
|
55
55
|
HistoricalInfraCandidate,
|
|
56
|
+
HandoffWithdrawal,
|
|
57
|
+
HandoffWithdrawalResult,
|
|
56
58
|
HeldNotice,
|
|
57
59
|
InterruptCategory,
|
|
58
60
|
HeldNoticeDraft,
|
|
@@ -430,6 +432,30 @@ interface ReportRow {
|
|
|
430
432
|
sentParts: number;
|
|
431
433
|
sentPartsHash: string | null;
|
|
432
434
|
lastError: string | null;
|
|
435
|
+
withdrawnAt: number | null;
|
|
436
|
+
withdrawnBy: string | null;
|
|
437
|
+
withdrawReason: string | null;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
interface HandoffWithdrawalRow {
|
|
441
|
+
id: string;
|
|
442
|
+
project: string;
|
|
443
|
+
target: string;
|
|
444
|
+
actor: string;
|
|
445
|
+
reason: string;
|
|
446
|
+
at: number;
|
|
447
|
+
summary: string;
|
|
448
|
+
detail: string;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
interface HeldNoticeLifecycleRow {
|
|
452
|
+
id: string;
|
|
453
|
+
project: string;
|
|
454
|
+
summary: string;
|
|
455
|
+
detail: string;
|
|
456
|
+
digestedAt: number | null;
|
|
457
|
+
digestReportId: string | null;
|
|
458
|
+
withdrawnAt: number | null;
|
|
433
459
|
}
|
|
434
460
|
|
|
435
461
|
/** The `material_events` table exactly as SQLite hands it back (#274). */
|
|
@@ -940,13 +966,30 @@ CREATE TABLE IF NOT EXISTS reports (
|
|
|
940
966
|
deliveredAt INTEGER,
|
|
941
967
|
sentParts INTEGER NOT NULL DEFAULT 0,
|
|
942
968
|
sentPartsHash TEXT,
|
|
943
|
-
lastError TEXT
|
|
969
|
+
lastError TEXT,
|
|
970
|
+
withdrawnAt INTEGER,
|
|
971
|
+
withdrawnBy TEXT,
|
|
972
|
+
withdrawReason TEXT
|
|
944
973
|
);
|
|
945
974
|
CREATE UNIQUE INDEX IF NOT EXISTS reports_dedupe
|
|
946
|
-
ON reports (project, dedupeKey)
|
|
975
|
+
ON reports (project, dedupeKey)
|
|
976
|
+
WHERE dedupeKey IS NOT NULL AND state <> 'failed' AND state <> 'withdrawn';
|
|
947
977
|
CREATE INDEX IF NOT EXISTS reports_project_state
|
|
948
978
|
ON reports (project, state, nextAttemptAt);
|
|
949
979
|
|
|
980
|
+
CREATE TABLE IF NOT EXISTS handoff_withdrawals (
|
|
981
|
+
id TEXT PRIMARY KEY,
|
|
982
|
+
project TEXT NOT NULL,
|
|
983
|
+
target TEXT NOT NULL,
|
|
984
|
+
actor TEXT NOT NULL,
|
|
985
|
+
reason TEXT NOT NULL,
|
|
986
|
+
at INTEGER NOT NULL,
|
|
987
|
+
summary TEXT NOT NULL,
|
|
988
|
+
detail TEXT NOT NULL
|
|
989
|
+
);
|
|
990
|
+
CREATE INDEX IF NOT EXISTS handoff_withdrawals_project_at
|
|
991
|
+
ON handoff_withdrawals (project, at DESC);
|
|
992
|
+
|
|
950
993
|
-- Ordinary material outcomes awaiting authorship in a deferred digest (#274).
|
|
951
994
|
-- Association happens when a digest is accepted into the outbox, not when the
|
|
952
995
|
-- session happens to remember the event or when Telegram later delivers it.
|
|
@@ -979,7 +1022,10 @@ CREATE TABLE IF NOT EXISTS held_notices (
|
|
|
979
1022
|
releaseOnAvailable INTEGER NOT NULL DEFAULT 0,
|
|
980
1023
|
urgent INTEGER NOT NULL DEFAULT 0,
|
|
981
1024
|
digestedAt INTEGER,
|
|
982
|
-
digestReportId TEXT REFERENCES reports(id)
|
|
1025
|
+
digestReportId TEXT REFERENCES reports(id),
|
|
1026
|
+
withdrawnAt INTEGER,
|
|
1027
|
+
withdrawnBy TEXT,
|
|
1028
|
+
withdrawReason TEXT
|
|
983
1029
|
);
|
|
984
1030
|
CREATE INDEX IF NOT EXISTS held_notices_project_undigested
|
|
985
1031
|
ON held_notices (project, digestedAt) WHERE digestedAt IS NULL;
|
|
@@ -1562,9 +1608,25 @@ function toReport(row: ReportRow): ReportRecord {
|
|
|
1562
1608
|
if (row.sentPartsHash !== null) record.sentPartsHash = row.sentPartsHash;
|
|
1563
1609
|
if (row.deliveredAt !== null) record.deliveredAt = row.deliveredAt;
|
|
1564
1610
|
if (row.lastError !== null) record.lastError = row.lastError;
|
|
1611
|
+
if (row.withdrawnAt !== null) record.withdrawnAt = row.withdrawnAt;
|
|
1612
|
+
if (row.withdrawnBy !== null) record.withdrawnBy = row.withdrawnBy;
|
|
1613
|
+
if (row.withdrawReason !== null) record.withdrawReason = row.withdrawReason;
|
|
1565
1614
|
return record;
|
|
1566
1615
|
}
|
|
1567
1616
|
|
|
1617
|
+
function toHandoffWithdrawal(row: HandoffWithdrawalRow): HandoffWithdrawal {
|
|
1618
|
+
return {
|
|
1619
|
+
id: row.id,
|
|
1620
|
+
project: row.project,
|
|
1621
|
+
summary: row.summary,
|
|
1622
|
+
detail: row.detail,
|
|
1623
|
+
target: row.target as HandoffWithdrawal["target"],
|
|
1624
|
+
actor: row.actor,
|
|
1625
|
+
reason: row.reason,
|
|
1626
|
+
at: row.at,
|
|
1627
|
+
};
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1568
1630
|
function toMaterialEvent(row: MaterialEventRow): MaterialEvent {
|
|
1569
1631
|
return {
|
|
1570
1632
|
id: row.id,
|
|
@@ -1905,20 +1967,20 @@ export function openStore(dbPath: string): Store {
|
|
|
1905
1967
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
1906
1968
|
db.exec("PRAGMA foreign_keys = ON;");
|
|
1907
1969
|
db.exec(SCHEMA);
|
|
1908
|
-
//
|
|
1909
|
-
//
|
|
1910
|
-
// one-time schema so owed ledger rows can be handed off again.
|
|
1970
|
+
// Failed or explicitly withdrawn handoffs no longer own a digest dedupe key:
|
|
1971
|
+
// both leave their underlying ledger rows owed for a replacement report.
|
|
1911
1972
|
const reportDedupeIndex = db
|
|
1912
1973
|
.query<{ sql: string | null }, []>(
|
|
1913
1974
|
`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'reports_dedupe'`,
|
|
1914
1975
|
)
|
|
1915
1976
|
.get();
|
|
1916
|
-
if (!(reportDedupeIndex?.sql ?? "").includes("state <> '
|
|
1977
|
+
if (!(reportDedupeIndex?.sql ?? "").includes("state <> 'withdrawn'")) {
|
|
1917
1978
|
db.exec(
|
|
1918
1979
|
`BEGIN IMMEDIATE;
|
|
1919
1980
|
DROP INDEX IF EXISTS reports_dedupe;
|
|
1920
1981
|
CREATE UNIQUE INDEX reports_dedupe
|
|
1921
|
-
ON reports (project, dedupeKey)
|
|
1982
|
+
ON reports (project, dedupeKey)
|
|
1983
|
+
WHERE dedupeKey IS NOT NULL AND state <> 'failed' AND state <> 'withdrawn';
|
|
1922
1984
|
COMMIT;`,
|
|
1923
1985
|
);
|
|
1924
1986
|
}
|
|
@@ -2112,6 +2174,15 @@ export function openStore(dbPath: string): Store {
|
|
|
2112
2174
|
}
|
|
2113
2175
|
// Urgent recovery notices bypass category batching only; they still wait for
|
|
2114
2176
|
// the configured availability window. Historical notices were not urgent.
|
|
2177
|
+
for (const [name, type] of [
|
|
2178
|
+
["withdrawnAt", "INTEGER"],
|
|
2179
|
+
["withdrawnBy", "TEXT"],
|
|
2180
|
+
["withdrawReason", "TEXT"],
|
|
2181
|
+
] as const) {
|
|
2182
|
+
if (!heldNoticeColumns.some((column) => column.name === name)) {
|
|
2183
|
+
db.exec(`ALTER TABLE held_notices ADD COLUMN ${name} ${type}`);
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2115
2186
|
if (!heldNoticeColumns.some((column) => column.name === "urgent")) {
|
|
2116
2187
|
db.exec("ALTER TABLE held_notices ADD COLUMN urgent INTEGER NOT NULL DEFAULT 0");
|
|
2117
2188
|
}
|
|
@@ -2129,6 +2200,15 @@ export function openStore(dbPath: string): Store {
|
|
|
2129
2200
|
if (!reportColumns.some((column) => column.name === "sentPartsHash")) {
|
|
2130
2201
|
db.exec("ALTER TABLE reports ADD COLUMN sentPartsHash TEXT");
|
|
2131
2202
|
}
|
|
2203
|
+
for (const [name, type] of [
|
|
2204
|
+
["withdrawnAt", "INTEGER"],
|
|
2205
|
+
["withdrawnBy", "TEXT"],
|
|
2206
|
+
["withdrawReason", "TEXT"],
|
|
2207
|
+
] as const) {
|
|
2208
|
+
if (!reportColumns.some((column) => column.name === name)) {
|
|
2209
|
+
db.exec(`ALTER TABLE reports ADD COLUMN ${name} ${type}`);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2132
2212
|
// The kind split (#459): a row is either a question a human must answer or a
|
|
2133
2213
|
// watch the orchestrator set for itself. Databases written before the split
|
|
2134
2214
|
// never marked it, and every such row was a question — `watch` has no
|
|
@@ -3012,7 +3092,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3012
3092
|
const selectReport = db.query<ReportRow, [string]>(`SELECT * FROM reports WHERE id = ?`);
|
|
3013
3093
|
const selectReportByDedupe = db.query<ReportRow, [string, string]>(
|
|
3014
3094
|
`SELECT * FROM reports
|
|
3015
|
-
WHERE project = ? AND dedupeKey = ? AND state
|
|
3095
|
+
WHERE project = ? AND dedupeKey = ? AND state NOT IN ('failed', 'withdrawn')
|
|
3016
3096
|
ORDER BY createdAt DESC, rowid DESC LIMIT 1`,
|
|
3017
3097
|
);
|
|
3018
3098
|
const selectDueReports = db.query<ReportRow, [string, number, number]>(
|
|
@@ -3021,6 +3101,81 @@ export function openStore(dbPath: string): Store {
|
|
|
3021
3101
|
ORDER BY createdAt ASC, rowid ASC
|
|
3022
3102
|
LIMIT ?`,
|
|
3023
3103
|
);
|
|
3104
|
+
const withdrawPendingReportRow = db.query<
|
|
3105
|
+
unknown,
|
|
3106
|
+
[number, string, string, number, string, string]
|
|
3107
|
+
>(
|
|
3108
|
+
`UPDATE reports
|
|
3109
|
+
SET state = 'withdrawn', withdrawnAt = ?, withdrawnBy = ?,
|
|
3110
|
+
withdrawReason = ?, updatedAt = ?, attemptId = NULL
|
|
3111
|
+
WHERE id = ? AND project = ? AND state = 'pending'`,
|
|
3112
|
+
);
|
|
3113
|
+
const selectReportNoticeIds = db.query<{ id: string }, [string, string]>(
|
|
3114
|
+
`SELECT id FROM held_notices WHERE digestReportId = ? AND project = ?`,
|
|
3115
|
+
);
|
|
3116
|
+
const unassignReportMaterialEvents = db.query<unknown, [string, string]>(
|
|
3117
|
+
`UPDATE material_events SET digestReportId = NULL
|
|
3118
|
+
WHERE digestReportId = ? AND project = ?`,
|
|
3119
|
+
);
|
|
3120
|
+
const markReportMaterialEventsPossibleRepeat = db.query<unknown, [string, string, string]>(
|
|
3121
|
+
`UPDATE material_events
|
|
3122
|
+
SET summary = CASE
|
|
3123
|
+
WHEN summary NOT LIKE 'POSSIBLE REPEAT%' THEN 'POSSIBLE REPEAT of report ' || ? || ': ' || summary
|
|
3124
|
+
ELSE summary
|
|
3125
|
+
END
|
|
3126
|
+
WHERE digestReportId = ? AND project = ?`,
|
|
3127
|
+
);
|
|
3128
|
+
const markReportHeldNoticesPossibleRepeat = db.query<unknown, [string, string, string]>(
|
|
3129
|
+
`UPDATE held_notices
|
|
3130
|
+
SET summary = CASE
|
|
3131
|
+
WHEN summary NOT LIKE 'POSSIBLE REPEAT%' THEN 'POSSIBLE REPEAT of report ' || ? || ': ' || summary
|
|
3132
|
+
ELSE summary
|
|
3133
|
+
END,
|
|
3134
|
+
detail = CASE
|
|
3135
|
+
WHEN detail NOT LIKE 'This handoff may already%' THEN
|
|
3136
|
+
'This handoff may already have reached Telegram before its outcome was lost.\n\n' || detail
|
|
3137
|
+
ELSE detail
|
|
3138
|
+
END
|
|
3139
|
+
WHERE digestReportId = ? AND project = ? AND withdrawnAt IS NULL`,
|
|
3140
|
+
);
|
|
3141
|
+
const unassignReportHeldNotices = db.query<unknown, [string, string]>(
|
|
3142
|
+
`UPDATE held_notices SET digestReportId = NULL, digestedAt = NULL
|
|
3143
|
+
WHERE digestReportId = ? AND project = ? AND withdrawnAt IS NULL`,
|
|
3144
|
+
);
|
|
3145
|
+
const selectHeldNoticeLifecycle = db.query<HeldNoticeLifecycleRow, [string]>(
|
|
3146
|
+
`SELECT id, project, summary, detail, digestedAt, digestReportId, withdrawnAt
|
|
3147
|
+
FROM held_notices WHERE id = ?`,
|
|
3148
|
+
);
|
|
3149
|
+
const withdrawHeldNoticeRow = db.query<
|
|
3150
|
+
unknown,
|
|
3151
|
+
[number, string, string, string, string]
|
|
3152
|
+
>(
|
|
3153
|
+
`UPDATE held_notices
|
|
3154
|
+
SET withdrawnAt = ?, withdrawnBy = ?, withdrawReason = ?
|
|
3155
|
+
WHERE id = ? AND project = ? AND withdrawnAt IS NULL AND (
|
|
3156
|
+
(digestReportId IS NULL AND digestedAt IS NULL)
|
|
3157
|
+
OR EXISTS (
|
|
3158
|
+
SELECT 1 FROM reports
|
|
3159
|
+
WHERE reports.id = held_notices.digestReportId
|
|
3160
|
+
AND reports.state IN ('failed', 'withdrawn')
|
|
3161
|
+
)
|
|
3162
|
+
)`,
|
|
3163
|
+
);
|
|
3164
|
+
const insertHandoffWithdrawal = db.query<
|
|
3165
|
+
unknown,
|
|
3166
|
+
[string, string, string, string, string, number, string, string]
|
|
3167
|
+
>(
|
|
3168
|
+
`INSERT INTO handoff_withdrawals
|
|
3169
|
+
(id, project, target, actor, reason, at, summary, detail)
|
|
3170
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
3171
|
+
);
|
|
3172
|
+
const selectHandoffWithdrawal = db.query<HandoffWithdrawalRow, [string]>(
|
|
3173
|
+
`SELECT * FROM handoff_withdrawals WHERE id = ?`,
|
|
3174
|
+
);
|
|
3175
|
+
const selectHandoffWithdrawals = db.query<HandoffWithdrawalRow, [string, number]>(
|
|
3176
|
+
`SELECT * FROM handoff_withdrawals
|
|
3177
|
+
WHERE project = ? ORDER BY at DESC, rowid DESC LIMIT ?`,
|
|
3178
|
+
);
|
|
3024
3179
|
const deletePendingMaterialReport = db.query<unknown, [string, string]>(
|
|
3025
3180
|
`DELETE FROM reports
|
|
3026
3181
|
WHERE id = ? AND project = ? AND kind = 'material' AND state = 'pending'`,
|
|
@@ -3051,7 +3206,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3051
3206
|
THEN 'POSSIBLE REPEAT of catch-up ' || ? || ': ' || summary
|
|
3052
3207
|
ELSE summary
|
|
3053
3208
|
END
|
|
3054
|
-
WHERE project = ? AND digestReportId = ? AND EXISTS (
|
|
3209
|
+
WHERE project = ? AND digestReportId = ? AND withdrawnAt IS NULL AND EXISTS (
|
|
3055
3210
|
SELECT 1 FROM reports
|
|
3056
3211
|
WHERE id = ? AND project = ? AND kind = 'digest' AND state = 'pending'
|
|
3057
3212
|
AND dedupeKey LIKE 'availability/%'
|
|
@@ -3059,12 +3214,13 @@ export function openStore(dbPath: string): Store {
|
|
|
3059
3214
|
);
|
|
3060
3215
|
const selectAvailabilityReservationCandidate = db.query<{ id: string }, [string]>(
|
|
3061
3216
|
`SELECT held_notices.id FROM held_notices
|
|
3062
|
-
WHERE held_notices.project = ? AND held_notices.releaseOnAvailable = 1
|
|
3217
|
+
WHERE held_notices.project = ? AND held_notices.releaseOnAvailable = 1
|
|
3218
|
+
AND held_notices.withdrawnAt IS NULL AND (
|
|
3063
3219
|
(held_notices.digestReportId IS NULL AND held_notices.digestedAt IS NULL)
|
|
3064
3220
|
OR EXISTS (
|
|
3065
3221
|
SELECT 1 FROM reports
|
|
3066
3222
|
WHERE reports.id = held_notices.digestReportId AND (
|
|
3067
|
-
reports.state
|
|
3223
|
+
reports.state IN ('failed', 'withdrawn')
|
|
3068
3224
|
OR (
|
|
3069
3225
|
reports.state = 'pending' AND reports.kind = 'digest'
|
|
3070
3226
|
AND reports.dedupeKey LIKE 'availability/%'
|
|
@@ -3180,9 +3336,9 @@ export function openStore(dbPath: string): Store {
|
|
|
3180
3336
|
releaseOnAvailable: number;
|
|
3181
3337
|
urgent: number;
|
|
3182
3338
|
};
|
|
3183
|
-
const undigestedNoticeWhere = `project = ? AND (
|
|
3339
|
+
const undigestedNoticeWhere = `project = ? AND withdrawnAt IS NULL AND (
|
|
3184
3340
|
(digestReportId IS NULL AND digestedAt IS NULL)
|
|
3185
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state
|
|
3341
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3186
3342
|
)`;
|
|
3187
3343
|
const selectUndigestedNotices = db.query<HeldNoticeRow, [string, number]>(
|
|
3188
3344
|
`SELECT id, category, summary, detail, createdAt, releaseOnAvailable, urgent FROM held_notices
|
|
@@ -3212,16 +3368,16 @@ export function openStore(dbPath: string): Store {
|
|
|
3212
3368
|
MIN(CASE WHEN releaseOnAvailable = 0 THEN createdAt END) AS digestOldestAt,
|
|
3213
3369
|
MIN(CASE WHEN releaseOnAvailable = 1 THEN createdAt END) AS availabilityOldestAt
|
|
3214
3370
|
FROM held_notices
|
|
3215
|
-
WHERE project = ? AND (
|
|
3371
|
+
WHERE project = ? AND withdrawnAt IS NULL AND (
|
|
3216
3372
|
(digestReportId IS NULL AND digestedAt IS NULL)
|
|
3217
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state
|
|
3373
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3218
3374
|
)`,
|
|
3219
3375
|
);
|
|
3220
3376
|
const assignHeldNoticeToDigest = db.query<unknown, [number, string, string, string]>(
|
|
3221
3377
|
`UPDATE held_notices SET digestedAt = ?, digestReportId = ?
|
|
3222
|
-
WHERE project = ? AND id = ? AND (
|
|
3378
|
+
WHERE project = ? AND id = ? AND withdrawnAt IS NULL AND (
|
|
3223
3379
|
(digestReportId IS NULL AND digestedAt IS NULL)
|
|
3224
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state
|
|
3380
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = held_notices.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3225
3381
|
)`,
|
|
3226
3382
|
);
|
|
3227
3383
|
const insertMaterialEvent = db.query<unknown, SqlValue[]>(
|
|
@@ -3236,7 +3392,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3236
3392
|
`SELECT * FROM material_events
|
|
3237
3393
|
WHERE project = ? AND (
|
|
3238
3394
|
digestReportId IS NULL
|
|
3239
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state
|
|
3395
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3240
3396
|
)
|
|
3241
3397
|
ORDER BY occurredAt ASC, recordedAt ASC, rowid ASC
|
|
3242
3398
|
LIMIT ?`,
|
|
@@ -3245,14 +3401,14 @@ export function openStore(dbPath: string): Store {
|
|
|
3245
3401
|
`SELECT COUNT(*) AS count, MIN(occurredAt) AS oldestAt FROM material_events
|
|
3246
3402
|
WHERE project = ? AND (
|
|
3247
3403
|
digestReportId IS NULL
|
|
3248
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state
|
|
3404
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3249
3405
|
)`,
|
|
3250
3406
|
);
|
|
3251
3407
|
const assignMaterialEventToDigest = db.query<unknown, [string, string, string]>(
|
|
3252
3408
|
`UPDATE material_events SET digestReportId = ?
|
|
3253
3409
|
WHERE project = ? AND id = ? AND (
|
|
3254
3410
|
digestReportId IS NULL
|
|
3255
|
-
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state
|
|
3411
|
+
OR EXISTS (SELECT 1 FROM reports WHERE reports.id = material_events.digestReportId AND reports.state IN ('failed', 'withdrawn'))
|
|
3256
3412
|
)`,
|
|
3257
3413
|
);
|
|
3258
3414
|
const insertIntake = db.query<unknown, SqlValue[]>(
|
|
@@ -3278,7 +3434,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3278
3434
|
// today's key under `digestDue` (#229).
|
|
3279
3435
|
const selectLastDigestKey = db.query<{ key: string | null }, [string]>(
|
|
3280
3436
|
`SELECT dedupeKey AS key FROM reports
|
|
3281
|
-
WHERE project = ? AND dedupeKey LIKE 'digest:%' AND state
|
|
3437
|
+
WHERE project = ? AND dedupeKey LIKE 'digest:%' AND state NOT IN ('failed', 'withdrawn')
|
|
3282
3438
|
ORDER BY createdAt DESC LIMIT 1`,
|
|
3283
3439
|
);
|
|
3284
3440
|
const claimReportRow = db.query<unknown, [string, number, string, number]>(
|
|
@@ -3404,6 +3560,109 @@ export function openStore(dbPath: string): Store {
|
|
|
3404
3560
|
}
|
|
3405
3561
|
return enqueued;
|
|
3406
3562
|
};
|
|
3563
|
+
|
|
3564
|
+
const withdrawHandoffTx = db.transaction(
|
|
3565
|
+
(
|
|
3566
|
+
id: string,
|
|
3567
|
+
project: string,
|
|
3568
|
+
actor: string,
|
|
3569
|
+
reason: string,
|
|
3570
|
+
at: number,
|
|
3571
|
+
): HandoffWithdrawalResult => {
|
|
3572
|
+
const report = selectReport.get(id);
|
|
3573
|
+
if (report !== null) {
|
|
3574
|
+
if (report.project !== project) return { kind: "not-found" };
|
|
3575
|
+
if (report.state !== "pending") {
|
|
3576
|
+
return { kind: "refused", target: "report", state: report.state };
|
|
3577
|
+
}
|
|
3578
|
+
const noticeIds = selectReportNoticeIds.all(id, project).map((row) => row.id);
|
|
3579
|
+
if (withdrawPendingReportRow.run(at, actor, reason, at, id, project).changes !== 1) {
|
|
3580
|
+
const current = selectReport.get(id);
|
|
3581
|
+
return {
|
|
3582
|
+
kind: "refused",
|
|
3583
|
+
target: "report",
|
|
3584
|
+
state: current?.state ?? "not-found",
|
|
3585
|
+
};
|
|
3586
|
+
}
|
|
3587
|
+
if (report.ambiguous !== 0) {
|
|
3588
|
+
markReportMaterialEventsPossibleRepeat.run(id, id, project);
|
|
3589
|
+
markReportHeldNoticesPossibleRepeat.run(id, id, project);
|
|
3590
|
+
}
|
|
3591
|
+
unassignReportMaterialEvents.run(id, project);
|
|
3592
|
+
unassignReportHeldNotices.run(id, project);
|
|
3593
|
+
for (const noticeId of noticeIds) {
|
|
3594
|
+
deleteAvailabilityReservationForNotice.run(project, noticeId);
|
|
3595
|
+
}
|
|
3596
|
+
const summary = report.body.split("\n", 1)[0]!.slice(0, 240);
|
|
3597
|
+
const withdrawal: HandoffWithdrawal = {
|
|
3598
|
+
id,
|
|
3599
|
+
project,
|
|
3600
|
+
target: "report",
|
|
3601
|
+
actor,
|
|
3602
|
+
reason,
|
|
3603
|
+
at,
|
|
3604
|
+
summary,
|
|
3605
|
+
detail: report.body,
|
|
3606
|
+
};
|
|
3607
|
+
insertHandoffWithdrawal.run(
|
|
3608
|
+
id,
|
|
3609
|
+
project,
|
|
3610
|
+
"report",
|
|
3611
|
+
actor,
|
|
3612
|
+
reason,
|
|
3613
|
+
at,
|
|
3614
|
+
summary,
|
|
3615
|
+
report.body,
|
|
3616
|
+
);
|
|
3617
|
+
return { kind: "withdrawn", withdrawal };
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
const prior = selectHandoffWithdrawal.get(id);
|
|
3621
|
+
if (prior !== null) {
|
|
3622
|
+
return prior.project === project
|
|
3623
|
+
? { kind: "refused", target: prior.target as HandoffWithdrawal["target"], state: "withdrawn" }
|
|
3624
|
+
: { kind: "not-found" };
|
|
3625
|
+
}
|
|
3626
|
+
const notice = selectHeldNoticeLifecycle.get(id);
|
|
3627
|
+
if (notice === null || notice.project !== project) return { kind: "not-found" };
|
|
3628
|
+
if (notice.withdrawnAt !== null) {
|
|
3629
|
+
return { kind: "refused", target: "notice", state: "withdrawn" };
|
|
3630
|
+
}
|
|
3631
|
+
if (notice.digestReportId !== null) {
|
|
3632
|
+
const owner = selectReport.get(notice.digestReportId);
|
|
3633
|
+
if (owner === null || (owner.state !== "failed" && owner.state !== "withdrawn")) {
|
|
3634
|
+
return { kind: "refused", target: "notice", state: "assigned" };
|
|
3635
|
+
}
|
|
3636
|
+
} else if (notice.digestedAt !== null) {
|
|
3637
|
+
return { kind: "refused", target: "notice", state: "digested" };
|
|
3638
|
+
}
|
|
3639
|
+
if (withdrawHeldNoticeRow.run(at, actor, reason, id, project).changes !== 1) {
|
|
3640
|
+
return { kind: "refused", target: "notice", state: "changed" };
|
|
3641
|
+
}
|
|
3642
|
+
deleteAvailabilityReservationForNotice.run(project, id);
|
|
3643
|
+
const withdrawal: HandoffWithdrawal = {
|
|
3644
|
+
id,
|
|
3645
|
+
project,
|
|
3646
|
+
target: "notice",
|
|
3647
|
+
actor,
|
|
3648
|
+
reason,
|
|
3649
|
+
at,
|
|
3650
|
+
summary: notice.summary,
|
|
3651
|
+
detail: notice.detail,
|
|
3652
|
+
};
|
|
3653
|
+
insertHandoffWithdrawal.run(
|
|
3654
|
+
id,
|
|
3655
|
+
project,
|
|
3656
|
+
"notice",
|
|
3657
|
+
actor,
|
|
3658
|
+
reason,
|
|
3659
|
+
at,
|
|
3660
|
+
notice.summary,
|
|
3661
|
+
notice.detail,
|
|
3662
|
+
);
|
|
3663
|
+
return { kind: "withdrawn", withdrawal };
|
|
3664
|
+
},
|
|
3665
|
+
);
|
|
3407
3666
|
const enqueueDigestReportTx = db.transaction(enqueueDigestReportRecord);
|
|
3408
3667
|
const enqueueAvailabilityReportTx = db.transaction(
|
|
3409
3668
|
(draft: ReportDraft, heldNoticeIds: readonly string[]): ReportEnqueue | undefined => {
|
|
@@ -4870,6 +5129,21 @@ export function openStore(dbPath: string): Store {
|
|
|
4870
5129
|
return row === null ? undefined : toReport(row);
|
|
4871
5130
|
},
|
|
4872
5131
|
|
|
5132
|
+
withdrawHandoff(
|
|
5133
|
+
id: string,
|
|
5134
|
+
project: string,
|
|
5135
|
+
actor: string,
|
|
5136
|
+
reason: string,
|
|
5137
|
+
at: number,
|
|
5138
|
+
): HandoffWithdrawalResult {
|
|
5139
|
+
return withdrawHandoffTx.immediate(id, project, actor, reason, at);
|
|
5140
|
+
},
|
|
5141
|
+
|
|
5142
|
+
handoffWithdrawals(project: string, limit = 5): HandoffWithdrawal[] {
|
|
5143
|
+
return selectHandoffWithdrawals.all(project, limit).map(toHandoffWithdrawal);
|
|
5144
|
+
},
|
|
5145
|
+
|
|
5146
|
+
|
|
4873
5147
|
deferPendingReportToNotice(id: string, notice: HeldNoticeDraft): boolean {
|
|
4874
5148
|
return deferPendingReportToNoticeTx(id, notice);
|
|
4875
5149
|
},
|
package/src/types.ts
CHANGED
|
@@ -275,8 +275,8 @@ export const RELEASE_SHAPES = [
|
|
|
275
275
|
"github-release",
|
|
276
276
|
"deploy",
|
|
277
277
|
/**
|
|
278
|
-
* Replace the running conductor itself: the
|
|
279
|
-
* and the Herdr plugin, pinned to one published release and executed
|
|
278
|
+
* Replace the running conductor itself: the discovered CLI package, the omp
|
|
279
|
+
* plugin and the Herdr plugin, pinned to one published release and executed
|
|
280
280
|
* detached. This is the "nobody patches the running conductor" boundary
|
|
281
281
|
* moved deliberately, not worked around — a session that may cut a release
|
|
282
282
|
* still cannot install one until an operator grants this shape too.
|
|
@@ -2354,7 +2354,7 @@ export type ReportKind = (typeof REPORT_KINDS)[number];
|
|
|
2354
2354
|
* `failed` is the bounded retry budget running out, and is itself news: a
|
|
2355
2355
|
* report nobody can deliver escalates as tier 2 in its own right.
|
|
2356
2356
|
*/
|
|
2357
|
-
export const REPORT_DELIVERY_STATES = ["pending", "sending", "delivered", "failed"] as const;
|
|
2357
|
+
export const REPORT_DELIVERY_STATES = ["pending", "sending", "delivered", "failed", "withdrawn"] as const;
|
|
2358
2358
|
|
|
2359
2359
|
export type ReportDeliveryState = (typeof REPORT_DELIVERY_STATES)[number];
|
|
2360
2360
|
|
|
@@ -2411,6 +2411,9 @@ export interface ReportRecord {
|
|
|
2411
2411
|
deliveredAt?: number;
|
|
2412
2412
|
/** Bounded text of the last known failure, surfaced verbatim by `status`. */
|
|
2413
2413
|
lastError?: string;
|
|
2414
|
+
withdrawnAt?: number;
|
|
2415
|
+
withdrawnBy?: string;
|
|
2416
|
+
withdrawReason?: string;
|
|
2414
2417
|
}
|
|
2415
2418
|
|
|
2416
2419
|
/** What a caller hands over. The store owns identity, state and every timestamp. */
|
|
@@ -2431,6 +2434,22 @@ export interface ReportEnqueue {
|
|
|
2431
2434
|
deduped: boolean;
|
|
2432
2435
|
}
|
|
2433
2436
|
|
|
2437
|
+
export interface HandoffWithdrawal {
|
|
2438
|
+
id: string;
|
|
2439
|
+
project: string;
|
|
2440
|
+
target: "report" | "notice";
|
|
2441
|
+
actor: string;
|
|
2442
|
+
reason: string;
|
|
2443
|
+
at: number;
|
|
2444
|
+
summary: string;
|
|
2445
|
+
detail: string;
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
export type HandoffWithdrawalResult =
|
|
2449
|
+
| { kind: "withdrawn"; withdrawal: HandoffWithdrawal }
|
|
2450
|
+
| { kind: "not-found" }
|
|
2451
|
+
| { kind: "refused"; target: "report" | "notice"; state: string };
|
|
2452
|
+
|
|
2434
2453
|
/** Maximum ledger rows of each kind placed in one digest prompt. Older rows
|
|
2435
2454
|
* stay first, so a busy day drains deterministically over later digests. */
|
|
2436
2455
|
export const DIGEST_BACKLOG_LIMIT = 20;
|
|
@@ -3076,6 +3095,16 @@ export interface Store {
|
|
|
3076
3095
|
categories?: readonly InterruptCategory[],
|
|
3077
3096
|
urgent?: boolean,
|
|
3078
3097
|
): HeldNotice[];
|
|
3098
|
+
/** Withdraw one unsent report or unassigned held notice, preserving an audit row. */
|
|
3099
|
+
withdrawHandoff(
|
|
3100
|
+
id: string,
|
|
3101
|
+
project: string,
|
|
3102
|
+
actor: string,
|
|
3103
|
+
reason: string,
|
|
3104
|
+
at: number,
|
|
3105
|
+
): HandoffWithdrawalResult;
|
|
3106
|
+
/** Recent successful withdrawals, newest first, for status/audit. */
|
|
3107
|
+
handoffWithdrawals(project: string, limit?: number): HandoffWithdrawal[];
|
|
3079
3108
|
/** Record the install identities this host carried on this pass (#919).
|
|
3080
3109
|
* One row, replaced: the question is "what is installed now", and a history
|
|
3081
3110
|
* of installs is what the upgrade journal already keeps. */
|
package/src/upgrade-journal.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
23
23
|
import { join } from "node:path";
|
|
24
24
|
import { stateDir } from "./config.ts";
|
|
25
|
+
import type { ResolvedGrants } from "./types.ts";
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* Where the fleet's upgrade journal lives: one JSONL file per host, because
|
|
@@ -74,19 +75,43 @@ export interface UpgradeJournalEntry {
|
|
|
74
75
|
pauseKey?: string;
|
|
75
76
|
/** Whether dispatch was already paused when the request began. */
|
|
76
77
|
initialPaused?: boolean;
|
|
78
|
+
/** Creation instant of the exact pause sentinel this transaction acquired. */
|
|
79
|
+
pauseSince?: number;
|
|
77
80
|
/** The fleet layers the install began under, for delta verification. */
|
|
78
81
|
initial?: JournalFleetState;
|
|
79
82
|
/** The projects the install covers, in the form the upgrade engine uses. */
|
|
80
83
|
selectors?: readonly (string | null)[];
|
|
84
|
+
/** Host-global install authority resolved before the detached unit started. */
|
|
85
|
+
installHolder?: ResolvedGrants["install"];
|
|
86
|
+
/** Every configured project affected by the host-global install, report-only. */
|
|
87
|
+
installProjects?: readonly string[];
|
|
81
88
|
/** Verification evidence when the post-restart tick closes the transaction. */
|
|
82
89
|
checks?: readonly UpgradeCheck[];
|
|
83
90
|
}
|
|
84
91
|
|
|
92
|
+
/** Paths and roots discovered before the first install mutation (#1019). */
|
|
93
|
+
export interface InstallSnapshotManagement {
|
|
94
|
+
cli: {
|
|
95
|
+
executable: string;
|
|
96
|
+
target: string;
|
|
97
|
+
root: string;
|
|
98
|
+
};
|
|
99
|
+
omp: {
|
|
100
|
+
root: string;
|
|
101
|
+
lock: string;
|
|
102
|
+
};
|
|
103
|
+
herdr: {
|
|
104
|
+
root?: string;
|
|
105
|
+
installRoot: string;
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
85
109
|
/** The installed identities a rollback must restore, verbatim. */
|
|
86
110
|
export interface InstallSnapshotSurfaces {
|
|
87
111
|
cliVersion: string;
|
|
88
112
|
ompVersion?: string;
|
|
89
113
|
herdrSource?: string;
|
|
114
|
+
management?: InstallSnapshotManagement;
|
|
90
115
|
}
|
|
91
116
|
|
|
92
117
|
export function upgradeJournalPath(root = stateDir()): string {
|
|
@@ -165,7 +190,12 @@ export type UnitSpawnResult = { code: number; stdout: string; stderr: string };
|
|
|
165
190
|
export type UnitSpawnFn = (argv: readonly string[]) => Promise<UnitSpawnResult>;
|
|
166
191
|
|
|
167
192
|
export type UnitCommand =
|
|
168
|
-
| {
|
|
193
|
+
| {
|
|
194
|
+
kind: "install";
|
|
195
|
+
version: string;
|
|
196
|
+
holder?: ResolvedGrants["install"];
|
|
197
|
+
projects?: readonly string[];
|
|
198
|
+
}
|
|
169
199
|
| { kind: "rollback"; version: string };
|
|
170
200
|
|
|
171
201
|
export interface LaunchedUnit {
|
|
@@ -197,6 +227,12 @@ export async function launchTransientUnit(
|
|
|
197
227
|
const setenv = [`PATH=${executorPath(env)}`, `OMP_CONDUCTOR_UNIT=${unit}`];
|
|
198
228
|
const session = env["HERDR_SESSION"];
|
|
199
229
|
if (session !== undefined && session.length > 0) setenv.push(`HERDR_SESSION=${session}`);
|
|
230
|
+
if (command.kind === "install" && command.holder !== undefined) {
|
|
231
|
+
setenv.push(`OMP_CONDUCTOR_INSTALL_HOLDER=${command.holder}`);
|
|
232
|
+
}
|
|
233
|
+
if (command.kind === "install" && command.projects !== undefined) {
|
|
234
|
+
setenv.push(`OMP_CONDUCTOR_INSTALL_PROJECTS=${command.projects.join(",")}`);
|
|
235
|
+
}
|
|
200
236
|
const argv = [
|
|
201
237
|
"systemd-run",
|
|
202
238
|
`--unit=${unit}`,
|