oysterun 1.6.0-beta.30 → 1.6.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/routec/dev/host_update_runtime_identity.md +36 -0
- package/host-service/cli/run-staged-lifecycle-recovery.mjs +7 -0
- package/host-service/host-lifecycle-recovery.mjs +9 -1
- package/host-service/lifecycle-compat-preload.mjs +1 -1
- package/host-service/lifecycle-compatibility.mjs +1 -1
- package/host-service/lifecycle-continuity-assets.mjs +53 -1
- package/host-service/lifecycle-continuity-projection.mjs +9 -3
- package/host-service/lifecycle-continuity-runner.mjs +34 -2
- package/host-service/lifecycle-continuity.mjs +55 -13
- package/host-service/lifecycle-user-path.mjs +1 -1
- package/host-service/package.json +1 -1
- package/host-service/routec-matrix-sqlite-store.mjs +34 -3
- package/host-service/server.mjs +17 -12
- package/host-service/sqlite-schema-compatibility.mjs +317 -0
- package/host-service/storage-service-binding.mjs +1 -1
- package/package.json +8 -1
- package/tool_scripts/oysterun_continuity_transaction.mjs +59 -15
- package/tool_scripts/oysterun_npm_package_manifest.mjs +1 -0
- package/tool_scripts/oysterun_update_restart_runner.mjs +62 -5
|
@@ -59,6 +59,42 @@ Synthetic update artifacts must use only a task-owned local registry. This
|
|
|
59
59
|
contract does not authorize npm publication, dist-tag changes, or production
|
|
60
60
|
Host mutation.
|
|
61
61
|
|
|
62
|
+
## Keep Or Roll Back (P412)
|
|
63
|
+
|
|
64
|
+
The continuity runner (`tool_scripts/oysterun_continuity_transaction.mjs`)
|
|
65
|
+
decides what happens after a failed check:
|
|
66
|
+
|
|
67
|
+
- **The target passed its Host check.** When a later Session, history or
|
|
68
|
+
route check fails while the same target process is still the sole, healthy
|
|
69
|
+
listener, the update is kept: the receipt records `TARGET_RETAINED` with the
|
|
70
|
+
failed check, cleanup removes the retained prior package, and the operation
|
|
71
|
+
ends as `update_complete` with the failure in `root_failure`. It does not
|
|
72
|
+
block Session start/stop or the next update.
|
|
73
|
+
- **The target cannot start or fails its Host check.** The runner rolls back.
|
|
74
|
+
Before it stops anything it proves that the restore can finish: the exact
|
|
75
|
+
package exchange (including an active target and a `failed-oysterun` that
|
|
76
|
+
both exist, `ambiguous_active_package`) and the source version reading every
|
|
77
|
+
startup-upgraded SQLite store (`docs/routec/dev/matrix_storage_sqlite.md`
|
|
78
|
+
**Startup Schema Compatibility**). When either is unprovable the Host is left
|
|
79
|
+
running and untouched and no Restore action is offered. Otherwise, after the
|
|
80
|
+
target stops, the SQLite fast recovery runs, then the prior package is put
|
|
81
|
+
back and started. When the restored older Host does not report whether a
|
|
82
|
+
user message ever reached a room, the updater reads that evidence from the
|
|
83
|
+
Host's Matrix store for the Session check.
|
|
84
|
+
|
|
85
|
+
**Restore and restart** is shown only when the cheap restore precheck
|
|
86
|
+
(`assessContinuityRestoreAvailability`) passes; the runner repeats the full
|
|
87
|
+
proof before stopping the Host. Otherwise only the diagnostic summary is
|
|
88
|
+
offered. The terminal `service:restore` path uses the same precheck.
|
|
89
|
+
|
|
90
|
+
Which code runs a rollback: the source Host prepares the operation and runs
|
|
91
|
+
the installer from its own package, then starts the restart runner by its path
|
|
92
|
+
in the active package root, which activation has already switched to the
|
|
93
|
+
target. The target's runner therefore performs the restart, qualification,
|
|
94
|
+
SQLite fast recovery and rollback, so a published source such as
|
|
95
|
+
1.6.0-beta.29 is rolled back by the newer updater, which knows the newer
|
|
96
|
+
schemas.
|
|
97
|
+
|
|
62
98
|
## One Lifecycle Coordinator
|
|
63
99
|
|
|
64
100
|
Host Preferences and terminal commands select a transaction kind in the same
|
|
@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
|
|
|
6
6
|
|
|
7
7
|
import { buildHostLifecycleRecoveryPlan } from "../host-lifecycle-recovery.mjs";
|
|
8
8
|
import { readContinuity, continuityReceiptPath } from "../lifecycle-continuity.mjs";
|
|
9
|
+
import { assessContinuityRestoreAvailability } from "../lifecycle-continuity-assets.mjs";
|
|
9
10
|
import {
|
|
10
11
|
verifyPersistedNodeNpmRuntimeIdentityAuthoritatively,
|
|
11
12
|
} from "../node-npm-runtime-identity.mjs";
|
|
@@ -48,6 +49,7 @@ export async function runStagedLifecycleRecovery({
|
|
|
48
49
|
readPreparation = parseJsonObject,
|
|
49
50
|
runnerIsAlive = isProcessRunning,
|
|
50
51
|
runTransaction = runHostLifecycleTransaction,
|
|
52
|
+
assessRestore = assessContinuityRestoreAvailability,
|
|
51
53
|
} = {}) {
|
|
52
54
|
const exactConfigDir = normalizeString(configDir);
|
|
53
55
|
if (!exactConfigDir || resolve(exactConfigDir) !== exactConfigDir) {
|
|
@@ -73,6 +75,11 @@ export async function runStagedLifecycleRecovery({
|
|
|
73
75
|
operation,
|
|
74
76
|
continuityReceipt: receipt,
|
|
75
77
|
runnerAlive,
|
|
78
|
+
restoreAvailability: assessRestore({
|
|
79
|
+
receipt,
|
|
80
|
+
configDir: exactConfigDir,
|
|
81
|
+
activePackageRoot: receipt.resources?.active_package_root || receipt.baseline?.host?.package_root,
|
|
82
|
+
}),
|
|
76
83
|
});
|
|
77
84
|
if (
|
|
78
85
|
recoveryPlan?.blocking !== true ||
|
|
@@ -75,6 +75,9 @@ export function buildHostLifecycleRecoveryPlan({
|
|
|
75
75
|
runnerAlive = false,
|
|
76
76
|
supersededEvaluation = null,
|
|
77
77
|
continuityReceipt = null,
|
|
78
|
+
// P412: assessContinuityRestoreAvailability() for this exact receipt.
|
|
79
|
+
// Restore is offered only when it can finish.
|
|
80
|
+
restoreAvailability = null,
|
|
78
81
|
} = {}) {
|
|
79
82
|
if (!operation?.operation_id) {
|
|
80
83
|
return {
|
|
@@ -101,7 +104,7 @@ export function buildHostLifecycleRecoveryPlan({
|
|
|
101
104
|
reason_code: "lifecycle_owner_active", package_mutated: false, host_restarted: false,
|
|
102
105
|
};
|
|
103
106
|
const restartOnly = r.transaction_kind === "restart_only";
|
|
104
|
-
const committed = r.state
|
|
107
|
+
const committed = ["COMMITTED", "TARGET_RETAINED"].includes(r.state);
|
|
105
108
|
const recovery = r.state === "RECOVERY_REQUIRED";
|
|
106
109
|
// A committed operation only has cleanup left; the Host continues it
|
|
107
110
|
// automatically once no runner is alive (no user action or restart).
|
|
@@ -109,6 +112,11 @@ export function buildHostLifecycleRecoveryPlan({
|
|
|
109
112
|
outcome: "automatic_cleanup", blocking: true, user_action: null,
|
|
110
113
|
reason_code: "continuity_committed", package_mutated: false, host_restarted: false,
|
|
111
114
|
};
|
|
115
|
+
if (recovery && !restartOnly && restoreAvailability?.available !== true) return {
|
|
116
|
+
outcome: "owner_action_required", blocking: true, user_action: { ...ACTIONS.export_diagnostics },
|
|
117
|
+
reason_code: "continuity_restore_unavailable", restore_unavailable_code: restoreAvailability?.code || null,
|
|
118
|
+
package_mutated: false, host_restarted: false,
|
|
119
|
+
};
|
|
112
120
|
const userAction = restartOnly ? action("restart_only", "Restart Host", "Restart did not finish",
|
|
113
121
|
"Restart Host continues the unfinished restart with its original Session snapshot.",
|
|
114
122
|
"Restart Host again and verify the original Sessions and history from the unfinished restart. No new snapshot is taken and active work will not be replayed.")
|
|
@@ -23,7 +23,7 @@ export function installResponseObserver(bound, dependencies = {}) {
|
|
|
23
23
|
readCompatibility(bound.path); // Changed descriptor/dependencies never authorize evidence.
|
|
24
24
|
const receipt = readContinuity(d.receipt_path, d.operation_id);
|
|
25
25
|
const expected = readContinuityUserPath(d.receipt_path, receipt);
|
|
26
|
-
if (!['COMMITTED','CLEANED','ROLLED_BACK_HEALTHY'].includes(receipt.state) || !receipt.snapshot_replay_retired_at) return;
|
|
26
|
+
if (!['COMMITTED','CLEANED','ROLLED_BACK_HEALTHY','TARGET_RETAINED'].includes(receipt.state) || !receipt.snapshot_replay_retired_at) return;
|
|
27
27
|
if (expected?.status === 'verified') { removeContinuityProbeGrant(d.config_dir, d.observation_id); unsubscribe(); return; }
|
|
28
28
|
if (expected?.status !== 'pending' || expected.pid !== process.pid || expected.origin !== d.origin) return;
|
|
29
29
|
ensureContinuityProbeGrant(d.config_dir, d.observation_id);
|
|
@@ -69,7 +69,7 @@ export function compatibilityRender(base, bundle) {
|
|
|
69
69
|
export function prepareCompatibility({ path, operationId, render }) {
|
|
70
70
|
let receipt = readContinuity(path, operationId);
|
|
71
71
|
if (receipt.compatibility) return readCompatibility(receipt.compatibility.descriptor_path);
|
|
72
|
-
if (['COMMITTED','CLEANED','ROLLED_BACK_HEALTHY'].includes(receipt.state)) fail('cannot add authority after closeout');
|
|
72
|
+
if (['COMMITTED','CLEANED','ROLLED_BACK_HEALTHY','TARGET_RETAINED'].includes(receipt.state)) fail('cannot add authority after closeout');
|
|
73
73
|
if (!receipt.resources || !receipt.source.definition_snapshot_digest) return null;
|
|
74
74
|
const source = existsSync(receipt.resources.previous_package_root) ? receipt.resources.previous_package_root : receipt.resources.active_package_root;
|
|
75
75
|
if (existsSync(join(source, 'host-service/lifecycle-user-path.mjs'))) return null;
|
|
@@ -15,6 +15,8 @@ import { restoreServiceDefinitionSnapshot, inspectServiceDefinition } from "./se
|
|
|
15
15
|
import { removeContinuityProbeGrant } from "./lifecycle-continuity-probe.mjs";
|
|
16
16
|
|
|
17
17
|
import { readCompatibility, installCompatibility, retireUnusedCompatibility } from "./lifecycle-compatibility.mjs";
|
|
18
|
+
import { dropSqliteSchemaDowngradeBackups, planSqliteSchemaDowngrade } from "./sqlite-schema-compatibility.mjs";
|
|
19
|
+
import { STORAGE_JOURNAL, resolveHostStorage } from "./storage-layout.mjs";
|
|
18
20
|
|
|
19
21
|
const fail = (check, message) => { throw continuityFailure("retained_assets", check, message); };
|
|
20
22
|
|
|
@@ -133,7 +135,7 @@ function preserveCopiedPackageModes(source, target) {
|
|
|
133
135
|
// safe; an unknown tree is never deleted to make recovery appear successful.
|
|
134
136
|
export function verifyContinuityAssets({ path, operationId, configDir, activePackageRoot }) {
|
|
135
137
|
const receipt = readContinuity(path, operationId);
|
|
136
|
-
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state)) fail("terminal_assets", "Terminal recovery resources cannot be rewritten.");
|
|
138
|
+
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state)) fail("terminal_assets", "Terminal recovery resources cannot be rewritten.");
|
|
137
139
|
const r = validateContinuityResources(receipt, configDir, activePackageRoot);
|
|
138
140
|
const sourceHash = receipt.source.package_tree_sha256;
|
|
139
141
|
if (!sourceHash) fail("source_hash", "Prior package has no retained tree identity.");
|
|
@@ -183,6 +185,54 @@ export function verifyContinuityAssets({ path, operationId, configDir, activePac
|
|
|
183
185
|
return { receipt, r, sourceHash, activeIsSource, snapshot, rendered, bindings, binDir, link };
|
|
184
186
|
}
|
|
185
187
|
|
|
188
|
+
function sourceStorageLayoutReadable(configDir, sourcePackageRoot) {
|
|
189
|
+
if (resolveHostStorage(configDir).version === 1) return true;
|
|
190
|
+
const supported = JSON.parse(readFileSync(join(sourcePackageRoot, "package.json"), "utf8")).oysterun_storage_layout_version;
|
|
191
|
+
return supported === 2 || ((supported === undefined || supported === 1) && existsSync(join(resolve(configDir), STORAGE_JOURNAL)));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function sourceSchemaPlan({ receipt, configDir, sourcePackageRoot, env }) {
|
|
195
|
+
const schema = planSqliteSchemaDowngrade({ configDir, sourcePackageRoot, sourceVersion: receipt.source.version, env });
|
|
196
|
+
if (!schema.feasible) throw continuityFailure("retained_assets", schema.blocked.code.replace(/^p412_/, ""), schema.blocked.message);
|
|
197
|
+
if (!sourceStorageLayoutReadable(configDir, sourcePackageRoot)) fail("storage_layout_unreadable", "The retained source cannot read the current Host storage layout.");
|
|
198
|
+
return schema;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// P412: prove, without changing anything, that a paired restore can finish:
|
|
202
|
+
// the exact package exchange (including both an active and a failed target
|
|
203
|
+
// already present) and the source version reading every startup SQLite store
|
|
204
|
+
// after its fast recovery. Run before the Host is stopped.
|
|
205
|
+
export function planContinuityRestore({ path, operationId, configDir, activePackageRoot, env = process.env }) {
|
|
206
|
+
const verified = verifyContinuityAssets({ path, operationId, configDir, activePackageRoot });
|
|
207
|
+
const { r, activeIsSource, receipt } = verified;
|
|
208
|
+
if (!activeIsSource && existsSync(r.active) && existsSync(r.failed_package_root)) {
|
|
209
|
+
fail("ambiguous_active_package", "Both an active target and failed target already exist; retain both for recovery review.");
|
|
210
|
+
}
|
|
211
|
+
const schema = sourceSchemaPlan({ receipt, configDir, sourcePackageRoot: activeIsSource ? r.active : r.previous_package_root, env });
|
|
212
|
+
return { passed: true, active_is_source: activeIsSource, schema };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Cheap form of planContinuityRestore for the recovery plan shown to a user:
|
|
216
|
+
// no package hashing. The runner still runs the full proof before stopping.
|
|
217
|
+
export function assessContinuityRestoreAvailability({ receipt, configDir, activePackageRoot, env = process.env }) {
|
|
218
|
+
try {
|
|
219
|
+
const r = validateContinuityResources(receipt, configDir, activePackageRoot);
|
|
220
|
+
const activeVersion = existsSync(join(r.active, "package.json"))
|
|
221
|
+
? JSON.parse(readFileSync(join(r.active, "package.json"), "utf8")).version : null;
|
|
222
|
+
const activeIsSource = activeVersion === receipt.source.version;
|
|
223
|
+
if (!activeIsSource && !(existsSync(r.previous_package_root) && lstatSync(r.previous_package_root).isDirectory())) {
|
|
224
|
+
fail("source_package_changed", "Retained prior package is missing.");
|
|
225
|
+
}
|
|
226
|
+
if (!activeIsSource && existsSync(r.active) && existsSync(r.failed_package_root)) {
|
|
227
|
+
fail("ambiguous_active_package", "Both an active target and failed target already exist; retain both for recovery review.");
|
|
228
|
+
}
|
|
229
|
+
sourceSchemaPlan({ receipt, configDir, sourcePackageRoot: activeIsSource ? r.active : r.previous_package_root, env });
|
|
230
|
+
return { available: true };
|
|
231
|
+
} catch (error) {
|
|
232
|
+
return { available: false, code: error.code || "p233_restore_unprovable", message: error.message };
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
186
236
|
// Disposable copy of the receipt-bound prior package for a rollback.
|
|
187
237
|
export function copyRetainedPackage(source, prepared) {
|
|
188
238
|
cpSync(source, prepared, { recursive: true, dereference: false, verbatimSymlinks: true, preserveTimestamps: true });
|
|
@@ -223,6 +273,8 @@ export function restoreContinuityAssets({ path, operationId, configDir, activePa
|
|
|
223
273
|
export function finalizeContinuityAssets({ path, operationId, configDir, activePackageRoot }) {
|
|
224
274
|
const receipt = assertContinuityFinalizable(path, operationId);
|
|
225
275
|
retireUnusedCompatibility(configDir);
|
|
276
|
+
// The kept version no longer needs a fast SQLite recovery path.
|
|
277
|
+
dropSqliteSchemaDowngradeBackups({ configDir });
|
|
226
278
|
if (receipt.operation_root) {
|
|
227
279
|
const r = validateContinuityResources(receipt, configDir, activePackageRoot);
|
|
228
280
|
rmSync(r.root, { recursive: true, force: true });
|
|
@@ -17,14 +17,20 @@ export function projectContinuityOperation(operation, receipt) {
|
|
|
17
17
|
};
|
|
18
18
|
if (!["CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state)) return common;
|
|
19
19
|
const recovered = receipt.state === "ROLLED_BACK_HEALTHY";
|
|
20
|
+
// P412: the target passed its Host check and was kept after a later check failed.
|
|
21
|
+
const retained = !recovered && receipt.target_retention?.host?.passed === true ? receipt.target_retention : null;
|
|
22
|
+
const retainedPhase = retained
|
|
23
|
+
? `Updated to ${retained.observed_version}; a post-update check failed and was recorded (${retained.failed_check.check}). The new version was kept and Sessions are available.`
|
|
24
|
+
: null;
|
|
20
25
|
return { ...common, status: restartOnly ? "restart_complete" : recovered ? "rolled_back" : "update_complete",
|
|
21
26
|
running: false, runner_pid: null, runner_lease_expires_at: null, lifecycle_owner_pid: null,
|
|
22
27
|
phase: (restartOnly ? "Host restart complete; Host and Sessions are healthy."
|
|
23
28
|
: recovered ? "Update failed; prior Host and Sessions are restored and usable."
|
|
24
|
-
: "Host and Sessions are healthy; automatic update checks and cleanup complete.") + routeMessage,
|
|
25
|
-
final_observed_version: recovered ? receipt.source.version : receipt.target.version,
|
|
29
|
+
: retainedPhase || "Host and Sessions are healthy; automatic update checks and cleanup complete.") + routeMessage,
|
|
30
|
+
final_observed_version: recovered ? receipt.source.version : retained?.observed_version || receipt.target.version,
|
|
26
31
|
operation_cleanup_completed: !recovered, rollback_completed: recovered && !restartOnly,
|
|
27
|
-
service_status: "healthy",
|
|
32
|
+
service_status: "healthy",
|
|
33
|
+
session_restore_status: retained?.failed_check ? "target_retained_after_failed_check" : "qualified_without_replay",
|
|
28
34
|
restart_restore_status: receipt.recovery_qualification?.sessions?.restore_required === false ? "not_required" : "consumed",
|
|
29
35
|
next_safe_action: null, recovery_required_at: null, recovery_reason: null, recovery_boundary: null,
|
|
30
36
|
};
|
|
@@ -5,8 +5,10 @@ import {
|
|
|
5
5
|
recordContinuityRosterPending,
|
|
6
6
|
} from "./lifecycle-continuity.mjs";
|
|
7
7
|
import { fetchContinuityObservation } from "./lifecycle-continuity-probe.mjs";
|
|
8
|
-
import { lstatSync, readFileSync } from "node:fs";
|
|
8
|
+
import { existsSync, lstatSync, readFileSync } from "node:fs";
|
|
9
9
|
import { join } from "node:path";
|
|
10
|
+
import Database from "better-sqlite3";
|
|
11
|
+
import { resolveStartupSqliteStorePaths } from "./sqlite-schema-compatibility.mjs";
|
|
10
12
|
|
|
11
13
|
export function resolveContinuityQualificationOrigin({
|
|
12
14
|
managedEndpoint,
|
|
@@ -59,6 +61,36 @@ function qualifyLegacyObservation(rows, peer, configDir) {
|
|
|
59
61
|
});
|
|
60
62
|
}
|
|
61
63
|
|
|
64
|
+
// P412: a Host older than the transcript-evidence baseline (for example a
|
|
65
|
+
// restored 1.6.0-beta.29) reports no `provider_conversation` for a Session
|
|
66
|
+
// whose provider already assigned an id. The updater then reads the same
|
|
67
|
+
// evidence the Host computes (a role=user message: semantic message.user or a
|
|
68
|
+
// human sender) from its Matrix store, read-only. Any problem answers
|
|
69
|
+
// "unknown", which grants no exception.
|
|
70
|
+
export function readStoreProviderConversationEvidence(configDir, roomId, env = process.env) {
|
|
71
|
+
if (typeof roomId !== "string" || !roomId) return "unknown";
|
|
72
|
+
let db;
|
|
73
|
+
try {
|
|
74
|
+
const path = resolveStartupSqliteStorePaths(configDir, env).routec_matrix;
|
|
75
|
+
if (!existsSync(path)) return "unknown";
|
|
76
|
+
db = new Database(path, { readonly: true, fileMustExist: true });
|
|
77
|
+
if (!db.prepare("SELECT 1 FROM matrix_rooms WHERE room_id = ?").get(roomId)) return "unknown";
|
|
78
|
+
const user = db.prepare(`SELECT 1 FROM matrix_events e WHERE e.room_id = ? AND e.is_state = 0 AND (
|
|
79
|
+
e.semantic_type = 'message.user' OR EXISTS (SELECT 1 FROM matrix_room_actors a
|
|
80
|
+
WHERE a.room_id = e.room_id AND a.matrix_user_id = e.sender AND a.actor_kind = 'human')) LIMIT 1`).get(roomId);
|
|
81
|
+
return user ? "present" : "none";
|
|
82
|
+
} catch {
|
|
83
|
+
return "unknown";
|
|
84
|
+
} finally {
|
|
85
|
+
db?.close();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function withStoreConversationEvidence(rows, configDir) {
|
|
90
|
+
return rows.map((row) => row?.provider_conversation == null && ["codex", "claude", "xai"].includes(row?.provider)
|
|
91
|
+
? { ...row, provider_conversation: readStoreProviderConversationEvidence(configDir, row.matrix_room_id) } : row);
|
|
92
|
+
}
|
|
93
|
+
|
|
62
94
|
/** Shared qualification used by the existing runner and the one-time adapter.
|
|
63
95
|
* Observations are live authenticated reads; the retained baseline is never
|
|
64
96
|
* replaced by a new observation after a restart or a failed qualification.
|
|
@@ -125,7 +157,7 @@ export async function qualifyLifecycleContinuity({
|
|
|
125
157
|
throw continuityFailure("session_qualification", "restart_not_consumed", "Waiting for the exact fresh Session restore receipt.");
|
|
126
158
|
}
|
|
127
159
|
const source = qualifyLegacyObservation(baseline.sessions, observation.sessions, configDir);
|
|
128
|
-
const target = qualifyLegacyObservation(observation.sessions, source, configDir);
|
|
160
|
+
const target = withStoreConversationEvidence(qualifyLegacyObservation(observation.sessions, source, configDir), configDir);
|
|
129
161
|
if (requireRestore && source.length > target.length) {
|
|
130
162
|
const expectedIds = new Set(source.map((row) => row.session_id));
|
|
131
163
|
const observedIds = target.map((row) => row.session_id);
|
|
@@ -17,6 +17,10 @@ export const CONTINUITY_STATES = Object.freeze([
|
|
|
17
17
|
"PREPARED", "PACKAGE_SWITCHED", "SERVICE_RESTARTED", "HOST_QUALIFIED",
|
|
18
18
|
"SESSIONS_QUALIFIED", "TUNNEL_QUALIFIED", "COMMITTED", "CLEANED",
|
|
19
19
|
]);
|
|
20
|
+
// P412: TARGET_RETAINED closes an update forward on the running target: a
|
|
21
|
+
// later check failed after the target passed Host qualification. Like
|
|
22
|
+
// COMMITTED, only cleanup remains; it then advances to CLEANED.
|
|
23
|
+
export const CONTINUITY_TARGET_RETAINED_STATE = "TARGET_RETAINED";
|
|
20
24
|
export const CONTINUITY_SCHEMA = "oysterun.lifecycle_continuity.v1";
|
|
21
25
|
export const RECOVERY_IDENTITY_SCHEMA = "oysterun.recovery_identity.v1";
|
|
22
26
|
const durableBoundaryChannel = channel("oysterun.lifecycle.continuity.durable");
|
|
@@ -135,13 +139,20 @@ function isUnstartedProviderConversationForNeverMessagedSession(sourceRow, befor
|
|
|
135
139
|
|
|
136
140
|
// P326: a Claude Session with no committed history (no anchors) whose restore found no Claude
|
|
137
141
|
// conversation for its exact retained id started a fresh one; only its provider ids may differ.
|
|
142
|
+
// P412: the same applies when the room holds history but the Host's own evidence shows that no user
|
|
143
|
+
// message ever reached the Session (Host and Loop notices are not user messages). A baseline taken
|
|
144
|
+
// by an older source Host recorded that evidence only for Sessions without provider ids; then the
|
|
145
|
+
// target's evidence for the same room stands in: a room with no user message now had none before,
|
|
146
|
+
// and every retained history anchor is still required to be present and unchanged.
|
|
138
147
|
function isFreshProviderConversationForNeverMessagedSession(sourceRow, before, targetRow) {
|
|
139
148
|
const fresh = targetRow?.provider_conversation_started_fresh;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
149
|
+
if (fresh?.reason !== "no_provider_conversation" || !before.identity.provider_resume_id ||
|
|
150
|
+
fresh.previous_provider_resume_id !== before.identity.provider_resume_id) return false;
|
|
151
|
+
const noHistory = before.identity.provider === "claude" &&
|
|
152
|
+
Array.isArray(sourceRow?.history_anchors) && sourceRow.history_anchors.length === 0;
|
|
153
|
+
const neverMessaged = sourceRow?.provider_conversation === "none" ||
|
|
154
|
+
(sourceRow?.provider_conversation == null && targetRow?.provider_conversation === "none");
|
|
155
|
+
return noHistory || neverMessaged;
|
|
145
156
|
}
|
|
146
157
|
|
|
147
158
|
function boundedBaseline(baseline) {
|
|
@@ -282,12 +293,12 @@ export function readContinuity(path, operationId) {
|
|
|
282
293
|
"receipt", "baseline_identity", "Retained continuity baseline changed.");
|
|
283
294
|
let previous = null;
|
|
284
295
|
for (const [index, transition] of receipt.transitions.entries()) {
|
|
285
|
-
const next = CONTINUITY_STATES[CONTINUITY_STATES.indexOf(previous) + 1];
|
|
286
|
-
const recovery = ["RECOVERY_REQUIRED", "ROLLED_BACK_HEALTHY"].includes(transition.state);
|
|
296
|
+
const next = previous === "TARGET_RETAINED" ? "CLEANED" : CONTINUITY_STATES[CONTINUITY_STATES.indexOf(previous) + 1];
|
|
297
|
+
const recovery = ["RECOVERY_REQUIRED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(transition.state);
|
|
287
298
|
requireValue(transition.sequence === index + 1 && Number.isFinite(Date.parse(transition.at)) &&
|
|
288
299
|
(index === 0 ? transition.state === "PREPARED" :
|
|
289
|
-
recovery ? !["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(previous) :
|
|
290
|
-
CONTINUITY_STATES.includes(previous) && transition.state === next),
|
|
300
|
+
recovery ? !["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(previous) :
|
|
301
|
+
(CONTINUITY_STATES.includes(previous) || previous === "TARGET_RETAINED") && transition.state === next),
|
|
291
302
|
"receipt", "transition_history", "Continuity transition history is inconsistent.");
|
|
292
303
|
previous = transition.state;
|
|
293
304
|
if (transition.state.endsWith("_QUALIFIED")) requireValue(transition.evidence?.passed === true &&
|
|
@@ -296,6 +307,8 @@ export function readContinuity(path, operationId) {
|
|
|
296
307
|
requireValue(previous === receipt.state, "receipt", "transition_history", "Continuity state differs from its durable history.");
|
|
297
308
|
if (receipt.state === "ROLLED_BACK_HEALTHY") requireValue(["host", "sessions", "tunnel"].every((key) =>
|
|
298
309
|
receipt.recovery_qualification?.[key]?.passed === true), "receipt", "rollback_evidence", "Healthy rollback requires the full prior-state qualification.");
|
|
310
|
+
if (receipt.transitions.some((transition) => transition.state === "TARGET_RETAINED")) requireValue(
|
|
311
|
+
receipt.target_retention?.host?.passed === true, "receipt", "target_retention_evidence", "A retained target requires its Host qualification.");
|
|
299
312
|
return receipt;
|
|
300
313
|
}
|
|
301
314
|
|
|
@@ -357,7 +370,8 @@ export function advanceContinuity(path, operationId, state, evidence = {}) {
|
|
|
357
370
|
const receipt = readContinuity(path, operationId);
|
|
358
371
|
if (receipt.state === state) return receipt;
|
|
359
372
|
const currentIndex = CONTINUITY_STATES.indexOf(receipt.state);
|
|
360
|
-
const expected =
|
|
373
|
+
const expected = receipt.state === "TARGET_RETAINED" ? "CLEANED"
|
|
374
|
+
: currentIndex >= 0 ? CONTINUITY_STATES[currentIndex + 1] : null;
|
|
361
375
|
requireValue(expected === state, "receipt", "transition_order",
|
|
362
376
|
`Cannot advance lifecycle from ${receipt.state} to ${state}.`);
|
|
363
377
|
if (state === "COMMITTED") {
|
|
@@ -384,7 +398,7 @@ export function advanceContinuity(path, operationId, state, evidence = {}) {
|
|
|
384
398
|
|
|
385
399
|
export function recordContinuityRestartAttempt(path, operationId) {
|
|
386
400
|
const receipt = readContinuity(path, operationId);
|
|
387
|
-
requireValue(!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state),
|
|
401
|
+
requireValue(!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state),
|
|
388
402
|
"restart", "terminal_operation", "A terminal operation cannot restart again.");
|
|
389
403
|
receipt.restart_attempted = true;
|
|
390
404
|
receipt.restart_attempted_at ||= new Date().toISOString();
|
|
@@ -402,7 +416,7 @@ function continuityUserPathExpectation(tunnel) {
|
|
|
402
416
|
export function recordContinuityRecovery(path, operationId, error, qualification = null) {
|
|
403
417
|
const receipt = readContinuity(path, operationId);
|
|
404
418
|
if (receipt.state === "ROLLED_BACK_HEALTHY") return receipt;
|
|
405
|
-
requireValue(!["COMMITTED", "CLEANED"].includes(receipt.state), "recovery", "already_committed", "A committed operation cannot roll back.");
|
|
419
|
+
requireValue(!["COMMITTED", "CLEANED", "TARGET_RETAINED"].includes(receipt.state), "recovery", "already_committed", "A committed operation cannot roll back.");
|
|
406
420
|
const diagnostic = error?.lifecycle_diagnostic;
|
|
407
421
|
const failure = {
|
|
408
422
|
stage: diagnostic?.stage || "recovery", check: diagnostic?.check || "transaction_failed",
|
|
@@ -426,9 +440,37 @@ export function recordContinuityRecovery(path, operationId, error, qualification
|
|
|
426
440
|
return writeContinuityAtomic(path, receipt);
|
|
427
441
|
}
|
|
428
442
|
|
|
443
|
+
// P412: keep the running target instead of rolling back. `qualification.host`
|
|
444
|
+
// must be a passed Host qualification of the target that is still the sole
|
|
445
|
+
// listener. The failed later check is recorded, not repaired.
|
|
446
|
+
export function retainContinuityTarget(path, operationId, { reason, failure, qualification, observedVersion }) {
|
|
447
|
+
const receipt = readContinuity(path, operationId);
|
|
448
|
+
requireValue(!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state),
|
|
449
|
+
"recovery", "terminal_operation", "A closed operation cannot retain its target again.");
|
|
450
|
+
requireValue(qualification?.host?.passed === true && reason === "post_host_qualification_check_failed" && failure,
|
|
451
|
+
"recovery", "target_retention_evidence", "Retaining the target requires its passed Host qualification and the failed check.");
|
|
452
|
+
requireValue(observedVersion === receipt.target.version,
|
|
453
|
+
"recovery", "target_retention_version", "Only the update target can be retained.");
|
|
454
|
+
const diagnostic = failure?.lifecycle_diagnostic;
|
|
455
|
+
const failed = failure ? { stage: diagnostic?.stage || "recovery", check: diagnostic?.check || "transaction_failed",
|
|
456
|
+
code: failure.code || "p233_transaction_failed", message: sanitizeLifecycleDiagnostic(failure.message || String(failure)),
|
|
457
|
+
restart_attempted: receipt.restart_attempted, at: new Date().toISOString() } : null;
|
|
458
|
+
receipt.first_failure ||= failed;
|
|
459
|
+
if (failed) receipt.latest_failure = failed;
|
|
460
|
+
receipt.target_retention = { reason, observed_version: observedVersion, failed_check: failed,
|
|
461
|
+
host: qualification.host, sessions: qualification.sessions || null, tunnel: qualification.tunnel || null,
|
|
462
|
+
at: new Date().toISOString() };
|
|
463
|
+
retireHostRestartRestoreState({ configDir: receipt.baseline.host.config_dir, restartId: receipt.restart_id });
|
|
464
|
+
receipt.snapshot_replay_retired_at = new Date().toISOString();
|
|
465
|
+
receipt.user_path_verification = continuityUserPathExpectation(qualification.tunnel);
|
|
466
|
+
receipt.state = "TARGET_RETAINED";
|
|
467
|
+
receipt.transitions.push({ sequence: receipt.transitions.length + 1, state: receipt.state, at: new Date().toISOString() });
|
|
468
|
+
return publishDurableBoundary(path, writeContinuityAtomic(path, receipt));
|
|
469
|
+
}
|
|
470
|
+
|
|
429
471
|
export function assertContinuityFinalizable(path, operationId) {
|
|
430
472
|
const receipt = readContinuity(path, operationId);
|
|
431
|
-
requireValue(["COMMITTED", "CLEANED"].includes(receipt.state), "cleanup", "commit_required",
|
|
473
|
+
requireValue(["COMMITTED", "CLEANED", "TARGET_RETAINED"].includes(receipt.state), "cleanup", "commit_required",
|
|
432
474
|
"Rollback resources are retained until the lifecycle receipt is COMMITTED.");
|
|
433
475
|
return receipt;
|
|
434
476
|
}
|
|
@@ -27,7 +27,7 @@ export function recordContinuityUserPath(path, operationId, { origin, pid, obser
|
|
|
27
27
|
path = resolveLifecycleStorageReference(path);
|
|
28
28
|
const receipt = readContinuity(path, operationId);
|
|
29
29
|
const expected = readContinuityUserPath(path, receipt);
|
|
30
|
-
if (!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state) ||
|
|
30
|
+
if (!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state) ||
|
|
31
31
|
!receipt.snapshot_replay_retired_at || expected?.status !== "pending" ||
|
|
32
32
|
expected.origin !== origin || expected.pid !== pid || !Number.isFinite(Date.parse(observed_at))) return false;
|
|
33
33
|
writeContinuityAtomic(`${path}.user-path`, {
|
|
@@ -4,6 +4,7 @@ import { preparePrivateStorageFile, writeAtomicJsonFile, syncStorageDirectory }
|
|
|
4
4
|
import { rebindStorageObject } from "./storage-references.mjs";
|
|
5
5
|
import { resolveHostStorage } from "./storage-layout.mjs";
|
|
6
6
|
import { routeCMatrixAuxiliaryPath } from "./routec-matrix-storage-paths.mjs";
|
|
7
|
+
import { minimumSqliteSchemaReaderVersion } from "./sqlite-schema-compatibility.mjs";
|
|
7
8
|
import { getConfigDir } from "./config.mjs";
|
|
8
9
|
import { createHash } from "crypto";
|
|
9
10
|
import Database from "better-sqlite3";
|
|
@@ -42,6 +43,8 @@ export const MATRIX_COMPLETED_TURN_COMPLETION_INDEX =
|
|
|
42
43
|
"idx_matrix_events_room_completed_turn_completion";
|
|
43
44
|
export const MATRIX_CALLBACK_SCHEMA_VERSION = 4;
|
|
44
45
|
const MATRIX_CALLBACK_SCHEMA_VERSION_META_KEY = "callback_schema_version";
|
|
46
|
+
// {"version": N, "min_reader": M}: the oldest reader that can open version N.
|
|
47
|
+
const MATRIX_CALLBACK_READER_COMPATIBILITY_META_KEY = "callback_schema_reader_compatibility";
|
|
45
48
|
export const MATRIX_CALLBACK_INDEX_NAMES = Object.freeze([
|
|
46
49
|
"idx_matrix_callbacks_request_idempotency_unique",
|
|
47
50
|
"idx_matrix_callbacks_source_state_updated",
|
|
@@ -732,9 +735,29 @@ export function ensureCallbackSchemaMigration(db) {
|
|
|
732
735
|
throw new Error("Route C Matrix SQLite callback schema version is corrupt");
|
|
733
736
|
}
|
|
734
737
|
if (priorVersion > MATRIX_CALLBACK_SCHEMA_VERSION) {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
+
// P412: a newer writer declares the oldest reader that can still open
|
|
739
|
+
// its data. Open it unchanged; never migrate the marker downward here.
|
|
740
|
+
let compatibility = null;
|
|
741
|
+
try {
|
|
742
|
+
compatibility = JSON.parse(db
|
|
743
|
+
.prepare("SELECT value FROM matrix_schema_meta WHERE key = ?")
|
|
744
|
+
.get(MATRIX_CALLBACK_READER_COMPATIBILITY_META_KEY)?.value ?? "null");
|
|
745
|
+
} catch { compatibility = null; }
|
|
746
|
+
if (compatibility?.version !== priorVersion || !Number.isSafeInteger(compatibility.min_reader) ||
|
|
747
|
+
compatibility.min_reader > MATRIX_CALLBACK_SCHEMA_VERSION) {
|
|
748
|
+
throw new Error(
|
|
749
|
+
`Route C Matrix SQLite callback schema ${priorVersion} is newer than supported ${MATRIX_CALLBACK_SCHEMA_VERSION}`
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
if (!MATRIX_CALLBACK_INDEX_NAMES.every((name) => hasNamedIndex(db, name))) {
|
|
753
|
+
throw new Error("Route C Matrix SQLite callback schema migration did not become ready");
|
|
754
|
+
}
|
|
755
|
+
return {
|
|
756
|
+
ready: true,
|
|
757
|
+
schema_version: priorVersion,
|
|
758
|
+
compatible_newer_schema: true,
|
|
759
|
+
index_names: [...MATRIX_CALLBACK_INDEX_NAMES],
|
|
760
|
+
};
|
|
738
761
|
}
|
|
739
762
|
}
|
|
740
763
|
const migrate = db.transaction(() => {
|
|
@@ -785,6 +808,14 @@ export function ensureCallbackSchemaMigration(db) {
|
|
|
785
808
|
MATRIX_CALLBACK_SCHEMA_VERSION_META_KEY,
|
|
786
809
|
MATRIX_CALLBACK_SCHEMA_VERSION
|
|
787
810
|
);
|
|
811
|
+
setSqliteMeta(
|
|
812
|
+
db,
|
|
813
|
+
MATRIX_CALLBACK_READER_COMPATIBILITY_META_KEY,
|
|
814
|
+
JSON.stringify({
|
|
815
|
+
version: MATRIX_CALLBACK_SCHEMA_VERSION,
|
|
816
|
+
min_reader: minimumSqliteSchemaReaderVersion("routec_matrix_callbacks", MATRIX_CALLBACK_SCHEMA_VERSION),
|
|
817
|
+
})
|
|
818
|
+
);
|
|
788
819
|
});
|
|
789
820
|
migrate.immediate();
|
|
790
821
|
const version = Number(
|
package/host-service/server.mjs
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
import { recordContinuityUserPath } from "./lifecycle-user-path.mjs";
|
|
23
23
|
import { projectContinuityOperation } from "./lifecycle-continuity-projection.mjs";
|
|
24
24
|
import { resolveContinuityQualificationOrigin } from "./lifecycle-continuity-runner.mjs";
|
|
25
|
-
import { retainContinuityFile, digestContinuityPackage } from "./lifecycle-continuity-assets.mjs";
|
|
25
|
+
import { retainContinuityFile, digestContinuityPackage, assessContinuityRestoreAvailability } from "./lifecycle-continuity-assets.mjs";
|
|
26
26
|
import {
|
|
27
27
|
createContinuityProbeGrant, verifyContinuityProbeGrant,
|
|
28
28
|
} from "./lifecycle-continuity-probe.mjs";
|
|
@@ -7488,7 +7488,7 @@ function reconcileOysterunUpdateOperation(currentVersion) {
|
|
|
7488
7488
|
if (operation.continuity_path) {
|
|
7489
7489
|
try {
|
|
7490
7490
|
const r = readContinuity(operation.continuity_path, operation.operation_id);
|
|
7491
|
-
if (!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(r.state)) {
|
|
7491
|
+
if (!["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(r.state)) {
|
|
7492
7492
|
const failed = recordContinuityRecovery(operation.continuity_path, operation.operation_id,
|
|
7493
7493
|
continuityFailure(restartOnly ? "restart_runner" : "update_runner", "runner_lease_expired", "The external lifecycle runner did not publish a terminal receipt before its bounded lease expired."));
|
|
7494
7494
|
operation = projectContinuityOperation(operation, failed);
|
|
@@ -7921,7 +7921,7 @@ function maybeCloseSupersededOysterunUpdateRecovery() {
|
|
|
7921
7921
|
// (no reinstall, restore or restart). The runner re-verifies the exact
|
|
7922
7922
|
// operation and resource ownership before removing anything; a failure
|
|
7923
7923
|
// keeps its reason and is retried by the next Host process.
|
|
7924
|
-
if (receipt?.state
|
|
7924
|
+
if (["COMMITTED", "TARGET_RETAINED"].includes(receipt?.state) && !operation.running && !isProcessRunning(operation.runner_pid) &&
|
|
7925
7925
|
!continuityAutomaticCleanups.has(operation.operation_id)) {
|
|
7926
7926
|
continuityAutomaticCleanups.add(operation.operation_id);
|
|
7927
7927
|
try {
|
|
@@ -7932,7 +7932,9 @@ function maybeCloseSupersededOysterunUpdateRecovery() {
|
|
|
7932
7932
|
}
|
|
7933
7933
|
}
|
|
7934
7934
|
const recoveryPlan = buildHostLifecycleRecoveryPlan({ operation, continuityReceipt: receipt,
|
|
7935
|
-
runnerAlive: isProcessRunning(operation.runner_pid)
|
|
7935
|
+
runnerAlive: isProcessRunning(operation.runner_pid),
|
|
7936
|
+
restoreAvailability: receipt?.state === "RECOVERY_REQUIRED" && receipt.transaction_kind !== "restart_only"
|
|
7937
|
+
? assessContinuityRestoreAvailability({ receipt, configDir: getConfigDir(), activePackageRoot: REPO_ROOT }) : null });
|
|
7936
7938
|
return { status: "continuity_operation_observed", reconciled: false,
|
|
7937
7939
|
recovery_boundary: recoveryPlan.reason_code, recovery_plan: recoveryPlan,
|
|
7938
7940
|
recovery_action: recoveryPlan.user_action, update_operation: serializeOysterunUpdateOperation(operation) };
|
|
@@ -8985,9 +8987,10 @@ function buildHostContinuityObservation(retained = null) {
|
|
|
8985
8987
|
alive: row.alive_before_restart, ready: row.ready_before_restart,
|
|
8986
8988
|
history_anchors: historyAnchors,
|
|
8987
8989
|
provider_conversation_started_fresh: row.provider_conversation_started_fresh || null,
|
|
8988
|
-
// P326 extension 2: transcript evidence
|
|
8989
|
-
|
|
8990
|
-
|
|
8990
|
+
// P326 extension 2 / P412: the Host's transcript evidence (a role=user message reached the
|
|
8991
|
+
// room or not; Host and Loop notices do not count) for every real-provider Session, including
|
|
8992
|
+
// one whose provider already assigned an id (Claude does at session/new).
|
|
8993
|
+
provider_conversation: ["codex", "claude", "xai"].includes(row.provider)
|
|
8991
8994
|
? readSessionProviderConversationEvidence({ session_id: row.session_id }) : null,
|
|
8992
8995
|
};
|
|
8993
8996
|
} catch (error) {
|
|
@@ -9052,10 +9055,11 @@ function recordContinuityRunnerHandoffFailure({ operationId, error, gateId = nul
|
|
|
9052
9055
|
const restartOnly = prior.transaction_kind === "restart_only";
|
|
9053
9056
|
const failure = continuityFailure(stage || (restartOnly ? "restart_handoff" : "update_handoff"), check || (restartOnly ? "restart_runner_spawn_failed" : "update_runner_spawn_failed"),
|
|
9054
9057
|
sanitizeLifecycleDiagnostic(error?.message || "The external lifecycle runner could not start."));
|
|
9055
|
-
const
|
|
9058
|
+
const committed = ["COMMITTED", "TARGET_RETAINED"].includes(prior.state);
|
|
9059
|
+
const receipt = committed ? prior : recordContinuityRecovery(current.continuity_path, operationId, failure);
|
|
9056
9060
|
writeUpdateOperationState({ ...projectContinuityOperation(current, receipt),
|
|
9057
9061
|
status: restartOnly ? "restart_recovery_required" : "update_recovery_required",
|
|
9058
|
-
phase:
|
|
9062
|
+
phase: committed ? "Committed cleanup requires continuation" : "External runner unavailable; recovery assets retained",
|
|
9059
9063
|
running: false, runner_pid: null, runner_lease_expires_at: null, lifecycle_owner_pid: null,
|
|
9060
9064
|
recovery_required_at: new Date().toISOString(), recovery_boundary: failure.code,
|
|
9061
9065
|
recovery_reason: failure.message, error: failure.message, error_redacted: true,
|
|
@@ -11283,10 +11287,11 @@ function scheduleContinuityRecovery({ kind, source, recheckOnly = false, automat
|
|
|
11283
11287
|
getHostRestartRestoreStatePath(getConfigDir()), join(receipt.operation_root, "restart-restore-state.pre-restart.json")) } });
|
|
11284
11288
|
}
|
|
11285
11289
|
const runner = resolveOysterunUpdateRestartRunner(toolchain);
|
|
11286
|
-
const
|
|
11290
|
+
const cleanupOnly = ["COMMITTED", "TARGET_RETAINED"].includes(receipt.state);
|
|
11291
|
+
const handoff = cleanupOnly || recheckOnly ? {} : createHostRestartHandoff();
|
|
11287
11292
|
const logPath = hostStoragePath(getConfigDir(), "operation_logs", `oysterun-continuity-recovery-${operation.operation_id}.log`);
|
|
11288
11293
|
operation = writeUpdateOperationState({ ...operation, ...handoff, status: "restart_scheduled",
|
|
11289
|
-
phase: recheckOnly ? "Checking the already-running Host and restored Sessions" :
|
|
11294
|
+
phase: recheckOnly ? "Checking the already-running Host and restored Sessions" : cleanupOnly ? "Finishing committed cleanup" : "Continuing the retained lifecycle operation",
|
|
11290
11295
|
running: true, lifecycle_owner_pid: process.pid, runner_pid: null, requested_action: kind,
|
|
11291
11296
|
runner_lease_expires_at: new Date(Date.now() + UPDATE_RESTART_RUNNER_LEASE_MS).toISOString(),
|
|
11292
11297
|
operation_log_path: logPath }, { configDir: getConfigDir() });
|
|
@@ -11313,7 +11318,7 @@ function scheduleContinuityRecovery({ kind, source, recheckOnly = false, automat
|
|
|
11313
11318
|
} catch (error) {
|
|
11314
11319
|
recordContinuityRunnerHandoffFailure({ operationId: operation.operation_id, error });
|
|
11315
11320
|
} finally { if (fd !== undefined) closeSync(fd); }
|
|
11316
|
-
},
|
|
11321
|
+
}, cleanupOnly || recheckOnly ? 0 : getHostRestartHandoffDelayMs(handoff.execute_not_before)).unref();
|
|
11317
11322
|
return recheckOnly ? null : buildOysterunUpdateStatusPayload({ status: "restart_scheduled", source, ...handoff });
|
|
11318
11323
|
}
|
|
11319
11324
|
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import Database from "better-sqlite3";
|
|
4
|
+
import { compareOysterunSemver } from "./npm-update-policy.mjs";
|
|
5
|
+
import { hostStoragePath } from "./storage-layout.mjs";
|
|
6
|
+
import { routeCMatrixJsonStoragePath, routeCMatrixSQLiteStoragePath } from "./routec-matrix-storage-paths.mjs";
|
|
7
|
+
|
|
8
|
+
// P412: every SQLite schema marker that a Host upgrades while it starts.
|
|
9
|
+
// A target can migrate these before its Sessions are checked, so a rollback
|
|
10
|
+
// must prove that the source version can open the data it leaves behind.
|
|
11
|
+
//
|
|
12
|
+
// Each version after the first declares how an older reader is served:
|
|
13
|
+
// - "marker_revert": the change only adds data an older reader ignores; the
|
|
14
|
+
// updater writes the previous version marker back and the next upgrade
|
|
15
|
+
// re-runs the (idempotent) migration.
|
|
16
|
+
// - "restore_backup_tables": the migration backs up the tables it rewrites in
|
|
17
|
+
// the same transaction (backupSqliteTablesForSchemaStep); the updater
|
|
18
|
+
// restores them. Never a copy of the whole database.
|
|
19
|
+
// - "not_reversible": no fast recovery; a rollback across it is refused.
|
|
20
|
+
// first_package_version is the first published package that writes the
|
|
21
|
+
// version. Later versions are read from the source package's declaration
|
|
22
|
+
// (SQLITE_SCHEMA_DECLARATION_FIELD) and may leave it null.
|
|
23
|
+
export const SQLITE_SCHEMA_DECLARATION_FIELD = "oysterun_sqlite_schema_versions";
|
|
24
|
+
export const SQLITE_SCHEMA_BACKUP_TABLE = "oysterun_schema_downgrade_backups";
|
|
25
|
+
export const SQLITE_OLD_READER_MODES = Object.freeze(["marker_revert", "restore_backup_tables", "not_reversible"]);
|
|
26
|
+
|
|
27
|
+
export const STARTUP_SQLITE_SCHEMAS = Object.freeze([
|
|
28
|
+
{
|
|
29
|
+
id: "routec_matrix_schema",
|
|
30
|
+
store: "routec_matrix",
|
|
31
|
+
meta_table: "matrix_schema_meta",
|
|
32
|
+
meta_key: "schema_version",
|
|
33
|
+
source_file: "host-service/routec-matrix-sqlite-store.mjs",
|
|
34
|
+
source_constant: "MATRIX_SQLITE_SCHEMA_VERSION",
|
|
35
|
+
versions: [{ version: 1, first_package_version: "1.5.7-beta.4" }],
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
id: "routec_matrix_stream_seq_index",
|
|
39
|
+
store: "routec_matrix",
|
|
40
|
+
meta_table: "matrix_schema_meta",
|
|
41
|
+
meta_key: "stream_seq_unique_index_version",
|
|
42
|
+
source_file: "host-service/routec-matrix-sqlite-store.mjs",
|
|
43
|
+
source_constant: "MATRIX_STREAM_SEQ_INDEX_VERSION",
|
|
44
|
+
versions: [{ version: 1, first_package_version: "1.5.7-beta.4" }],
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: "routec_matrix_completed_turn_navigation_index",
|
|
48
|
+
store: "routec_matrix",
|
|
49
|
+
meta_table: "matrix_schema_meta",
|
|
50
|
+
meta_key: "completed_turn_navigation_index_version",
|
|
51
|
+
source_file: "host-service/routec-matrix-sqlite-store.mjs",
|
|
52
|
+
source_constant: "MATRIX_COMPLETED_TURN_NAVIGATION_INDEX_VERSION",
|
|
53
|
+
versions: [{ version: 1, first_package_version: "1.5.7-beta.15" }],
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
id: "routec_matrix_callbacks",
|
|
57
|
+
store: "routec_matrix",
|
|
58
|
+
meta_table: "matrix_schema_meta",
|
|
59
|
+
meta_key: "callback_schema_version",
|
|
60
|
+
min_reader_meta_key: "callback_schema_reader_compatibility",
|
|
61
|
+
source_file: "host-service/routec-matrix-sqlite-store.mjs",
|
|
62
|
+
source_constant: "MATRIX_CALLBACK_SCHEMA_VERSION",
|
|
63
|
+
versions: [
|
|
64
|
+
{ version: 1, first_package_version: "1.5.7-beta.15" },
|
|
65
|
+
{ version: 2, first_package_version: "1.5.7-beta.15", old_reader: "not_reversible" },
|
|
66
|
+
{ version: 3, first_package_version: "1.6.0-beta.7", old_reader: "not_reversible" },
|
|
67
|
+
// Added nullable/defaulted recovery columns and backfilled only them;
|
|
68
|
+
// 1.6.0-beta.29 inserts with an explicit column list.
|
|
69
|
+
{ version: 4, first_package_version: "1.6.0-beta.30", old_reader: "marker_revert", tables: ["matrix_callbacks"] },
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
id: "routec_tool_events",
|
|
74
|
+
store: "routec_tool_events",
|
|
75
|
+
meta_table: "tool_event_store_meta",
|
|
76
|
+
meta_key: "schema_version",
|
|
77
|
+
source_file: "host-service/large-tool-call-store.mjs",
|
|
78
|
+
source_constant: "SQLITE_SCHEMA_VERSION",
|
|
79
|
+
versions: [{ version: 2, first_package_version: "1.5.7-beta.4" }],
|
|
80
|
+
},
|
|
81
|
+
]);
|
|
82
|
+
|
|
83
|
+
const fail = (code, message) => {
|
|
84
|
+
const error = new Error(message);
|
|
85
|
+
error.code = `p412_${code}`;
|
|
86
|
+
return error;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
function entry(id, schemas = STARTUP_SQLITE_SCHEMAS) {
|
|
90
|
+
const found = schemas.find((schema) => schema.id === id);
|
|
91
|
+
if (!found) throw fail("schema_unknown", `Unknown startup SQLite schema ${id}.`);
|
|
92
|
+
return found;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function currentSqliteSchemaVersion(id) {
|
|
96
|
+
return entry(id).versions.at(-1).version;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function currentSqliteSchemaVersions() {
|
|
100
|
+
return Object.fromEntries(STARTUP_SQLITE_SCHEMAS.map((schema) => [schema.id, schema.versions.at(-1).version]));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// The oldest reader version that can open `version` data with no fast
|
|
104
|
+
// recovery at all: additive steps keep every earlier reader's tables intact.
|
|
105
|
+
export function minimumSqliteSchemaReaderVersion(id, version = currentSqliteSchemaVersion(id)) {
|
|
106
|
+
const versions = entry(id).versions;
|
|
107
|
+
let reader = version;
|
|
108
|
+
for (let index = versions.findIndex((row) => row.version === version); index > 0; index -= 1) {
|
|
109
|
+
if (versions[index].old_reader !== "marker_revert") break;
|
|
110
|
+
reader = versions[index - 1].version;
|
|
111
|
+
}
|
|
112
|
+
return reader;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function resolveStartupSqliteStorePaths(configDir, env = process.env) {
|
|
116
|
+
return {
|
|
117
|
+
routec_matrix: routeCMatrixSQLiteStoragePath(routeCMatrixJsonStoragePath(configDir, env.OYSTERUN_ROUTEC_MATRIX_STORAGE_PATH)),
|
|
118
|
+
routec_tool_events: hostStoragePath(configDir, "routec-tool-events.sqlite"),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Which version of each schema the source package opens. A package that
|
|
123
|
+
// declares its versions is exact. An older package predates the declaration,
|
|
124
|
+
// so the release history above decides; a source older than a schema's first
|
|
125
|
+
// version never opens that schema at all.
|
|
126
|
+
export function resolveSourceSqliteSchemaSupport({ sourcePackageRoot, sourceVersion, schemas = STARTUP_SQLITE_SCHEMAS }) {
|
|
127
|
+
const manifest = JSON.parse(readFileSync(join(sourcePackageRoot, "package.json"), "utf8"));
|
|
128
|
+
const version = manifest.version || sourceVersion;
|
|
129
|
+
const declared = manifest[SQLITE_SCHEMA_DECLARATION_FIELD];
|
|
130
|
+
if (declared !== undefined && (!declared || typeof declared !== "object" || Array.isArray(declared))) {
|
|
131
|
+
throw fail("source_declaration", "The retained source package has an invalid SQLite schema declaration.");
|
|
132
|
+
}
|
|
133
|
+
return Object.fromEntries(schemas.map((schema) => {
|
|
134
|
+
if (declared) {
|
|
135
|
+
const value = declared[schema.id];
|
|
136
|
+
if (value === undefined) return [schema.id, { supported: null, basis: "declared_absent" }];
|
|
137
|
+
if (!Number.isSafeInteger(value) || value < 1) throw fail("source_declaration", `The retained source declares an invalid ${schema.id} version.`);
|
|
138
|
+
return [schema.id, { supported: value, basis: "declared" }];
|
|
139
|
+
}
|
|
140
|
+
let supported = null;
|
|
141
|
+
for (const row of schema.versions) {
|
|
142
|
+
const comparison = row.first_package_version ? compareOysterunSemver(version, row.first_package_version) : -1;
|
|
143
|
+
if (comparison === null) throw fail("source_version", "The retained source version cannot be compared.");
|
|
144
|
+
if (comparison >= 0) supported = row.version;
|
|
145
|
+
}
|
|
146
|
+
return [schema.id, { supported, basis: "release_history" }];
|
|
147
|
+
}));
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function openReadOnly(path) {
|
|
151
|
+
return new Database(path, { readonly: true, fileMustExist: true });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function tableExists(db, name) {
|
|
155
|
+
return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function readMarker(db, schema) {
|
|
159
|
+
if (!tableExists(db, schema.meta_table)) return null;
|
|
160
|
+
const raw = db.prepare(`SELECT value FROM ${schema.meta_table} WHERE key = ?`).get(schema.meta_key)?.value;
|
|
161
|
+
if (raw === undefined) return null;
|
|
162
|
+
const value = Number(raw);
|
|
163
|
+
if (!Number.isSafeInteger(value) || value < 1) throw fail("marker_corrupt", `${schema.id} has a corrupt version marker.`);
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function readBackupRows(db, schemaId, fromVersion) {
|
|
168
|
+
if (!tableExists(db, SQLITE_SCHEMA_BACKUP_TABLE)) return [];
|
|
169
|
+
return db.prepare(`SELECT * FROM ${SQLITE_SCHEMA_BACKUP_TABLE} WHERE schema_id = ? AND from_version = ?`).all(schemaId, fromVersion);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Read-only: which downgrade steps would let the source open every startup
|
|
173
|
+
// SQLite store as it is now. The data stays unchanged; nothing is stopped.
|
|
174
|
+
export function planSqliteSchemaDowngrade({ configDir, sourcePackageRoot, sourceVersion, env = process.env, schemas = STARTUP_SQLITE_SCHEMAS }) {
|
|
175
|
+
const support = resolveSourceSqliteSchemaSupport({ sourcePackageRoot, sourceVersion, schemas });
|
|
176
|
+
const paths = resolveStartupSqliteStorePaths(configDir, env);
|
|
177
|
+
const stores = [];
|
|
178
|
+
let blocked = null;
|
|
179
|
+
const handles = new Map();
|
|
180
|
+
try {
|
|
181
|
+
for (const schema of schemas) {
|
|
182
|
+
const path = paths[schema.store];
|
|
183
|
+
const supported = support[schema.id].supported;
|
|
184
|
+
if (!existsSync(path)) { stores.push({ id: schema.id, path, current: null, supported, steps: [] }); continue; }
|
|
185
|
+
if (!handles.has(path)) handles.set(path, openReadOnly(path));
|
|
186
|
+
const db = handles.get(path);
|
|
187
|
+
const current = readMarker(db, schema);
|
|
188
|
+
const row = { id: schema.id, path, current, supported, steps: [],
|
|
189
|
+
meta_table: schema.meta_table, meta_key: schema.meta_key, min_reader_meta_key: schema.min_reader_meta_key || null };
|
|
190
|
+
stores.push(row);
|
|
191
|
+
// An unversioned store, a schema the source never opens, or data the
|
|
192
|
+
// source already supports needs no recovery.
|
|
193
|
+
if (current === null || supported === null || current <= supported) continue;
|
|
194
|
+
if (current > schema.versions.at(-1).version) {
|
|
195
|
+
blocked ||= { id: schema.id, code: "data_newer_than_updater", message: `${schema.id} version ${current} is newer than this updater knows.` };
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
for (let version = current; version > supported; version -= 1) {
|
|
199
|
+
const declared = schema.versions.find((candidate) => candidate.version === version);
|
|
200
|
+
const previous = schema.versions.find((candidate) => candidate.version === version - 1);
|
|
201
|
+
if (!declared || !previous || !SQLITE_OLD_READER_MODES.includes(declared.old_reader) || declared.old_reader === "not_reversible") {
|
|
202
|
+
blocked ||= { id: schema.id, code: "schema_step_not_reversible", message: `${schema.id} ${version - 1} -> ${version} has no fast recovery for the source version.` };
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
if (declared.old_reader === "restore_backup_tables") {
|
|
206
|
+
const backups = readBackupRows(db, schema.id, version - 1);
|
|
207
|
+
const tables = declared.tables || [];
|
|
208
|
+
if (!tables.length || tables.some((table) => !backups.some((backup) => backup.table_name === table && tableExists(db, backup.backup_table)))) {
|
|
209
|
+
blocked ||= { id: schema.id, code: "schema_backup_missing", message: `${schema.id} ${version - 1} -> ${version} has no retained table backup.` };
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
row.steps.push({ from: version, to: version - 1, mode: declared.old_reader, tables: declared.tables || [] });
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
} finally {
|
|
217
|
+
for (const db of handles.values()) db.close();
|
|
218
|
+
}
|
|
219
|
+
return { feasible: !blocked, blocked, source_version: sourceVersion, support, stores };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function setMarker(db, schema, value) {
|
|
223
|
+
db.prepare(`UPDATE ${schema.meta_table} SET value = ? WHERE key = ?`).run(String(value), schema.meta_key);
|
|
224
|
+
if (schema.min_reader_meta_key) {
|
|
225
|
+
db.prepare(`DELETE FROM ${schema.meta_table} WHERE key = ?`).run(schema.min_reader_meta_key);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function restoreBackupTables(db, schemaId, fromVersion, tables) {
|
|
230
|
+
for (const table of tables) {
|
|
231
|
+
const backup = readBackupRows(db, schemaId, fromVersion).find((row) => row.table_name === table);
|
|
232
|
+
if (!backup || !tableExists(db, backup.backup_table)) throw fail("schema_backup_missing", `${schemaId} backup for ${table} is missing.`);
|
|
233
|
+
db.exec(`DROP TABLE IF EXISTS "${table}"`);
|
|
234
|
+
db.exec(backup.table_sql);
|
|
235
|
+
db.exec(`INSERT INTO "${table}" SELECT * FROM "${backup.backup_table}"`);
|
|
236
|
+
for (const sql of JSON.parse(backup.index_sql_json)) db.exec(sql);
|
|
237
|
+
db.exec(`DROP TABLE "${backup.backup_table}"`);
|
|
238
|
+
db.prepare(`DELETE FROM ${SQLITE_SCHEMA_BACKUP_TABLE} WHERE schema_id = ? AND from_version = ? AND table_name = ?`).run(schemaId, fromVersion, table);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Runs only while no Host holds the stores (after the target is stopped and
|
|
243
|
+
// before the source starts). Each store changes in one transaction and only
|
|
244
|
+
// if its marker still equals the planned value.
|
|
245
|
+
export function applySqliteSchemaDowngrade(plan) {
|
|
246
|
+
if (!plan?.feasible) throw fail("schema_downgrade_unplanned", plan?.blocked?.message || "No feasible SQLite downgrade plan.");
|
|
247
|
+
const applied = [];
|
|
248
|
+
const byPath = new Map();
|
|
249
|
+
for (const row of plan.stores) if (row.steps.length) byPath.set(row.path, [...(byPath.get(row.path) || []), row]);
|
|
250
|
+
for (const [path, rows] of byPath) {
|
|
251
|
+
const db = new Database(path, { fileMustExist: true });
|
|
252
|
+
try {
|
|
253
|
+
db.pragma("busy_timeout = 5000");
|
|
254
|
+
db.transaction(() => {
|
|
255
|
+
for (const row of rows) {
|
|
256
|
+
const schema = { id: row.id, meta_table: row.meta_table, meta_key: row.meta_key, min_reader_meta_key: row.min_reader_meta_key };
|
|
257
|
+
if (readMarker(db, schema) !== row.current) throw fail("schema_marker_changed", `${row.id} changed after the downgrade was planned.`);
|
|
258
|
+
for (const step of row.steps) {
|
|
259
|
+
if (step.mode === "restore_backup_tables") restoreBackupTables(db, row.id, step.to, step.tables);
|
|
260
|
+
setMarker(db, schema, step.to);
|
|
261
|
+
}
|
|
262
|
+
applied.push({ id: row.id, from: row.current, to: row.steps.at(-1).to, modes: row.steps.map((step) => step.mode) });
|
|
263
|
+
}
|
|
264
|
+
}).immediate();
|
|
265
|
+
} finally {
|
|
266
|
+
db.close();
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return { applied, at: new Date().toISOString() };
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// For a future "restore_backup_tables" step: call inside the migration's
|
|
273
|
+
// transaction, before the tables change. Keeps the exact table/index SQL so
|
|
274
|
+
// the restore rebuilds the source shape. Only the named tables are copied.
|
|
275
|
+
export function backupSqliteTablesForSchemaStep(db, { schemaId, fromVersion, tables, schemas = STARTUP_SQLITE_SCHEMAS }) {
|
|
276
|
+
const declared = entry(schemaId, schemas).versions.find((row) => row.version === fromVersion + 1);
|
|
277
|
+
if (declared?.old_reader !== "restore_backup_tables" || JSON.stringify(declared.tables) !== JSON.stringify(tables)) {
|
|
278
|
+
throw fail("schema_backup_undeclared", `${schemaId} ${fromVersion} -> ${fromVersion + 1} does not declare these backup tables.`);
|
|
279
|
+
}
|
|
280
|
+
db.exec(`CREATE TABLE IF NOT EXISTS ${SQLITE_SCHEMA_BACKUP_TABLE} (
|
|
281
|
+
schema_id TEXT NOT NULL, from_version INTEGER NOT NULL, table_name TEXT NOT NULL,
|
|
282
|
+
backup_table TEXT NOT NULL, table_sql TEXT NOT NULL, index_sql_json TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
283
|
+
PRIMARY KEY(schema_id, from_version, table_name))`);
|
|
284
|
+
for (const table of tables) {
|
|
285
|
+
const tableSql = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(table)?.sql;
|
|
286
|
+
if (!tableSql) throw fail("schema_backup_table", `${table} does not exist.`);
|
|
287
|
+
const indexSql = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL").all(table).map((row) => row.sql);
|
|
288
|
+
const backupTable = `oysterun_backup__${schemaId}__v${fromVersion}__${table}`;
|
|
289
|
+
db.exec(`DROP TABLE IF EXISTS "${backupTable}"`);
|
|
290
|
+
db.exec(`CREATE TABLE "${backupTable}" AS SELECT * FROM "${table}"`);
|
|
291
|
+
db.prepare(`INSERT OR REPLACE INTO ${SQLITE_SCHEMA_BACKUP_TABLE} VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
292
|
+
.run(schemaId, fromVersion, table, backupTable, tableSql, JSON.stringify(indexSql), new Date().toISOString());
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// After an update is kept, its table backups are no longer a rollback path.
|
|
297
|
+
export function dropSqliteSchemaDowngradeBackups({ configDir, env = process.env }) {
|
|
298
|
+
const dropped = [];
|
|
299
|
+
for (const path of new Set(Object.values(resolveStartupSqliteStorePaths(configDir, env)))) {
|
|
300
|
+
if (!existsSync(path)) continue;
|
|
301
|
+
const db = new Database(path, { fileMustExist: true });
|
|
302
|
+
try {
|
|
303
|
+
if (!tableExists(db, SQLITE_SCHEMA_BACKUP_TABLE)) continue;
|
|
304
|
+
db.pragma("busy_timeout = 5000");
|
|
305
|
+
db.transaction(() => {
|
|
306
|
+
for (const row of db.prepare(`SELECT backup_table FROM ${SQLITE_SCHEMA_BACKUP_TABLE}`).all()) {
|
|
307
|
+
db.exec(`DROP TABLE IF EXISTS "${row.backup_table}"`);
|
|
308
|
+
dropped.push(row.backup_table);
|
|
309
|
+
}
|
|
310
|
+
db.exec(`DROP TABLE ${SQLITE_SCHEMA_BACKUP_TABLE}`);
|
|
311
|
+
}).immediate();
|
|
312
|
+
} finally {
|
|
313
|
+
db.close();
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return dropped;
|
|
317
|
+
}
|
|
@@ -169,7 +169,7 @@ export function finalizeHostStorageServiceBinding(
|
|
|
169
169
|
if (!match) continue;
|
|
170
170
|
const path = join(operations, name),
|
|
171
171
|
receipt = readContinuity(path, match[1]);
|
|
172
|
-
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state))
|
|
172
|
+
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state))
|
|
173
173
|
continue;
|
|
174
174
|
if (
|
|
175
175
|
resolve(receipt.baseline.host.config_dir) !== root ||
|
package/package.json
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oysterun",
|
|
3
3
|
"oysterun_storage_layout_version": 2,
|
|
4
|
-
"
|
|
4
|
+
"oysterun_sqlite_schema_versions": {
|
|
5
|
+
"routec_matrix_schema": 1,
|
|
6
|
+
"routec_matrix_stream_seq_index": 1,
|
|
7
|
+
"routec_matrix_completed_turn_navigation_index": 1,
|
|
8
|
+
"routec_matrix_callbacks": 4,
|
|
9
|
+
"routec_tool_events": 2
|
|
10
|
+
},
|
|
11
|
+
"version": "1.6.0-beta.31",
|
|
5
12
|
"type": "module",
|
|
6
13
|
"description": "Oysterun Host installer and local service",
|
|
7
14
|
"bin": {
|
|
@@ -11,11 +11,13 @@ import { spawnSync } from "node:child_process";
|
|
|
11
11
|
import {
|
|
12
12
|
advanceContinuity, continuityDigest, continuityFailure, continuityReceiptPath, readContinuity,
|
|
13
13
|
recordContinuityRecovery, writeContinuityAtomic, writeContinuityBytesAtomic, recordContinuityRestartAttempt,
|
|
14
|
-
qualifyContinuityHost, qualifyContinuitySessions, resolveRestartRecoveryPackageVersion,
|
|
14
|
+
qualifyContinuityHost, qualifyContinuitySessions, resolveRestartRecoveryPackageVersion, retainContinuityTarget,
|
|
15
15
|
} from "../host-service/lifecycle-continuity.mjs";
|
|
16
16
|
import {
|
|
17
17
|
finalizeContinuityAssets, restoreContinuityAssets, validateContinuityResources, verifyContinuityAssets, digestContinuityPackage,
|
|
18
|
+
planContinuityRestore,
|
|
18
19
|
} from "../host-service/lifecycle-continuity-assets.mjs";
|
|
20
|
+
import { applySqliteSchemaDowngrade } from "../host-service/sqlite-schema-compatibility.mjs";
|
|
19
21
|
import { qualifyLifecycleContinuity } from "../host-service/lifecycle-continuity-runner.mjs";
|
|
20
22
|
import { fetchContinuityObservation, ensureContinuityProbeGrant } from "../host-service/lifecycle-continuity-probe.mjs";
|
|
21
23
|
import { verifyPersistedNodeNpmRuntimeIdentity, resolveNodeNpmRuntimeIdentity } from "../host-service/node-npm-runtime-identity.mjs";
|
|
@@ -117,7 +119,7 @@ export async function runContinuityTransaction(args, overrides = {}) {
|
|
|
117
119
|
const current = readUpdateOperationState({ configDir: args.config_dir });
|
|
118
120
|
const currentPath = continuityReceiptPath(args.config_dir, args.operation_id);
|
|
119
121
|
const receipt = readContinuity(currentPath, args.operation_id);
|
|
120
|
-
if (current.operation_id === args.operation_id && !["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(receipt.state)) {
|
|
122
|
+
if (current.operation_id === args.operation_id && !["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(receipt.state)) {
|
|
121
123
|
const failed = recordContinuityRecovery(currentPath, args.operation_id, error);
|
|
122
124
|
writeUpdateOperationState({ ...current,
|
|
123
125
|
status: failed.transaction_kind === "restart_only" ? "restart_recovery_required" : "update_recovery_required",
|
|
@@ -134,6 +136,13 @@ export async function runContinuityTransaction(args, overrides = {}) {
|
|
|
134
136
|
async function continueTransaction(args, overrides, path) {
|
|
135
137
|
const get = () => readContinuity(path, args.operation_id);
|
|
136
138
|
const restorePairedAssets = () => {
|
|
139
|
+
// P412 (2a/2c): the Host is stopped here. Prove the exchange again, then
|
|
140
|
+
// let the source open the data before its package is put back.
|
|
141
|
+
const restorePlan = planContinuityRestore({ path, operationId: args.operation_id, configDir: args.config_dir, activePackageRoot: args.active_package_root });
|
|
142
|
+
if (restorePlan.schema.stores.some((row) => row.steps.length)) {
|
|
143
|
+
const schemaRecovery = applySqliteSchemaDowngrade(restorePlan.schema);
|
|
144
|
+
writeContinuityAtomic(path, { ...get(), schema_recovery: schemaRecovery });
|
|
145
|
+
}
|
|
137
146
|
const verified = verifyContinuityAssets({ path, operationId: args.operation_id, configDir: args.config_dir, activePackageRoot: args.active_package_root });
|
|
138
147
|
reverseStorageForRetainedSource({ root: args.config_dir, stack: args.stack, port: verified.receipt.baseline.host.port,
|
|
139
148
|
serviceLabel: args.stack === "production" ? "com.oysterun.host" : `com.oysterun.host.${args.stack}`,
|
|
@@ -175,7 +184,7 @@ async function continueTransaction(args, overrides, path) {
|
|
|
175
184
|
: "Verifying the existing authenticated Host URL",
|
|
176
185
|
runner_lease_expires_at: new Date(Date.now() + UPDATE_RESTART_RUNNER_LEASE_MS).toISOString() });
|
|
177
186
|
const runService = async (command, options = {}) => {
|
|
178
|
-
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY"].includes(get().state)) {
|
|
187
|
+
if (["COMMITTED", "CLEANED", "ROLLED_BACK_HEALTHY", "TARGET_RETAINED"].includes(get().state)) {
|
|
179
188
|
throw continuityFailure("recovery", "terminal_operation", "A completed operation cannot mutate the Host again.");
|
|
180
189
|
}
|
|
181
190
|
if (args.recheck_only === "1") throw continuityFailure("recovery", "read_only_recheck", "Requalification cannot restart or restore the Host.");
|
|
@@ -210,7 +219,8 @@ async function continueTransaction(args, overrides, path) {
|
|
|
210
219
|
const rolledBack = receipt.state === "ROLLED_BACK_HEALTHY";
|
|
211
220
|
const result = {
|
|
212
221
|
status, operation_id: args.operation_id, continuity_path: path,
|
|
213
|
-
completed_at: new Date().toISOString(), observed_version: rolledBack ? receipt.source.version
|
|
222
|
+
completed_at: new Date().toISOString(), observed_version: rolledBack ? receipt.source.version
|
|
223
|
+
: receipt.target_retention?.observed_version || receipt.target.version,
|
|
214
224
|
rollback_available: false, rollback_completed: rolledBack && !restartOnly, restart_attempted: receipt.restart_attempted,
|
|
215
225
|
operation_cleanup_completed: receipt.state === "CLEANED", root_failure: receipt.first_failure,
|
|
216
226
|
retained_recovery_assets: receipt.state !== "CLEANED", error_redacted: true,
|
|
@@ -231,6 +241,9 @@ async function continueTransaction(args, overrides, path) {
|
|
|
231
241
|
: "Previous package, service, every Session, history and authenticated Host route restored and verified." },
|
|
232
242
|
rollback_result: { result: restartOnly ? "not_required" : "passed", code: restartOnly ? "same_package_restart_recovered" : "paired_source_restored",
|
|
233
243
|
safe_message: restartOnly ? "The original package remains installed." : "The receipt-bound prior package and service definition are restored." } } : {}),
|
|
244
|
+
...(receipt.target_retention ? { rollback_result: { result: "not_required", code: "target_retained",
|
|
245
|
+
safe_message: receipt.target_retention.failed_check ? "The new version passed its Host check and was kept; the later failed check is recorded."
|
|
246
|
+
: "The new version passed its Host check and was kept." } } : {}),
|
|
234
247
|
cleanup_result: { result: receipt.state === "CLEANED" ? "passed" : "incomplete",
|
|
235
248
|
code: receipt.state === "CLEANED" ? "operation_root_removed" : "operation_root_retained",
|
|
236
249
|
safe_message: receipt.state === "CLEANED" ? "Committed operation resources were removed." : "Recovery assets remain retained." },
|
|
@@ -242,9 +255,37 @@ async function continueTransaction(args, overrides, path) {
|
|
|
242
255
|
};
|
|
243
256
|
const finalize = () => terminal(restartOnly ? "restart_complete" : "update_complete",
|
|
244
257
|
finalizeContinuityAssets({ path, operationId: args.operation_id, configDir: args.config_dir, activePackageRoot: args.active_package_root }));
|
|
245
|
-
|
|
258
|
+
const cleanupFailed = (error) => {
|
|
259
|
+
patch({ status: "restart_recovery_required", phase: "Qualification committed; cleanup requires continuation",
|
|
260
|
+
running: false, operation_cleanup_completed: false, runner_pid: null, runner_lease_expires_at: null,
|
|
261
|
+
error: sanitizeLifecycleDiagnostic(error.message), next_safe_action: "complete_update_and_restart" });
|
|
262
|
+
throw error;
|
|
263
|
+
};
|
|
264
|
+
const finalizeRetained = () => { try { return finalize(); } catch (error) { return cleanupFailed(error); } };
|
|
265
|
+
// P412 (2b): the target passed its Host qualification and is still the
|
|
266
|
+
// sole, healthy listener: keep it, record the failed check, close forward.
|
|
267
|
+
const retainQualifiedTarget = async (failure) => {
|
|
268
|
+
const r = get();
|
|
269
|
+
const hostProof = r.qualifications?.HOST_QUALIFIED;
|
|
270
|
+
if (restartOnly || failure?.lifecycle_diagnostic?.check === "recovery_requested" || hostProof?.passed !== true) return null;
|
|
271
|
+
const pids = listeners(r.baseline.host.port);
|
|
272
|
+
if (pids.length !== 1 || pids[0] !== hostProof.pid) return null;
|
|
273
|
+
let host;
|
|
274
|
+
try {
|
|
275
|
+
const live = await observe({ configDir: args.config_dir, operationId: args.operation_id, origin: new URL(args.health_url).origin });
|
|
276
|
+
if (live.host.pid !== pids[0]) return null;
|
|
277
|
+
host = qualifyContinuityHost(r.baseline.host, live.host, { version: r.target.version,
|
|
278
|
+
runtimeIdentityDigest: r.source.runtime_identity_digest, expectNewPid: true,
|
|
279
|
+
definitionDigest: r.target.service_definition_sha256 || r.baseline.host.service_definition_sha256 });
|
|
280
|
+
} catch { return null; }
|
|
281
|
+
if (host.pid !== hostProof.pid) return null;
|
|
282
|
+
retainContinuityTarget(path, args.operation_id, { reason: "post_host_qualification_check_failed", failure,
|
|
283
|
+
qualification: { host }, observedVersion: r.target.version });
|
|
284
|
+
return finalizeRetained();
|
|
285
|
+
};
|
|
286
|
+
if (["COMMITTED", "CLEANED", "TARGET_RETAINED"].includes(get().state) &&
|
|
246
287
|
(args.restore_before_restart === "1" || args.rollback_before_restart === "1")) throw continuityFailure("recovery", "already_committed", "Rollback is closed after COMMITTED; use the explicit finalize command for cleanup.");
|
|
247
|
-
if (["COMMITTED", "CLEANED"].includes(get().state)) return finalize();
|
|
288
|
+
if (["COMMITTED", "CLEANED", "TARGET_RETAINED"].includes(get().state)) return finalize();
|
|
248
289
|
if (get().state === "ROLLED_BACK_HEALTHY") return terminal(restartOnly ? "restart_complete" : "rolled_back", get());
|
|
249
290
|
|
|
250
291
|
if (args.recheck_only === "1") {
|
|
@@ -330,15 +371,12 @@ async function continueTransaction(args, overrides, path) {
|
|
|
330
371
|
return finalize();
|
|
331
372
|
} catch (error) {
|
|
332
373
|
// Failure during cleanup is a committed cleanup failure, never rollback.
|
|
333
|
-
if (["COMMITTED", "CLEANED"].includes(get().state))
|
|
334
|
-
patch({ status: "restart_recovery_required", phase: "Qualification committed; cleanup requires continuation",
|
|
335
|
-
running: false, operation_cleanup_completed: false, runner_pid: null, runner_lease_expires_at: null,
|
|
336
|
-
error: sanitizeLifecycleDiagnostic(error.message), next_safe_action: "complete_update_and_restart" });
|
|
337
|
-
throw error;
|
|
338
|
-
}
|
|
374
|
+
if (["COMMITTED", "CLEANED", "TARGET_RETAINED"].includes(get().state)) cleanupFailed(error);
|
|
339
375
|
failure = error;
|
|
340
376
|
recordContinuityRecovery(path, args.operation_id, error);
|
|
341
377
|
}
|
|
378
|
+
const retained = await retainQualifiedTarget(failure);
|
|
379
|
+
if (retained) return retained;
|
|
342
380
|
patch({ status: restartOnly ? "restart_recovery_required" : "post_restart_health_failed",
|
|
343
381
|
phase: restartOnly ? "Restart qualification failed; recovery required" : "Target qualification failed; restoring the previous version",
|
|
344
382
|
running: !restartOnly, error: sanitizeLifecycleDiagnostic(failure.message) });
|
|
@@ -350,6 +388,7 @@ async function continueTransaction(args, overrides, path) {
|
|
|
350
388
|
recovery_reason: get().first_failure.message, next_safe_action: null, operation_cleanup_completed: false });
|
|
351
389
|
throw failure;
|
|
352
390
|
}
|
|
391
|
+
let restoreUnprovable = false;
|
|
353
392
|
try {
|
|
354
393
|
const r = get();
|
|
355
394
|
const pids = listeners(r.baseline.host.port);
|
|
@@ -364,7 +403,10 @@ async function continueTransaction(args, overrides, path) {
|
|
|
364
403
|
}
|
|
365
404
|
validateContinuityResources(r, args.config_dir, args.active_package_root);
|
|
366
405
|
prepareCompatibility({ path, operationId: args.operation_id, render: renderServiceDefinition });
|
|
367
|
-
|
|
406
|
+
// P412 (3): prove the package exchange and the source reading the current
|
|
407
|
+
// data before any Host is stopped; otherwise leave the Host untouched.
|
|
408
|
+
try { planContinuityRestore({ path, operationId: args.operation_id, configDir: args.config_dir, activePackageRoot: args.active_package_root }); }
|
|
409
|
+
catch (error) { restoreUnprovable = true; throw error; }
|
|
368
410
|
const restartPrior = !originalStillLive || existsSync(r.resources.previous_package_root);
|
|
369
411
|
if (pids.length && !originalStillLive) {
|
|
370
412
|
const live = await observe({ configDir: args.config_dir, operationId: args.operation_id, origin: new URL(args.health_url).origin });
|
|
@@ -432,9 +474,11 @@ async function continueTransaction(args, overrides, path) {
|
|
|
432
474
|
return terminal("rolled_back", recovered);
|
|
433
475
|
} catch (error) {
|
|
434
476
|
recordContinuityRecovery(path, args.operation_id, failure);
|
|
435
|
-
patch({ status: "rollback_failed", phase:
|
|
477
|
+
patch({ status: "rollback_failed", phase: restoreUnprovable
|
|
478
|
+
? "The previous version cannot be restored safely; the Host was not stopped or changed"
|
|
479
|
+
: "Recovery incomplete; retained assets require canonical continuation",
|
|
436
480
|
running: false, terminal_result_status: "rollback_failed", runner_pid: null, runner_lease_expires_at: null,
|
|
437
|
-
rollback_completed: false, operation_cleanup_completed: false, next_safe_action: "restore_and_restart",
|
|
481
|
+
rollback_completed: false, operation_cleanup_completed: false, next_safe_action: restoreUnprovable ? null : "restore_and_restart",
|
|
438
482
|
recovery_required_at: new Date().toISOString(), recovery_boundary: error.code || "paired_recovery_incomplete",
|
|
439
483
|
recovery_reason: sanitizeLifecycleDiagnostic(error.message), error: sanitizeLifecycleDiagnostic(error.message) });
|
|
440
484
|
throw error;
|
|
@@ -118,6 +118,7 @@ export const npmRequiredPackagePaths = Object.freeze([
|
|
|
118
118
|
"host-service/lifecycle-compat-cli.mjs",
|
|
119
119
|
"host-service/lifecycle-continuity-assets.mjs",
|
|
120
120
|
"host-service/lifecycle-continuity-probe.mjs",
|
|
121
|
+
"host-service/sqlite-schema-compatibility.mjs",
|
|
121
122
|
"host-service/lifecycle-continuity-projection.mjs",
|
|
122
123
|
"host-service/lifecycle-continuity-runner.mjs",
|
|
123
124
|
"host-service/lifecycle-service-render-bundle.mjs",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { writeAtomicJsonFile } from "../host-service/atomic-file.mjs";
|
|
3
3
|
import { compatibilityEnvironment } from "../host-service/lifecycle-compatibility.mjs";
|
|
4
4
|
import { runContinuityTransaction } from "./oysterun_continuity_transaction.mjs";
|
|
5
|
+
import { planSqliteSchemaDowngrade, applySqliteSchemaDowngrade } from "../host-service/sqlite-schema-compatibility.mjs";
|
|
5
6
|
|
|
6
7
|
import { spawnSync } from "node:child_process";
|
|
7
8
|
import {
|
|
@@ -622,12 +623,45 @@ function recordTerminalOutcomeDiagnostics(args, receipt, summary) {
|
|
|
622
623
|
return receipt.diagnostic_summary;
|
|
623
624
|
}
|
|
624
625
|
|
|
626
|
+
// P412: the prior package must be able to open the data the target left.
|
|
627
|
+
// Planned read-only before the Host is stopped; applied once it is stopped.
|
|
628
|
+
function planPriorPackageSchema({ receipt, paths, configDir }) {
|
|
629
|
+
const plan = planSqliteSchemaDowngrade({
|
|
630
|
+
configDir,
|
|
631
|
+
sourcePackageRoot: paths.previousRoot,
|
|
632
|
+
sourceVersion: receipt.previous_version,
|
|
633
|
+
});
|
|
634
|
+
if (!plan.feasible) {
|
|
635
|
+
const error = new Error(`The previous version cannot open the current Host data: ${plan.blocked.message}`);
|
|
636
|
+
error.code = plan.blocked.code;
|
|
637
|
+
error.restore_unprovable = true;
|
|
638
|
+
throw error;
|
|
639
|
+
}
|
|
640
|
+
return plan;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
function schemaRevertNeeded(plan) {
|
|
644
|
+
return plan.stores.some((row) => row.steps.length);
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// `revertSchema` only after the Host that holds the stores has been stopped.
|
|
625
648
|
function restorePriorPackage({
|
|
626
649
|
receipt,
|
|
627
650
|
paths,
|
|
628
651
|
restartSnapshotBackup,
|
|
629
652
|
configDir,
|
|
653
|
+
revertSchema = false,
|
|
630
654
|
}) {
|
|
655
|
+
const schemaPlan = planPriorPackageSchema({ receipt, paths, configDir });
|
|
656
|
+
if (schemaRevertNeeded(schemaPlan)) {
|
|
657
|
+
if (!revertSchema) {
|
|
658
|
+
const error = new Error("The previous version needs a data revert that cannot run while the Host holds the data.");
|
|
659
|
+
error.code = "p412_schema_revert_requires_stopped_host";
|
|
660
|
+
error.restore_unprovable = true;
|
|
661
|
+
throw error;
|
|
662
|
+
}
|
|
663
|
+
applySqliteSchemaDowngrade(schemaPlan);
|
|
664
|
+
}
|
|
631
665
|
if (existsSync(paths.failedRoot)) {
|
|
632
666
|
rmSync(paths.failedRoot, { recursive: true, force: true });
|
|
633
667
|
}
|
|
@@ -970,6 +1004,14 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
970
1004
|
error: null,
|
|
971
1005
|
});
|
|
972
1006
|
try {
|
|
1007
|
+
// This path restarts a still-running Host: refuse before any change
|
|
1008
|
+
// when the previous version would need a data revert first.
|
|
1009
|
+
if (schemaRevertNeeded(planPriorPackageSchema({ receipt, paths, configDir: args.config_dir }))) {
|
|
1010
|
+
const error = new Error("The previous version needs a data revert that cannot run while the Host holds the data.");
|
|
1011
|
+
error.code = "p412_schema_revert_requires_stopped_host";
|
|
1012
|
+
error.restore_unprovable = true;
|
|
1013
|
+
throw error;
|
|
1014
|
+
}
|
|
973
1015
|
const restoredVersion = restorePriorPackage({
|
|
974
1016
|
receipt,
|
|
975
1017
|
paths,
|
|
@@ -1100,7 +1142,7 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
1100
1142
|
status: "rollback_failed",
|
|
1101
1143
|
requested_action: "restore_and_restart",
|
|
1102
1144
|
completed_at: new Date().toISOString(),
|
|
1103
|
-
rollback_available: true,
|
|
1145
|
+
rollback_available: error?.restore_unprovable !== true,
|
|
1104
1146
|
rollback_completed: false,
|
|
1105
1147
|
operation_cleanup_completed: false,
|
|
1106
1148
|
root_failure: diagnosticSummary.root_failure,
|
|
@@ -1121,7 +1163,7 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
1121
1163
|
rollback_completed: false,
|
|
1122
1164
|
runner_pid: null,
|
|
1123
1165
|
runner_lease_expires_at: null,
|
|
1124
|
-
next_safe_action: "restore_and_restart",
|
|
1166
|
+
next_safe_action: error?.restore_unprovable === true ? null : "restore_and_restart",
|
|
1125
1167
|
error: safeError,
|
|
1126
1168
|
root_failure: failedReceipt.root_failure,
|
|
1127
1169
|
rollback_result: failedReceipt.rollback_result,
|
|
@@ -1326,16 +1368,29 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
1326
1368
|
});
|
|
1327
1369
|
|
|
1328
1370
|
try {
|
|
1371
|
+
// Never stop the target for a rollback the prior package cannot open.
|
|
1372
|
+
planPriorPackageSchema({ receipt, paths, configDir: args.config_dir });
|
|
1329
1373
|
try {
|
|
1330
1374
|
runService("service:stop");
|
|
1331
1375
|
} catch {
|
|
1332
1376
|
// A failed restart commonly leaves no target Host to stop.
|
|
1333
1377
|
}
|
|
1378
|
+
if (schemaRevertNeeded(planPriorPackageSchema({ receipt, paths, configDir: args.config_dir }))) {
|
|
1379
|
+
let stillServing = false;
|
|
1380
|
+
try { await probeHealth(); stillServing = true; } catch { /* stopped, as required */ }
|
|
1381
|
+
if (stillServing) {
|
|
1382
|
+
const error = new Error("The Host still serves after stop; its data cannot be reverted for the previous version.");
|
|
1383
|
+
error.code = "p412_schema_revert_requires_stopped_host";
|
|
1384
|
+
error.restore_unprovable = true;
|
|
1385
|
+
throw error;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1334
1388
|
const restoredVersion = restorePriorPackage({
|
|
1335
1389
|
receipt,
|
|
1336
1390
|
paths,
|
|
1337
1391
|
restartSnapshotBackup,
|
|
1338
1392
|
configDir: args.config_dir,
|
|
1393
|
+
revertSchema: true,
|
|
1339
1394
|
});
|
|
1340
1395
|
runService("service:restart");
|
|
1341
1396
|
const health = await probeHealth();
|
|
@@ -1460,7 +1515,7 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
1460
1515
|
...receipt,
|
|
1461
1516
|
status: "rollback_failed",
|
|
1462
1517
|
completed_at: new Date().toISOString(),
|
|
1463
|
-
rollback_available: true,
|
|
1518
|
+
rollback_available: rollbackError?.restore_unprovable !== true,
|
|
1464
1519
|
rollback_completed: false,
|
|
1465
1520
|
operation_cleanup_completed: false,
|
|
1466
1521
|
target_failure: safeTargetError,
|
|
@@ -1476,14 +1531,16 @@ export async function runUpdateRestartTransaction(args, overrides = {}) {
|
|
|
1476
1531
|
writeJsonAtomic(args.result_path, failedReceipt);
|
|
1477
1532
|
updateOperation(args.config_dir, args.operation_id, {
|
|
1478
1533
|
status: "rollback_failed",
|
|
1479
|
-
phase:
|
|
1534
|
+
phase: rollbackError?.restore_unprovable === true
|
|
1535
|
+
? "The previous version cannot open the current Host data; the Host was not stopped"
|
|
1536
|
+
: "Rollback failed; operator action required",
|
|
1480
1537
|
running: false,
|
|
1481
1538
|
terminal_result_status: "rollback_failed",
|
|
1482
1539
|
operation_cleanup_completed: false,
|
|
1483
1540
|
rollback_completed: false,
|
|
1484
1541
|
runner_pid: null,
|
|
1485
1542
|
runner_lease_expires_at: null,
|
|
1486
|
-
next_safe_action: "restore_and_restart",
|
|
1543
|
+
next_safe_action: rollbackError?.restore_unprovable === true ? null : "restore_and_restart",
|
|
1487
1544
|
error: safeRollbackError,
|
|
1488
1545
|
restart_attempted: true,
|
|
1489
1546
|
root_failure: failedReceipt.root_failure,
|