@zq-silk/yui 0.11.3 → 0.12.1

package/dist/storage/upgrade/sqliteStateMigration.js

@@ -22,6 +22,7 @@
  * dropped, duplicated, or mutated record.
  */
 import { createHash } from "node:crypto";
+import { existsSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import Database from "better-sqlite3";
 import { mailboxTargetKey } from "../../coordination/workMailbox.js";
@@ -33,6 +34,50 @@ import { CURRENT_STORED_TASK_SCHEMA_VERSION } from "../taskStore.js";
 export const STAGED_DATABASE_FILENAME = "yui.db.staged";
 /** The committed database filename. */
 export const COMMITTED_DATABASE_FILENAME = "yui.db";
+/** Disposable, schema-current snapshot used to reconstruct an older source. */
+export const MIGRATION_SOURCE_DATABASE_FILENAME = "yui.db.migration-source";
+/**
+ * Reconstruct a migration snapshot without advancing the authoritative DB.
+ *
+ * `VACUUM INTO` takes a consistent SQLite snapshot, including committed WAL
+ * state, without writing the source. Pending physical migrations are then
+ * applied only to that disposable copy before the current reader enumerates
+ * record families that may not exist in the older source schema.
+ */
+export function readMigrationSourceStateFromSqlite(home) {
+    const sourcePath = join(home, COMMITTED_DATABASE_FILENAME);
+    const snapshotPath = join(home, MIGRATION_SOURCE_DATABASE_FILENAME);
+    if (existsSync(snapshotPath)) {
+        throw new Error(`Refusing to overwrite an existing SQLite migration-source snapshot: ${snapshotPath}. `
+            + "Discard it and retry.");
+    }
+    try {
+        const source = new Database(sourcePath, { readonly: true });
+        try {
+            source.prepare("VACUUM INTO ?").run(snapshotPath);
+        }
+        finally {
+            source.close();
+        }
+        // Physical migration 14 rewrites the WorkMailbox family while adding its
+        // current columns. Preserve the source rows before applying it so the
+        // logical record migration still sees and transforms v1 exactly once.
+        const sourceMailboxes = readWorkMailboxesFromSqlite(snapshotPath);
+        const stagedStore = new SqliteTaskStore(home, {
+            databaseFilename: MIGRATION_SOURCE_DATABASE_FILENAME,
+            migration: true
+        });
+        stagedStore.close();
+        const state = readStateFromSqlite(home, MIGRATION_SOURCE_DATABASE_FILENAME);
+        state.mailboxes = sourceMailboxes;
+        return state;
+    }
+    finally {
+        rmSync(snapshotPath, { force: true });
+        rmSync(`${snapshotPath}-wal`, { force: true });
+        rmSync(`${snapshotPath}-shm`, { force: true });
+    }
+}
 /**
  * Preserve SQLite-owned durable state that intentionally sits outside the
  * state.json-shaped Task aggregate snapshot. Volatile coordination locks and
@@ -351,9 +396,6 @@ export function populateSqliteFromState(home, state, databaseFilename) {
                 if (stored.leaderFailure !== null) {
                     store.saveLeaderFailure(stored.leaderFailure);
                 }
-                if (stored.operatorNotification !== null) {
-                    store.saveOperatorNotification(stored.operatorNotification);
-                }
                 // Per-task ID high-water marks.
                 for (const [kind, highWater] of Object.entries(stored.idHighWaterMarks)) {
                     if (typeof highWater === "number" && Number.isFinite(highWater)) {
@@ -595,6 +637,20 @@ function readWorkMailboxRows(db) {
     const hasInputDelivery = db.prepare("PRAGMA table_info(mailboxes)").all().some(({ name }) => name === "input_delivery");
     return db.prepare(`SELECT target_kind, task_id, role_name, next_sequence, processing, pending${hasInputDelivery ? ", input_delivery" : ""} FROM mailboxes ORDER BY target_key`).all();
 }
+function readWorkMailboxesFromSqlite(dbPath) {
+    const db = new Database(dbPath, { readonly: true });
+    try {
+        const mailboxes = {};
+        for (const row of readWorkMailboxRows(db)) {
+            const mailbox = rowToMailbox(row);
+            mailboxes[mailboxTargetKey(mailbox.target)] = mailbox;
+        }
+        return mailboxes;
+    }
+    finally {
+        db.close();
+    }
+}
 /**
  * Compute per-family checksums from the SQLite database. The database is
  * opened read-only; this is the independent verification read.
@@ -692,8 +748,8 @@ export function verifySqliteChecksums(state, home, databaseFilename) {
  *
  * The database is opened read-only; this function never writes.
  */
-export function readStateFromSqlite(home) {
-    const dbPath = join(home, COMMITTED_DATABASE_FILENAME);
+export function readStateFromSqlite(home, databaseFilename = COMMITTED_DATABASE_FILENAME) {
+    const dbPath = join(home, databaseFilename);
     const db = new Database(dbPath, { readonly: true });
     try {
         const manifest = readStorageSchemaManifest(home);
@@ -737,7 +793,7 @@ export function readStateFromSqlite(home) {
         const tasks = state.tasks;
         const taskRows = db.prepare("SELECT task_id, payload, brief FROM task_records").all();
         for (const row of taskRows) {
-            tasks[row.task_id] = {
+            const aggregate = {
                 schemaVersion: storedTaskVersion,
                 task: JSON.parse(row.payload),
                 brief: row.brief === null ? null : JSON.parse(row.brief),
@@ -761,12 +817,14 @@ export function readStateFromSqlite(home) {
                 events: {},
                 wakes: {},
                 leaderFailure: null,
-                operatorNotification: null,
                 idHighWaterMarks: {},
                 capabilityGrants: {},
                 releaseWorkflows: {},
                 publicationReferences: {}
             };
+            if (storedTaskVersion <= 17)
+                aggregate.operatorNotification = null;
+            tasks[row.task_id] = aggregate;
         }
         // Per-task record families, keyed by their state.json map keys.
         loadTaskPayloadMap(db, "task_roles", tasks, "roles", (record) => record.name);
@@ -808,7 +866,7 @@ export function readStateFromSqlite(home) {
             if (row.kind === "leader-failure") {
                 stored.leaderFailure = JSON.parse(row.payload);
             }
-            else {
+            else if (storedTaskVersion <= 17) {
                 stored.operatorNotification = JSON.parse(row.payload);
             }
         }

package/dist/storage/upgrade/upgradeOrchestrator.js

@@ -34,14 +34,17 @@
  * authoritative input byte-for-byte unchanged.
  */
 import { spawn } from "node:child_process";
-import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
+import Database from "better-sqlite3";
 import { describeReport, runMigration } from "../migration/index.js";
 import { validateCompatibleFileTaskStore } from "../compatibleTaskStore.js";
 import { stopFileTaskController, ensureFileTaskController, ensureFileTaskControllerIdentity } from "../../controller/clientRuntime.js";
 import { callController } from "../../core/controllerClient.js";
 import { FileTaskStore, STORAGE_STATE_FILE, withStorageWriteLock } from "../taskStore.js";
 import { SqliteTaskStore } from "../sqliteStore.js";
+import { migrateSqliteSchema } from "../sqliteSchema.js";
 import { clearUpgradeFence, placeUpgradeFence, readUpgradeFence, UpgradeFenceError } from "../upgradeFence.js";
 import { withUpgradeCoordinationLock } from "../upgradeCoordination.js";
 import { clearUpgradeReceipt, writeUpgradeReceipt, upgradeReceiptPath } from "./upgradeReceipt.js";
@@ -52,7 +55,7 @@ import { repairPseudoLayout7 } from "./pseudoLayoutRepair.js";
 import { createSqliteMigrationTarget } from "./sqliteMigrationTarget.js";
 import { createSqliteRecordMigrationTarget } from "./sqliteRecordMigrationTarget.js";
 import { COMMITTED_DATABASE_FILENAME } from "./sqliteStateMigration.js";
-import { inspectOfflineUpgradeInventory } from "./offlineUpgradeInventory.js";
+import { inspectOfflineUpgradeInventory, inspectSqliteDurableUpgradeInventory } from "./offlineUpgradeInventory.js";
 /**
  * Run the storage upgrade for one Home. Never throws for an expected blocker;
  * it returns a structured `blocked` result instead. It only throws on a truly
@@ -95,6 +98,9 @@ export async function runStorageUpgrade(options) {
             action: blocker.action
         }, classification);
     }
+    if (classification.sqliteMigration !== undefined) {
+        return runSqliteInPlaceUpgrade(options, classification, now, callerPid);
+    }
     // A pseudo-layout-7 Home (manifest 7, no yui.db, readable state.json) needs
     // the deterministic staged state.json→SQLite repair, not the version
     // migration engine (Issue 01). The update preflight reports it as
@@ -357,6 +363,329 @@ function offlineInventoryBlocker(inventory, sceneUnchanged) {
         ...(sceneUnchanged ? { sceneUnchanged: true } : {})
     };
 }
+/**
+ * A valid SQLite migration prefix advances the authoritative database in
+ * place. It still uses the offline runtime gate because the old Controller
+ * cannot safely resume after the schema transaction commits, but it never
+ * snapshots, copies, rebuilds, or swaps the database.
+ */
+async function runSqliteInPlaceUpgrade(options, classification, now, callerPid) {
+    const migration = classification.sqliteMigration;
+    const inventory = await readOfflineInventory(options, options.home);
+    if (inventory.total > 0) {
+        return withClassification(offlineInventoryBlocker(inventory, true), classification);
+    }
+    if (options.mode === "update-preflight") {
+        return {
+            outcome: "update-preflight",
+            status: "in-place-migration",
+            stepCount: migration.pendingVersions.length,
+            classification,
+            sqliteMigration: migration
+        };
+    }
+    if (options.mode === "dry-run") {
+        return validateSqliteInPlaceDryRun(options, classification, now);
+    }
+    return executeSqliteInPlaceUpgrade(options, classification, now, callerPid);
+}
+/**
+ * Exercise every pending SQLite migration against a consistent disposable
+ * snapshot, then open that snapshot through the current production loader.
+ * The authoritative database is opened read-only and is never migrated.
+ */
+async function validateSqliteInPlaceDryRun(options, classification, now) {
+    const migration = classification.sqliteMigration;
+    const stagingRoot = mkdtempSync(join(tmpdir(), "yui-sqlite-upgrade-dry-run-"));
+    const stagedDatabasePath = join(stagingRoot, COMMITTED_DATABASE_FILENAME);
+    let source;
+    let staged;
+    let loader;
+    try {
+        source = new Database(join(options.home, COMMITTED_DATABASE_FILENAME), {
+            readonly: true,
+            fileMustExist: true
+        });
+        source.pragma("busy_timeout = 5000");
+        await source.backup(stagedDatabasePath);
+        source.close();
+        source = undefined;
+        staged = new Database(stagedDatabasePath);
+        staged.pragma("journal_mode = WAL");
+        staged.pragma("synchronous = FULL");
+        staged.pragma("foreign_keys = ON");
+        const applied = migrateSqliteSchema(staged, { mode: "apply" }).applied;
+        if (applied.join(",") !== migration.pendingVersions.join(",")) {
+            throw new Error(`staged SQLite migration applied versions ${applied.join(",") || "none"}; `
+                + `expected ${migration.pendingVersions.join(",")}`);
+        }
+        staged.close();
+        staged = undefined;
+        loader = new SqliteTaskStore(stagingRoot);
+        const quickCheck = loader.databaseHandle().pragma("quick_check", { simple: true });
+        if (quickCheck !== "ok") {
+            throw new Error(`staged SQLite quick_check returned ${String(quickCheck)}`);
+        }
+        loader.close();
+        loader = undefined;
+        return {
+            outcome: "dry-run",
+            classification,
+            report: sqliteInPlaceReport(options.latest, migration, "dry-run", now)
+        };
+    }
+    catch (error) {
+        return withClassification({
+            outcome: "blocked",
+            stage: "validate",
+            message: `SQLite dry-run validation failed: ${messageOf(error)}`,
+            action: "The authoritative database was not changed. Resolve the reported migration or loader failure, then retry the dry run."
+        }, classification);
+    }
+    finally {
+        try {
+            loader?.close();
+        }
+        catch { /* best-effort disposable cleanup */ }
+        try {
+            staged?.close();
+        }
+        catch { /* best-effort disposable cleanup */ }
+        try {
+            source?.close();
+        }
+        catch { /* best-effort disposable cleanup */ }
+        rmSync(stagingRoot, { recursive: true, force: true });
+    }
+}
+/** Acquire a local write fence or prove the exact parent-owned fence exists. */
+function acquireUpgradeAdmission(options, reason, now, callerPid) {
+    const externalOwnerPid = options.externalUpgradeFenceOwnerPid;
+    if (externalOwnerPid !== undefined) {
+        const fence = readUpgradeFence(options.home);
+        if (!isPositivePid(externalOwnerPid) || fence?.ownerPid !== externalOwnerPid) {
+            throw new UpgradeFenceError(`expected parent-owned upgrade fence for PID ${String(externalOwnerPid)}`);
+        }
+        return () => { };
+    }
+    return placeUpgradeFence(options.home, {
+        reason,
+        createdAt: now().toISOString(),
+        ownerPid: callerPid
+    });
+}
+async function executeSqliteInPlaceUpgrade(options, classification, now, callerPid) {
+    const { home } = options;
+    const migration = classification.sqliteMigration;
+    let releaseFence;
+    try {
+        releaseFence = acquireUpgradeAdmission(options, "SQLite schema migration in progress", now, callerPid);
+    }
+    catch (error) {
+        if (!(error instanceof UpgradeFenceError))
+            throw error;
+        return withClassification({
+            outcome: "blocked",
+            stage: "coordination",
+            message: `Upgrade coordination could not be acquired: ${error.message}`,
+            action: "Wait for the current maintenance operation to finish, then retry."
+        }, classification);
+    }
+    const externallyQuiesced = options.controllerLifecycle === "externally-quiesced";
+    let controllerWasRunning = false;
+    let controllerStopConfirmed = false;
+    let controllerIdentity;
+    let committed = false;
+    let result;
+    let unexpected;
+    try {
+        if (!externallyQuiesced) {
+            const controllerStatus = options.controllerStatus
+                ?? ((targetHome) => defaultControllerStatus(targetHome, options.controllerOptions));
+            let status;
+            try {
+                status = await controllerStatus(home);
+            }
+            catch (error) {
+                result = withClassification(controllerLifecycleBlocker("Controller status could not be verified", error), classification);
+            }
+            if (result === undefined && !isControllerLifecycleStatus(status)) {
+                result = withClassification(controllerLifecycleBlocker("Controller status was malformed", new Error("expected a boolean running field")), classification);
+            }
+            if (result === undefined && status.running && !isControllerLaunchIdentity(status.identity)) {
+                result = withClassification(controllerLifecycleBlocker("Controller launch identity could not be authenticated", new Error("executable/argv/version identity is unavailable")), classification);
+            }
+            if (result === undefined && status.running && !isPositivePid(status.pid)) {
+                result = withClassification(controllerLifecycleBlocker("Controller PID could not be authenticated", new Error("a positive status PID is unavailable for fenced stop")), classification);
+            }
+            if (result === undefined) {
+                controllerWasRunning = status.running;
+                controllerIdentity = status.running ? status.identity : undefined;
+                controllerStopConfirmed = !controllerWasRunning;
+            }
+            if (result === undefined && controllerWasRunning) {
+                const stopController = options.stopController
+                    ?? ((targetHome, expectedPid) => defaultStopController(targetHome, expectedPid, options.controllerOptions));
+                try {
+                    const expectedPid = status.pid;
+                    const stopped = await stopController(home, expectedPid);
+                    if (!confirmedControllerStopped(stopped, expectedPid)) {
+                        result = withClassification(controllerLifecycleBlocker("Controller stop did not confirm a drained process", new Error(`stop did not confirm captured PID ${expectedPid} with stopped:true`)), classification);
+                    }
+                    else {
+                        controllerStopConfirmed = true;
+                    }
+                }
+                catch (error) {
+                    result = withClassification(controllerLifecycleBlocker("Controller stop/drain failed", error), classification);
+                }
+            }
+        }
+        else {
+            controllerStopConfirmed = true;
+        }
+        // The first inventory is a cheap/read-only preflight. Re-read after the
+        // Controller has fully drained so a lifecycle request that committed while
+        // the fence was being placed cannot slip through on stale evidence.
+        if (result === undefined) {
+            const finalInventory = await readOfflineInventory(options, home);
+            if (finalInventory.total > 0) {
+                result = withClassification(offlineInventoryBlocker(finalInventory, true), classification);
+            }
+        }
+        if (result === undefined) {
+            try {
+                result = withUpgradeCoordinationLock(home, () => {
+                    const quiesce = verifyQuiesced(home, callerPid);
+                    if (quiesce !== null)
+                        return withClassification(quiesce, classification);
+                    const db = new Database(join(home, COMMITTED_DATABASE_FILENAME));
+                    try {
+                        db.pragma("journal_mode = WAL");
+                        db.pragma("synchronous = FULL");
+                        db.pragma("foreign_keys = ON");
+                        db.pragma("busy_timeout = 5000");
+                        const apply = db.transaction(() => {
+                            // BEGIN IMMEDIATE waits for every older SQLite writer. With the
+                            // upgrade fence still held, the state read here is the final
+                            // authoritative durable gate for the same schema transaction.
+                            const durableInventory = inspectSqliteDurableUpgradeInventory(home, db);
+                            if (durableInventory.total > 0) {
+                                return withClassification(offlineInventoryBlocker(durableInventory, true), classification);
+                            }
+                            migrateSqliteSchema(db, { mode: "apply" });
+                            return {
+                                outcome: "upgraded",
+                                classification,
+                                migrationMode: "in-place",
+                                report: sqliteInPlaceReport(options.latest, migration, "execute", now)
+                            };
+                        });
+                        const applied = apply.immediate();
+                        if (applied.outcome === "upgraded")
+                            committed = true;
+                        return applied;
+                    }
+                    finally {
+                        db.close();
+                    }
+                });
+            }
+            catch (error) {
+                unexpected = error;
+            }
+        }
+    }
+    finally {
+        try {
+            releaseFence();
+        }
+        catch (error) {
+            if (unexpected === undefined)
+                unexpected = error;
+        }
+    }
+    if (unexpected !== undefined) {
+        if (committed) {
+            return withClassification({
+                outcome: "blocked",
+                stage: "post-verify",
+                message: `SQLite migration committed but completion failed: ${messageOf(unexpected)}`,
+                action: "Do not restore the old Controller. Re-run this Yui version; the SQLite migration ledger is the commit record.",
+                storageCommitted: true
+            }, classification);
+        }
+        if (controllerWasRunning && controllerStopConfirmed) {
+            await restoreController(home, options, unexpected, controllerIdentity);
+        }
+        throw unexpected;
+    }
+    if (result === undefined)
+        throw new Error("SQLite in-place upgrade did not produce a result.");
+    if (result.outcome === "upgraded" && !externallyQuiesced && controllerWasRunning) {
+        try {
+            await (options.startController
+                ?? ((targetHome) => defaultStartController(targetHome, options.controllerOptions)))(home);
+        }
+        catch (error) {
+            return withClassification({
+                outcome: "blocked",
+                stage: "post-verify",
+                message: `SQLite migration committed but the replacement Controller did not start: ${messageOf(error)}`,
+                action: "Do not restore the old Controller. Start the current Yui Controller; the SQLite migration ledger already committed.",
+                storageCommitted: true
+            }, classification);
+        }
+        return result;
+    }
+    if (result.outcome === "blocked"
+        && !committed
+        && !externallyQuiesced
+        && controllerWasRunning
+        && controllerStopConfirmed) {
+        await restoreController(home, options, new Error(result.message), controllerIdentity);
+    }
+    return result;
+}
+function sqliteInPlaceReport(latest, migration, mode, now) {
+    const detail = `SQLite schema ${migration.currentVersion}->${migration.targetVersion}; `
+        + `pending versions ${migration.pendingVersions.join(", ")}`;
+    if (mode === "dry-run") {
+        return {
+            outcome: "dry-run",
+            mode,
+            source: latest,
+            target: latest,
+            steps: [],
+            effects: [],
+            derived: { rebuiltEffects: [] },
+            validation: {
+                checks: [{
+                        name: "SQLite staged migration and loader gate",
+                        outcome: "passed",
+                        detail: `${detail}; snapshot migrated and reopened by the current SQLite loader`
+                    }]
+            }
+        };
+    }
+    return {
+        outcome: "migrated",
+        mode,
+        source: latest,
+        target: latest,
+        steps: [],
+        effects: [],
+        derived: { rebuiltEffects: [] },
+        validation: {
+            checks: [{ name: "SQLite migration transaction", outcome: "passed", detail }]
+        },
+        switch: {
+            status: "switched",
+            detail: "SQLite schema migrated in place; no database copy or rebuild was created."
+        },
+        completedAt: now().toISOString()
+    };
+}
 /** Dry run: validate through the staged gate, then discard; never switch. */
 function dryRun(options, classification, target) {
     // Refuse to reuse a stale staging directory from an interrupted run.
@@ -390,11 +719,7 @@ async function execute(options, classification, target, callerPid, now) {
     // the other upgrader's fence.
     let releaseFence;
     try {
-        releaseFence = placeUpgradeFence(home, {
-            reason: "storage upgrade in progress",
-            createdAt: now().toISOString(),
-            ownerPid: callerPid
-        });
+        releaseFence = acquireUpgradeAdmission(options, "storage upgrade in progress", now, callerPid);
     }
     catch (error) {
         if (!(error instanceof UpgradeFenceError))
@@ -615,11 +940,7 @@ async function executePseudoLayout7Repair(options, classification, now) {
     const callerPid = options.callerPid ?? process.pid;
     let releaseFence;
     try {
-        releaseFence = placeUpgradeFence(home, {
-            reason: "storage repair in progress",
-            createdAt: now().toISOString(),
-            ownerPid: callerPid
-        });
+        releaseFence = acquireUpgradeAdmission(options, "storage repair in progress", now, callerPid);
     }
     catch (error) {
         if (!(error instanceof UpgradeFenceError))

package/dist/task/completionReadiness.js

@@ -4,6 +4,7 @@ import { deltaRecheckBlocksAcceptance } from "../review/reviewRound.js";
 import { isSemanticReviewRound } from "../review/reviewOutcomeClassifier.js";
 import { reviewFindingLedgerWriteFailedFromEvents } from "../review/reviewFindingLedger.js";
 import { resolveRecordedTaskFinalReviewContract } from "../review/taskFinalReviewContractRebind.js";
+import { sameTaskFinalReviewContract } from "../review/taskFinalReviewContract.js";
 import { blockingProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
 const ACTIVE_JOB_STATUSES = new Set([
     "queued",
@@ -22,7 +23,8 @@ export function projectCompletionReadiness(facts, options = {}) {
     const advisories = [];
     const { task } = facts;
     const findingsGate = options.findingsGate ?? true;
-    const taskFinalReviewRequired = resolveRecordedTaskFinalReviewContract(task.id, facts.workItems, facts.reviewRounds, facts.taskFinalReviewContractEvents) !== undefined;
+    const taskFinalReviewContract = resolveRecordedTaskFinalReviewContract(task.id, facts.workItems, facts.reviewRounds, facts.taskFinalReviewContractEvents)?.effective;
+    const taskFinalReviewRequired = taskFinalReviewContract !== undefined;
     // A pending/running Task-final Review must be resumed or blocked first.
     for (const round of facts.reviewRounds) {
         if ((round.scope ?? "work-item") !== "task")
@@ -43,11 +45,13 @@ export function projectCompletionReadiness(facts, options = {}) {
     // escalations remain audit evidence, but a later completed full Review (or
     // accepted delta) supersedes them instead of blocking completion forever.
     const latestCompletedTaskReview = facts.reviewRounds
-        .filter((round) => ((round.scope ?? "work-item") === "task" && isSemanticReviewRound(round, {
-        listAgentRuns: () => facts.agentRuns,
-        listReviewFindings: () => facts.reviewFindings,
-        listEvents: () => facts.events
-    })))
+        .filter((round) => ((round.scope ?? "work-item") === "task"
+        && (taskFinalReviewContract === undefined || sameTaskFinalReviewContract(round.taskFinalReviewContract, taskFinalReviewContract))
+        && isSemanticReviewRound(round, {
+            listAgentRuns: () => facts.agentRuns,
+            listReviewFindings: () => facts.reviewFindings,
+            listEvents: () => facts.events
+        })))
         .slice()
         .sort((left, right) => (left.createdAt.localeCompare(right.createdAt)
         || left.id.localeCompare(right.id, undefined, { numeric: true })))