@flame0510/project-aether 1.5.0 → 1.5.2

package/lib/agent-edit-state.ts

@@ -2,9 +2,10 @@
  * Persisted state of agent edits: the `agent_edits` table (lib/db-bootstrap.mjs).
  *
  * Kept apart from lib/agent-edit.ts so busy checks (lib/agent-busy.ts) can ask which
- * agents are being edited without importing the edit action.
+ * agents are being edited without importing the edit action. The machinery is
+ * lib/agent-job-state.ts; this file holds the table's types and names.
  */
-import { openDb } from '@/lib/db';
+import { createJobState } from '@/lib/agent-job-state';
 
 export type EditStatus = 'rebuilding' | 'done' | 'failed' | 'interrupted';
 
@@ -25,84 +26,42 @@ export interface AgentEditRow {
 
 const FINAL: readonly EditStatus[] = ['done', 'failed', 'interrupted'];
 
-export function insertEdit(agentId: string, displayName: string | null, portRange: string | null): number {
-  const db = openDb(false);
-  try {
-    const now = Date.now();
-    const info = db.prepare(
-      `INSERT INTO agent_edits (agent_id, status, display_name, port_range, started_at, updated_at)
-       VALUES (?, 'rebuilding', ?, ?, ?, ?)`,
-    ).run(agentId, displayName, portRange, now, now);
-    return Number(info.lastInsertRowid);
-  } finally {
-    db.close();
-  }
-}
+const state = createJobState({
+  table: 'agent_edits',
+  initialStatus: 'rebuilding',
+  activeStatuses: ACTIVE_EDIT_STATUSES,
+  finalStatuses: FINAL,
+  insertColumns: ['display_name', 'port_range'],
+  writableColumns: [],
+});
 
 type Writable = Partial<Pick<AgentEditRow, 'status' | 'error'>>;
 
+export function insertEdit(agentId: string, displayName: string | null, portRange: string | null): number {
+  return state.insert(agentId, displayName, portRange);
+}
+
 export function updateEditRow(id: number, fields: Writable): void {
-  const entries = Object.entries(fields).filter(([, v]) => v !== undefined);
-  const now = Date.now();
-  const sets = [...entries.map(([k]) => `${k} = ?`), 'updated_at = ?'];
-  const values: unknown[] = [...entries.map(([, v]) => v), now];
-  if (fields.status && FINAL.includes(fields.status)) {
-    sets.push('finished_at = ?');
-    values.push(now);
-  } else if (fields.status) {
-    sets.push('finished_at = NULL');
-  }
-  const db = openDb(false);
-  try {
-    db.prepare(`UPDATE agent_edits SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
-  } finally {
-    db.close();
-  }
+  state.update(id, fields);
 }
 
 export function latestEdit(agentId: string): AgentEditRow | null {
-  const db = openDb(true);
-  try {
-    return (db.prepare('SELECT * FROM agent_edits WHERE agent_id = ? ORDER BY id DESC LIMIT 1').get(agentId) as AgentEditRow | undefined) ?? null;
-  } finally {
-    db.close();
-  }
+  return state.latest<AgentEditRow>(agentId);
 }
 
 export function isEditActive(agentId: string): boolean {
-  const row = latestEdit(agentId);
-  return !!row && ACTIVE_EDIT_STATUSES.includes(row.status);
+  return state.isActive(agentId);
 }
 
-/** AGENT_IDs with an edit in an active status. */
 export function activeEditAgentIds(): Set<string> {
-  const db = openDb(true);
-  try {
-    const placeholders = ACTIVE_EDIT_STATUSES.map(() => '?').join(', ');
-    const rows = db.prepare(`SELECT DISTINCT agent_id FROM agent_edits WHERE status IN (${placeholders})`).all(...ACTIVE_EDIT_STATUSES) as { agent_id: string }[];
-    return new Set(rows.map((r) => r.agent_id));
-  } finally {
-    db.close();
-  }
+  return state.activeAgentIds();
 }
 
-/**
- * At startup no edit job can be running: a row still active was cut off by the restart.
- * It becomes `interrupted`, keeping the name and port it was applying.
- */
 export function markInterruptedEdits(): number {
-  const db = openDb(false);
-  try {
-    const placeholders = ACTIVE_EDIT_STATUSES.map(() => '?').join(', ');
-    const now = Date.now();
-    const info = db.prepare(
-      `UPDATE agent_edits
-         SET error = COALESCE(error, 'Rev4a restarted while this step was running: ' || status),
-             status = 'interrupted', updated_at = ?, finished_at = ?
-       WHERE status IN (${placeholders})`,
-    ).run(now, now, ...ACTIVE_EDIT_STATUSES);
-    return info.changes;
-  } finally {
-    db.close();
-  }
+  return state.markInterrupted();
+}
+
+/** Keep the newest `keepPerAgent` edits of each agent; the rest is history. */
+export function pruneEdits(keepPerAgent: number): number {
+  return state.prune(keepPerAgent);
 }

package/lib/agent-job-state.ts

@@ -0,0 +1,145 @@
+/**
+ * The four lifecycle job tables — `agent_upgrades`, `agent_recreates`, `agent_restores`,
+ * `agent_edits` — are the same machine: an insert that starts a row in one status, an
+ * update that stamps `updated_at` and `finished_at`, a latest lookup, an "is something
+ * running" check, an at-startup sweep that turns interrupted rows into `interrupted`, and
+ * a retention prune. They used to be four copies of that code, which drifted; this factory
+ * is the single implementation, and each table keeps its own types and function names.
+ *
+ * The table and column names come from this file's callers, never from a request, and
+ * `update` only writes the columns its spec allows: nothing user-supplied can reach the
+ * generated SQL.
+ */
+import { openDb } from '@/lib/db';
+
+export interface JobStateSpec {
+  /** Table name (created by lib/db-bootstrap.mjs). */
+  table: string;
+  /** The status an inserted row starts in. */
+  initialStatus: string;
+  /** Statuses during which the agent must not be touched by anything else. */
+  activeStatuses: readonly string[];
+  /** Statuses that end a job: `finished_at` is stamped when one is set. */
+  finalStatuses: readonly string[];
+  /** Extra columns `insert` writes, after agent_id/status/started_at/updated_at. */
+  insertColumns: readonly string[];
+  /** Extra columns `update` may set, besides status and error. */
+  writableColumns: readonly string[];
+}
+
+export interface JobState {
+  insert(agentId: string, ...values: (string | number | null)[]): number;
+  update(id: number, fields: Record<string, string | null | undefined>): void;
+  latest<T>(agentId: string): T | null;
+  isActive(agentId: string): boolean;
+  activeAgentIds(): Set<string>;
+  markInterrupted(): number;
+  /** Delete all but the newest `keepPerAgent` rows of each agent. Returns the count. */
+  prune(keepPerAgent: number): number;
+}
+
+export function createJobState(spec: JobStateSpec): JobState {
+  const { table, initialStatus, activeStatuses, finalStatuses } = spec;
+  const insertColumns = ['agent_id', 'status', ...spec.insertColumns, 'started_at', 'updated_at'];
+  const writable = new Set(['status', 'error', ...spec.writableColumns]);
+  const activePlaceholders = activeStatuses.map(() => '?').join(', ');
+
+  const latest = <T,>(agentId: string): T | null => {
+    const db = openDb(true);
+    try {
+      return (db.prepare(`SELECT * FROM ${table} WHERE agent_id = ? ORDER BY id DESC LIMIT 1`).get(agentId) as T | undefined) ?? null;
+    } finally {
+      db.close();
+    }
+  };
+
+  return {
+    insert(agentId, ...values) {
+      if (values.length !== spec.insertColumns.length) {
+        throw new Error(`${table}: insert expects ${spec.insertColumns.length} value(s), got ${values.length}`);
+      }
+      const db = openDb(false);
+      try {
+        const now = Date.now();
+        const info = db.prepare(
+          `INSERT INTO ${table} (${insertColumns.join(', ')})
+           VALUES (${insertColumns.map(() => '?').join(', ')})`,
+        ).run(agentId, initialStatus, ...values, now, now);
+        return Number(info.lastInsertRowid);
+      } finally {
+        db.close();
+      }
+    },
+
+    update(id, fields) {
+      const entries = Object.entries(fields).filter(([key, value]) => value !== undefined && writable.has(key));
+      const now = Date.now();
+      const sets = [...entries.map(([key]) => `${key} = ?`), 'updated_at = ?'];
+      const values: unknown[] = [...entries.map(([, value]) => value), now];
+      const status = fields.status;
+      if (status && finalStatuses.includes(status)) {
+        sets.push('finished_at = ?');
+        values.push(now);
+      } else if (status) {
+        sets.push('finished_at = NULL');
+      }
+      const db = openDb(false);
+      try {
+        db.prepare(`UPDATE ${table} SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
+      } finally {
+        db.close();
+      }
+    },
+
+    latest,
+
+    isActive(agentId) {
+      const row = latest<{ status: string }>(agentId);
+      return !!row && activeStatuses.includes(row.status);
+    },
+
+    activeAgentIds() {
+      const db = openDb(true);
+      try {
+        const rows = db.prepare(
+          `SELECT DISTINCT agent_id FROM ${table} WHERE status IN (${activePlaceholders})`,
+        ).all(...activeStatuses) as { agent_id: string }[];
+        return new Set(rows.map((r) => r.agent_id));
+      } finally {
+        db.close();
+      }
+    },
+
+    markInterrupted() {
+      const db = openDb(false);
+      try {
+        const now = Date.now();
+        const info = db.prepare(
+          `UPDATE ${table}
+              SET error = COALESCE(error, 'Rev4a restarted while this step was running: ' || status),
+                  status = 'interrupted', updated_at = ?, finished_at = ?
+            WHERE status IN (${activePlaceholders})`,
+        ).run(now, now, ...activeStatuses);
+        return info.changes;
+      } finally {
+        db.close();
+      }
+    },
+
+    prune(keepPerAgent) {
+      const db = openDb(false);
+      try {
+        const info = db.prepare(
+          `DELETE FROM ${table} WHERE id IN (
+             SELECT id FROM (
+               SELECT id, ROW_NUMBER() OVER (PARTITION BY agent_id ORDER BY id DESC) AS rn FROM ${table}
+             ) WHERE rn > ?
+           )`,
+        ).run(keepPerAgent);
+        return info.changes;
+      } finally {
+        db.close();
+      }
+    },
+  };
+}

package/lib/agent-jobs-maintenance.ts

@@ -0,0 +1,34 @@
+/**
+ * Retention for the lifecycle job tables (`agent_upgrades`, `agent_recreates`,
+ * `agent_restores`, `agent_edits`). They record every update, recreate, restore and edit
+ * an agent has been through, and nothing pruned them: the rows grew for the life of the
+ * installation while the panel only ever reads the latest one of each agent. Each table
+ * keeps the newest `KEEP_PER_AGENT` rows per agent; the rest is history.
+ *
+ * Best-effort: a failure is logged and does not stop the others or the startup.
+ */
+import { pruneUpdates } from '@/lib/agent-update-state';
+import { pruneRecreates } from '@/lib/agent-recreate-state';
+import { pruneRestores } from '@/lib/agent-restore-state';
+import { pruneEdits } from '@/lib/agent-edit-state';
+
+/** Rows kept per agent, per table. Enough to see the recent history of an agent. */
+const KEEP_PER_AGENT = 20;
+
+export function pruneLifecycleRows(keepPerAgent: number = KEEP_PER_AGENT): { table: string; removed: number }[] {
+  const prunes: [string, (keep: number) => number][] = [
+    ['agent_upgrades', pruneUpdates],
+    ['agent_recreates', pruneRecreates],
+    ['agent_restores', pruneRestores],
+    ['agent_edits', pruneEdits],
+  ];
+  const result: { table: string; removed: number }[] = [];
+  for (const [table, prune] of prunes) {
+    try {
+      result.push({ table, removed: prune(keepPerAgent) });
+    } catch (e) {
+      console.warn(`[rev4a] could not prune ${table}: ${(e as Error).message}`);
+    }
+  }
+  return result;
+}

package/lib/agent-recreate-state.ts

@@ -3,8 +3,9 @@
  *
  * Kept apart from lib/agent-recreate.ts so busy checks (lib/agent-busy.ts, the provider
  * sync) can ask which agents are being recreated without importing the recreate action.
+ * The machinery is lib/agent-job-state.ts; this file holds the table's types and names.
  */
-import { openDb } from '@/lib/db';
+import { createJobState } from '@/lib/agent-job-state';
 
 export type RecreateStatus = 'backing_up' | 'recreating' | 'done' | 'failed' | 'interrupted';
 
@@ -25,84 +26,42 @@ export interface AgentRecreateRow {
 
 const FINAL: readonly RecreateStatus[] = ['done', 'failed', 'interrupted'];
 
-export function insertRecreate(agentId: string, image: string): number {
-  const db = openDb(false);
-  try {
-    const now = Date.now();
-    const info = db.prepare(
-      `INSERT INTO agent_recreates (agent_id, status, image, started_at, updated_at)
-       VALUES (?, 'backing_up', ?, ?, ?)`,
-    ).run(agentId, image, now, now);
-    return Number(info.lastInsertRowid);
-  } finally {
-    db.close();
-  }
-}
+const state = createJobState({
+  table: 'agent_recreates',
+  initialStatus: 'backing_up',
+  activeStatuses: ACTIVE_RECREATE_STATUSES,
+  finalStatuses: FINAL,
+  insertColumns: ['image'],
+  writableColumns: ['backup_file'],
+});
 
 type Writable = Partial<Pick<AgentRecreateRow, 'status' | 'backup_file' | 'error'>>;
 
+export function insertRecreate(agentId: string, image: string): number {
+  return state.insert(agentId, image);
+}
+
 export function updateRecreateRow(id: number, fields: Writable): void {
-  const entries = Object.entries(fields).filter(([, v]) => v !== undefined);
-  const now = Date.now();
-  const sets = [...entries.map(([k]) => `${k} = ?`), 'updated_at = ?'];
-  const values: unknown[] = [...entries.map(([, v]) => v), now];
-  if (fields.status && FINAL.includes(fields.status)) {
-    sets.push('finished_at = ?');
-    values.push(now);
-  } else if (fields.status) {
-    sets.push('finished_at = NULL');
-  }
-  const db = openDb(false);
-  try {
-    db.prepare(`UPDATE agent_recreates SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
-  } finally {
-    db.close();
-  }
+  state.update(id, fields);
 }
 
 export function latestRecreate(agentId: string): AgentRecreateRow | null {
-  const db = openDb(true);
-  try {
-    return (db.prepare('SELECT * FROM agent_recreates WHERE agent_id = ? ORDER BY id DESC LIMIT 1').get(agentId) as AgentRecreateRow | undefined) ?? null;
-  } finally {
-    db.close();
-  }
+  return state.latest<AgentRecreateRow>(agentId);
 }
 
 export function isRecreateActive(agentId: string): boolean {
-  const row = latestRecreate(agentId);
-  return !!row && ACTIVE_RECREATE_STATUSES.includes(row.status);
+  return state.isActive(agentId);
 }
 
-/** AGENT_IDs with a recreate in an active status. */
 export function activeRecreateAgentIds(): Set<string> {
-  const db = openDb(true);
-  try {
-    const placeholders = ACTIVE_RECREATE_STATUSES.map(() => '?').join(', ');
-    const rows = db.prepare(`SELECT DISTINCT agent_id FROM agent_recreates WHERE status IN (${placeholders})`).all(...ACTIVE_RECREATE_STATUSES) as { agent_id: string }[];
-    return new Set(rows.map((r) => r.agent_id));
-  } finally {
-    db.close();
-  }
+  return state.activeAgentIds();
 }
 
-/**
- * At startup no recreate job can be running: any row still active was cut off by the
- * restart. It becomes `interrupted`, keeping the step it was in.
- */
 export function markInterruptedRecreates(): number {
-  const db = openDb(false);
-  try {
-    const placeholders = ACTIVE_RECREATE_STATUSES.map(() => '?').join(', ');
-    const now = Date.now();
-    const info = db.prepare(
-      `UPDATE agent_recreates
-         SET error = COALESCE(error, 'Rev4a restarted while this step was running: ' || status),
-             status = 'interrupted', updated_at = ?, finished_at = ?
-       WHERE status IN (${placeholders})`,
-    ).run(now, now, ...ACTIVE_RECREATE_STATUSES);
-    return info.changes;
-  } finally {
-    db.close();
-  }
+  return state.markInterrupted();
+}
+
+/** Keep the newest `keepPerAgent` recreates of each agent; the rest is history. */
+export function pruneRecreates(keepPerAgent: number): number {
+  return state.prune(keepPerAgent);
 }

package/lib/agent-restore-state.ts

@@ -2,9 +2,10 @@
  * Persisted state of agent restores: the `agent_restores` table (lib/db-bootstrap.mjs).
  *
  * Kept apart from lib/agent-restore.ts so busy checks (lib/agent-busy.ts) can ask which
- * agents are being restored without importing the restore action.
+ * agents are being restored without importing the restore action. The machinery is
+ * lib/agent-job-state.ts; this file holds the table's types and names.
  */
-import { openDb } from '@/lib/db';
+import { createJobState } from '@/lib/agent-job-state';
 
 export type RestoreStatus = 'restoring' | 'done' | 'failed' | 'interrupted';
 
@@ -24,84 +25,42 @@ export interface AgentRestoreRow {
 
 const FINAL: readonly RestoreStatus[] = ['done', 'failed', 'interrupted'];
 
-export function insertRestore(agentId: string, file: string): number {
-  const db = openDb(false);
-  try {
-    const now = Date.now();
-    const info = db.prepare(
-      `INSERT INTO agent_restores (agent_id, status, file, started_at, updated_at)
-       VALUES (?, 'restoring', ?, ?, ?)`,
-    ).run(agentId, file, now, now);
-    return Number(info.lastInsertRowid);
-  } finally {
-    db.close();
-  }
-}
+const state = createJobState({
+  table: 'agent_restores',
+  initialStatus: 'restoring',
+  activeStatuses: ACTIVE_RESTORE_STATUSES,
+  finalStatuses: FINAL,
+  insertColumns: ['file'],
+  writableColumns: [],
+});
 
 type Writable = Partial<Pick<AgentRestoreRow, 'status' | 'error'>>;
 
+export function insertRestore(agentId: string, file: string): number {
+  return state.insert(agentId, file);
+}
+
 export function updateRestoreRow(id: number, fields: Writable): void {
-  const entries = Object.entries(fields).filter(([, v]) => v !== undefined);
-  const now = Date.now();
-  const sets = [...entries.map(([k]) => `${k} = ?`), 'updated_at = ?'];
-  const values: unknown[] = [...entries.map(([, v]) => v), now];
-  if (fields.status && FINAL.includes(fields.status)) {
-    sets.push('finished_at = ?');
-    values.push(now);
-  } else if (fields.status) {
-    sets.push('finished_at = NULL');
-  }
-  const db = openDb(false);
-  try {
-    db.prepare(`UPDATE agent_restores SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
-  } finally {
-    db.close();
-  }
+  state.update(id, fields);
 }
 
 export function latestRestore(agentId: string): AgentRestoreRow | null {
-  const db = openDb(true);
-  try {
-    return (db.prepare('SELECT * FROM agent_restores WHERE agent_id = ? ORDER BY id DESC LIMIT 1').get(agentId) as AgentRestoreRow | undefined) ?? null;
-  } finally {
-    db.close();
-  }
+  return state.latest<AgentRestoreRow>(agentId);
 }
 
 export function isRestoreActive(agentId: string): boolean {
-  const row = latestRestore(agentId);
-  return !!row && ACTIVE_RESTORE_STATUSES.includes(row.status);
+  return state.isActive(agentId);
 }
 
-/** AGENT_IDs with a restore in an active status. */
 export function activeRestoreAgentIds(): Set<string> {
-  const db = openDb(true);
-  try {
-    const placeholders = ACTIVE_RESTORE_STATUSES.map(() => '?').join(', ');
-    const rows = db.prepare(`SELECT DISTINCT agent_id FROM agent_restores WHERE status IN (${placeholders})`).all(...ACTIVE_RESTORE_STATUSES) as { agent_id: string }[];
-    return new Set(rows.map((r) => r.agent_id));
-  } finally {
-    db.close();
-  }
+  return state.activeAgentIds();
 }
 
-/**
- * At startup no restore job can be running: a row still active was cut off by the
- * restart. It becomes `interrupted`, keeping the archive it was restoring.
- */
 export function markInterruptedRestores(): number {
-  const db = openDb(false);
-  try {
-    const placeholders = ACTIVE_RESTORE_STATUSES.map(() => '?').join(', ');
-    const now = Date.now();
-    const info = db.prepare(
-      `UPDATE agent_restores
-         SET error = COALESCE(error, 'Rev4a restarted while this step was running: ' || status),
-             status = 'interrupted', updated_at = ?, finished_at = ?
-       WHERE status IN (${placeholders})`,
-    ).run(now, now, ...ACTIVE_RESTORE_STATUSES);
-    return info.changes;
-  } finally {
-    db.close();
-  }
+  return state.markInterrupted();
+}
+
+/** Keep the newest `keepPerAgent` restores of each agent; the rest is history. */
+export function pruneRestores(keepPerAgent: number): number {
+  return state.prune(keepPerAgent);
 }

package/lib/agent-restore.ts

@@ -123,19 +123,42 @@ export function agentRestoreView(agentId: string): AgentRestoreView | null {
 
 /**
  * At startup no restore job can be running: rows still active were cut off by the
- * restart. They become `interrupted`; a container left stopped is started again,
- * best-effort, so nothing stays down.
+ * restart. A restore interrupted **mid-extract** leaves a half-written volume, so
+ * starting the container would present a partial workspace as if it were the restore: the
+ * extract is run again from the same archive (it is the source of truth and the file was
+ * checked before the job started), in the background, and the row lands in its final
+ * status. When the archive is gone the container is started on whatever the volume holds,
+ * and the row says so.
  */
 export async function recoverInterruptedRestores(): Promise<number> {
   const ids = activeRestoreAgentIds();
   if (ids.size === 0) return 0;
   const interrupted = markInterruptedRestores();
   for (const agentId of ids) {
-    const container = await inspectAgentContainer(agentId).catch(() => null);
-    const name = container?.Name?.replace(/^\//, '');
-    if (!name || container?.State?.Running) continue;
+    const row = latestRestore(agentId);
+    const name = (await inspectAgentContainer(agentId).catch(() => null))?.Name?.replace(/^\//, '');
+    if (row?.file && (await backupFileExists(agentId, row.file))) {
+      console.log(`[agent:restore] ${agentId}: re-running the interrupted restore from ${row.file}`);
+      void runRestore(row.id, agentId, row.file).finally(() => undefined);
+      continue;
+    }
+    if (!name) continue;
     const started = await startAndVerifyContainer(agentId, name);
     if (!started.ok) console.warn(`[agent:restore] ${agentId} did not come back after an interrupted restore: ${started.error}`);
   }
   return interrupted;
 }
+
+/** Whether the archive still exists in the backup volume. */
+async function backupFileExists(agentId: string, file: string): Promise<boolean> {
+  if (!isValidBackupFile(agentId, file)) return false;
+  try {
+    await runDocker(
+      ['run', '--rm', '-v', `${BACKUP_VOLUME}:/backup`, 'alpine', 'sh', '-c', 'test -f "/backup/$FILE"'],
+      { timeoutMs: 30_000, env: { FILE: file } },
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}