@starci/hfs 4.0.8 → 4.0.10

package/runtime/engine/db/machine-connection.mjs

@@ -0,0 +1,135 @@
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { sleepSync as scaledSleepSync } from '../../scripts/lib/sleep-sync.mjs';
+
+const require = createRequire(import.meta.url);
+export const MACHINE_BUSY_TIMEOUT_MS = 15000;
+export const MACHINE_BUSY_CODE = 'STARCI_MACHINE_BUSY';
+export const MACHINE_CORRUPT_CODE = 'STARCI_MACHINE_CORRUPT';
+const OPEN_RETRY_DELAYS_MS = Object.freeze([0, 300, 900]);
+const BUSY_RETRY_DELAYS_MS = Object.freeze([100, 300, 900, 2000]);
+export const CORRUPT_RETRY_DELAYS_MS = Object.freeze([25, 150, 600]);
+export const waitForRetry = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+export const isMachineBusy = (error) => error?.code === MACHINE_BUSY_CODE;
+export const isBusyError = (error) => error?.errcode === 5 || error?.errcode === 6 || /SQLITE_BUSY|database is (?:locked|busy)/i.test(String(error?.message ?? error));
+export const isCorruptError = (error) => {
+  if (!error) return false;
+  if (error.code === MACHINE_CORRUPT_CODE) return true;
+  const code = Number(error.errcode);
+  if (Number.isInteger(code) && ((code & 0xff) === 11 || (code & 0xff) === 26)) return true;
+  return /database disk image is malformed|file is not a database|SQLITE_CORRUPT|SQLITE_NOTADB/i.test(String(error.message ?? error));
+};
+export const errText = (error) => String(error?.message ?? error).slice(0, 500);
+const busyError = (file, error, { retries, where }) => Object.assign(Error(`machine-db-busy: ${file} ${where}: database still locked after busy_timeout and ${retries} retries: ${String(error?.message ?? error).slice(0, 200)}`),
+  { code: MACHINE_BUSY_CODE, cause: error, file, retries, where });
+
+/** Persistent corruption diagnostics observe the store and stderr without recording an incident. */
+export function corruptDiagnostic(file, error, { retries, where }) {
+  if (error?.code === MACHINE_CORRUPT_CODE) return error;
+  const out = Object.assign(Error(`machine-db-corrupt: ${file} ${where}: SQLITE_CORRUPT persisted after ${retries} reopen(s): ${errText(error)}`),
+    { code: MACHINE_CORRUPT_CODE, cause: error, file, retries, where });
+  let check = null;
+  try {
+    const { DatabaseSync } = require('node:sqlite');
+    const db = new DatabaseSync(file, { readOnly: true, timeout: MACHINE_BUSY_TIMEOUT_MS });
+    try { check = db.prepare('PRAGMA quick_check').all().map((r) => r.quick_check).slice(0, 20); } finally { db.close(); }
+  } catch (e) { check = [`quick_check failed: ${errText(e)}`]; }
+  out.quickCheck = check;
+  process.stderr.write(`[machine-db] INCIDENT ${out.message} (quick_check: ${JSON.stringify(check)})\n`);
+  return out;
+}
+
+/** Connection retries share one policy; the machine owner supplies operational incident persistence. */
+export function machineConnectionMethods({ reportIncident }) {
+  function openWithRetry(openRaw, { file, onCorrupt = reportIncident }) {
+    let lastError, busyOpens = 0, failures = 0;
+    for (;;) {
+      if (failures && OPEN_RETRY_DELAYS_MS[failures]) waitForRetry(OPEN_RETRY_DELAYS_MS[failures]);
+      try { return openRaw(); }
+      catch (error) {
+        const busy = isBusyError(error) && !isCorruptError(error);
+        if (!busy && !/unable to open/i.test(String(error?.message ?? '')) && !(isCorruptError(error) && error.code !== MACHINE_CORRUPT_CODE)) throw error;
+        lastError = error;
+        if (busy) {
+          if (busyOpens >= BUSY_RETRY_DELAYS_MS.length) throw busyError(path.resolve(file), error, { retries: busyOpens, where: 'open' });
+          scaledSleepSync(BUSY_RETRY_DELAYS_MS[busyOpens]);
+          busyOpens += 1;
+        } else if ((failures += 1) >= OPEN_RETRY_DELAYS_MS.length) break;
+      }
+    }
+    if (isCorruptError(lastError)) throw onCorrupt(path.resolve(file), lastError, { retries: OPEN_RETRY_DELAYS_MS.length - 1, where: 'open' });
+    throw lastError;
+  }
+
+  function resilientConnection(openRaw, { file, inTransaction, onRecovered, onCorrupt }) {
+    let raw = openRaw(), generation = 0;
+    const reopen = () => { try { raw.close(); } catch { /* closed */ } raw = openRaw(); generation += 1; };
+    const retrying = (where, op) => {
+      let busyRetries = 0;
+      for (let retries = 0; ; retries += 1) {
+        try {
+          const out = op();
+          if (retries) onRecovered({ where, retries });
+          return out;
+        } catch (error) {
+          if (isBusyError(error) && !isMachineBusy(error) && !isCorruptError(error)) {
+            if (inTransaction() || raw.isTransaction) throw error;
+            if (busyRetries >= BUSY_RETRY_DELAYS_MS.length) throw busyError(file, error, { retries: busyRetries, where });
+            scaledSleepSync(BUSY_RETRY_DELAYS_MS[busyRetries]);
+            busyRetries += 1;
+            retries -= 1;
+            continue;
+          }
+          if (!isCorruptError(error) || error.code === MACHINE_CORRUPT_CODE) throw error;
+          if (inTransaction() || raw.isTransaction) throw error;
+          if (retries >= CORRUPT_RETRY_DELAYS_MS.length) throw onCorrupt(file, error, { retries, where });
+          waitForRetry(CORRUPT_RETRY_DELAYS_MS[retries]);
+          try { reopen(); } catch (openError) { if (!isCorruptError(openError)) throw openError; }
+        }
+      }
+    };
+    const prepare = (sql) => {
+      let stmt = null, gen = -1;
+      const settings = [];
+      const current = () => {
+        if (gen !== generation) {
+          stmt = raw.prepare(sql);
+          for (const [k, a] of settings) stmt[k](...a);
+          gen = generation;
+        }
+        return stmt;
+      };
+      retrying(`prepare ${sql.slice(0, 80)}`, current);
+      return new Proxy({}, {
+        get(_, prop) {
+          if (prop === 'run' || prop === 'get' || prop === 'all' || prop === 'iterate') return (...args) => retrying(`${prop} ${sql.slice(0, 80)}`, () => current()[prop](...args));
+          if (typeof prop === 'string' && /^set[A-Z]/.test(prop)) return (...args) => { settings.push([prop, args]); return current()[prop](...args); };
+          const value = current()[prop];
+          return typeof value === 'function' ? value.bind(current()) : value;
+        },
+      });
+    };
+    return new Proxy({}, {
+      get(_, prop) {
+        if (prop === 'prepare') return prepare;
+        if (prop === 'exec') return (sql) => retrying(`exec ${String(sql).slice(0, 80)}`, () => raw.exec(sql));
+        if (prop === 'reopen') return reopen;
+        if (prop === 'raw') return raw;
+        const value = raw[prop];
+        return typeof value === 'function' ? value.bind(raw) : value;
+      },
+    });
+  }
+
+  function connectionState(openRaw, { file, inTransaction = () => false, onCorrupt = reportIncident }) {
+    const recovered = [];
+    const noteRecovered = (r) => {
+      recovered.push({ ...r, at: Date.now() });
+      process.stderr.write(`[machine-db] transient SQLITE_CORRUPT recovered after ${r.retries} reopen(s) at ${r.where} (${file})\n`);
+    };
+    const db = resilientConnection(openRaw, { file, inTransaction, onRecovered: noteRecovered, onCorrupt });
+    return { db, recovered, noteRecovered, close() { try { db.close(); } catch { /* closed */ } } };
+  }
+
+  return { openWithRetry, connectionState };
+}

package/runtime/engine/db/machine-schema.mjs

@@ -0,0 +1,84 @@
+// Machine schema lifecycle runs only through the machine store's connection owner.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { artifactRoot } from './blob.mjs';
+import { sha256 } from '../digest.mjs';
+import { hasTable } from '../../scripts/lib/sqlite.mjs';
+
+const require = createRequire(import.meta.url);
+// SQLite ALTER RENAME quotes identifiers. Preserve literals/constraints while ignoring that spelling and formatting.
+function sqlIdentity(sql) {
+  const tokens = String(sql).match(/--[^\r\n]*|\/\*[\s\S]*?\*\/|'(?:''|[^'])*'|"(?:[^"]|"")*"|`(?:[^`]|``)*`|\[[^\]]*\]|[a-z_][a-z_0-9]*|\d+(?:\.\d+)?|[^\s]/gi) ?? [];
+  const normalized = tokens.filter(token => !token.startsWith('--') && !token.startsWith('/*')).map(token => {
+    if (token.startsWith("'")) return token;
+    if (token.startsWith('"')) return token.slice(1, -1).replaceAll('""', '"').toLowerCase();
+    if (token.startsWith('`')) return token.slice(1, -1).replaceAll('``', '`').toLowerCase();
+    if (token.startsWith('[')) return token.slice(1, -1).toLowerCase();
+    return token.toLowerCase();
+  });
+  for (const at of [2, 3]) if (normalized.slice(at, at + 3).join(' ') === 'if not exists') normalized.splice(at, 3);
+  return JSON.stringify(normalized);
+}
+const objects = db => db.prepare("SELECT type,name,sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT GLOB 'sqlite_*' ORDER BY type,name").all();
+
+/** Create or validate the current host schema; an existing store's journal is historical evidence, never a rewrite target. */
+export function machineSchemaMethods({ schema: MACHINE_SCHEMA, version: MACHINE_VERSION, initSqlFile: INIT_SQL_FILE, controllers: CONTROLLERS, runtimeRev, pragma }) {
+  let requiredObjects;
+  function expectedObjects() {
+    if (requiredObjects) return requiredObjects;
+    const { DatabaseSync } = require('node:sqlite'), reference = new DatabaseSync(':memory:');
+    try {
+      reference.exec(fs.readFileSync(INIT_SQL_FILE, 'utf8'));
+      requiredObjects = objects(reference).map(row => ({ ...row, identity: sqlIdentity(row.sql) }));
+      return requiredObjects;
+    } finally { reference.close(); }
+  }
+  /** Refuse unsupported identity or a missing/changed current object without replacing the caller's store. */
+  function refuseOld(file, why) {
+    throw Object.assign(Error(`machine-schema-old: ${path.resolve(file)} ${why}; expected '${MACHINE_SCHEMA}' (user_version ${MACHINE_VERSION}) with its current physical schema; the refused store remains unchanged`),
+      { code: 'STARCI_MACHINE_SCHEMA_OLD' });
+  }
+  function checkSchema(db, file) {
+    const version = Number(pragma(db, 'user_version'));
+    if (!hasTable(db, 'machine_meta')) refuseOld(file, `has no machine_meta (user_version ${version})`);
+    const schema = db.prepare("SELECT value FROM machine_meta WHERE key='schema'").get()?.value;
+    if (schema !== MACHINE_SCHEMA) refuseOld(file, `is schema '${schema ?? 'none'}'`);
+    if (version !== MACHINE_VERSION) refuseOld(file, `is user_version ${version}`);
+    const actual = new Map(objects(db).map(row => [`${row.type}:${row.name}`, sqlIdentity(row.sql)]));
+    for (const row of expectedObjects()) if (actual.get(`${row.type}:${row.name}`) !== row.identity)
+      refuseOld(file, `has a missing or changed ${row.type} '${row.name}'`);
+  }
+  function rollbackSchema(db, error) {
+    try { db.exec('ROLLBACK'); }
+    catch (rollbackError) { throw new AggregateError([error, rollbackError], 'machine schema failed and rollback failed', { cause: error }); }
+    throw error;
+  }
+  function createSchema(db, { file, env, now }) {
+    const sql = fs.readFileSync(INIT_SQL_FILE, 'utf8'), at = now();
+    db.exec('BEGIN IMMEDIATE');
+    try {
+      if (Number(pragma(db, 'user_version')) !== 0 || db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT GLOB 'sqlite_*' LIMIT 1").get()) {
+        // Another initializer may have committed while this connection waited for BEGIN; keep its current rows/journal.
+        checkSchema(db, file); db.exec('COMMIT'); return;
+      }
+      db.exec(sql);
+      const meta = db.prepare('INSERT INTO machine_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value');
+      for (const [key, value] of Object.entries({ host_id: os.hostname(), schema: MACHINE_SCHEMA, created_at: String(at), blob_root: artifactRoot(env),
+        runtime_rev: runtimeRev(), sqlite_version: db.prepare('select sqlite_version() v').get().v,
+        node_version: process.version, journal_mode: 'wal' })) meta.run(key, String(value));
+      db.prepare("INSERT INTO schema_migrations(version,name,runtime_rev,sql_sha256,started_at,finished_at,status) VALUES(?,'0001-init',?,?,?,?,'done')")
+        .run(MACHINE_VERSION, runtimeRev(), sha256(sql), at, now());
+      // Every controller starts in shadow until the owner switches it.
+      for (const controller of CONTROLLERS) {
+        db.prepare("INSERT INTO mode_changes(controller,from_mode,to_mode,by,reason,at) VALUES(?,NULL,'shadow','machine-db:init','0001-init',?)").run(controller, at);
+        db.prepare("INSERT INTO controller_modes(controller,mode,set_at,set_by) VALUES(?,'shadow',?,'machine-db:init')").run(controller, at);
+      }
+      db.exec(`PRAGMA user_version=${MACHINE_VERSION}`);
+      checkSchema(db, file);
+      db.exec('COMMIT');
+    } catch (error) { rollbackSchema(db, error); }
+  }
+  return { checkSchema, createSchema };
+}