@starci/hfs 4.7.31 → 4.7.33

package/runtime/engine/db/machine-connection.mjs

@@ -0,0 +1,150 @@
+import { installRefResolver } from './ref-value.mjs';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { sleepSync as scaledSleepSync } from '../../scripts/lib/sleep-sync.mjs';
+
+const require = createRequire(import.meta.url);
+export const MACHINE_BUSY_TIMEOUT_MS = 15000;
+export const MACHINE_BUSY_CODE = 'STARCI_MACHINE_BUSY';
+export const MACHINE_CORRUPT_CODE = 'STARCI_MACHINE_CORRUPT';
+const OPEN_RETRY_DELAYS_MS = Object.freeze([0, 300, 900]);
+const BUSY_RETRY_DELAYS_MS = Object.freeze([100, 300, 900, 2000]);
+export const CORRUPT_RETRY_DELAYS_MS = Object.freeze([25, 150, 600]);
+export const waitForRetry = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+export const isMachineBusy = (error) => error?.code === MACHINE_BUSY_CODE;
+export const isBusyError = (error) => error?.errcode === 5 || error?.errcode === 6 || /SQLITE_BUSY|database is (?:locked|busy)/i.test(String(error?.message ?? error));
+export const isCorruptError = (error) => {
+  if (!error) return false;
+  if (error.code === MACHINE_CORRUPT_CODE) return true;
+  const code = Number(error.errcode);
+  if (Number.isInteger(code) && ((code & 0xff) === 11 || (code & 0xff) === 26)) return true;
+  return /database disk image is malformed|file is not a database|SQLITE_CORRUPT|SQLITE_NOTADB/i.test(String(error.message ?? error));
+};
+export const errText = (error) => String(error?.message ?? error).slice(0, 500);
+const busyError = (file, error, { retries, where }) => Object.assign(new Error(`machine-db-busy: ${file} ${where}: database still locked after busy_timeout and ${retries} retries: ${String(error?.message ?? error).slice(0, 200)}`),
+  { code: MACHINE_BUSY_CODE, cause: error, file, retries, where });
+
+/** Persistent corruption diagnostics observe the store and stderr without recording an incident. */
+export function corruptDiagnostic(file, error, { retries, where }) {
+  if (error?.code === MACHINE_CORRUPT_CODE) return error;
+  const out = Object.assign(new Error(`machine-db-corrupt: ${file} ${where}: SQLITE_CORRUPT persisted after ${retries} reopen(s): ${errText(error)}`),
+    { code: MACHINE_CORRUPT_CODE, cause: error, file, retries, where });
+  let check = null;
+  try {
+    const { DatabaseSync } = require('node:sqlite');
+    const db = installRefResolver(new DatabaseSync(file, { readOnly: true, timeout: MACHINE_BUSY_TIMEOUT_MS }));
+    try { check = db.prepare('PRAGMA quick_check').all().map((r) => r.quick_check).slice(0, 20); } finally { db.close(); }
+  } catch (e) { check = [`quick_check failed: ${errText(e)}`]; }
+  out.quickCheck = check;
+  process.stderr.write(`[machine-db] INCIDENT ${out.message} (quick_check: ${JSON.stringify(check)})\n`);
+  return out;
+}
+
+/** Which open failure is retryable: a busy wait, an 'unable to open' reopen, or a first corrupt hit; null rethrows. */
+const openFailureKind = (error) => {
+  if (isBusyError(error) && !isCorruptError(error)) return 'busy';
+  if (/unable to open/i.test(String(error?.message ?? ''))) return 'open';
+  if (isCorruptError(error) && error.code !== MACHINE_CORRUPT_CODE) return 'corrupt';
+  return null;
+};
+
+const transientBusy = (error) => isBusyError(error) && !isMachineBusy(error) && !isCorruptError(error);
+
+function resilientConnection(openRaw, { file, inTransaction, onRecovered, onCorrupt }) {
+  let raw = openRaw(), generation = 0;
+  const reopen = () => { try { raw.close(); } catch { /* closed */ } raw = openRaw(); generation += 1; };
+  const recoverCorrupt = (error, retries, where) => {
+    if (!isCorruptError(error) || error.code === MACHINE_CORRUPT_CODE) throw error;
+    if (inTransaction() || raw.isTransaction) throw error;
+    if (retries >= CORRUPT_RETRY_DELAYS_MS.length) throw onCorrupt(file, error, { retries, where });
+    waitForRetry(CORRUPT_RETRY_DELAYS_MS[retries]);
+    try { reopen(); } catch (openError) { if (!isCorruptError(openError)) throw openError; }
+  };
+  const retrying = (where, op) => {
+    let busyRetries = 0;
+    for (let retries = 0; ; retries += 1) {
+      try {
+        const out = op();
+        if (retries) onRecovered({ where, retries });
+        return out;
+      } catch (error) {
+        if (!transientBusy(error)) { recoverCorrupt(error, retries, where); continue; }
+        if (inTransaction() || raw.isTransaction) throw error;
+        if (busyRetries >= BUSY_RETRY_DELAYS_MS.length) throw busyError(file, error, { retries: busyRetries, where });
+        scaledSleepSync(BUSY_RETRY_DELAYS_MS[busyRetries]);
+        busyRetries += 1;
+        retries -= 1;
+      }
+    }
+  };
+  const prepare = (sql) => {
+    let stmt = null, gen = -1;
+    const settings = [];
+    const current = () => {
+      if (gen !== generation) {
+        stmt = raw.prepare(sql);
+        for (const [k, a] of settings) stmt[k](...a);
+        gen = generation;
+      }
+      return stmt;
+    };
+    retrying(`prepare ${sql.slice(0, 80)}`, current);
+    return new Proxy({}, {
+      get(_, prop) {
+        if (prop === 'run' || prop === 'get' || prop === 'all' || prop === 'iterate') return (...args) => retrying(`${prop} ${sql.slice(0, 80)}`, () => current()[prop](...args));
+        if (typeof prop === 'string' && /^set[A-Z]/.test(prop)) return (...args) => { settings.push([prop, args]); return current()[prop](...args); };
+        const value = current()[prop];
+        return typeof value === 'function' ? value.bind(current()) : value;
+      },
+    });
+  };
+  return new Proxy({}, {
+    get(_, prop) {
+      if (prop === 'prepare') return prepare;
+      if (prop === 'exec') return (sql) => retrying(`exec ${String(sql).slice(0, 80)}`, () => raw.exec(sql));
+      if (prop === 'reopen') return reopen;
+      if (prop === 'raw') return raw;
+      const value = raw[prop];
+      return typeof value === 'function' ? value.bind(raw) : value;
+    },
+  });
+}
+
+/** Connection retries share one policy; the machine owner supplies operational incident persistence. */
+export function machineConnectionMethods({ reportIncident }) {
+  function openWithRetry(openRaw, { file, onCorrupt = reportIncident }) {
+    let lastError, busyOpens = 0, failures = 0;
+    const retryOpenFailure = (error) => {
+      lastError = error;
+      const kind = openFailureKind(error);
+      if (kind === null) throw error;
+      if (kind === 'busy') {
+        if (busyOpens >= BUSY_RETRY_DELAYS_MS.length) throw busyError(path.resolve(file), error, { retries: busyOpens, where: 'open' });
+        scaledSleepSync(BUSY_RETRY_DELAYS_MS[busyOpens]);
+        busyOpens += 1;
+        return true;
+      }
+      failures += 1;
+      return failures < OPEN_RETRY_DELAYS_MS.length;
+    };
+    for (;;) {
+      if (failures && OPEN_RETRY_DELAYS_MS[failures]) waitForRetry(OPEN_RETRY_DELAYS_MS[failures]);
+      try { return openRaw(); }
+      catch (error) { if (!retryOpenFailure(error)) break; }
+    }
+    if (isCorruptError(lastError)) throw onCorrupt(path.resolve(file), lastError, { retries: OPEN_RETRY_DELAYS_MS.length - 1, where: 'open' });
+    throw lastError;
+  }
+
+  function connectionState(openRaw, { file, inTransaction = () => false, onCorrupt = reportIncident }) {
+    const recovered = [];
+    const noteRecovered = (r) => {
+      recovered.push({ ...r, at: Date.now() });
+      process.stderr.write(`[machine-db] transient SQLITE_CORRUPT recovered after ${r.retries} reopen(s) at ${r.where} (${file})\n`);
+    };
+    const db = resilientConnection(openRaw, { file, inTransaction, onRecovered: noteRecovered, onCorrupt });
+    return { db, recovered, noteRecovered, close() { try { db.close(); } catch { /* closed */ } } };
+  }
+
+  return { openWithRetry, connectionState };
+}

package/runtime/engine/db/machine-open.mjs

@@ -0,0 +1,60 @@
+import { createRequire } from 'node:module';
+import fs from 'node:fs';
+import path from 'node:path';
+import { installRefResolver, openReadOnlyDb } from './ref-value.mjs';
+import { MACHINE_BUSY_TIMEOUT_MS, isBusyError } from './machine-connection.mjs';
+
+// engine/db/machine-open.mjs — how machine.sqlite is opened: the connection policy of the machine.mjs header, a reference-resolving handle.
+const require = createRequire(import.meta.url);
+const need = (ok, message, code = 'STARCI_MACHINE_DB') => { if (!ok) throw Object.assign(new Error(message), { code }); };
+const busyTimeoutOf = (env = process.env) => { const n = Number(env?.STARCI_MACHINE_BUSY_TIMEOUT_MS); return Number.isInteger(n) && n > 0 ? n : MACHINE_BUSY_TIMEOUT_MS; };
+const writerPragmas = (env) => ({ synchronous: 'NORMAL', busy_timeout: busyTimeoutOf(env), temp_store: 'MEMORY', cache_size: -16000,
+  journal_size_limit: 67108864, trusted_schema: 'OFF' });
+export const pragma = (db, name) => { const row = db.prepare(`PRAGMA ${name}`).get(); return row ? Object.values(row)[0] : null; };
+
+function recordFacts(db) {
+  const facts = { sqlite_version: db.prepare('select sqlite_version() v').get().v, node_version: process.version, journal_mode: String(pragma(db, 'journal_mode')).toLowerCase() };
+  const stale = Object.entries(facts).filter(([k, v]) => db.prepare('SELECT value FROM machine_meta WHERE key=?').get(k)?.value !== v);
+  if (!stale.length) return;
+  const put = db.prepare('INSERT INTO machine_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value');
+  try { for (const [k, v] of stale) put.run(k, v); } catch (error) { if (!isBusyError(error)) throw error; /* another writer records them */ }
+}
+
+/** The connection opener bound to the schema methods and the corruption reporter of machine.mjs. */
+export function machineOpenMethods({ checkSchema, createSchema, openWithRetry, corruptIncident }) {
+  function openConnection(file, { readOnly = false, env = process.env, now = Date.now, onCorrupt = corruptIncident } = {}) {
+    const { DatabaseSync } = require('node:sqlite');
+    need(typeof file === 'string' && file.trim(), 'openMachine needs a file');
+    if (!readOnly) fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
+    return openWithRetry(() => {
+      let db;
+      try {
+        if (readOnly) {
+          db = openReadOnlyDb(file, MACHINE_BUSY_TIMEOUT_MS);
+          db.exec('PRAGMA query_only=ON; PRAGMA temp_store=MEMORY; PRAGMA cache_size=-16000;');
+          checkSchema(db, file);
+          return db;
+        }
+        db = installRefResolver(new DatabaseSync(file, { timeout: busyTimeoutOf(env) }));
+        const empty = Number(pragma(db, 'user_version')) === 0 && !db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT GLOB 'sqlite_*' LIMIT 1").get();
+        if (!empty) checkSchema(db, file); // Refuse a store that is not exactly the current schema before persistent WAL/facts changes.
+        if (empty) db.exec('PRAGMA page_size=4096; PRAGMA auto_vacuum=INCREMENTAL;');
+        const mode = String(pragma(db, 'journal_mode=WAL')).toLowerCase();
+        need(mode === 'wal', `machine.sqlite journal_mode is '${mode}', not wal (${file})`, 'STARCI_MACHINE_NOT_WAL');
+        db.exec('PRAGMA foreign_keys=ON;');
+        for (const [k, v] of Object.entries(writerPragmas(env))) db.exec(`PRAGMA ${k}=${v};`);
+        // Never an automatic checkpoint: the engine leader's fenced checkpoint() is the only one (header, G17).
+        db.exec('PRAGMA wal_autocheckpoint=0;');
+        if (empty) createSchema(db, { file, env, now });
+        checkSchema(db, file);
+        recordFacts(db);
+        return db;
+      } catch (error) {
+        try { db?.close(); } catch { /* closed */ }
+        throw error;
+      }
+    }, { file, onCorrupt });
+  }
+
+  return { openConnection };
+}

package/runtime/engine/db/machine-schema.mjs

@@ -0,0 +1,105 @@
+// Machine schema lifecycle runs only through the machine store's connection owner.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { artifactRoot } from './blob.mjs';
+import { hasTable } from '../../scripts/lib/sqlite.mjs';
+
+const require = createRequire(import.meta.url);
+// SQLite ALTER RENAME quotes identifiers. Preserve literals/constraints while ignoring that spelling and formatting.
+const SQL_IDENTITY_TOKEN_PARTS = [
+  String.raw`--[^\r\n]*`, String.raw`\/\*[\s\S]*?\*\/`, "'[^']*(?:''[^']*)*'",
+  '"[^"]*(?:""[^"]*)*"', '`[^`]*(?:``[^`]*)*`', String.raw`\[[^\]]*\]`,
+  '[a-z_][a-z_0-9]*', String.raw`\d+(?:\.\d+)?`, String.raw`[^\s]`,
+];
+const SQL_IDENTITY_TOKENS = new RegExp(SQL_IDENTITY_TOKEN_PARTS.join('|'), 'gi');
+
+function sqlIdentity(sql) {
+  const tokens = Array.from(String(sql).matchAll(SQL_IDENTITY_TOKENS), (found) => found[0]);
+  const normalized = tokens.filter(token => !token.startsWith('--') && !token.startsWith('/*')).map(token => {
+    if (token.startsWith("'")) return token;
+    if (token.startsWith('"')) return token.slice(1, -1).replaceAll('""', '"').toLowerCase();
+    if (token.startsWith('`')) return token.slice(1, -1).replaceAll('``', '`').toLowerCase();
+    if (token.startsWith('[')) return token.slice(1, -1).toLowerCase();
+    return token.toLowerCase();
+  });
+  for (const at of [2, 3]) if (normalized.slice(at, at + 3).join(' ') === 'if not exists') normalized.splice(at, 3);
+  return JSON.stringify(normalized);
+}
+/** Canonical object identities include attached indexes and triggers, regardless of their names. */
+export const readSchemaObjects = db => db.prepare("SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name NOT GLOB 'sqlite_*' ORDER BY type,name")
+  .all().map(row => ({ ...row, identity: sqlIdentity(row.sql) }));
+
+/** Describe canonical DDL in an isolated reference; no project or host store is opened. */
+export function describeSchema(sql) {
+  const { DatabaseSync } = require('node:sqlite'), reference = new DatabaseSync(':memory:');
+  try {
+    reference.exec(sql);
+    return { objects: readSchemaObjects(reference),
+      blobRefs: reference.prepare('SELECT table_name,column_name FROM blob_ref_columns ORDER BY table_name,column_name').all() };
+  } finally { reference.close(); }
+}
+function rollbackSchema(db, error) {
+  try { db.exec('ROLLBACK'); }
+  catch (rollbackError) { throw new AggregateError([error, rollbackError], 'machine schema failed and rollback failed', { cause: error }); }
+  throw error;
+}
+
+/** Create the one host schema on an empty file, or validate that an existing store is exactly that schema; there is no upgrade path. */
+export function machineSchemaMethods({ schema: MACHINE_SCHEMA, version: MACHINE_VERSION, initSqlFile: INIT_SQL_FILE, controllers: CONTROLLERS, runtimeRev, pragma }) {
+  let requiredObjects;
+  function expectedObjects() {
+    if (requiredObjects) return requiredObjects;
+    requiredObjects = describeSchema(fs.readFileSync(INIT_SQL_FILE, 'utf8')).objects;
+    return requiredObjects;
+  }
+  /** Refuse any store that is not exactly the current schema, leaving it unchanged; the operator replaces it with a fresh store. */
+  function refuseOld(file, why) {
+    throw Object.assign(new Error(`machine-schema-old: ${path.resolve(file)} ${why}; expected '${MACHINE_SCHEMA}' (user_version ${MACHINE_VERSION}) with its current physical schema; the store is not the current schema and has been left unchanged: replace it with a fresh store`),
+      { code: 'STARCI_MACHINE_SCHEMA_OLD' });
+  }
+  function checkIdentity(db, file) {
+    const version = Number(pragma(db, 'user_version'));
+    if (!hasTable(db, 'machine_meta')) refuseOld(file, 'has no machine_meta (user_version '+version+')');
+    const schema = db.prepare("SELECT value FROM machine_meta WHERE key='schema'").get()?.value;
+    if (schema !== MACHINE_SCHEMA) refuseOld(file, "is schema '"+(schema ?? 'none')+"'");
+    if (version !== MACHINE_VERSION) refuseOld(file, 'is user_version '+version);
+  }
+  function checkObjects(db, file, expected) {
+    const rows = readSchemaObjects(db), actual = new Map(rows.map(row => [row.type+':'+row.name, row.identity]));
+    for (const row of expected) if (actual.get(row.type+':'+row.name) !== row.identity)
+      refuseOld(file, 'has a missing or changed '+row.type+" '"+row.name+"'");
+    const names = new Set(expected.map(row => row.type+':'+row.name));
+    for (const row of rows) if (!names.has(row.type+':'+row.name))
+      refuseOld(file, 'has an unexpected '+row.type+" '"+row.name+"'");
+  }
+  function checkSchema(db, file) {
+    checkIdentity(db, file);
+    checkObjects(db, file, expectedObjects());
+  }
+  function createSchema(db, { file, env, now }) {
+    const sql = fs.readFileSync(INIT_SQL_FILE, 'utf8'), at = now();
+    db.exec('BEGIN IMMEDIATE');
+    try {
+      if (Number(pragma(db, 'user_version')) !== 0 || db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT GLOB 'sqlite_*' LIMIT 1").get()) {
+        // Another initializer may have committed while this connection waited for BEGIN; keep its current rows.
+        checkSchema(db, file); db.exec('COMMIT'); return;
+      }
+      db.exec(sql);
+      const meta = db.prepare('INSERT INTO machine_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value');
+      for (const [key, value] of Object.entries({ host_id: os.hostname(), schema: MACHINE_SCHEMA, created_at: String(at), blob_root: artifactRoot(env),
+        runtime_rev: runtimeRev(), sqlite_version: db.prepare('select sqlite_version() v').get().v,
+        node_version: process.version, journal_mode: 'wal' })) meta.run(key, String(value));
+      // Every controller starts in shadow until the owner switches it.
+      for (const controller of CONTROLLERS) {
+        db.prepare("INSERT INTO mode_changes(controller,from_mode,to_mode,by,reason,at) VALUES(?,NULL,'shadow','machine-db:init','init',?)").run(controller, at);
+        db.prepare("INSERT INTO controller_modes(controller,mode,set_at,set_by) VALUES(?,'shadow',?,'machine-db:init')").run(controller, at);
+      }
+      db.exec(`PRAGMA user_version=${MACHINE_VERSION}`);
+      checkSchema(db, file);
+      db.exec('COMMIT');
+    } catch (error) { rollbackSchema(db, error); }
+  }
+  return { checkSchema, createSchema };
+}