@logicseek/hello-cc 0.1.9 → 1.0.0

package/lib/runtime/state.mjs

@@ -1,15 +1,198 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import process from 'node:process';
+import { compareProcessIdentity, inspectProcessIdentity } from '../process/identity.mjs';
 import { CliError } from '../shared/errors.mjs';
+import { withFileLock } from '../shared/file-lock.mjs';
+import { writeJsonSafe } from '../shared/json-file.mjs';
 import {
   globalRuntimePath,
   runtimePath
 } from './paths.mjs';
-import { runtimeApiUrl } from '../web/runtime.mjs';
+import { runtimeHttpRequest } from '../web/runtime.mjs';
+import { API_VERSION } from '../web/api-version.mjs';
 
 const DEFAULT_PRODUCT_NAME = 'hello-cc';
 const DEFAULT_CLI_NAME = 'hcc';
+export const RUNTIME_POINTER_UNKNOWN_GRACE_MS = 120_000;
+
+function completeProcessIdentity(value) {
+  return compareProcessIdentity(value, value) === 'live';
+}
+
+export function runtimeProcessIdentity({
+  pid = process.pid,
+  inspect = inspectProcessIdentity
+} = {}) {
+  const observed = inspect(pid);
+  return observed?.state === 'live' && completeProcessIdentity(observed.identity)
+    ? observed.identity
+    : null;
+}
+
+export function classifyRuntimePointer(runtime, {
+  inspect = inspectProcessIdentity,
+  ageMs = 0,
+  unknownGraceMs = RUNTIME_POINTER_UNKNOWN_GRACE_MS
+} = {}) {
+  if (!Number.isFinite(ageMs) || ageMs < 0 ||
+      !Number.isFinite(unknownGraceMs) || unknownGraceMs < 0) {
+    throw new TypeError('runtime pointer ages must be non-negative finite numbers');
+  }
+  const pid = Number(runtime?.pid);
+  const storedEvidence = runtimeIdentityEvidence(runtime);
+  const stored = storedEvidence.present ? storedEvidence.identity : null;
+  if (!Number.isInteger(pid) || pid <= 0 || !completeProcessIdentity(stored) || stored.pid !== pid) {
+    return { state: 'unknown', reclaimable: ageMs >= unknownGraceMs };
+  }
+  const current = inspect(pid);
+  if (current?.state === 'dead') return { state: 'dead', reclaimable: true };
+  if (current?.state !== 'live' || !completeProcessIdentity(current.identity)) {
+    return { state: 'unknown', reclaimable: ageMs >= unknownGraceMs };
+  }
+  return compareProcessIdentity(stored, current.identity) === 'live'
+    ? { state: 'alive', reclaimable: false }
+    : { state: 'dead', reclaimable: true };
+}
+
+function runtimePointerAge(stat, currentMs, unknownGraceMs) {
+  const timestampMs = Math.floor(Math.max(Number(stat.mtimeMs), Number(stat.ctimeMs)));
+  if (!Number.isFinite(currentMs) || !Number.isFinite(timestampMs)) {
+    throw new TypeError('runtime pointer timestamps must be finite numbers');
+  }
+  return {
+    ageMs: currentMs < timestampMs ? 0 : currentMs - timestampMs,
+    cutoffSec: Math.max(0, Math.floor(timestampMs / 1000) + Math.ceil(unknownGraceMs / 1000))
+  };
+}
+
+export function reclaimRuntimePointerFiles(files, {
+  nowMs = Date.now,
+  inspect = inspectProcessIdentity,
+  unknownGraceMs = RUNTIME_POINTER_UNKNOWN_GRACE_MS,
+  reclaimUnknown = true,
+  dryRun = false,
+  onReclaim = null,
+  withLock = withFileLock
+} = {}) {
+  if (typeof reclaimUnknown !== 'boolean') {
+    throw new TypeError('reclaimUnknown must be a boolean');
+  }
+  let blocked = false;
+  let reclaimed = 0;
+  const outcomes = [];
+  for (const suppliedFile of new Set((files || []).map((file) => path.resolve(file)))) {
+    if (!fs.existsSync(suppliedFile)) continue;
+    let recorded = false;
+    const record = (outcome) => {
+      recorded = true;
+      outcomes.push({ file: suppliedFile, ...outcome });
+    };
+    try {
+      withLock(suppliedFile, (file) => {
+        if (!fs.existsSync(file)) {
+          record({ state: 'missing', action: 'skipped' });
+          return;
+        }
+        const stat = fs.lstatSync(file);
+        if (stat.isSymbolicLink() || !stat.isFile()) {
+          blocked = true;
+          record({ state: 'unknown', action: 'blocked', reason: 'invalid_pointer' });
+          return;
+        }
+        const raw = fs.readFileSync(file, 'utf8');
+        let runtime = null;
+        try { runtime = JSON.parse(raw); } catch {}
+        const timing = runtimePointerAge(stat, nowMs(), unknownGraceMs);
+        const evidence = classifyRuntimePointer(runtime, {
+          inspect,
+          ageMs: timing.ageMs,
+          unknownGraceMs
+        });
+        if (!evidence.reclaimable || (evidence.state === 'unknown' && !reclaimUnknown)) {
+          blocked = true;
+          record({ state: evidence.state, action: 'blocked', reason: 'owner_not_confirmed_dead' });
+          return;
+        }
+        const current = fs.lstatSync(file);
+        const currentRaw = fs.readFileSync(file, 'utf8');
+        if (current.isSymbolicLink() || !current.isFile() ||
+            current.dev !== stat.dev || current.ino !== stat.ino ||
+            current.size !== stat.size || current.mtimeMs !== stat.mtimeMs ||
+            current.ctimeMs !== stat.ctimeMs || currentRaw !== raw) {
+          blocked = true;
+          record({ state: 'unknown', action: 'blocked', reason: 'pointer_changed' });
+          return;
+        }
+        if (!dryRun) fs.unlinkSync(file);
+        reclaimed += 1;
+        record({ state: evidence.state, action: dryRun ? 'would-reclaim' : 'reclaimed' });
+        onReclaim?.({
+          file,
+          state: evidence.state,
+          gcCutoff: evidence.state === 'unknown' ? timing.cutoffSec : null
+        });
+      });
+    } catch {
+      blocked = true;
+      if (!recorded) record({ state: 'unknown', action: 'blocked', reason: 'lock_unavailable' });
+    }
+  }
+  return { blocked, reclaimed, outcomes };
+}
+
+// A runtime pointer carries the runtime process pid. If that pid is no longer
+// alive, the pointer is stale and must not be returned as a reachable runtime
+// (bg-01): otherwise commands and shims fail against a dead runtime until
+// someone manually clears the file. A pointer without a pid is left alone
+// (we cannot determine liveness, so keep the previous behavior).
+function pidAlive(pid) {
+  if (pid === undefined || pid === null || pid === '') return true;
+  const numeric = Number(pid);
+  if (!Number.isInteger(numeric)) return true;
+  try { process.kill(numeric, 0); return true; }
+  catch (e) { return e.code === 'EPERM'; }
+}
+
+function runtimeIdentityEvidence(runtime) {
+  const snakeCase = Object.hasOwn(runtime || {}, 'process_identity');
+  const camelCase = Object.hasOwn(runtime || {}, 'processIdentity');
+  if (!snakeCase && !camelCase) return { present: false, identity: null };
+  if (snakeCase && camelCase) return { present: true, identity: null };
+  return {
+    present: true,
+    identity: snakeCase ? runtime.process_identity : runtime.processIdentity
+  };
+}
+
+function runtimePointerMatchesCurrentProcess(runtime, {
+  inspect = inspectProcessIdentity
+} = {}) {
+  const storedEvidence = runtimeIdentityEvidence(runtime);
+  if (!storedEvidence.present) return pidAlive(runtime?.pid);
+  const stored = storedEvidence.identity;
+  if (!completeProcessIdentity(stored)) return false;
+  const pid = Number(runtime?.pid);
+  if (!Number.isInteger(pid) || pid <= 0 || stored.pid !== pid) return false;
+  return classifyRuntimePointer(runtime, { ageMs: 0, inspect }).state === 'alive';
+}
+
+function runtimeMetadataMatchesPointer(runtime, metadata) {
+  if (Object.hasOwn(runtime || {}, 'product') &&
+      (typeof runtime.product !== 'string' || !runtime.product ||
+       metadata?.product !== runtime.product)) return false;
+  const storedEvidence = runtimeIdentityEvidence(runtime);
+  if (!storedEvidence.present) return true;
+  const stored = storedEvidence.identity;
+  const observedEvidence = runtimeIdentityEvidence(metadata);
+  if (!completeProcessIdentity(stored) || !observedEvidence.present) return false;
+  const pointerPid = Number(runtime?.pid);
+  const metadataPid = Number(metadata?.pid);
+  const observed = observedEvidence.identity;
+  return Number.isInteger(pointerPid) && pointerPid > 0 && stored.pid === pointerPid &&
+    metadataPid === pointerPid && completeProcessIdentity(observed) &&
+    observed.pid === metadataPid && compareProcessIdentity(stored, observed) === 'live';
+}
 
 export function readGlobalRuntimeFile() {
   const file = globalRuntimePath();
@@ -17,33 +200,34 @@ export function readGlobalRuntimeFile() {
   try {
     return JSON.parse(fs.readFileSync(file, 'utf8'));
   } catch {
-    try { fs.rmSync(file, { force: true }); } catch {}
+    // Do not delete on a parse failure: a torn/partial write must not orphan a
+    // healthy runtime pointer. Leave it for health probing / restart to reclaim.
     return null;
   }
 }
 
 export function writeGlobalRuntime(runtime) {
   const file = globalRuntimePath();
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, JSON.stringify(runtime, null, 2), { mode: 0o600 });
-  fs.chmodSync(file, 0o600);
+  withFileLock(file, (lockedFile) => writeJsonSafe(lockedFile, runtime, { mode: 0o600 }));
   return file;
 }
 
 export function writeRuntime(ctx, runtime) {
   const file = runtimePath(ctx);
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, JSON.stringify(runtime, null, 2), { mode: 0o600 });
-  fs.chmodSync(file, 0o600);
+  withFileLock(file, (lockedFile) => writeJsonSafe(lockedFile, runtime, { mode: 0o600 }));
   return file;
 }
 
 export function readRuntime(ctx, opts = {}) {
   const localOnly = Boolean(opts.localOnly || process.env.HCC_RUNTIME_LOCAL_ONLY === '1');
+  const inspectRuntimeProcessIdentity = typeof opts.inspectProcessIdentity === 'function'
+    ? opts.inspectProcessIdentity
+    : inspectProcessIdentity;
   if (process.env.HCC_RUNTIME_URL) {
     return {
       base_url: process.env.HCC_RUNTIME_URL,
       token: process.env.HCC_RUNTIME_TOKEN || '',
+      tls_ca_file: process.env.HCC_RUNTIME_CA || undefined,
       source: 'env'
     };
   }
@@ -52,14 +236,20 @@ export function readRuntime(ctx, opts = {}) {
     try {
       const runtime = JSON.parse(fs.readFileSync(file, 'utf8'));
       if (!runtime.base_url) throw new Error('missing base_url');
+      if (!runtimePointerMatchesCurrentProcess(runtime, { inspect: inspectRuntimeProcessIdentity })) {
+        throw new Error('runtime process identity does not match');
+      }
       return { ...runtime, source: file };
     } catch {
-      try { fs.rmSync(file, { force: true }); } catch {}
+      // Do not delete on a torn/partial read; fall through to the global
+      // runtime and let health probing / restart reclaim a truly dead file.
     }
   }
   if (!localOnly) {
     const global = readGlobalRuntimeFile();
-    if (global?.base_url) {
+    if (global?.base_url && runtimePointerMatchesCurrentProcess(global, {
+      inspect: inspectRuntimeProcessIdentity
+    })) {
       return { ...global, source: globalRuntimePath(), global: true };
     }
   }
@@ -77,12 +267,14 @@ export function readRuntimeFile(ctx) {
 
 export async function probeRuntime(runtime) {
   if (!runtime?.base_url) return false;
-  const url = runtimeApiUrl(runtime, '/api/runtime');
   const headers = {};
   if (runtime.token) headers.Authorization = `Bearer ${runtime.token}`;
   try {
-    const response = await fetch(url, { headers });
-    return response.ok;
+    // xx-03: probing must not hang the CLI indefinitely against a stalled runtime.
+    const response = await runtimeHttpRequest(runtime, '/api/runtime', { headers, timeoutMs: 3000 });
+    if (!response.ok) return false;
+    const metadata = JSON.parse(response.text);
+    return metadata?.api_version === API_VERSION && runtimeMetadataMatchesPointer(runtime, metadata);
   } catch {
     return false;
   }
@@ -112,20 +304,28 @@ export async function readHealthyGlobalRuntime() {
 
 export function clearRuntime(ctx, pid = process.pid) {
   const file = runtimePath(ctx);
-  if (fs.existsSync(file)) {
-    try {
-      const runtime = JSON.parse(fs.readFileSync(file, 'utf8'));
-      if (!runtime.pid || runtime.pid === pid) fs.rmSync(file, { force: true });
-    } catch {
-      fs.rmSync(file, { force: true });
-    }
-  }
+  try {
+    withFileLock(file, (lockedFile) => {
+      if (!fs.existsSync(lockedFile)) return;
+      try {
+        const runtime = JSON.parse(fs.readFileSync(lockedFile, 'utf8'));
+        if (runtime.pid === pid) fs.rmSync(lockedFile, { force: true });
+      } catch {
+        // Leave an unparseable pointer in place: only remove when we can confirm
+        // it belongs to this pid, so a torn write never deletes another runtime's file.
+      }
+    });
+  } catch {}
   const globalFile = globalRuntimePath();
-  if (!fs.existsSync(globalFile)) return;
   try {
-    const runtime = JSON.parse(fs.readFileSync(globalFile, 'utf8'));
-    if (!runtime.pid || runtime.pid === pid) fs.rmSync(globalFile, { force: true });
-  } catch {
-    fs.rmSync(globalFile, { force: true });
-  }
+    withFileLock(globalFile, (lockedFile) => {
+      if (!fs.existsSync(lockedFile)) return;
+      try {
+        const runtime = JSON.parse(fs.readFileSync(lockedFile, 'utf8'));
+        if (runtime.pid === pid) fs.rmSync(lockedFile, { force: true });
+      } catch {
+        // Same as above: do not blind-delete a torn global runtime pointer.
+      }
+    });
+  } catch {}
 }

package/lib/shared/cleanup-error.mjs

@@ -0,0 +1,54 @@
+const CLEANUP_FAILURE = Symbol('cleanup-failure');
+
+export function aggregateCleanupFailure(
+  primaryError,
+  cleanupError,
+  message,
+  cleanup = {}
+) {
+  const error = new AggregateError(
+    [primaryError, cleanupError],
+    message,
+    { cause: primaryError }
+  );
+  error[CLEANUP_FAILURE] = true;
+  error.cleanup = Object.freeze({ ...cleanup });
+  return error;
+}
+
+export function cleanupFailureRoot(error) {
+  let current = error;
+  let foundCleanup = false;
+  const seen = new Set();
+  while (current instanceof AggregateError && current[CLEANUP_FAILURE] === true) {
+    if (seen.has(current) || current.errors?.[0] !== current.cause) return null;
+    seen.add(current);
+    foundCleanup = true;
+    current = current.cause;
+  }
+  return foundCleanup ? current : null;
+}
+
+export function replaceCleanupFailureRoot(error, replacement) {
+  const seen = new Set();
+  function rebuild(current) {
+    if (!(current instanceof AggregateError) || current[CLEANUP_FAILURE] !== true) {
+      return replacement;
+    }
+    if (seen.has(current) || current.errors?.length !== 2 || current.errors[0] !== current.cause) {
+      return null;
+    }
+    seen.add(current);
+    const primaryError = rebuild(current.cause);
+    if (primaryError === null) return null;
+    return aggregateCleanupFailure(
+      primaryError,
+      current.errors[1],
+      current.message,
+      current.cleanup
+    );
+  }
+
+  if (!(error instanceof AggregateError) || error[CLEANUP_FAILURE] !== true) return null;
+  return rebuild(error);
+}

package/lib/shared/clock-grace.mjs

@@ -0,0 +1,106 @@
+// Wall-clock jump protection (hb-05). All age/TTL comparisons use wall-clock
+// seconds (last_seen_at / expires_at are persisted across processes), so a
+// forward jump (macOS sleep, NTP step) makes every peer look stale and every
+// lock look expired at once. The long-lived web runtime detects the jump and
+// persists a grace deadline in the DB meta table; every process (runtime and
+// short-lived CLIs, which share the same system clock) then suppresses
+// age-based decisions until the grace window passes. This is deliberately
+// conservative: keeping a truly-dead peer alive for one grace window is safe;
+// reaping/taking-over a live peer is not.
+
+export const CLOCK_GRACE_SEC = 120;
+
+function parsePersistedDeadline(value) {
+  if (typeof value !== 'string' || !/^(0|[1-9][0-9]*)$/.test(value)) {
+    throw new Error('invalid persisted deadline for clock grace');
+  }
+  const persisted = Number(value);
+  if (!Number.isSafeInteger(persisted) || persisted < 0) {
+    throw new Error('invalid persisted deadline for clock grace');
+  }
+  return persisted;
+}
+
+function missingMetaTable(error) {
+  return /no such table:\s*(?:main\.)?meta\b/i.test(String(error?.message || ''));
+}
+
+export function readClockGraceUntil(db) {
+  let row;
+  try {
+    row = db.prepare("SELECT value FROM meta WHERE key = 'clock_grace_until'").get();
+  } catch (error) {
+    if (missingMetaTable(error)) return 0;
+    throw error;
+  }
+  if (!row) return 0;
+  try {
+    return parsePersistedDeadline(row.value);
+  } catch {
+    return 0;
+  }
+}
+
+export function writeClockGraceUntil(db, until) {
+  if (typeof until !== 'number' || !Number.isSafeInteger(until) || until < 0) {
+    throw new TypeError('clock grace deadline must be a non-negative safe integer');
+  }
+
+  let row = db.prepare(`
+    INSERT INTO meta(key, value) VALUES ('clock_grace_until', ?)
+    ON CONFLICT(key) DO UPDATE SET value = CAST(
+      MAX(CAST(meta.value AS INTEGER), CAST(excluded.value AS INTEGER)) AS TEXT
+    )
+    WHERE (meta.value = '0' OR (
+        meta.value GLOB '[1-9]*'
+        AND meta.value NOT GLOB '*[^0-9]*'
+      ))
+      AND CAST(meta.value AS INTEGER) BETWEEN 0 AND 9007199254740991
+    RETURNING value AS deadline
+  `).get(String(until));
+
+  if (!row) {
+    // CS-08: the stored value is non-canonical (corrupt or legacy). Repair it by
+    // overwriting with the new deadline instead of failing the whole owning
+    // transaction.
+    row = db.prepare(`
+      INSERT INTO meta(key, value) VALUES ('clock_grace_until', ?)
+      ON CONFLICT(key) DO UPDATE SET value = excluded.value
+      RETURNING value AS deadline
+    `).get(String(until));
+  }
+  if (!row) throw new Error('clock grace deadline could not be persisted');
+  return parsePersistedDeadline(row.deadline);
+}
+
+// True while the grace window is active at wall-clock second `nowSec`.
+export function clockGraceSuppressed(nowSec, graceUntil) {
+  return Number(graceUntil) > Number(nowSec);
+}
+
+// Classify a wall-clock probe delta (ms between two ticks of a known-interval
+// poller). Returns null when the delta is normal, or { kind } for a forward
+// jump (sleep / NTP step forward) or backward jump (NTP step back).
+export function classifyClockJump(deltaMs, { tickMs = 30000, forwardFactor = 3, backwardMs = 5000 } = {}) {
+  if (deltaMs > tickMs * forwardFactor) return { kind: 'forward', deltaMs };
+  if (deltaMs < -backwardMs) return { kind: 'backward', deltaMs };
+  return null;
+}
+
+export function classifyClockDrift({ wallDeltaMs, monotonicDeltaMs } = {}, {
+  forwardMs = 60000,
+  backwardMs = 5000
+} = {}) {
+  if (![wallDeltaMs, monotonicDeltaMs, forwardMs, backwardMs].every(Number.isFinite) ||
+      monotonicDeltaMs < 0 || forwardMs < 0 || backwardMs < 0) {
+    throw new TypeError('clock drift inputs must be finite non-negative intervals');
+  }
+  const driftMs = wallDeltaMs - monotonicDeltaMs;
+  if (driftMs > forwardMs) {
+    return { kind: 'forward', driftMs, wallDeltaMs, monotonicDeltaMs };
+  }
+  if (driftMs < -backwardMs) {
+    return { kind: 'backward', driftMs, wallDeltaMs, monotonicDeltaMs };
+  }
+  return null;
+}

package/lib/shared/errors.mjs

@@ -1,3 +1,5 @@
+import { cleanupFailureRoot } from './cleanup-error.mjs';
+
 export class CliError extends Error {
   constructor(code, message, extra = {}) {
     super(message);
@@ -5,3 +7,10 @@ export class CliError extends Error {
     this.extra = extra;
   }
 }
+
+export function publicCliFailure(error) {
+  if (error instanceof CliError) return { error, cleanupFailed: false };
+  const rootError = cleanupFailureRoot(error);
+  if (rootError instanceof CliError) return { error: rootError, cleanupFailed: true };
+  return null;
+}