@logicseek/hello-cc 0.1.9 → 1.0.0

package/lib/core/coordination/lease-renewal.mjs

@@ -0,0 +1,102 @@
+import { CliError } from '../../shared/errors.mjs';
+
+function requireSafeTimestamp(value, name) {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new TypeError(`${name} must be a non-negative safe integer`);
+  }
+  return value;
+}
+
+export function leaseDeadline(nowSec, ttl, { option = '--ttl' } = {}) {
+  const observedAt = requireSafeTimestamp(nowSec, 'nowSec');
+  const duration = requirePositiveSafeInteger(ttl, 'ttl');
+  const deadline = observedAt + duration;
+  if (!Number.isSafeInteger(deadline)) {
+    throw new CliError('BAD_ARGS', `${option} produces an unsafe lease deadline`);
+  }
+  return deadline;
+}
+
+function requirePositiveSafeInteger(value, name) {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new TypeError(`${name} must be a positive safe integer`);
+  }
+  return value;
+}
+
+function persistedLockTtls(db, owner, observedAt, includeExpired) {
+  const expiryPredicate = includeExpired ? '' : ' AND expires_at > ?';
+  const args = includeExpired ? [owner] : [owner, observedAt];
+  return db.prepare(`
+    SELECT resource, ttl_sec
+    FROM locks
+    WHERE owner = ?${expiryPredicate}
+  `).all(...args);
+}
+
+function requireRenewablePersistedTtls(db, owner, observedAt, includeExpired, { cap = null } = {}) {
+  for (const row of persistedLockTtls(db, owner, observedAt, includeExpired)) {
+    const ttl = Number(row.ttl_sec);
+    try {
+      requirePositiveSafeInteger(ttl, 'persisted lock TTL');
+      leaseDeadline(observedAt, cap === null ? ttl : Math.min(ttl, cap), {
+        option: 'persisted lock TTL'
+      });
+    } catch {
+      throw new CliError('BAD_STATE', `Lock ${row.resource} has an unsafe persisted TTL`);
+    }
+  }
+}
+
+export function renewOwnedLocks(db, {
+  owner,
+  nowSec,
+  ttlOverride = null,
+  ttlCap = null,
+  includeExpired = false
+} = {}) {
+  if (!db || typeof db.prepare !== 'function') throw new TypeError('db is required');
+  if (typeof owner !== 'string' || !owner) throw new TypeError('owner is required');
+  const observedAt = requireSafeTimestamp(nowSec, 'nowSec');
+  const expiryPredicate = includeExpired ? '' : ' AND expires_at > ?';
+
+  if (ttlOverride !== null) {
+    const ttl = requirePositiveSafeInteger(ttlOverride, 'ttlOverride');
+    const target = leaseDeadline(observedAt, ttl);
+    const args = includeExpired
+      ? [target, ttl, owner]
+      : [target, ttl, owner, observedAt];
+    return db.prepare(`
+      UPDATE locks
+      SET expires_at = MAX(expires_at, ?), ttl_sec = ?
+      WHERE owner = ?${expiryPredicate}
+    `).run(...args).changes;
+  }
+
+  if (ttlCap !== null) {
+    const cap = requirePositiveSafeInteger(ttlCap, 'ttlCap');
+    leaseDeadline(observedAt, cap, { option: 'hook TTL cap' });
+    requireRenewablePersistedTtls(db, owner, observedAt, includeExpired, { cap });
+    const args = includeExpired
+      ? [observedAt, cap, owner]
+      : [observedAt, cap, owner, observedAt];
+    return db.prepare(`
+      UPDATE locks
+      SET expires_at = MAX(expires_at, ? + MIN(ttl_sec, ?))
+      WHERE owner = ?${expiryPredicate}
+    `).run(...args).changes;
+  }
+
+  requireRenewablePersistedTtls(db, owner, observedAt, includeExpired);
+  const maxTtl = Number.MAX_SAFE_INTEGER - observedAt;
+  const args = includeExpired
+    ? [observedAt, owner, maxTtl]
+    : [observedAt, owner, observedAt, maxTtl];
+  return db.prepare(`
+    UPDATE locks
+    SET expires_at = MAX(expires_at, ? + ttl_sec)
+    WHERE owner = ?${expiryPredicate}
+      AND typeof(ttl_sec) = 'integer'
+      AND ttl_sec BETWEEN 1 AND ?
+  `).run(...args).changes;
+}

package/lib/core/coordination/lock-evidence.mjs

@@ -0,0 +1,77 @@
+import { readClockGraceUntil } from '../../shared/clock-grace.mjs';
+import { locksConflict } from './locks.mjs';
+
+export function captureLockAcquireSubject(db, options = {}) {
+  const taskId = options.taskId || null;
+  const requested = options.requested || null;
+  const observedAt = Number(options.now);
+  const task = taskId ? db.prepare('SELECT * FROM tasks WHERE id = ?').get(taskId) || null : null;
+  const lockRows = requested
+    ? db.prepare(`
+        SELECT * FROM locks
+        WHERE base_resource = ?
+           OR (base_resource IS NULL AND resource = ?)
+        ORDER BY resource ASC
+      `).all(requested.base_resource, requested.base_resource)
+    : db.prepare('SELECT * FROM locks ORDER BY resource ASC').all();
+  const locks = lockRows
+    .filter((lock) => !requested || locksConflict(lock, requested));
+  const expiredLocks = locks.filter((lock) => Number(lock.expires_at) <= observedAt);
+  const owners = [];
+  for (const owner of [...new Set(expiredLocks.map((lock) => lock.owner).filter(Boolean))].sort()) {
+    const peer = db.prepare(`
+      SELECT id, status, pid, pid_start_token, pid_command_hash, last_seen_at
+      FROM peers WHERE id = ?
+    `).get(owner) || null;
+    const binding = db.prepare(`
+      SELECT peer, transport, runtime_target, updated_at
+      FROM peer_bindings WHERE peer = ?
+    `).get(owner) || null;
+    owners.push({ owner, peer, binding });
+  }
+  return { task, locks, owners, graceUntil: readClockGraceUntil(db), observedAt };
+}
+
+export function sameLockAcquireSubject(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+export function observeLockOwnerEvidence(subject, observePeer) {
+  return new Map((subject.owners || []).map(({ owner, peer, binding }) => [
+    owner,
+    peer ? observePeer(peer, binding) : { state: 'unknown', reason: 'peer_missing' }
+  ]));
+}
+
+export function captureLockRenewalSubject(db, { owner, now }) {
+  const peer = db.prepare(`
+    SELECT id, status, pid, pid_start_token, pid_command_hash, last_seen_at
+    FROM peers WHERE id = ?
+  `).get(owner) || null;
+  const binding = db.prepare(`
+    SELECT peer, transport, runtime_target, updated_at
+    FROM peer_bindings WHERE peer = ?
+  `).get(owner) || null;
+  const locks = db.prepare(`
+    SELECT * FROM locks WHERE owner = ? ORDER BY resource ASC
+  `).all(owner);
+  return {
+    owner,
+    peer,
+    binding,
+    locks,
+    graceUntil: readClockGraceUntil(db),
+    observedAt: Number(now)
+  };
+}
+
+export function clockCandidatesFromLocks(subject, evidenceByOwner, owner = null) {
+  return (subject.locks || [])
+    .filter((lock) => Number(lock.expires_at) <= subject.observedAt && (!owner || lock.owner === owner))
+    .map((lock) => ({
+      boundary: Number(lock.expires_at),
+      evidence: evidenceByOwner.get(lock.owner)?.state || 'unknown',
+      owner: lock.owner,
+      resource: lock.resource
+    }));
+}

package/lib/core/coordination/optimistic-evidence.mjs

@@ -0,0 +1,26 @@
+import { tx } from '../../db/schema.mjs';
+import { CliError } from '../../shared/errors.mjs';
+
+export function runOptimisticEvidenceMutation(db, options = {}) {
+  const attempts = Number(options.attempts || 2);
+  for (let attempt = 0; attempt < attempts; attempt += 1) {
+    const subject = options.capture(db);
+    const evidence = options.observe(subject);
+    let subjectChanged = false;
+    const result = tx(db, () => {
+      const current = options.capture(db);
+      if (!options.same(subject, current)) {
+        subjectChanged = true;
+        return null;
+      }
+      const prepared = typeof options.beforeMutate === 'function'
+        ? options.beforeMutate(current, evidence)
+        : undefined;
+      return options.mutate(current, evidence, prepared);
+    });
+    if (!subjectChanged) return result;
+  }
+  throw new CliError('SUBJECT_CHANGED', options.changedMessage || 'Coordination subject changed while evidence was being observed; retry', {
+    retryable: true
+  });
+}

package/lib/core/coordination/tasks.mjs

@@ -1,7 +1,11 @@
 import { tx } from '../../db/schema.mjs';
 import { CliError } from '../../shared/errors.mjs';
+import { clockGraceSuppressed, readClockGraceUntil } from '../../shared/clock-grace.mjs';
+import { inspectProcessIdentity } from '../../process/identity.mjs';
+import { peerEvidenceAllowsReap, resolvePeerEvidence } from '../peers/evidence.mjs';
 
 const DEFAULT_ACTIVE_PEER_TTL = 600;
+const UNKNOWN_EVIDENCE_GRACE_SEC = 120;
 
 function defaultNow() {
   return Math.floor(Date.now() / 1000);
@@ -13,10 +17,28 @@ function noopMessage() {
   return null;
 }
 
+function processEvidenceForPeer(row) {
+  return resolvePeerEvidence({
+    peer: row,
+    processes: row?.pid ? [{
+      name: 'peer',
+      storedIdentity: {
+        pid: Number(row.pid),
+        startToken: row.pid_start_token,
+        commandHash: row.pid_command_hash
+      },
+      current: inspectProcessIdentity(row.pid)
+    }] : []
+  });
+}
+
 export function createTaskStore(deps = {}) {
   const now = typeof deps.now === 'function' ? deps.now : defaultNow;
   const addEvent = typeof deps.addEvent === 'function' ? deps.addEvent : noopEvent;
   const sendMessage = typeof deps.sendMessage === 'function' ? deps.sendMessage : noopMessage;
+  const observeClockSafety = typeof deps.observeClockSafety === 'function'
+    ? deps.observeClockSafety
+    : null;
   const activePeerTtl = Number(deps.activePeerTtl || DEFAULT_ACTIVE_PEER_TTL);
 
   function taskOwnedError(peer, row, action) {
@@ -29,6 +51,23 @@ export function createTaskStore(deps = {}) {
     });
   }
 
+  function takeoverSubject(db, id) {
+    const row = db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) || null;
+    const ownerRow = row?.owner ? db.prepare(`
+      SELECT id, status, pid, pid_start_token, pid_command_hash, last_seen_at
+      FROM peers WHERE id = ?
+    `).get(row.owner) || null : null;
+    const binding = row?.owner ? db.prepare(`
+      SELECT peer, transport, runtime_target, updated_at
+      FROM peer_bindings WHERE peer = ?
+    `).get(row.owner) || null : null;
+    return { row, ownerRow, binding, graceUntil: readClockGraceUntil(db) };
+  }
+
+  function sameTakeoverSubject(left, right) {
+    return JSON.stringify(left) === JSON.stringify(right);
+  }
+
   function claimTaskRowsForPeer(db, peer, ids, { force = false, source = null } = {}) {
     return tx(db, () => {
       const tasks = [];
@@ -61,7 +100,13 @@ export function createTaskStore(deps = {}) {
     });
   }
 
-  function takeoverPolicyDetails(db, row, peer, { policy = 'any', staleAfter = activePeerTtl } = {}) {
+  function takeoverPolicyDetails(db, row, peer, {
+    policy = 'blocked-or-stale',
+    staleAfter = activePeerTtl,
+    ownerEvidence = null,
+    ownerRow: suppliedOwnerRow = undefined,
+    graceUntil: suppliedGraceUntil = undefined
+  } = {}) {
     const normalized = String(policy || 'any').toLowerCase();
     const allowed = new Set(['any', 'blocked', 'stale', 'blocked-or-stale']);
     if (!allowed.has(normalized)) {
@@ -69,58 +114,133 @@ export function createTaskStore(deps = {}) {
     }
     const previousOwner = row.owner || null;
     const blocked = row.status === 'blocked';
-    const ownerRow = previousOwner ? db.prepare('SELECT id, last_seen_at FROM peers WHERE id = ?').get(previousOwner) : null;
+    const ownerRow = suppliedOwnerRow === undefined && previousOwner ? db.prepare(`
+      SELECT id, status, pid, pid_start_token, pid_command_hash, last_seen_at
+      FROM peers WHERE id = ?
+    `).get(previousOwner) : (suppliedOwnerRow || null);
     const ownerAge = ownerRow ? now() - Number(ownerRow.last_seen_at || 0) : null;
-    const ownerStale = Boolean(previousOwner && previousOwner !== peer && (!ownerRow || ownerAge > staleAfter));
+    const evidence = ownerEvidence || { state: 'unknown', reason: ownerRow ? 'evidence_not_observed' : 'peer_missing' };
+    const ownerDead = evidence.state === 'dead';
+    const ownerLive = evidence.state === 'live';
+    // hb-05: during a wall-clock grace window (sleep/NTP jump) age is
+    // untrustworthy; never treat the owner as stale so a live owner's task is
+    // not taken over right after the machine wakes.
+    const graceUntil = suppliedGraceUntil === undefined ? readClockGraceUntil(db) : suppliedGraceUntil;
+    const graceActive = clockGraceSuppressed(now(), graceUntil);
+    const unknownExpired = ownerRow && peerEvidenceAllowsReap(evidence, {
+      nowSec: now(),
+      lastSeenAt: Number(ownerRow.last_seen_at || 0),
+      staleAfterSec: Math.max(staleAfter, UNKNOWN_EVIDENCE_GRACE_SEC),
+      graceUntil
+    });
+    const ownerStale = Boolean(previousOwner && previousOwner !== peer && (
+      !ownerRow || ownerDead ||
+      (!ownerLive && !graceActive && unknownExpired)
+    ));
     const alreadyOwner = previousOwner === peer;
     const ok = alreadyOwner ||
       normalized === 'any' ||
       (normalized === 'blocked' && blocked) ||
       (normalized === 'stale' && ownerStale) ||
       (normalized === 'blocked-or-stale' && (blocked || ownerStale));
-    return { policy: normalized, blocked, owner_stale: ownerStale, owner_age_sec: ownerAge, stale_after_sec: staleAfter, ok };
+    return {
+      policy: normalized,
+      blocked,
+      owner_stale: ownerStale,
+      owner_age_sec: ownerAge,
+      owner_evidence_state: evidence.state,
+      owner_evidence_reason: evidence.reason,
+      stale_after_sec: staleAfter,
+      ok
+    };
   }
 
-  function takeOverTaskForPeer(db, peer, id, { reason, policy = 'any', staleAfter = activePeerTtl, source = null } = {}) {
-    return tx(db, () => {
-      const row = db.prepare('SELECT * FROM tasks WHERE id = ?').get(id);
+  function takeOverTaskForPeer(db, peer, id, {
+    reason,
+    policy = 'blocked-or-stale',
+    staleAfter = activePeerTtl,
+    source = null,
+    ownerEvidence = null,
+    ownerEvidenceFor = null
+  } = {}) {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      const subject = takeoverSubject(db, id);
+      const row = subject.row;
       if (!row) throw new CliError('NOT_FOUND', `Task #${id} does not exist`);
       if (['done', 'abandoned'].includes(row.status)) {
         throw new CliError('BAD_STATE', `Task #${id} is ${row.status}`);
       }
-      const policyDetails = takeoverPolicyDetails(db, row, peer, { policy, staleAfter });
-      if (!policyDetails.ok) {
-        throw new CliError('TAKEOVER_POLICY', `Task #${id} does not match takeover policy ${policyDetails.policy}`, {
-          task_id: id,
-          status: row.status,
-          owner: row.owner || null,
+      const observedEvidence = row.owner && typeof ownerEvidenceFor === 'function'
+        ? ownerEvidenceFor(row.owner, row, subject.ownerRow, subject.binding)
+        : (ownerEvidence || (subject.ownerRow
+            ? processEvidenceForPeer(subject.ownerRow)
+            : { state: 'unknown', reason: 'peer_missing' }));
+      if (observeClockSafety) {
+        observeClockSafety({
+          db,
+          row,
+          ownerRow: subject.ownerRow,
+          binding: subject.binding,
+          evidence: observedEvidence,
+          staleAfter
+        });
+      }
+      let subjectChanged = false;
+      const result = tx(db, () => {
+        const current = takeoverSubject(db, id);
+        if (!sameTakeoverSubject(subject, current)) {
+          subjectChanged = true;
+          return null;
+        }
+        const policyDetails = takeoverPolicyDetails(db, current.row, peer, {
+          policy,
+          staleAfter,
+          ownerEvidence: observedEvidence,
+          ownerRow: current.ownerRow,
+          graceUntil: current.graceUntil
+        });
+        if (!policyDetails.ok) {
+          throw new CliError('TAKEOVER_POLICY', `Task #${id} does not match takeover policy ${policyDetails.policy}`, {
+            task_id: id,
+            status: current.row.status,
+            owner: current.row.owner || null,
+            policy: policyDetails.policy,
+            owner_stale: policyDetails.owner_stale,
+            owner_age_sec: policyDetails.owner_age_sec,
+            owner_evidence_state: policyDetails.owner_evidence_state,
+            owner_evidence_reason: policyDetails.owner_evidence_reason,
+            stale_after_sec: policyDetails.stale_after_sec
+          });
+        }
+        const previousOwner = current.row.owner || null;
+        const previousAssignee = current.row.assignee || null;
+        const t = now();
+        db.prepare(`
+          UPDATE tasks
+          SET owner = ?, status = 'claimed', claimed_at = COALESCE(claimed_at, ?), updated_at = ?
+          WHERE id = ?
+        `).run(peer, t, t, id);
+        addEvent(db, 'task.takeover', peer, id, {
+          previous_owner: previousOwner,
+          previous_assignee: previousAssignee,
+          reason,
           policy: policyDetails.policy,
           owner_stale: policyDetails.owner_stale,
-          owner_age_sec: policyDetails.owner_age_sec,
-          stale_after_sec: policyDetails.stale_after_sec
+          owner_evidence_state: policyDetails.owner_evidence_state,
+          owner_evidence_reason: policyDetails.owner_evidence_reason,
+          stale_after_sec: staleAfter,
+          ...(source ? { source } : {})
         });
-      }
-      const previousOwner = row.owner || null;
-      const previousAssignee = row.assignee || null;
-      const t = now();
-      db.prepare(`
-        UPDATE tasks
-        SET owner = ?, status = 'claimed', claimed_at = COALESCE(claimed_at, ?), updated_at = ?
-        WHERE id = ?
-      `).run(peer, t, t, id);
-      addEvent(db, 'task.takeover', peer, id, {
-        previous_owner: previousOwner,
-        previous_assignee: previousAssignee,
-        reason,
-        policy: policyDetails.policy,
-        owner_stale: policyDetails.owner_stale,
-        stale_after_sec: staleAfter,
-        ...(source ? { source } : {})
+        if (previousOwner && previousOwner !== peer) {
+          sendMessage(db, peer, previousOwner, id, 'task.takeover', `Task #${id} taken over by ${peer}: ${reason}`);
+        }
+        return db.prepare('SELECT * FROM tasks WHERE id = ?').get(id);
       });
-      if (previousOwner && previousOwner !== peer) {
-        sendMessage(db, peer, previousOwner, id, 'task.takeover', `Task #${id} taken over by ${peer}: ${reason}`);
-      }
-      return db.prepare('SELECT * FROM tasks WHERE id = ?').get(id);
+      if (!subjectChanged) return result;
+    }
+    throw new CliError('SUBJECT_CHANGED', `Task #${id} changed while owner evidence was being observed; retry`, {
+      task_id: id,
+      retryable: true
     });
   }