harnery 0.43.0 → 0.44.0

package/src/core/inbox/peer-messages.ts

@@ -0,0 +1,94 @@
+import { resolveBinName } from "../config.ts";
+import type { HarneryInboxLimits, HarneryInboxMessageRecordV1 } from "./contract.ts";
+import { HarneryInboxService } from "./service.ts";
+import { type MessageTargetInput, resolveMessageTarget } from "./targets.ts";
+
+const DAY = 24 * 60 * 60_000;
+
+/** Preserve the existing ping body/count bounds. Pending messages have no TTL;
+ * the lifecycle planner protects them through the idle finalization window. */
+export const PEER_INBOX_LIMITS: HarneryInboxLimits = {
+  max_message_body_bytes: 4_000,
+  max_pending_count: 25,
+  max_pending_bytes: 100_000,
+  max_history_bytes: 1024 * 1024,
+  max_history_records: 10_000,
+  warning_pressure_ratio: 0.8,
+  max_surface_count: 10,
+  max_surface_bytes: 16_000,
+  max_surface_tokens: 4_000,
+  surfaced_grace_ms: 7 * DAY,
+  terminal_grace_ms: 7 * DAY,
+};
+
+export function peerInbox(coordRoot: string): HarneryInboxService {
+  return new HarneryInboxService({ coord_root: coordRoot, limits: PEER_INBOX_LIMITS });
+}
+
+/** CLI and web use this one path. A receipt proves a synced append, never that
+ * a process is awake or that the recipient has noticed the message. */
+export function sendPeerMessage(input: {
+  coordRoot: string;
+  target: MessageTargetInput;
+  senderInstanceId: string | null;
+  senderName: string;
+  body: string;
+}) {
+  const target = resolveMessageTarget(input.coordRoot, input.target);
+  if (!target.ok) return target;
+  const receipt = peerInbox(input.coordRoot).send({
+    sender_instance_id: input.senderInstanceId,
+    sender_display_name: input.senderName,
+    recipient_instance_id: target.target.instance_id,
+    recipient_display_name: target.target.name,
+    body: input.body.trim(),
+  });
+  return { ok: true as const, target: target.target, ...receipt, state: target.target.delivery };
+}
+
+export function formatPeerInbox(
+  coordRoot: string,
+  messages: readonly HarneryInboxMessageRecordV1[],
+): string {
+  if (messages.length === 0) return "";
+  const onlyAgents = messages.every(({ sender_instance_id }) => sender_instance_id !== null);
+  const header = onlyAgents
+    ? messages.length === 1
+      ? "Message from a peer agent:"
+      : `${messages.length} messages from peer agents:`
+    : messages.length === 1
+      ? "Message for this task:"
+      : `${messages.length} messages for this task:`;
+  const body = messages
+    .map(
+      (message) =>
+        `  ${message.message_id} from ${message.sender_instance_id === null ? "Operator" : `agent-${message.sender_display_name}`} (${message.created_at}):\n` +
+        message.body
+          .split("\n")
+          .map((line) => `    ${line}`)
+          .join("\n"),
+    )
+    .join("\n\n");
+  const replyTarget = messages.find(({ sender_instance_id }) => sender_instance_id !== null);
+  return (
+    `${header}\n\n${body}\n\nMessages are bound to this task instance.` +
+    (replyTarget
+      ? ` Reply with \`${resolveBinName(coordRoot)} agents ping ${replyTarget.sender_display_name} ` +
+        `--instance-id ${replyTarget.sender_instance_id} "<your reply>"\`.`
+      : "")
+  );
+}
+
+/** Wait for every earlier stdout write to finish before acknowledging context.
+ * If output fails, the caller throws and leaves the batch pending. */
+export async function acknowledgeEmittedPeerInbox(
+  coordRoot: string,
+  instanceId: string,
+  messageIds: readonly string[],
+): Promise<void> {
+  if (messageIds.length === 0) return;
+  await new Promise<void>((resolve, reject) => {
+    process.stdout.write("", (error) => (error ? reject(error) : resolve()));
+  });
+  peerInbox(coordRoot).acknowledgeSurface(instanceId, messageIds);
+}

package/src/core/inbox/service.ts

@@ -1,7 +1,12 @@
 import { randomUUID } from "node:crypto";
 import { existsSync, statSync } from "node:fs";
 import { join, resolve } from "node:path";
-import { readCrashSafeJsonlFile, updateCrashSafeJsonlFile } from "../storage/durable-history.ts";
+import {
+  type CrashSafeJsonlUpdate,
+  type DurableHistoryReadOptions,
+  readCrashSafeJsonlFile,
+  updateCrashSafeJsonlFile,
+} from "../storage/durable-history.ts";
 import {
   HARNERY_INBOX_RECORD_SCHEMA,
   HARNERY_INBOX_STATUS_SCHEMA,
@@ -22,7 +27,7 @@ export interface HarneryInboxServiceOptions {
 }
 
 export interface HarneryInboxSendInput {
-  sender_instance_id: string;
+  sender_instance_id: string | null;
   sender_display_name: string;
   recipient_instance_id: string;
   recipient_display_name: string;
@@ -59,7 +64,7 @@ export class HarneryInboxService {
       schema: HARNERY_INBOX_RECORD_SCHEMA,
       kind: "message",
       message_id: `msg_${this.#options.id?.() ?? randomUUID()}`,
-      sender_instance_id: opaqueId(input.sender_instance_id, "sender instance"),
+      sender_instance_id: senderId(input.sender_instance_id),
       sender_display_name: boundedDisplay(input.sender_display_name),
       recipient_instance_id: recipient,
       recipient_display_name: boundedDisplay(input.recipient_display_name),
@@ -68,7 +73,7 @@ export class HarneryInboxService {
       body_bytes: bodyBytes,
     };
     const path = inboxPath(this.#options.coord_root, recipient);
-    return updateCrashSafeJsonlFile<HarneryInboxRecordV1, HarneryInboxSendReceipt>(
+    return updateInboxHistory<HarneryInboxRecordV1, HarneryInboxSendReceipt>(
       path,
       readOptions(limits),
       (records) => {
@@ -88,10 +93,12 @@ export class HarneryInboxService {
   }
 
   records(recipientInstanceId: string): HarneryInboxRecordV1[] {
-    return readCrashSafeJsonlFile<HarneryInboxRecordV1>(
+    const records = readCrashSafeJsonlFile<HarneryInboxRecordV1>(
       inboxPath(this.#options.coord_root, opaqueId(recipientInstanceId, "recipient instance")),
       readOptions(this.#options.limits),
     );
+    for (const record of records) validateRecord(record);
+    return records;
   }
 
   pending(recipientInstanceId: string): HarneryInboxMessageRecordV1[] {
@@ -133,13 +140,45 @@ export class HarneryInboxService {
       max_tokens: this.#options.limits.max_surface_tokens,
     },
   ): Promise<HarneryInboxSurfaceResult> {
-    validateSurfaceBudget(budget);
     const recipient = opaqueId(recipientInstanceId, "recipient instance");
-    const selected = selectOldestFitting(this.pending(recipient), budget);
+    const selected = this.prepareSurface(recipient, budget);
     await emit(selected);
-    if (selected.length > 0) {
+    this.acknowledgeSurface(
+      recipient,
+      selected.map(({ message_id }) => message_id),
+    );
+    const remaining = this.pending(recipient);
+    return {
+      emitted: selected,
+      remaining_pending_count: remaining.length,
+      remaining_pending_bytes: remaining.reduce((sum, record) => sum + record.body_bytes, 0),
+      repeated_after_crash_possible: true,
+    };
+  }
+
+  /** Preparing prompt text is read-only. A failed or abandoned emission must
+   * leave the same messages pending for the recipient's next prompt. */
+  prepareSurface(
+    recipientInstanceId: string,
+    budget: HarneryInboxSurfaceBudget = {
+      max_count: this.#options.limits.max_surface_count,
+      max_bytes: this.#options.limits.max_surface_bytes,
+      max_tokens: this.#options.limits.max_surface_tokens,
+    },
+  ): HarneryInboxMessageRecordV1[] {
+    validateSurfaceBudget(budget);
+    return selectOldestFitting(this.pending(recipientInstanceId), budget);
+  }
+
+  /** Call only after the selected IDs have actually been emitted. A message
+   * appended while output is in flight remains pending rather than being
+   * included in an acknowledgement of an older batch. */
+  acknowledgeSurface(recipientInstanceId: string, messageIds: readonly string[]): void {
+    const recipient = opaqueId(recipientInstanceId, "recipient instance");
+    const selected = new Set(messageIds);
+    if (selected.size > 0) {
       const surfacedAt = now(this.#options).toISOString();
-      updateCrashSafeJsonlFile<HarneryInboxRecordV1, void>(
+      updateInboxHistory<HarneryInboxRecordV1, void>(
         inboxPath(this.#options.coord_root, recipient),
         readOptions(this.#options.limits),
         (records) => {
@@ -147,9 +186,9 @@ export class HarneryInboxService {
             project(records).pending.map(({ message_id }) => message_id),
           );
           return {
-            append: selected
-              .filter(({ message_id }) => stillPending.has(message_id))
-              .map(({ message_id }) => ({
+            append: [...selected]
+              .filter((message_id) => stillPending.has(message_id))
+              .map((message_id) => ({
                 schema: HARNERY_INBOX_RECORD_SCHEMA,
                 kind: "surfaced" as const,
                 message_id,
@@ -160,13 +199,32 @@ export class HarneryInboxService {
         },
       );
     }
-    const remaining = this.pending(recipient);
-    return {
-      emitted: selected,
-      remaining_pending_count: remaining.length,
-      remaining_pending_bytes: remaining.reduce((sum, record) => sum + record.body_bytes, 0),
-      repeated_after_crash_possible: true,
-    };
+  }
+}
+
+/** The shared writer refuses an occupied lease before invoking the update.
+ * Briefly retry only that exact refusal so concurrent peer senders both get
+ * a chance to append. All validation, authority and I/O failures still fail
+ * immediately, and an occupied live lease is never removed or overridden. */
+function updateInboxHistory<T, R>(
+  path: string,
+  options: DurableHistoryReadOptions,
+  update: (records: readonly T[]) => CrashSafeJsonlUpdate<R>,
+): R {
+  const deadline = performance.now() + 2_000;
+  const pause = new Int32Array(new SharedArrayBuffer(4));
+  while (true) {
+    try {
+      return updateCrashSafeJsonlFile(path, options, update);
+    } catch (error) {
+      if (
+        !(error instanceof Error) ||
+        error.message !== `durable history lease busy: ${path}.lease` ||
+        performance.now() >= deadline
+      )
+        throw error;
+      Atomics.wait(pause, 0, 0, Math.min(10, deadline - performance.now()));
+    }
   }
 }
 
@@ -270,10 +328,15 @@ function now(options: HarneryInboxServiceOptions): Date {
 }
 
 function opaqueId(value: string, field: string): string {
-  if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(value)) throw new Error(`invalid ${field} id`);
+  if (typeof value !== "string" || !/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(value))
+    throw new Error(`invalid ${field} id`);
   return value;
 }
 
+function senderId(value: string | null): string | null {
+  return value === null ? null : opaqueId(value, "sender instance");
+}
+
 function boundedDisplay(value: string): string {
   const result = value.trim();
   if (result.length === 0 || Buffer.byteLength(result) > 256)
@@ -283,8 +346,11 @@ function boundedDisplay(value: string): string {
 
 function validateRecord(record: HarneryInboxRecordV1): void {
   if (record.schema !== HARNERY_INBOX_RECORD_SCHEMA) throw new Error("unknown inbox record schema");
-  if (record.kind === "message" && Buffer.byteLength(record.body) !== record.body_bytes) {
-    throw new Error(`inbox message ${record.message_id} has invalid body_bytes`);
+  if (record.kind === "message") {
+    senderId(record.sender_instance_id);
+    if (Buffer.byteLength(record.body) !== record.body_bytes) {
+      throw new Error(`inbox message ${record.message_id} has invalid body_bytes`);
+    }
   }
 }
 

package/src/core/inbox/targets.ts

@@ -0,0 +1,148 @@
+import { resolveName } from "../agents/state/names.ts";
+import { coordFreshnessSeconds } from "../config.ts";
+import {
+  type CoordinationGenerationViewV3,
+  type CoordinationViewV3,
+  readCoordinationViewV3,
+  requireAuthoritySafeCoordinationViewV3,
+} from "../events/v3/coordination-view.ts";
+import {
+  liveInstanceIdV3,
+  nativeInstanceIdV3,
+  observeLiveEventLedgerRouteV3,
+} from "../events/v3/live-route-observer.ts";
+
+export interface MessageTargetInput {
+  name?: string;
+  instanceId?: string;
+  requireLive?: boolean;
+  now?: Date;
+}
+
+export type MessageTargetResult =
+  | {
+      ok: true;
+      target: { instance_id: string; name: string; delivery: "written-live" | "queued" };
+    }
+  | {
+      ok: false;
+      code: "no_peer" | "ambiguous_peer" | "not_live" | "authority_unsafe";
+      message: string;
+      candidates?: string[];
+    };
+
+/** Resolve one immutable recipient without extending presence or claiming attention. */
+export function resolveMessageTarget(
+  coordRoot: string,
+  input: MessageTargetInput,
+): MessageTargetResult {
+  const requestedName = normalizeName(input.name);
+  const requestedInstance = input.instanceId?.trim();
+  if (!requestedName && !requestedInstance) {
+    return { ok: false, code: "no_peer", message: "Supply an exact agent name or instance id." };
+  }
+  const now = input.now?.getTime() ?? Date.now();
+  if (!Number.isFinite(now)) return unsafe("The message addressing clock is invalid.");
+
+  let view: CoordinationViewV3;
+  try {
+    if (observeLiveEventLedgerRouteV3(coordRoot).state === "blocked") {
+      return unsafe("The event ledger cannot establish message recipient authority.");
+    }
+    view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(coordRoot));
+    if (!view.source_complete) return unsafe("The event ledger read is incomplete.");
+  } catch {
+    return unsafe("The event ledger cannot establish message recipient authority.");
+  }
+
+  // Address historical instances without reopening them. A current generation
+  // wins over its own terminal history; otherwise use its last ledger position.
+  const byInstance = new Map<string, CoordinationGenerationViewV3>();
+  for (const generation of Object.values(view.terminal_generations)) {
+    const prior = byInstance.get(generation.instance_id);
+    if (!prior || laterPosition(generation, prior))
+      byInstance.set(generation.instance_id, generation);
+  }
+  for (const generation of Object.values(view.instances))
+    byInstance.set(generation.instance_id, generation);
+  const freshnessMs = coordFreshnessSeconds(coordRoot) * 1_000;
+  const candidates = [...byInstance.values()].flatMap((generation) => {
+    const instanceId = nativeInstanceIdV3(generation.instance_id);
+    const name = resolveName(coordRoot, instanceId)?.name;
+    if (!name) return [];
+    if (requestedInstance && liveInstanceIdV3(requestedInstance) !== generation.instance_id)
+      return [];
+    if (requestedName && normalizeName(name) !== requestedName) return [];
+    const age = now - Date.parse(generation.last_observed_at);
+    const diagnostics = view.diagnostics_by_generation[generation.generation_id] ?? [];
+    return [
+      {
+        instanceId,
+        name,
+        safe: !diagnostics.some((diagnostic) => diagnostic.authority_blocking),
+        fresh:
+          generation.phase === "live" && Number.isFinite(age) && age >= 0 && age <= freshnessMs,
+      },
+    ];
+  });
+
+  if (candidates.length === 0) {
+    return {
+      ok: false,
+      code: "no_peer",
+      message: "No known agent matches that exact name and instance.",
+    };
+  }
+  if (candidates.some((candidate) => !candidate.safe)) {
+    return unsafe("A matching agent generation does not have safe coordination authority.");
+  }
+  const fresh = candidates.filter((candidate) => candidate.fresh);
+  const selected = requestedInstance ? candidates : fresh.length > 0 ? fresh : candidates;
+  if (selected.length !== 1) {
+    return {
+      ok: false,
+      code: "ambiguous_peer",
+      message: "Several agent instances hold that exact name; specify --instance-id.",
+      candidates: selected.map(({ instanceId }) => instanceId).sort(),
+    };
+  }
+  const target = selected[0]!;
+  if (input.requireLive && !target.fresh) {
+    return {
+      ok: false,
+      code: "not_live",
+      message:
+        "That agent is known but outside live presence; omit --require-live to queue its message.",
+    };
+  }
+  return {
+    ok: true,
+    target: {
+      instance_id: target.instanceId,
+      name: target.name,
+      delivery: target.fresh ? "written-live" : "queued",
+    },
+  };
+}
+
+function laterPosition(
+  left: CoordinationGenerationViewV3,
+  right: CoordinationGenerationViewV3,
+): boolean {
+  return (
+    left.source_position.segment_ordinal > right.source_position.segment_ordinal ||
+    (left.source_position.segment_ordinal === right.source_position.segment_ordinal &&
+      left.source_position.byte_offset > right.source_position.byte_offset)
+  );
+}
+
+function normalizeName(name: string | undefined): string {
+  return (name ?? "")
+    .trim()
+    .replace(/^agent-/i, "")
+    .toLowerCase();
+}
+
+function unsafe(message: string): MessageTargetResult {
+  return { ok: false, code: "authority_unsafe", message };
+}

package/src/core/inbox/watcher.ts

@@ -23,14 +23,13 @@ export async function* watchInbox(
 
 function delay(ms: number, signal: AbortSignal): Promise<void> {
   return new Promise((resolve) => {
-    const timer = setTimeout(resolve, ms);
-    signal.addEventListener(
-      "abort",
-      () => {
-        clearTimeout(timer);
-        resolve();
-      },
-      { once: true },
-    );
+    const finish = () => {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", finish);
+      resolve();
+    };
+    const timer = setTimeout(finish, ms);
+    signal.addEventListener("abort", finish, { once: true });
+    if (signal.aborted) finish();
   });
 }

package/src/core/journal/index.ts

@@ -20,6 +20,14 @@ import {
   readLiveCoordinationRow,
   readLiveCoordinationRows,
 } from "../agents/state/live-coordination-view.ts";
+import {
+  readCoordinationViewV3,
+  requireAuthoritySafeCoordinationViewV3,
+} from "../events/v3/coordination-view.ts";
+import {
+  liveInstanceIdV3,
+  observeLiveEventLedgerRouteV3,
+} from "../events/v3/live-route-observer.ts";
 
 /**
  * Agent journal: per-agent markdown journal at `.harnery/journal/<instance_id>.md`.
@@ -421,11 +429,20 @@ export function sweepOrphanJournals(): string[] {
   const dir = journalDir();
   const root = monorepoRoot();
   const archived: string[] = [];
+  if (!root) return archived;
+  let view: ReturnType<typeof readCoordinationViewV3>;
+  try {
+    if (observeLiveEventLedgerRouteV3(root).state === "blocked") return archived;
+    view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(root));
+    if (!view.source_complete) return archived;
+  } catch {
+    // A missing or unsafe read does not prove that an owner has ended.
+    return archived;
+  }
   for (const f of readdirSync(dir)) {
     if (!f.endsWith(".md")) continue;
     const instanceId = f.replace(/\.md$/, "");
-    const hb = root ? readLiveCoordinationRow(root, instanceId) : null;
-    if (!hb) {
+    if (!view.instances[liveInstanceIdV3(instanceId)]) {
       const dest = archiveJournal(instanceId);
       if (dest) archived.push(dest);
     }

package/src/core/resources/contract.ts

@@ -1,3 +1,5 @@
+import type { ProcessGenerationVerdict } from "../agents/state/proc-start.ts";
+
 export const RESOURCE_SNAPSHOT_SCHEMA_VERSION = 2;
 export const RESOURCE_SERVICE_STATUS_SCHEMA_VERSION = 1;
 
@@ -154,6 +156,8 @@ export interface ResourceServiceStatusRecord {
 export interface ResourceServiceStatus {
   running: boolean;
   stale: boolean;
+  generation: ProcessGenerationVerdict;
+  liveness: "running" | "stopped" | "unknown";
   record?: ResourceServiceStatusRecord;
   status_path: string;
   snapshot_path: string;

package/src/core/resources/service-status.ts

@@ -1,5 +1,9 @@
+import { existsSync } from "node:fs";
 import { hostname } from "node:os";
-import { checkPidToken } from "../agents/state/proc-start.ts";
+import {
+  type ProcessGenerationVerdict,
+  verifyProcessGeneration,
+} from "../agents/state/proc-start.ts";
 import {
   RESOURCE_SERVICE_STATUS_SCHEMA_VERSION,
   type ResourceServiceStatus,
@@ -9,9 +13,15 @@ import { readResourceServiceRecord, resourcePaths } from "./storage.ts";
 
 const STALE_HEARTBEAT_MS = 15_000;
 
+export interface ResourceProcessDependencies {
+  readToken?: (pid: number) => string | null;
+  probePid?: (pid: number) => unknown;
+}
+
 export function readResourceServiceStatus(
   coordRoot: string,
   nowMs = Date.now(),
+  dependencies: ResourceProcessDependencies = {},
 ): ResourceServiceStatus {
   const paths = resourcePaths(coordRoot);
   const record = readResourceServiceRecord(coordRoot);
@@ -19,21 +29,22 @@ export function readResourceServiceStatus(
     return {
       running: false,
       stale: false,
+      generation: "unknown",
+      liveness: existsSync(paths.service) ? "unknown" : "stopped",
       status_path: paths.service,
       snapshot_path: paths.snapshot,
     };
   }
   const heartbeatAt = Date.parse(record.heartbeat_at);
   const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
-  const local = record.host === hostname();
-  const running =
-    local &&
-    !stale &&
-    (record.state === "starting" || record.state === "running" || record.state === "stopping") &&
-    pidStillMatches(record.pid, record.start_token);
+  const owner = resourceProcessLiveness(record, dependencies);
+  const liveness =
+    owner.generation === "same" && record.state === "stopped" ? "stopped" : owner.liveness;
   return {
-    running,
+    running: liveness === "running" && !stale,
     stale,
+    generation: owner.generation,
+    liveness,
     record,
     status_path: paths.service,
     snapshot_path: paths.snapshot,
@@ -52,11 +63,23 @@ function validRecord(
   );
 }
 
-function pidStillMatches(pid: number, startToken: string | undefined): boolean {
+export function resourceProcessLiveness(
+  owner: { pid: number; host: string; start_token?: string },
+  dependencies: ResourceProcessDependencies = {},
+): { generation: ProcessGenerationVerdict; liveness: ResourceServiceStatus["liveness"] } {
+  if (owner.host !== hostname()) return { generation: "unknown", liveness: "unknown" };
   try {
-    process.kill(pid, 0);
+    (dependencies.probePid ?? ((pid) => process.kill(pid, 0)))(owner.pid);
   } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
+    return {
+      generation: "unknown",
+      liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
+    };
   }
-  return checkPidToken(pid, startToken) !== "mismatch";
+  const generation = verifyProcessGeneration(owner.pid, owner.start_token, dependencies.readToken);
+  return {
+    generation,
+    liveness:
+      generation === "same" ? "running" : generation === "different" ? "stopped" : "unknown",
+  };
 }