harnery 0.32.0 → 0.34.0

package/src/core/agents/events/consume.ts

@@ -1,254 +0,0 @@
-/**
- * Incremental consumer for the canonical event stream at
- * `.harnery/events.ndjson`.
- *
- * A small `.harnery/.events-cursor` file persists the last-projected event_id;
- * subsequent runs skip everything up to and including that id and return only
- * the new tail.
- *
- * Read strategy: the global cursor is advanced on every agent's turn.stop, so
- * it almost always sits within the last few KB of the stream. We exploit that
- * with a bounded **tail read** (reading only the last `tailBytes` of the file
- * and locating the cursor there) which turns an O(file-size) read into
- * O(window). If the cursor is older than the window (a long-idle system, or a
- * cursor that's been rotated out) we fall back to a wider read that is itself
- * capped (`fallbackCapBytes`): the stream is an append-only ledger that grows
- * without bound, so a whole-file `readFileSync` throws V8's max-string-length
- * error ("Cannot create a string longer than 0x1fffffe8 characters") once it
- * passes ~512MB, which would abort the projection. Events older than the cap are
- * stale for coord-state purposes, so a bounded replay is the correct fallback.
- *
- * Idempotency: projectors are idempotent by `event_id`. The
- * cursor file makes this cheap (we never replay an already-projected event by
- * accident) and the projector tolerates the rare full-replay (cursor rotated
- * out) without double-applying.
- */
-
-import {
-  closeSync,
-  existsSync,
-  mkdirSync,
-  openSync,
-  readFileSync,
-  readSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { dirname, join } from "node:path";
-import { coordEnv } from "../../../lib/env.ts";
-
-const STREAM_REL = ".harnery/events.ndjson";
-const CURSOR_REL = ".harnery/.events-cursor";
-const DEFAULT_TAIL_BYTES = 2 * 1024 * 1024; // 2 MiB, thousands of events of headroom
-/** Cap for the fall-through read when the cursor misses the tail window. Well
- * under V8's ~512MB max string length so it never throws on the unbounded
- * ledger; comfortably larger than any realistic cursor drift (the global cursor
- * advances on every agent's turn.stop, so it sits near EOF in practice). */
-const DEFAULT_FALLBACK_CAP_BYTES = 64 * 1024 * 1024; // 64 MiB
-
-export interface CanonicalEvent {
-  schema_version: number;
-  event_id: string;
-  event_type: string;
-  ts: string;
-  instance_id: string;
-  session_id: string;
-  parent_session_id?: string;
-  turn_id?: string;
-  parent_turn_id?: string;
-  adapter: string;
-  source: string;
-  data: Record<string, unknown>;
-}
-
-export interface ConsumeResult {
-  events: CanonicalEvent[];
-  /** event_id of the last event the caller successfully projected. */
-  lastEventId: string | null;
-  /** Total bytes in the stream file at read time (for diagnostics). */
-  streamBytes: number;
-}
-
-export interface ConsumeOpts {
-  /** Ignore the cursor and return everything, useful for backfill. */
-  replayAll?: boolean;
-  /**
-   * Tail-window size in bytes for the fast-path read. Defaults to
-   * `HARNERY_AGENT_COORD_TAIL_BYTES` env or 2 MiB. Mainly a test seam; production
-   * callers should leave it unset.
-   */
-  tailBytes?: number;
-  /**
-   * Cap in bytes for the fall-through read (cursor missed the tail window).
-   * Defaults to `HARNERY_AGENT_COORD_FALLBACK_CAP_BYTES` env or 64 MiB. Bounds
-   * the read so the unbounded ledger can never overflow V8's max string length.
-   * Mainly a test seam; production callers should leave it unset.
-   */
-  fallbackCapBytes?: number;
-}
-
-/**
- * Read all events after the persisted cursor, in chronological (file) order.
- */
-export function consumeSince(coordRoot: string, opts: ConsumeOpts = {}): ConsumeResult {
-  const streamPath = join(coordRoot, STREAM_REL);
-  if (!existsSync(streamPath)) {
-    return { events: [], lastEventId: null, streamBytes: 0 };
-  }
-
-  const cursor = opts.replayAll ? null : readCursor(coordRoot);
-  const fileSize = statSync(streamPath).size;
-  const windowBytes = resolveTailBytes(opts.tailBytes);
-
-  // Tail fast-path: a cursor exists and the file is larger than the window.
-  // Read only the trailing window, drop the (likely partial) first line, and
-  // look for the cursor. A hit returns just the new events without ever
-  // touching the bulk of the file.
-  if (cursor !== null && fileSize > windowBytes) {
-    const tail = readTailUtf8(streamPath, fileSize, windowBytes);
-    const firstNl = tail.indexOf("\n");
-    const usable = firstNl >= 0 ? tail.slice(firstNl + 1) : "";
-    const parsed = eventsAfterCursor(parseLines(usable), cursor);
-    if (parsed.foundCursor) {
-      return {
-        events: parsed.events,
-        lastEventId: parsed.events.at(-1)?.event_id ?? cursor,
-        streamBytes: fileSize,
-      };
-    }
-    // Cursor older than the tail window → fall through to a full read.
-  }
-
-  // Bounded fall-through: first run, replayAll, small file, or cursor not found
-  // in the tail. This must NEVER read the whole file: the stream grows without
-  // bound and a >512MB readFileSync throws V8's max-string-length error, which
-  // would abort the projection. Read at most `cap` bytes from the tail; when we
-  // start mid-file, drop the (likely partial) first line.
-  const cap = resolveFallbackCap(opts.fallbackCapBytes);
-  const readBytes = Math.min(fileSize, cap);
-  let text = readTailUtf8(streamPath, fileSize, readBytes);
-  if (readBytes < fileSize) {
-    const firstNl = text.indexOf("\n");
-    text = firstNl >= 0 ? text.slice(firstNl + 1) : "";
-  }
-  const all = parseLines(text);
-  const parsed = eventsAfterCursor(all, cursor);
-  if (parsed.foundCursor) {
-    return {
-      events: parsed.events,
-      lastEventId: parsed.events.at(-1)?.event_id ?? cursor,
-      streamBytes: fileSize,
-    };
-  }
-
-  // The cursor names an event older than the capped window (or rotated out), so
-  // replay what the window holds (safer than silent state drift; the projector
-  // is idempotent by event_id). lastEventId null keeps the cursor put.
-  return { events: all, lastEventId: null, streamBytes: fileSize };
-}
-
-function resolveFallbackCap(override?: number): number {
-  if (override !== undefined && override > 0) return override;
-  const env = coordEnv("AGENT_COORD_FALLBACK_CAP_BYTES");
-  const n = env ? Number(env) : Number.NaN;
-  return Number.isFinite(n) && n > 0 ? n : DEFAULT_FALLBACK_CAP_BYTES;
-}
-
-function resolveTailBytes(override?: number): number {
-  if (override !== undefined && override > 0) return override;
-  const env = coordEnv("AGENT_COORD_TAIL_BYTES");
-  const n = env ? Number(env) : Number.NaN;
-  return Number.isFinite(n) && n > 0 ? n : DEFAULT_TAIL_BYTES;
-}
-
-/**
- * Read at most `capBytes` from the tail of the event stream as UTF-8, dropping
- * the (partial) leading line when the file is larger than the cap. Shared
- * bounded reader for CLI consumers (`agents trace` / `agents health`) that must
- * never `readFileSync` the whole unbounded ledger — a >512MB read throws V8's
- * max-string-length error. `truncated` is true when older bytes were skipped, so
- * callers can surface the cap rather than silently under-reporting.
- */
-export function readStreamTailBounded(
-  streamPath: string,
-  capBytes: number,
-): { text: string; truncated: boolean } {
-  if (!existsSync(streamPath)) return { text: "", truncated: false };
-  const fileSize = statSync(streamPath).size;
-  const readBytes = Math.min(fileSize, capBytes);
-  let text = readTailUtf8(streamPath, fileSize, readBytes);
-  const truncated = readBytes < fileSize;
-  if (truncated) {
-    const nl = text.indexOf("\n");
-    text = nl >= 0 ? text.slice(nl + 1) : "";
-  }
-  return { text, truncated };
-}
-
-/** Read the trailing `windowBytes` of a file as UTF-8 without loading the rest. */
-function readTailUtf8(streamPath: string, fileSize: number, windowBytes: number): string {
-  const start = Math.max(0, fileSize - windowBytes);
-  const len = fileSize - start;
-  const buf = Buffer.allocUnsafe(len);
-  const fd = openSync(streamPath, "r");
-  try {
-    let read = 0;
-    while (read < len) {
-      const n = readSync(fd, buf, read, len - read, start + read);
-      if (n === 0) break;
-      read += n;
-    }
-    return buf.toString("utf8", 0, read);
-  } finally {
-    closeSync(fd);
-  }
-}
-
-function parseLines(raw: string): CanonicalEvent[] {
-  const out: CanonicalEvent[] = [];
-  for (const line of raw.split("\n")) {
-    if (!line.trim()) continue;
-    try {
-      out.push(JSON.parse(line) as CanonicalEvent);
-    } catch {
-      /* skip malformed line */
-    }
-  }
-  return out;
-}
-
-/**
- * Events strictly after `cursor` (or all events when `cursor` is null). When a
- * non-null cursor isn't present in `all`, returns `foundCursor: false` so the
- * caller can decide between a wider read and a full replay.
- */
-function eventsAfterCursor(
-  all: CanonicalEvent[],
-  cursor: string | null,
-): { events: CanonicalEvent[]; foundCursor: boolean } {
-  if (cursor === null) return { events: all, foundCursor: true };
-  const idx = all.findIndex((e) => e.event_id === cursor);
-  if (idx < 0) return { events: [], foundCursor: false };
-  return { events: all.slice(idx + 1), foundCursor: true };
-}
-
-export function readCursor(coordRoot: string): string | null {
-  const path = join(coordRoot, CURSOR_REL);
-  if (!existsSync(path)) return null;
-  try {
-    const v = readFileSync(path, "utf8").trim();
-    return v.length > 0 ? v : null;
-  } catch {
-    return null;
-  }
-}
-
-export function writeCursor(coordRoot: string, eventId: string): void {
-  const path = join(coordRoot, CURSOR_REL);
-  try {
-    mkdirSync(dirname(path), { recursive: true });
-    writeFileSync(path, eventId, "utf8");
-  } catch {
-    /* swallow; next run replays */
-  }
-}

package/src/core/agents/events/emit.ts

@@ -1,115 +0,0 @@
-/**
- * Canonical-event writer for agent-coord. Mirrors the agent-hooks emitter:
- * both modules write the same envelope shape to the same
- * `.harnery/events.ndjson` file but stay independent (shared code limited to
- * event/schema types).
- *
- * Phase 4: agent-coord uses this from CLI handlers (state.task_set,
- * state.status_checked, state.journal_append, council.*, decision.*,
- * presence.*).
- */
-
-import { appendFileSync, closeSync, mkdirSync, openSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { maybeRotateEventStream } from "../../hooks/events/rotate.ts";
-import { ulid } from "./ulid.ts";
-
-const SCHEMA_VERSION = 1 as const;
-const STREAM_FILE = ".harnery/events.ndjson";
-const LOCK_FILE = ".harnery/events.ndjson.lock";
-const MAX_LINE_BYTES = 64 * 1024;
-
-export type { Adapter, Source } from "../../hooks/events/schema.ts";
-
-import type { Adapter, Source } from "../../hooks/events/schema.ts";
-
-export interface Envelope {
-  schema_version: typeof SCHEMA_VERSION;
-  event_id: string;
-  event_type: string;
-  ts: string;
-  instance_id: string;
-  session_id: string;
-  parent_session_id?: string;
-  turn_id?: string;
-  parent_turn_id?: string;
-  adapter: Adapter;
-  source: Source;
-  data: Record<string, unknown>;
-}
-
-export interface EmitInput {
-  event_type: string;
-  instance_id: string;
-  session_id: string;
-  adapter: Adapter;
-  source?: Source;
-  parent_session_id?: string;
-  turn_id?: string;
-  parent_turn_id?: string;
-  ts?: string;
-  data: Record<string, unknown>;
-}
-
-export function buildEnvelope(input: EmitInput): Envelope {
-  return {
-    schema_version: SCHEMA_VERSION,
-    event_id: ulid(),
-    event_type: input.event_type,
-    ts: input.ts ?? new Date().toISOString(),
-    instance_id: input.instance_id,
-    session_id: input.session_id,
-    parent_session_id: input.parent_session_id,
-    turn_id: input.turn_id,
-    parent_turn_id: input.parent_turn_id,
-    adapter: input.adapter,
-    source: input.source ?? "agent-coord",
-    data: input.data,
-  };
-}
-
-export function emit(coordRoot: string, input: EmitInput): Envelope {
-  const envelope = buildEnvelope(input);
-  const streamPath = join(coordRoot, STREAM_FILE);
-  const lockPath = join(coordRoot, LOCK_FILE);
-
-  // Size-triggered rotation (shared with the agent-hooks emitter): roll the
-  // active file to a dated archive when it crosses the cap. Fail-soft.
-  maybeRotateEventStream(coordRoot);
-
-  ensureDir(dirname(streamPath));
-  ensureFile(lockPath);
-
-  let line = `${JSON.stringify(envelope)}\n`;
-  if (Buffer.byteLength(line, "utf8") > MAX_LINE_BYTES) {
-    line = `${JSON.stringify({
-      ...envelope,
-      data: { __over_size_limit: true, original_bytes: Buffer.byteLength(line, "utf8") },
-    })}\n`;
-  }
-
-  const lockFd = openSync(lockPath, "r+");
-  try {
-    appendFileSync(streamPath, line, { encoding: "utf8", flag: "a" });
-  } finally {
-    closeSync(lockFd);
-  }
-
-  return envelope;
-}
-
-function ensureDir(path: string): void {
-  try {
-    mkdirSync(path, { recursive: true });
-  } catch {
-    /* swallow */
-  }
-}
-
-function ensureFile(path: string): void {
-  try {
-    closeSync(openSync(path, "a"));
-  } catch {
-    /* swallow */
-  }
-}

package/src/core/agents/events/ulid.ts

@@ -1,51 +0,0 @@
-/**
- * Tiny ULID generator. Crockford-base32, 10-char timestamp (48-bit ms epoch)
- * + 16-char random (80-bit). 26 chars total, monotonically-sortable.
- *
- * Intentionally duplicated from the agent-hooks ULID generator: agent-hooks
- * and agent-coord stay strictly-separated packages, sharing only event/schema
- * types. The writer side reimplements per-module so neither depends on the
- * other's runtime code.
- */
-
-const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
-
-export function ulid(): string {
-  const ts = encodeTime(Date.now());
-  const rnd = encodeRandom();
-  return ts + rnd;
-}
-
-function encodeTime(ms: number): string {
-  let out = "";
-  let n = BigInt(ms);
-  const base = BigInt(32);
-  for (let i = 0; i < 10; i++) {
-    const mod = Number(n % base);
-    out = ALPHABET[mod] + out;
-    n = n / base;
-  }
-  return out;
-}
-
-function encodeRandom(): string {
-  const bytes = new Uint8Array(10);
-  crypto.getRandomValues(bytes);
-  let out = "";
-  let buf = 0n;
-  let bits = 0;
-  for (const b of bytes) {
-    buf = (buf << 8n) | BigInt(b);
-    bits += 8;
-    while (bits >= 5) {
-      bits -= 5;
-      const idx = Number((buf >> BigInt(bits)) & 0x1fn);
-      out += ALPHABET[idx];
-    }
-  }
-  if (bits > 0) {
-    const idx = Number((buf << BigInt(5 - bits)) & 0x1fn);
-    out += ALPHABET[idx];
-  }
-  return out.slice(0, 16);
-}