@yeaft/webchat-agent 0.1.482 → 0.1.483

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.482",
+  "version": "0.1.483",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/conversation/migrate-messages-threadid.js

@@ -0,0 +1,116 @@
+/**
+ * migrate-messages-threadid.js — task-307 one-shot migration.
+ *
+ * For every message under <yeaftDir>/conversation/messages/ and cold/ that
+ * does NOT yet carry a `threadId:` frontmatter field, stamp it with
+ * `threadId: main`. This matches design §5 semantics: pre-Phase-2 messages
+ * predate threading and belong to the root conversation.
+ *
+ * Idempotency:
+ *   A marker file `<yeaftDir>/conversation/.migrations/messagesThreadId`
+ *   is written on successful completion. Subsequent runs observe the
+ *   marker and short-circuit with `{ ran: false }`.
+ *
+ * Safety:
+ *   - Messages that already carry a threadId are left untouched.
+ *   - If a file is unreadable / unparseable it is skipped (never rewritten).
+ *   - Write errors are swallowed so a half-successful migration can resume
+ *     on the next boot by re-running.
+ *
+ * Usage (programmatic):
+ *   import { migrateMessagesThreadId } from './migrate-messages-threadid.js';
+ *   const res = migrateMessagesThreadId('/home/me/.yeaft');
+ *   // { ran: true, migrated: 42, skipped: 3 }
+ *
+ * Usage (CLI):
+ *   node agent/unify/conversation/migrate-messages-threadid.js [yeaftDir]
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+} from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+export function migrateMessagesThreadId(yeaftDir) {
+  const root = yeaftDir || join(homedir(), '.yeaft');
+  const convDir = join(root, 'conversation');
+  const markerDir = join(convDir, '.migrations');
+  const markerPath = join(markerDir, 'messagesThreadId');
+
+  if (!existsSync(convDir)) {
+    return { ran: false, reason: 'no conversation dir', migrated: 0, skipped: 0 };
+  }
+  try {
+    if (existsSync(markerPath)) {
+      return { ran: false, reason: 'already migrated', migrated: 0, skipped: 0 };
+    }
+  } catch { /* best-effort */ }
+
+  const dirs = [join(convDir, 'messages'), join(convDir, 'cold')];
+  let migrated = 0;
+  let skipped = 0;
+
+  for (const dir of dirs) {
+    if (!existsSync(dir)) continue;
+    let files;
+    try {
+      files = readdirSync(dir).filter(f => f.endsWith('.md'));
+    } catch {
+      continue;
+    }
+    for (const f of files) {
+      const path = join(dir, f);
+      let raw;
+      try {
+        raw = readFileSync(path, 'utf8');
+      } catch {
+        skipped += 1;
+        continue;
+      }
+      if (!raw || !raw.startsWith('---')) { skipped += 1; continue; }
+      const endIdx = raw.indexOf('\n---', 3);
+      if (endIdx === -1) { skipped += 1; continue; }
+      const frontmatter = raw.slice(4, endIdx);
+      const body = raw.slice(endIdx); // leading '\n---' ...
+      if (/^threadId:/m.test(frontmatter)) {
+        skipped += 1;
+        continue; // already has threadId
+      }
+      const newFm = frontmatter.replace(/\s*$/, '') + '\nthreadId: main';
+      const rebuilt = '---\n' + newFm.trimStart() + body;
+      try {
+        writeFileSync(path, rebuilt, { encoding: 'utf8', mode: 0o644 });
+        migrated += 1;
+      } catch {
+        skipped += 1;
+      }
+    }
+  }
+
+  try {
+    mkdirSync(markerDir, { recursive: true, mode: 0o755 });
+    writeFileSync(
+      markerPath,
+      `migrated: ${new Date().toISOString()}\ncount: ${migrated}\nskipped: ${skipped}\n`,
+      { encoding: 'utf8', mode: 0o644 },
+    );
+  } catch {
+    // Without the marker the migration will re-run; it's idempotent per-file
+    // (files already carrying threadId are skipped) so that's acceptable.
+  }
+
+  return { ran: true, migrated, skipped };
+}
+
+// CLI entry point.
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const dir = process.argv[2];
+  const res = migrateMessagesThreadId(dir);
+  // eslint-disable-next-line no-console
+  console.log(JSON.stringify(res, null, 2));
+}

package/unify/conversation/persist.js

@@ -56,6 +56,10 @@ function serializeMessage(msg) {
   if (msg.turnNumber != null) fm.push(`turnNumber: ${msg.turnNumber}`);
   if (msg.toolCallId) fm.push(`toolCallId: ${msg.toolCallId}`);
   if (msg.isError) fm.push(`isError: true`);
+  // task-307: every message is stamped with a threadId so multi-thread
+  // routing can filter/replay by thread without rescanning JSON blobs.
+  // Defaults to 'main' for legacy messages (see migrate-messages-threadid.js).
+  fm.push(`threadId: ${msg.threadId || 'main'}`);
 
   // Token estimate
   const content = msg.content || '';
@@ -111,10 +115,14 @@ export function parseMessage(raw) {
       case 'toolCallId': msg.toolCallId = value; break;
       case 'isError': msg.isError = value === 'true'; break;
       case 'tokens_est': msg.tokens_est = parseInt(value, 10); break;
+      case 'threadId': msg.threadId = value; break;
       // toolCalls are multi-line YAML — handled separately below
     }
   }
 
+  // task-307: legacy messages written before threadId existed default to 'main'.
+  if (!msg.threadId) msg.threadId = 'main';
+
   // Parse toolCalls if present (simplified multi-line YAML)
   if (frontmatter.includes('toolCalls:')) {
     const toolCalls = [];

package/unify/input-queue/store.js

@@ -0,0 +1,322 @@
+/**
+ * input-queue/store.js — Persistent FIFO input queue for Yeaft Unify.
+ *
+ * task-307b (Phase 2): a disk-backed queue of user inputs waiting to be
+ * routed to a thread/engine. Each entry is one JSON file at
+ *   <yeaftDir>/input-queue/<id>.json
+ * so that crashing during a routing decision never loses a queued input.
+ *
+ * Schema follows design doc §5:
+ *   {
+ *     id:         'iq-xxxxxxxx',
+ *     text:       '<the user-typed input>',
+ *     createdAt:  1234567890123,            // epoch ms
+ *     status:     'pending' | 'routing' | 'dispatched',
+ *     routedTo:   '<threadId>' | null,      // set when status === 'dispatched'
+ *     routedAt:   1234567890456 | null,
+ *     error:      '<message>' | null,       // optional — populated when a
+ *                                           //   routing attempt fails and
+ *                                           //   the entry is put back to
+ *                                           //   'pending' for retry.
+ *   }
+ *
+ * State machine:
+ *        enqueue(text)
+ *             │
+ *             ▼
+ *         pending ─────claim()─────► routing ─────markRouted()────► dispatched
+ *             ▲                        │
+ *             └──────markFailed()──────┘  (status back to pending; error field recorded)
+ *
+ *   - pending    : waiting for a dispatcher
+ *   - routing    : a dispatcher has taken responsibility; crash here means
+ *                  boot-time recovery will still see the entry and can
+ *                  re-claim it (the row is still on disk).
+ *   - dispatched : routed to a thread — the file is removed because the
+ *                  durable audit trail now lives in the messages.
+ *
+ * API (task-307b per PM):
+ *   - enqueue(text)                → entry          Create + persist pending.
+ *   - dequeue()                    → entry | null   Peek at oldest pending
+ *                                                   (non-mutating — see
+ *                                                   claim() for the mutating
+ *                                                   transition).
+ *   - claim()                      → entry | null   Oldest pending → 'routing'
+ *                                                   (atomic w.r.t. disk).
+ *   - list(status?)                → entry[]        Snapshot filtered by status.
+ *   - markRouted(id, routedTo)     → entry | null   routing → dispatched;
+ *                                                   file deleted.
+ *   - markFailed(id, err)          → entry | null   routing → pending; error
+ *                                                   recorded; kept on disk.
+ *   - peek() / get(id) / remove(id) / size() / pendingCount()
+ *
+ * Persistence:
+ *   - Writes are synchronous. The queue is a durability boundary, not a
+ *     hot path (one write per state transition).
+ *   - Permission/FS failures don't throw: they are swallowed and logged once
+ *     per process (matches ConversationStore's philosophy).
+ *   - If `yeaftDir` is omitted, the store operates purely in memory.
+ */
+
+import { randomUUID } from 'crypto';
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  unlinkSync,
+} from 'fs';
+import { join } from 'path';
+
+/** All valid status values, per design §5. */
+export const INPUT_QUEUE_STATUSES = ['pending', 'routing', 'dispatched'];
+
+export class InputQueueStore {
+  /** @type {Map<string, object>} */
+  #entries;
+  /** @type {string|null} */
+  #dir;
+  /** @type {boolean} */
+  #persistent;
+  /** @type {boolean} */
+  #warned;
+
+  /**
+   * @param {string|null} [yeaftDir]
+   */
+  constructor(yeaftDir = null) {
+    this.#entries = new Map();
+    this.#warned = false;
+
+    if (yeaftDir) {
+      this.#dir = join(yeaftDir, 'input-queue');
+      this.#persistent = true;
+      this.#ensureDir();
+      this.#loadAll();
+    } else {
+      this.#dir = null;
+      this.#persistent = false;
+    }
+  }
+
+  get persistent() { return this.#persistent; }
+  size() { return this.#entries.size; }
+
+  pendingCount() {
+    let n = 0;
+    for (const e of this.#entries.values()) if (e.status === 'pending') n += 1;
+    return n;
+  }
+
+  /**
+   * Append a new pending entry and persist it.
+   * @param {string} text
+   * @returns {object} the entry
+   */
+  enqueue(text) {
+    if (typeof text !== 'string') throw new Error('text must be a string');
+    const id = `iq-${randomUUID().slice(0, 8)}`;
+    const entry = {
+      id,
+      text,
+      createdAt: Date.now(),
+      status: 'pending',
+      routedTo: null,
+      routedAt: null,
+      error: null,
+    };
+    this.#entries.set(id, entry);
+    this.#writeEntry(entry);
+    return entry;
+  }
+
+  /** Oldest pending entry (no state change). null if empty. */
+  peek() {
+    let oldest = null;
+    for (const e of this.#entries.values()) {
+      if (e.status !== 'pending') continue;
+      if (!oldest || e.createdAt < oldest.createdAt) oldest = e;
+    }
+    return oldest;
+  }
+
+  /**
+   * Return the oldest pending entry without mutating state. Kept as a
+   * separate method from claim() because some consumers only want to
+   * observe the head of the queue (e.g. UI preview).
+   */
+  dequeue() {
+    return this.peek();
+  }
+
+  /**
+   * Transition the oldest pending entry → 'routing' and persist. This is
+   * the real consumer-facing take: after claim() the caller must eventually
+   * call markRouted() (success) or markFailed() (→ put back as pending).
+   *
+   * @returns {object|null} the claimed entry, or null if nothing pending
+   */
+  claim() {
+    const e = this.peek();
+    if (!e) return null;
+    e.status = 'routing';
+    this.#writeEntry(e);
+    return e;
+  }
+
+  /**
+   * Mark an entry as successfully dispatched to a thread. Persists the
+   * updated state then removes the file — the authoritative record now
+   * lives in the conversation/messages log.
+   *
+   * @param {string} id
+   * @param {string} routedTo — thread id (e.g. 'main' or 'thr-xxxxxxxx')
+   * @returns {object|null}
+   */
+  markRouted(id, routedTo) {
+    const e = this.#entries.get(id);
+    if (!e) return null;
+    if (!routedTo || typeof routedTo !== 'string') throw new Error('routedTo required');
+    e.status = 'dispatched';
+    e.routedTo = routedTo;
+    e.routedAt = Date.now();
+    // Persist the transition before removing (crash-safe ordering).
+    this.#writeEntry(e);
+    this.#removeFile(id);
+    this.#entries.delete(id);
+    return e;
+  }
+
+  /**
+   * Mark a routing attempt failed. The entry returns to 'pending' so the
+   * next claim() can retry it; the error string is retained for diagnostics.
+   *
+   * @param {string} id
+   * @param {string|Error} err
+   */
+  markFailed(id, err) {
+    const e = this.#entries.get(id);
+    if (!e) return null;
+    e.status = 'pending';
+    e.error = typeof err === 'string' ? err : (err?.message || String(err));
+    this.#writeEntry(e);
+    return e;
+  }
+
+  /** Remove an entry entirely (memory + disk). */
+  remove(id) {
+    if (!this.#entries.has(id)) return false;
+    this.#entries.delete(id);
+    this.#removeFile(id);
+    return true;
+  }
+
+  /**
+   * Snapshot of all entries, optionally filtered by status. Chronological.
+   * @param {'pending'|'routing'|'dispatched'} [status]
+   */
+  list(status) {
+    let arr = [...this.#entries.values()];
+    if (status) arr = arr.filter(e => e.status === status);
+    arr.sort((a, b) => a.createdAt - b.createdAt);
+    return arr;
+  }
+
+  get(id) { return this.#entries.get(id) || null; }
+
+  // ─── Persistence internals ────────────────────────────
+
+  #ensureDir() {
+    try {
+      if (!existsSync(this.#dir)) mkdirSync(this.#dir, { recursive: true, mode: 0o755 });
+    } catch (err) {
+      this.#warn(`Cannot create input-queue dir: ${err?.code || err?.message}`);
+    }
+  }
+
+  #loadAll() {
+    if (!existsSync(this.#dir)) return;
+    let files;
+    try {
+      files = readdirSync(this.#dir);
+    } catch (err) {
+      this.#warn(`Cannot read input-queue dir: ${err?.code || err?.message}`);
+      return;
+    }
+    for (const f of files) {
+      if (!f.endsWith('.json')) continue;
+      const path = join(this.#dir, f);
+      try {
+        const raw = readFileSync(path, 'utf8');
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== 'object' || !parsed.id) continue;
+        if (!INPUT_QUEUE_STATUSES.includes(parsed.status)) continue;
+        // Crash recovery: an entry left in 'routing' at startup had a
+        // dispatcher claim it right before the crash. Put it back to
+        // 'pending' so the next claim() can retry it.
+        const status = parsed.status === 'routing' ? 'pending' : parsed.status;
+        this.#entries.set(parsed.id, {
+          id: parsed.id,
+          text: typeof parsed.text === 'string' ? parsed.text : '',
+          createdAt: Number(parsed.createdAt) || Date.now(),
+          status,
+          routedTo: parsed.routedTo || null,
+          routedAt: parsed.routedAt || null,
+          error: parsed.error || null,
+        });
+      } catch {
+        // Skip corrupt file.
+      }
+    }
+  }
+
+  #writeEntry(entry) {
+    if (!this.#persistent) return;
+    const path = join(this.#dir, `${entry.id}.json`);
+    try {
+      writeFileSync(path, JSON.stringify(entry, null, 2), { encoding: 'utf8', mode: 0o644 });
+    } catch (err) {
+      this.#warn(`Cannot write ${entry.id}: ${err?.code || err?.message}`);
+    }
+  }
+
+  #removeFile(id) {
+    if (!this.#persistent) return;
+    const path = join(this.#dir, `${id}.json`);
+    if (!existsSync(path)) return;
+    try {
+      unlinkSync(path);
+    } catch (err) {
+      this.#warn(`Cannot remove ${id}: ${err?.code || err?.message}`);
+    }
+  }
+
+  #warn(msg) {
+    if (this.#warned) return;
+    this.#warned = true;
+    // eslint-disable-next-line no-console
+    console.warn(`[Yeaft InputQueue] ${msg}`);
+  }
+}
+
+// ─── Singleton helpers ────────────────────────────────────
+
+/** @type {InputQueueStore|null} */
+let inputQueueStore = null;
+
+export function initInputQueueStore(opts = {}) {
+  if (!inputQueueStore || opts.force) {
+    inputQueueStore = new InputQueueStore(opts.yeaftDir || null);
+  }
+  return inputQueueStore;
+}
+
+export function getInputQueueStore() {
+  if (!inputQueueStore) inputQueueStore = new InputQueueStore(null);
+  return inputQueueStore;
+}
+
+export function _resetInputQueueStoreForTests() {
+  inputQueueStore = null;
+}