@adaptic/maestro 1.10.5 → 1.10.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adaptic/maestro",
-  "version": "1.10.5",
+  "version": "1.10.6",
   "description": "Maestro — Autonomous AI agent operating system. Deploy AI employees on dedicated Mac minis.",
   "type": "module",
   "bin": {
@@ -46,7 +46,7 @@
   },
   "always-build-npm": true,
   "scripts": {
-    "test": "node --test lib/cadence-bus.test.mjs scripts/cadence/enqueue-cadence-tick.test.mjs scripts/daemon/cadence-consumer.test.mjs scripts/daemon/dispatcher-cooldown.test.mjs scripts/daemon/lib/session-router.test.mjs scripts/local-triggers/generate-plists.test.mjs scripts/poller/slack-socket-mode.test.mjs bin/maestro.test.mjs",
+    "test": "node --test lib/cadence-bus.test.mjs scripts/cadence/enqueue-cadence-tick.test.mjs scripts/daemon/cadence-consumer.test.mjs scripts/daemon/dispatcher-cooldown.test.mjs scripts/daemon/inbox-deferral.test.mjs scripts/daemon/lib/session-router.test.mjs scripts/local-triggers/generate-plists.test.mjs scripts/poller/slack-socket-mode.test.mjs bin/maestro.test.mjs",
     "test:cadence": "node --test lib/cadence-bus.test.mjs scripts/cadence/enqueue-cadence-tick.test.mjs scripts/daemon/cadence-consumer.test.mjs",
     "test:cli": "node --test bin/maestro.test.mjs",
     "test:plists": "node --test scripts/local-triggers/generate-plists.test.mjs",

package/scripts/daemon/agent-daemon.mjs

@@ -47,7 +47,8 @@ import { dispatch, getStatus, availableSlots, canDispatchBacklog, resetActiveSes
 import { buildPrompt } from "./prompt-builder.mjs";
 import { sendQuickResponse, sendHoldingMessage, isQuickReply } from "./responder.mjs";
 import { recordPoll, recordClassification, recordSession, writeHealthDashboard } from "./health.mjs";
-import { acquireLock, updateLock, scanStaleLocks, acquireThreadLock, claimRequest, hasActiveClaim, sweepStaleItemClaims } from "./session-lock.mjs";
+import { acquireLock, releaseLock, updateLock, scanStaleLocks, acquireThreadLock, claimRequest, hasActiveClaim, sweepStaleItemClaims } from "./session-lock.mjs";
+import { markDeferred } from "./inbox-deferral.mjs";
 
 // ---------------------------------------------------------------------------
 // Configuration
@@ -266,15 +267,28 @@ async function processItem(item, service) {
       if (threadTs && channel) {
         const threadCheck = acquireThreadLock(channel, threadTs);
         if (!threadCheck.allowed) {
-          console.log(`[daemon] Thread/channel dedup: skipping item from ${item.sender} — ${threadCheck.reason}`);
+          // Defer instead of dropping: rename to .deferred so the file
+          // is preserved and re-promoted to the live inbox when the
+          // active session for this channel releases its lock. The
+          // dispatcher calls promoteDeferred() from its session-close
+          // path; the next poll cycle then picks the latest deferred
+          // item up and dispatches a single session that sees the full
+          // thread context (older bursts are bundled).
+          //
+          // We also release the per-item lock that the poll filter
+          // acquired at the top of the cycle — otherwise the next
+          // poll's acquireLock would block re-processing of this
+          // exact item after we promote it back to live.
+          const deferred = markDeferred(item, service, AGENT_REPO_DIR);
+          releaseLock(itemId);
+          console.log(`[daemon] Thread/channel dedup: deferring item from ${item.sender} — ${threadCheck.reason}${deferred ? "" : " (no inbox file to defer)"}`);
           logEvent("classifications", {
             item_id: itemId,
             sender: item.sender,
             service,
-            skipped: true,
+            deferred: deferred > 0,
             reason: `thread_dedup: ${threadCheck.reason}`,
           });
-          markProcessed(item, service);
           return;
         }
       }

package/scripts/daemon/dispatcher.mjs

@@ -6,6 +6,7 @@ import { spawn } from "child_process";
 import { appendFileSync, mkdirSync, writeFileSync, readFileSync, renameSync } from "fs";
 import { join, dirname } from "path";
 import { releaseLock, releaseThreadLock, releaseRequestClaim, claimItem, releaseItemClaim } from "./session-lock.mjs";
+import { promoteDeferred } from "./inbox-deferral.mjs";
 import { recordSession } from "./health.mjs";
 
 const AGENT_REPO_DIR = process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
@@ -384,10 +385,35 @@ function spawnSession(entry) {
     const itemId = item.raw_ref || item.id || item.title;
     if (itemId) releaseLock(itemId);
 
-    // Release thread lock so new messages in this thread can be processed
-    if (item.thread_id) {
+    // Release thread/channel lock so new messages in this thread/DM can
+    // be processed. We must release unconditionally when a channel is
+    // known — the previous `if (item.thread_id)` gate skipped DMs (which
+    // always have empty thread_id), so DM-channel locks never cleared
+    // until the 60min TTL fired. acquireThreadLock normalises DM
+    // channels to `dm-channel` internally; releaseThreadLock mirrors
+    // that normalisation, so calling it with an empty thread_id on a
+    // DM channel does the right thing.
+    {
       const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
-      if (channel) releaseThreadLock(channel, item.thread_id);
+      if (channel) {
+        releaseThreadLock(channel, item.thread_id);
+        // Now that the lock is gone, promote any messages that were
+        // deferred behind it. Latest-wins: a burst of N messages
+        // collapses into ONE re-dispatch carrying the most recent
+        // message (its thread_context already includes the earlier
+        // ones), so the user gets a single coherent reply rather than
+        // N replies serialised over N session-durations.
+        const promo = promoteDeferred(channel, AGENT_REPO_DIR);
+        if (promo.promoted > 0) {
+          logSession({
+            event: "deferred_promoted",
+            channel,
+            promoted: promo.promoted,
+            bundled: promo.bundled,
+            service: promo.service,
+          });
+        }
+      }
     }
 
     // Release request claim so the same type of request can be processed again
@@ -462,10 +488,24 @@ function spawnSession(entry) {
     const itemId = item.raw_ref || item.id || item.title;
     if (itemId) releaseLock(itemId);
 
-    // Release thread lock so new messages in this thread can be processed.
-    if (item.thread_id) {
+    // Release thread/channel lock so new messages can be processed.
+    // Mirror of close-handler logic — unconditional when channel is
+    // known, since releaseThreadLock handles the DM normalization.
+    {
       const channel = item.channel_id || (item.raw_ref ? (item.raw_ref.match(/slack:([^:]+):/) || [])[1] : null) || item.channel;
-      if (channel) releaseThreadLock(channel, item.thread_id);
+      if (channel) {
+        releaseThreadLock(channel, item.thread_id);
+        const promo = promoteDeferred(channel, AGENT_REPO_DIR);
+        if (promo.promoted > 0) {
+          logSession({
+            event: "deferred_promoted_on_error",
+            channel,
+            promoted: promo.promoted,
+            bundled: promo.bundled,
+            service: promo.service,
+          });
+        }
+      }
     }
 
     // Release request claim + emit explicit claim_released event so

package/scripts/daemon/inbox-deferral.mjs

@@ -0,0 +1,171 @@
+/**
+ * Maestro — Inbox deferral utilities
+ *
+ * When the dispatcher hits a thread/channel-level dedup lock, the
+ * incoming message used to be marked `.processed` and dropped on the
+ * floor. That meant rapid-fire DMs (or thread bursts) silently lost
+ * every message after the first.
+ *
+ * This module replaces that with a defer-then-promote pattern:
+ *
+ *   markDeferred(item, service, agentRoot)
+ *     Renames the inbox file from `<name>` to `<name>.deferred`.
+ *     The daemon's inbox scanner skips `.deferred` files, so the
+ *     message is parked but not lost.
+ *
+ *   promoteDeferred(channel, agentRoot)
+ *     Called from the dispatcher's session-close release path. Finds
+ *     every `.deferred` file in any service inbox dir whose body
+ *     references `channel`. If exactly one is found, renames it back
+ *     to its original name (next poll picks it up). If N>1 are found,
+ *     keeps only the LATEST (by timestamp), promotes that one, and
+ *     marks the others `.processed-bundled` for the audit trail —
+ *     the latest item's `thread_context` already contains the prior
+ *     messages as conversation history, so Claude sees everything
+ *     and can compose a single coherent reply.
+ *
+ * The file format is the YAML emitted by the slack/gmail/calendar
+ * pollers (a flat top-level object with quoted scalar fields). We
+ * deliberately avoid pulling in a YAML parser — regex extraction of
+ * `channel_id:` and `timestamp:` is sufficient for the routing
+ * decision and keeps this module dependency-free.
+ */
+
+import { readdirSync, readFileSync, renameSync, existsSync } from "node:fs";
+import { join } from "node:path";
+
+const SERVICE_DIRS = ["slack", "gmail", "calendar", "internal", "sms", "whatsapp"];
+
+/**
+ * Rename a live inbox file to its `.deferred` suffix so the scanner
+ * skips it until the channel/thread lock releases.
+ *
+ * Mirrors markProcessed's file-matching strategy (id or raw_ref
+ * substring) so we hit the same file that markProcessed would have.
+ *
+ * Idempotent: if the file is already `.deferred` or `.processed`, no-op.
+ *
+ * @returns {number} number of files renamed (usually 1, occasionally 0
+ *                   for backlog items with no inbox file)
+ */
+export function markDeferred(item, service, agentRoot) {
+  if (!item || !service || !agentRoot) return 0;
+  const inboxDir = join(agentRoot, "state", "inbox", service);
+  if (!existsSync(inboxDir)) return 0;
+  let renamed = 0;
+  try {
+    const needle = item.id || item.raw_ref;
+    if (!needle) return 0;
+    const files = readdirSync(inboxDir).filter(
+      (f) => !f.endsWith(".processed") &&
+             !f.endsWith(".deferred") &&
+             !f.endsWith(".processed-bundled") &&
+             (f.includes(item.id || "") || f.includes(item.raw_ref || ""))
+    );
+    for (const file of files) {
+      renameSync(join(inboxDir, file), join(inboxDir, `${file}.deferred`));
+      renamed++;
+    }
+  } catch {
+    // Inbox dir may not exist for non-poller-backed items (e.g. backlog).
+  }
+  return renamed;
+}
+
+/**
+ * Extract a top-level scalar value from a poller-written inbox YAML
+ * file. Returns `null` if the field is absent.
+ *
+ * We only need this for `channel_id` (routing) and `timestamp`
+ * (latest-wins selection) — both are written as quoted single-line
+ * scalars by every poller, so a one-line regex is enough.
+ */
+function readScalar(body, field) {
+  // Matches: `  field: "value"`  or  `field: value`  (leading space ok)
+  const re = new RegExp(`^\\s*${field}\\s*:\\s*"?([^"\\n]+?)"?\\s*$`, "m");
+  const m = body.match(re);
+  return m ? m[1].trim() : null;
+}
+
+/**
+ * Promote every `.deferred` item targeting `channel` back into the
+ * live inbox. Bundles bursts: if multiple deferred items exist for
+ * the same channel, only the latest is re-queued (the others are
+ * marked `.processed-bundled` since their content is already present
+ * in the latest item's thread_context).
+ *
+ * Called from the dispatcher's session-close path, AFTER the channel
+ * lock has been released, so the next poll cycle is free to re-acquire.
+ *
+ * @returns {{promoted: number, bundled: number, service: string|null}}
+ */
+export function promoteDeferred(channel, agentRoot) {
+  const result = { promoted: 0, bundled: 0, service: null };
+  if (!channel || !agentRoot) return result;
+
+  for (const service of SERVICE_DIRS) {
+    const inboxDir = join(agentRoot, "state", "inbox", service);
+    if (!existsSync(inboxDir)) continue;
+
+    let deferredFiles;
+    try {
+      deferredFiles = readdirSync(inboxDir).filter((f) => f.endsWith(".deferred"));
+    } catch {
+      continue;
+    }
+    if (deferredFiles.length === 0) continue;
+
+    // Filter to items whose body references this channel. Reading
+    // every .deferred file is cheap (a handful at most) — we stay
+    // dependency-free by regex-scanning rather than parsing YAML.
+    const matches = [];
+    for (const file of deferredFiles) {
+      let body;
+      try {
+        body = readFileSync(join(inboxDir, file), "utf-8");
+      } catch {
+        continue;
+      }
+      const fileChannel = readScalar(body, "channel_id");
+      if (fileChannel !== channel) continue;
+      matches.push({
+        file,
+        timestamp: readScalar(body, "timestamp") || "",
+      });
+    }
+    if (matches.length === 0) continue;
+
+    // Latest-wins: lex-sort ISO timestamps, take the most recent.
+    matches.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+    const latest = matches[matches.length - 1];
+    const older = matches.slice(0, -1);
+
+    // Promote the latest back to live inbox.
+    try {
+      const live = latest.file.replace(/\.deferred$/, "");
+      renameSync(join(inboxDir, latest.file), join(inboxDir, live));
+      result.promoted++;
+      result.service = service;
+    } catch {
+      // If the rename failed (race with another process), leave it
+      // as .deferred — the next session-close will retry.
+    }
+
+    // Mark older bursts as bundled — their content is already part of
+    // the latest item's thread_context, so they don't need their own
+    // session, but we preserve the file for audit.
+    for (const m of older) {
+      try {
+        renameSync(
+          join(inboxDir, m.file),
+          join(inboxDir, m.file.replace(/\.deferred$/, ".processed-bundled"))
+        );
+        result.bundled++;
+      } catch {
+        // Best-effort; missing file is fine.
+      }
+    }
+  }
+
+  return result;
+}

package/scripts/daemon/inbox-deferral.test.mjs

@@ -0,0 +1,154 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+import { markDeferred, promoteDeferred } from "./inbox-deferral.mjs";
+
+function makeAgentRoot() {
+  const root = mkdtempSync(join(tmpdir(), "maestro-deferral-"));
+  mkdirSync(join(root, "state", "inbox", "slack"), { recursive: true });
+  return root;
+}
+
+function writeInboxItem(root, name, { channel_id, timestamp, id = "test-id" }) {
+  const body = [
+    `id: "${id}"`,
+    `service: "slack"`,
+    `channel_id: "${channel_id}"`,
+    `timestamp: "${timestamp}"`,
+    `content: |`,
+    `  body`,
+    "",
+  ].join("\n");
+  writeFileSync(join(root, "state", "inbox", "slack", name), body);
+}
+
+test("markDeferred renames live file to .deferred", () => {
+  const root = makeAgentRoot();
+  try {
+    writeInboxItem(root, "msg-A.yaml", {
+      channel_id: "D001",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "msg-A",
+    });
+    const n = markDeferred({ id: "msg-A" }, "slack", root);
+    assert.equal(n, 1);
+    const files = readdirSync(join(root, "state", "inbox", "slack"));
+    assert.deepEqual(files, ["msg-A.yaml.deferred"]);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("markDeferred is idempotent — already-deferred files are skipped", () => {
+  const root = makeAgentRoot();
+  try {
+    writeInboxItem(root, "msg-B.yaml.deferred", {
+      channel_id: "D001",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "msg-B",
+    });
+    const n = markDeferred({ id: "msg-B" }, "slack", root);
+    assert.equal(n, 0);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("markDeferred is no-op when item has no matching file (e.g. backlog)", () => {
+  const root = makeAgentRoot();
+  try {
+    const n = markDeferred({ id: "no-such-id" }, "slack", root);
+    assert.equal(n, 0);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("promoteDeferred with single item: renames back to live", () => {
+  const root = makeAgentRoot();
+  try {
+    writeInboxItem(root, "msg-C.yaml.deferred", {
+      channel_id: "D099N1JEA10",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "msg-C",
+    });
+    const r = promoteDeferred("D099N1JEA10", root);
+    assert.equal(r.promoted, 1);
+    assert.equal(r.bundled, 0);
+    assert.equal(r.service, "slack");
+    const files = readdirSync(join(root, "state", "inbox", "slack"));
+    assert.deepEqual(files, ["msg-C.yaml"]);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("promoteDeferred with multiple items: keeps latest, bundles rest", () => {
+  const root = makeAgentRoot();
+  try {
+    writeInboxItem(root, "msg-1.yaml.deferred", {
+      channel_id: "D001",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "msg-1",
+    });
+    writeInboxItem(root, "msg-2.yaml.deferred", {
+      channel_id: "D001",
+      timestamp: "2026-05-13T00:01:00Z",
+      id: "msg-2",
+    });
+    writeInboxItem(root, "msg-3.yaml.deferred", {
+      channel_id: "D001",
+      timestamp: "2026-05-13T00:02:00Z",
+      id: "msg-3",
+    });
+    const r = promoteDeferred("D001", root);
+    assert.equal(r.promoted, 1);
+    assert.equal(r.bundled, 2);
+    const files = readdirSync(join(root, "state", "inbox", "slack")).sort();
+    assert.deepEqual(files, [
+      "msg-1.yaml.processed-bundled",
+      "msg-2.yaml.processed-bundled",
+      "msg-3.yaml", // latest survives as live
+    ]);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("promoteDeferred ignores items in other channels", () => {
+  const root = makeAgentRoot();
+  try {
+    writeInboxItem(root, "for-A.yaml.deferred", {
+      channel_id: "D-A",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "for-A",
+    });
+    writeInboxItem(root, "for-B.yaml.deferred", {
+      channel_id: "D-B",
+      timestamp: "2026-05-13T00:00:00Z",
+      id: "for-B",
+    });
+    const r = promoteDeferred("D-A", root);
+    assert.equal(r.promoted, 1);
+    assert.equal(r.bundled, 0);
+    const files = readdirSync(join(root, "state", "inbox", "slack")).sort();
+    assert.deepEqual(files, ["for-A.yaml", "for-B.yaml.deferred"]);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("promoteDeferred no-op when nothing to promote", () => {
+  const root = makeAgentRoot();
+  try {
+    const r = promoteDeferred("D999", root);
+    assert.equal(r.promoted, 0);
+    assert.equal(r.bundled, 0);
+    assert.equal(r.service, null);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

package/scripts/poller/utils.mjs

@@ -17,11 +17,19 @@ export function writeInboxItem(service, item) {
   const filename = `${ts}-${item.id}.yaml`;
   const filePath = join(dir, filename);
 
-  // Skip if this item was already written or already processed.
-  // The daemon renames files to .processed after handling them;
-  // without this check the 30-min thread lookback re-creates the
-  // file every poll cycle, causing duplicate responses.
-  if (existsSync(filePath) || existsSync(filePath + ".processed")) {
+  // Skip if this item was already written or already processed/deferred.
+  // The daemon renames files to .processed after handling, to .deferred
+  // when parked behind an active channel lock, and to .processed-bundled
+  // when older entries are folded into a newer message during promotion.
+  // Without all four checks the 30-min thread lookback re-creates the
+  // file every poll cycle, causing duplicate responses or surprise
+  // re-promotion of work that was already bundled.
+  if (
+    existsSync(filePath) ||
+    existsSync(filePath + ".processed") ||
+    existsSync(filePath + ".deferred") ||
+    existsSync(filePath + ".processed-bundled")
+  ) {
     return;
   }