@melaya/runner 1.0.96 → 1.0.98

package/dist/assistantHost.py

@@ -37,6 +37,52 @@ import traceback
 _EVENT_PREFIX = "MELASSIST "
 _IDLE_EXIT_SECONDS = int(os.environ.get("MEL_ASSISTANT_IDLE_SECONDS", "900"))  # 15 min
 
+# PR4 rehydration: this host instance id + the DB generation it booted under.
+import uuid as _uuid
+_HOST_ID = _uuid.uuid4().hex
+try:
+    _GENERATION = int(os.environ.get("MEL_ASSISTANT_GENERATION", "0") or 0)
+except Exception:
+    _GENERATION = 0
+
+
+def _run_async(coro):
+    """Run a coroutine to completion from the synchronous main loop (no running
+    loop here — turns use their own asyncio.run)."""
+    import asyncio
+    return asyncio.run(coro)
+
+
+def _agent_memory(agent):
+    """Best-effort handle to the agent's BoundedMemory (for watermark + restore)."""
+    try:
+        mem = getattr(agent, "memory", None)
+        if mem is not None and hasattr(mem, "memory_watermark"):
+            return mem
+    except Exception:
+        pass
+    return None
+
+
+def _memory_watermark(agent) -> int:
+    mem = _agent_memory(agent)
+    try:
+        return int(mem.memory_watermark()) if mem is not None else 0
+    except Exception:
+        return 0
+
+
+def _render_summary_text(summary) -> str:
+    """Render a server AssistantSummary ({facts, provenance}) to a data-only text
+    block for the compressed-summary slot. Never policy — reference only."""
+    if not isinstance(summary, dict):
+        return ""
+    facts = summary.get("facts") or []
+    lines = [f"- {str(f)[:240]}" for f in facts if str(f).strip()]
+    if not lines:
+        return ""
+    return "[Earlier conversation summary — reference only]\n" + "\n".join(lines)
+
 # Streaming state for the CURRENT turn — the pre_print / post_acting hooks read
 # this to emit delta / tool events keyed to the turn in flight. `cancel` is the
 # STOP flag: set by the stdin-reader thread, polled by the running turn.
@@ -177,10 +223,15 @@ def _build_agent():
     model = os.environ.get("MEL_ASSISTANT_MODEL", "") or None
     language = os.environ.get("MEL_ASSISTANT_LANGUAGE", "en")
 
-    # Phone control is offered ONLY on the native mobile app surface — the same
-    # gate as the server-side melaya_run_phone_task tool. A desktop / web chat
-    # never gets the phone_* tools, even if a phone is paired.
-    phone_enabled = os.environ.get("MEL_ASSISTANT_SURFACE", "") == "mobile-native"
+    # Phone control unlocks on the native mobile app surface OR whenever the user
+    # has a PAIRED phone (MEL_ASSISTANT_PHONE_READY, set by the server from
+    # getPhonePresence). A desktop / web chat drives the paired phone REMOTELY
+    # through the same Redis queue, so it must not be surface-gated — without this,
+    # desktop chats had zero phone_* tools and the model refused phone tasks.
+    phone_enabled = (
+        os.environ.get("MEL_ASSISTANT_SURFACE", "") == "mobile-native"
+        or os.environ.get("MEL_ASSISTANT_PHONE_READY", "") in ("1", "true", "True")
+    )
 
     # Read-only platform tools (melaya_agent); phone control (Device Control) on
     # mobile only. These POST to /api/v1/private/assistant-tool + /phone/command
@@ -201,10 +252,20 @@ def _build_agent():
     try:
         if connector_services:
             from shared.orchestration.lazy_registry import build_lazy_toolkit
+            # Phone tools must ALWAYS be fully passed, never scoped: a phone task
+            # can't afford search_tools/activate_tool round-trips per tap (that is
+            # the "why did it restrict the tools + it's slow" regression). The lazy
+            # toolkit pins active_categories only UP TO `budget` (default 25), so
+            # phone (~23) + melaya_agent got bumped into the deferred pool. When
+            # phone is enabled, widen the budget so the WHOLE phone + base set stays
+            # active/pinned; the connectors still lazy-defer beyond that.
+            _budget = int(os.environ.get("MEL_LAZY_BUDGET", "25"))
+            if phone_enabled:
+                _budget = max(_budget, 64)
             toolkit = build_lazy_toolkit(
                 active_categories=categories,
                 include_categories=categories + connector_services,
-                budget=int(os.environ.get("MEL_LAZY_BUDGET", "25")),
+                budget=_budget,
             )
         else:
             toolkit = build_toolkit(categories=categories)
@@ -487,7 +548,20 @@ def main() -> int:
         _emit("", "error", message=f"assistant host failed to start: {exc}")
         return 1
 
-    _emit("", "ready")
+    # PR4: report memory watermark + host identity + generation so the server can
+    # decide whether to offer a generation-bound rehydrate snapshot.
+    # Surface context compaction to the chat UI on the RUNNER path too: the
+    # BoundedMemory fires pre_compact / post_compact around a summarise pass.
+    try:
+        from shared.orchestration.hooks import pipeline_hooks
+        pipeline_hooks.register("pre_compact", lambda ctx: _emit(str(_stream.get("turnId") or ""), "compacting"))
+        pipeline_hooks.register("post_compact", lambda ctx: _emit(
+            str(_stream.get("turnId") or ""), "compacted",
+            dropped=int((ctx or {}).get("messages_compressed") or 0)))
+    except Exception:
+        _log("compaction hooks unavailable (non-fatal)")
+
+    _emit("", "ready", memoryWatermark=_memory_watermark(agent), hostInstanceId=_HOST_ID, generation=_GENERATION)
     _log("ready")
 
     q: "queue.Queue[str]" = queue.Queue()
@@ -510,6 +584,38 @@ def main() -> int:
         except Exception:
             _log(f"bad stdin line (not json): {line[:120]}")
             continue
+        # PR4: generation-bound rehydrate. Seed a FRESH host's memory from the
+        # server snapshot (task pin + summary + recent tail) via restoreSnapshot —
+        # watermark-guarded so a re-offer never double-seeds. Never memory.add().
+        if isinstance(req, dict) and req.get("kind") == "restore":
+            try:
+                gen = int(req.get("generation") or 0)
+            except Exception:
+                gen = 0
+            if _GENERATION and gen and gen != _GENERATION:
+                _log(f"restore for stale generation {gen} (mine={_GENERATION}) — ignored")
+                continue
+            mem = _agent_memory(agent)
+            if mem is None:
+                continue
+            try:
+                from agentscope.message import Msg as _Msg
+                recent = req.get("recent") or []
+                recent_msgs = [
+                    _Msg(str(m.get("role") or "user"), str(m.get("content") or ""), str(m.get("role") or "user"))
+                    for m in recent if isinstance(m, dict) and str(m.get("content") or "").strip()
+                ]
+                applied = _run_async(mem.restore_snapshot(
+                    task_pin=str(req.get("taskPin") or ""),
+                    summary_text=_render_summary_text(req.get("summary")),
+                    recent_msgs=recent_msgs,
+                    watermark=int(req.get("watermark") or 0),
+                ))
+                _emit("", "rehydrated", applied=bool(applied), watermark=_memory_watermark(agent))
+                _log(f"restore applied={applied} watermark={_memory_watermark(agent)}")
+            except Exception:
+                _log("restore failed:\n" + traceback.format_exc())
+            continue
         turn_id = str(req.get("turnId") or "")
         message = str(req.get("message") or "")
         # Per-turn autonomy mode: the TS side carries an updated `hitl_mode`

package/dist/connection.js

@@ -29,6 +29,13 @@ import { startLumaBrowserBridge } from "./lumaBrowserBridge.js";
 const HEARTBEAT_INTERVAL = 30_000;
 const activeProcesses = new Map();
 const activeAssistants = new Map();
+// PR4: sid-keyed boot reservation. The guard-then-spawn in runner:assistant_start
+// has an async gap (ensureSharedModules / ensurePythonEnv awaits) between the
+// activeAssistants.has() check and the .set(). A second assistant_start in that
+// window — a concurrent SSE, or a RESTARTED server re-emitting start while the
+// host is still booting — would double-spawn two python hosts. Reserving the sid
+// SYNCHRONOUSLY here closes that window.
+const bootingAssistants = new Set();
 const ASSISTANT_IDLE_MS = 15 * 60 * 1000;
 export async function connect(opts) {
     const spinner = ora("Connecting to Melaya...").start();
@@ -876,11 +883,19 @@ export async function connect(opts) {
         const sid = String(payload.sessionId || "");
         if (!sid)
             return;
+        const generation = Number(payload.generation ?? 0) || 0;
         const emitEv = (ev) => socket.emit("runner:assistant_event", { sessionId: sid, turnId: "", ...ev });
+        // A live host already exists — report ready (with its watermark so the server
+        // can decide whether to re-offer a rehydrate). No second spawn.
         if (activeAssistants.has(sid)) {
             emitEv({ kind: "ready" });
             return;
         }
+        // A boot is already in flight for this sid (concurrent start / server restart
+        // mid-boot). Drop this duplicate — the in-flight boot will emit `ready`.
+        if (bootingAssistants.has(sid))
+            return;
+        bootingAssistants.add(sid); // SYNCHRONOUS reservation before any await
         try {
             const sharedVersion = String(payload.sharedVersion ?? "latest");
             await ensureSharedModules(opts.serverUrl, sharedVersion, opts.token);
@@ -932,6 +947,10 @@ export async function connect(opts) {
                 MEL_ASSISTANT_MODEL: String(payload.model || ""),
                 MEL_ASSISTANT_LANGUAGE: String(payload.language || "en"),
                 MEL_ASSISTANT_SURFACE: String(payload.surface || ""),
+                // Phone toolkit unlocks when the user has a paired phone, on ANY surface
+                // (desktop drives the paired phone remotely). The host ORs this with the
+                // mobile-native surface check. Empty ⇒ not paired ⇒ no phone tools.
+                MEL_ASSISTANT_PHONE_READY: payload.phoneReady ? "1" : "",
                 // Connector tool sets the user enabled for this chat (comma-joined service
                 // ids). The host seeds a lazy toolkit from these so ANY connector's tools
                 // are reachable without exploding context.
@@ -942,6 +961,10 @@ export async function connect(opts) {
                 // reads this at start; each turn's frame can flip it (see assistant_turn).
                 // Fail-safe: unknown ⇒ "safe".
                 MEL_ASSISTANT_HITL_MODE: (payload.hitlMode === "autonomous" || payload.hitlMode === "payments_only") ? payload.hitlMode : "safe",
+                // PR4: the DB-authoritative generation this host boots under. The host
+                // echoes it in `ready` and stamps events; an older boot self-terminates
+                // on a generation mismatch.
+                MEL_ASSISTANT_GENERATION: String(generation),
                 // Run context for phone-driving assistant turns. The assistant host has
                 // no pipeline MEL_RUN_ID, so without this phone.py._cmd omits `run_id`,
                 // the server's arm block (phone.ts, `if (runId)`) never fires, and the
@@ -964,7 +987,7 @@ export async function connect(opts) {
                 ...(payload.credentials || {}),
             };
             const proc = spawn(envResult.pythonPath, ["-u", stagedHost], { cwd: workDir, env, stdio: ["pipe", "pipe", "pipe"] });
-            const session = { proc, lastActivity: Date.now(), stdoutBuf: "" };
+            const session = { proc, lastActivity: Date.now(), stdoutBuf: "", generation };
             activeAssistants.set(sid, session);
             console.log(chalk.hex("#7c6ff0")(`  ◆ Assistant session ${sid.slice(0, 10)}… (${payload.provider})`));
             proc.stdout?.on("data", (data) => {
@@ -1008,6 +1031,39 @@ export async function connect(opts) {
         catch (e) {
             emitEv({ kind: "error", message: `assistant start failed: ${e?.message || e}` });
         }
+        finally {
+            // Release the boot reservation on EVERY path (spawned, or errored/returned
+            // early). By now the host is either in activeAssistants or dead.
+            bootingAssistants.delete(sid);
+        }
+    });
+    // PR4: generation-bound rehydrate. The server sends this AFTER `ready`, only
+    // when the DB watermark is newer than the host's. Forward it to the host as a
+    // `restore` frame; the host applies it via restoreSnapshot ONLY if its own
+    // memory watermark is older (idempotent — a re-offer never double-seeds).
+    socket.on("runner:assistant_rehydrate", (payload) => {
+        const sid = String(payload.sessionId || "");
+        const s = activeAssistants.get(sid);
+        if (!s)
+            return;
+        // Bind to the boot generation — a frame for an older/newer generation than the
+        // live host is dropped (the host also re-checks).
+        const gen = Number(payload.generation ?? 0) || 0;
+        if (gen && s.generation && gen !== s.generation)
+            return;
+        s.lastActivity = Date.now();
+        try {
+            s.proc.stdin?.write(JSON.stringify({
+                kind: "restore",
+                generation: gen,
+                watermark: Number(payload.watermark ?? 0) || 0,
+                taskPin: payload.taskPin ?? null,
+                summary: payload.summary ?? null,
+                summaryUptoSeq: Number(payload.summaryUptoSeq ?? 0) || 0,
+                recent: Array.isArray(payload.recent) ? payload.recent : [],
+            }) + "\n");
+        }
+        catch { /* best-effort — host stays as-is on a write failure */ }
     });
     socket.on("runner:assistant_turn", (payload) => {
         const s = activeAssistants.get(String(payload.sessionId || ""));

package/package.json

@@ -1,42 +1,42 @@
-{
-  "name": "@melaya/runner",
-  "version": "1.0.96",
-  "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
-  "license": "UNLICENSED",
-  "private": false,
-  "type": "module",
-  "bin": {
-    "melaya-runner": "dist/cli.js"
-  },
-  "main": "dist/index.js",
-  "files": [
-    "dist/**/*.js",
-    "dist/**/*.d.ts",
-    "dist/**/*.py",
-    "localRagIngest.py",
-    "localRagRetrieve.py",
-    "nltk_data/**",
-    "README.md"
-  ],
-  "scripts": {
-    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
-    "prepublishOnly": "npm run build"
-  },
-  "dependencies": {
-    "chalk": "^5.3.0",
-    "commander": "^12.0.0",
-    "ora": "^8.0.0",
-    "playwright": "^1.47.0",
-    "socket.io-client": "^4.8.0"
-  },
-  "devDependencies": {
-    "@types/node": "^20.0.0",
-    "typescript": "^5.5.0"
-  },
-  "engines": {
-    "node": ">=18"
-  },
-  "publishConfig": {
-    "access": "public"
-  }
-}
+{
+  "name": "@melaya/runner",
+  "version": "1.0.98",
+  "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
+  "license": "UNLICENSED",
+  "private": false,
+  "type": "module",
+  "bin": {
+    "melaya-runner": "dist/cli.js"
+  },
+  "main": "dist/index.js",
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.py",
+    "localRagIngest.py",
+    "localRagRetrieve.py",
+    "nltk_data/**",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "commander": "^12.0.0",
+    "ora": "^8.0.0",
+    "playwright": "^1.47.0",
+    "socket.io-client": "^4.8.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.5.0"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}