bazilion 0.1.0 → 0.2.0

package/dist/worker.js

@@ -76,9 +76,34 @@ import { existsSync as existsSync5, rmSync as rmSync2 } from "fs";
 import { writeFileSync as writeFileSync3 } from "fs";
 import { join as join7 } from "path";
 
+// ../daemon/src/core/profile-group/spawn.ts
+import { readdirSync as readdirSync3, rmSync as rmSync4 } from "fs";
+
+// ../daemon/src/core/profile-group/rm-with-retry.ts
+import { rmSync as rmSync3 } from "fs";
+
 // ../daemon/src/core/services.ts
 var SERVICES = [
   // --- LLM providers (configured via API keys / URLs) ---
+  // Top 3: openai-codex (ChatGPT OAuth), openai (API key), anthropic.
+  // Everything else in rough popularity order; locals last.
+  {
+    id: "openai-codex",
+    displayName: "OpenAI ChatGPT (OAuth)",
+    category: "provider",
+    hint: "Use your ChatGPT Plus/Pro/Team account (same login as Codex CLI)",
+    // No form fields — credentials come from an OAuth flow. The /config page
+    // renders a Connect/Disconnect card using /api/auth/openai instead of the
+    // standard field inputs.
+    fields: []
+  },
+  {
+    id: "openai",
+    displayName: "OpenAI",
+    category: "provider",
+    hint: "GPT models \xB7 platform.openai.com",
+    fields: [{ envVar: "OPENAI_API_KEY", kind: "secret", label: "API key", placeholder: "sk-..." }]
+  },
   {
     id: "anthropic",
     displayName: "Anthropic",
@@ -94,23 +119,6 @@ var SERVICES = [
       }
     ]
   },
-  {
-    id: "openai",
-    displayName: "OpenAI",
-    category: "provider",
-    hint: "GPT models \xB7 platform.openai.com",
-    fields: [{ envVar: "OPENAI_API_KEY", kind: "secret", label: "API key", placeholder: "sk-..." }]
-  },
-  {
-    id: "openai-codex",
-    displayName: "OpenAI ChatGPT (OAuth)",
-    category: "provider",
-    hint: "Use your ChatGPT Plus/Pro/Team account (same login as Codex CLI)",
-    // No form fields — credentials come from an OAuth flow. The /config page
-    // renders a Connect/Disconnect card using /api/auth/openai instead of the
-    // standard field inputs.
-    fields: []
-  },
   {
     id: "google",
     displayName: "Google (Gemini)",
@@ -153,6 +161,27 @@ var SERVICES = [
     hint: "Authenticates via AWS SDK env (AWS_PROFILE or AWS_ACCESS_KEY_ID/SECRET)",
     fields: []
   },
+  {
+    id: "github-copilot",
+    displayName: "GitHub Copilot",
+    category: "provider",
+    hint: "Use a GitHub Copilot subscription to call Claude/GPT/Gemini via Copilot",
+    fields: [
+      {
+        envVar: "COPILOT_GITHUB_TOKEN",
+        kind: "secret",
+        label: "GitHub token",
+        description: "Generic GH_TOKEN/GITHUB_TOKEN are ignored \u2014 set this scoped variable explicitly (or run `bazilion auth copilot login` once available)."
+      }
+    ]
+  },
+  {
+    id: "deepseek",
+    displayName: "DeepSeek",
+    category: "provider",
+    hint: "DeepSeek V4 Flash / Pro \xB7 platform.deepseek.com",
+    fields: [{ envVar: "DEEPSEEK_API_KEY", kind: "secret", label: "API key" }]
+  },
   {
     id: "mistral",
     displayName: "Mistral",
@@ -160,6 +189,13 @@ var SERVICES = [
     hint: "mistral.ai",
     fields: [{ envVar: "MISTRAL_API_KEY", kind: "secret", label: "API key" }]
   },
+  {
+    id: "xai",
+    displayName: "xAI",
+    category: "provider",
+    hint: "Grok \xB7 x.ai",
+    fields: [{ envVar: "XAI_API_KEY", kind: "secret", label: "API key" }]
+  },
   {
     id: "groq",
     displayName: "Groq",
@@ -174,11 +210,46 @@ var SERVICES = [
     fields: [{ envVar: "CEREBRAS_API_KEY", kind: "secret", label: "API key" }]
   },
   {
-    id: "xai",
-    displayName: "xAI",
+    id: "fireworks",
+    displayName: "Fireworks AI",
     category: "provider",
-    hint: "Grok \xB7 x.ai",
-    fields: [{ envVar: "XAI_API_KEY", kind: "secret", label: "API key" }]
+    hint: "DeepSeek/GLM/Kimi via fireworks.ai",
+    fields: [{ envVar: "FIREWORKS_API_KEY", kind: "secret", label: "API key" }]
+  },
+  {
+    id: "together",
+    displayName: "Together AI",
+    category: "provider",
+    hint: "Open-weight models \xB7 together.ai",
+    fields: [{ envVar: "TOGETHER_API_KEY", kind: "secret", label: "API key" }]
+  },
+  {
+    id: "moonshotai",
+    displayName: "Moonshot AI",
+    category: "provider",
+    hint: "Kimi K2/K2.5/K2.6 \xB7 platform.moonshot.ai",
+    fields: [{ envVar: "MOONSHOT_API_KEY", kind: "secret", label: "API key" }]
+  },
+  {
+    id: "kimi-coding",
+    displayName: "Kimi Coding",
+    category: "provider",
+    hint: "Coding-tuned Kimi endpoint \xB7 platform.moonshot.cn",
+    fields: [{ envVar: "KIMI_API_KEY", kind: "secret", label: "API key" }]
+  },
+  {
+    id: "minimax",
+    displayName: "MiniMax",
+    category: "provider",
+    hint: "MiniMax M2 family \xB7 platform.minimaxi.com",
+    fields: [{ envVar: "MINIMAX_API_KEY", kind: "secret", label: "API key" }]
+  },
+  {
+    id: "xiaomi",
+    displayName: "Xiaomi MiMo",
+    category: "provider",
+    hint: "API billing endpoint \xB7 platform.xiaomimimo.com",
+    fields: [{ envVar: "XIAOMI_API_KEY", kind: "secret", label: "API key" }]
   },
   {
     id: "zai",
@@ -193,6 +264,27 @@ var SERVICES = [
     hint: "Inference endpoints \xB7 huggingface.co",
     fields: [{ envVar: "HF_TOKEN", kind: "secret", label: "Access token", placeholder: "hf_..." }]
   },
+  {
+    id: "cloudflare-ai-gateway",
+    displayName: "Cloudflare AI Gateway",
+    category: "provider",
+    hint: "Per-gateway routing to OpenAI/Anthropic/Workers AI",
+    fields: [
+      { envVar: "CLOUDFLARE_API_KEY", kind: "secret", label: "API key" },
+      { envVar: "CLOUDFLARE_ACCOUNT_ID", kind: "config", label: "Account ID" },
+      { envVar: "CLOUDFLARE_GATEWAY_ID", kind: "config", label: "Gateway ID" }
+    ]
+  },
+  {
+    id: "cloudflare-workers-ai",
+    displayName: "Cloudflare Workers AI",
+    category: "provider",
+    hint: "Inference on Cloudflare Workers \xB7 ai.cloudflare.com",
+    fields: [
+      { envVar: "CLOUDFLARE_API_KEY", kind: "secret", label: "API key" },
+      { envVar: "CLOUDFLARE_ACCOUNT_ID", kind: "config", label: "Account ID" }
+    ]
+  },
   {
     id: "openrouter",
     displayName: "OpenRouter",
@@ -216,6 +308,13 @@ var SERVICES = [
       }
     ]
   },
+  {
+    id: "opencode",
+    displayName: "OpenCode",
+    category: "provider",
+    hint: "OpenAI-compatible proxy from the OpenCode CLI",
+    fields: [{ envVar: "OPENCODE_API_KEY", kind: "secret", label: "API key" }]
+  },
   {
     id: "lmstudio",
     displayName: "LM Studio",
@@ -254,11 +353,54 @@ var SERVICES = [
       }
     ]
   },
+  {
+    id: "llamacpp",
+    displayName: "llama.cpp",
+    category: "provider",
+    hint: "Local inference \xB7 llama.cpp llama-server (OpenAI-compat /v1 endpoint)",
+    fields: [
+      {
+        envVar: "LLAMACPP_URL",
+        kind: "config",
+        label: "Endpoint URL",
+        placeholder: "http://127.0.0.1:8080/v1"
+      },
+      {
+        envVar: "LLAMACPP_API_KEY",
+        kind: "secret",
+        label: "API key (only if started with --api-key)",
+        description: "llama-server runs without auth by default. Set this only if you launched the server with the `--api-key KEY` flag."
+      }
+    ]
+  },
   // --- Ancillary services (web search, etc) ---
+  {
+    id: "firecrawl",
+    displayName: "Firecrawl",
+    category: "service",
+    group: "Web Search",
+    hint: "web_fetch fallback for JS-heavy/blocked pages \xB7 firecrawl.dev (free tier available)",
+    fields: [
+      {
+        envVar: "FIRECRAWL_API_KEY",
+        kind: "secret",
+        label: "API key",
+        placeholder: "fc-...",
+        description: "When set, web_fetch automatically falls back to Firecrawl if the primary Readability extraction yields too little content."
+      },
+      {
+        envVar: "FIRECRAWL_URL",
+        kind: "config",
+        label: "Base URL (optional, for self-hosted)",
+        placeholder: "https://api.firecrawl.dev"
+      }
+    ]
+  },
   {
     id: "brave-search",
     displayName: "Brave Search",
     category: "service",
+    group: "Web Search",
     hint: "Web search tool \xB7 free tier at brave.com/search/api/",
     fields: [{ envVar: "BRAVE_API_KEY", kind: "secret", label: "API key", placeholder: "BSA..." }]
   },
@@ -266,6 +408,7 @@ var SERVICES = [
     id: "searxng",
     displayName: "SearXNG",
     category: "service",
+    group: "Web Search",
     hint: "Self-hosted meta-search engine \xB7 searxng.org",
     fields: [
       {
@@ -309,7 +452,7 @@ import { createHash, randomBytes as randomBytes2, randomUUID as randomUUID4 } fr
 import { existsSync as existsSync6, readFileSync as readFileSync4 } from "fs";
 
 // ../daemon/src/core/skills/import.ts
-import { cpSync, existsSync as existsSync7, mkdtempSync, readdirSync as readdirSync3, rmSync as rmSync3, statSync as statSync2 } from "fs";
+import { cpSync, existsSync as existsSync7, mkdtempSync, readdirSync as readdirSync4, rmSync as rmSync5, statSync as statSync2 } from "fs";
 import { tmpdir } from "os";
 import { basename, join as join8, resolve as resolve2, sep } from "path";
 import AdmZip from "adm-zip";
@@ -322,9 +465,9 @@ import { parse as parseYaml } from "yaml";
 import {
   existsSync as existsSync8,
   mkdirSync as mkdirSync4,
-  readdirSync as readdirSync4,
+  readdirSync as readdirSync5,
   readFileSync as readFileSync6,
-  rmSync as rmSync4,
+  rmSync as rmSync6,
   statSync as statSync3,
   writeFileSync as writeFileSync4
 } from "fs";
@@ -357,7 +500,7 @@ function safeKey(root, key) {
 }
 function walkMd(dir, prefix, out) {
   if (!existsSync8(dir)) return;
-  for (const e of readdirSync4(dir, { withFileTypes: true })) {
+  for (const e of readdirSync5(dir, { withFileTypes: true })) {
     if (e.name.startsWith(".")) continue;
     const full = join9(dir, e.name);
     const key = prefix ? `${prefix}/${e.name}` : e.name;
@@ -432,7 +575,7 @@ function qmdBackend(root) {
     },
     async remove(key) {
       const path = safeKey(root, key);
-      if (existsSync8(path)) rmSync4(path);
+      if (existsSync8(path)) rmSync6(path);
       const store = await getStore(root);
       await store.update();
     }
@@ -558,7 +701,7 @@ function piMessagesToProviderView(messages) {
 }
 
 // ../daemon/src/runtime/pi/session.ts
-import { existsSync as existsSync11, mkdirSync as mkdirSync5, readdirSync as readdirSync6, statSync as statSync5 } from "fs";
+import { existsSync as existsSync11, mkdirSync as mkdirSync5, readdirSync as readdirSync7, statSync as statSync5 } from "fs";
 import { basename as basename2, join as join13 } from "path";
 import {
   AuthStorage,
@@ -567,14 +710,14 @@ import {
   ModelRegistry,
   SessionManager,
   SettingsManager
-} from "@mariozechner/pi-coding-agent";
+} from "@earendil-works/pi-coding-agent";
 
 // ../daemon/src/runtime/providers/pi-adapter.ts
 import {
   getModel,
   streamSimple,
   Type
-} from "@mariozechner/pi-ai";
+} from "@earendil-works/pi-ai";
 function defaultBaseUrlFor(providerName) {
   switch (providerName) {
     case "lmstudio":
@@ -616,7 +759,7 @@ function resolveModel(cfg, modelId) {
 }
 
 // ../daemon/src/runtime/auth/openai-codex.ts
-import { loginOpenAICodex, refreshOpenAICodexToken } from "@mariozechner/pi-ai/oauth";
+import { loginOpenAICodex, refreshOpenAICodexToken } from "@earendil-works/pi-ai/oauth";
 
 // ../daemon/src/runtime/session/prompt.ts
 import { existsSync as existsSync9, readFileSync as readFileSync7 } from "fs";
@@ -626,13 +769,11 @@ var CONTEXT_FILE_ORDER = [
   "SOUL.md",
   "TOOLS.md",
   "IDENTITY.md",
-  "HEARTBEAT.md",
-  "BOOTSTRAP.md"
+  "HEARTBEAT.md"
 ];
 function buildSystemPrompt(agent) {
   const parts = [];
   const contextBlocks = [];
-  let bootstrapPresent = false;
   for (const file of CONTEXT_FILE_ORDER) {
     const path = join10(agent.agent.dir, file);
     if (!existsSync9(path)) continue;
@@ -641,23 +782,39 @@ function buildSystemPrompt(agent) {
     contextBlocks.push(`## ${file}
 
 ${content}`);
-    if (file === "BOOTSTRAP.md") bootstrapPresent = true;
   }
   if (contextBlocks.length > 0) {
     parts.push(`# Project Context
 
 ${contextBlocks.join("\n\n")}`);
   }
-  if (bootstrapPresent) {
-    parts.push(
-      "NOTE: This is your first session. After you have completed the bootstrap conversation, call the `bootstrap_done` tool to delete BOOTSTRAP.md."
-    );
+  const bootstrapPath = join10(agent.agent.dir, "BOOTSTRAP.md");
+  if (existsSync9(bootstrapPath)) {
+    const bootstrap = readFileSync7(bootstrapPath, "utf8").trimEnd();
+    if (bootstrap) {
+      parts.push(
+        [
+          "# First-Run Ritual",
+          "",
+          "This is your first session. The document below is **conversational guidance**, not a checklist to execute in one shot. It describes a multi-turn Q&A you should have with the human, one question per turn.",
+          "",
+          "## Hard rules",
+          "- Your first reply is ONLY a greeting + ONE question. No tool calls. Wait for the human to answer.",
+          "- Each subsequent turn: at most one new question. Wait between turns.",
+          "- Only after the ritual is complete (you have enough to write IDENTITY.md): call `home_write` once, then `bootstrap_done`.",
+          "",
+          "## BOOTSTRAP.md",
+          "",
+          bootstrap
+        ].join("\n")
+      );
+    }
   }
   parts.push(
     [
       "# Agent Home",
       "",
-      "Your private home holds who you are \u2014 identity, soul, behaviour rules, wake-up routine. It is not shared with other agents and cannot be overwritten by them. The files above (IDENTITY.md, SOUL.md, AGENTS.md, TOOLS.md, HEARTBEAT.md, BOOTSTRAP.md) live in this home.",
+      "Your private home holds who you are \u2014 identity, soul, behaviour rules, wake-up routine. It is not shared with other agents and cannot be overwritten by them. The files above (IDENTITY.md, SOUL.md, AGENTS.md, TOOLS.md, HEARTBEAT.md) live in this home, plus BOOTSTRAP.md when you are still in your first-run ritual.",
       "",
       "- To change who you are (name, vibe, personality, how you behave): use `home_write`.",
       "- To inspect exact wording of your own files: use `home_read` or `home_list`.",
@@ -684,13 +841,19 @@ You have access to the following skills: ${agent.skills.join(", ")}.`
     parts.push(
       `# About the User
 
-Read-only context about the human you're working with in this group. You cannot edit this \u2014 if it's wrong, say so and they will update it.
+Shared context about the human you're working with in this group. Both you and the human curate it. To update: call \`user_md_get\` (returns current content + an etag), merge your change into the full text, then call \`user_md_write\` with the merged content and the etag. Use this for STABLE user-specific facts (preferences, role, working hours, how they like to be addressed) \u2014 and to CORRECT stale entries when the human tells you something different from what's recorded. For project knowledge use \`memory_write\` instead; for personal notes about yourself use \`home_write\` on IDENTITY.md. **Do NOT send peer messages announcing USER.md changes \u2014 every agent in the group sees the new content in their system prompt on their next turn automatically.**
 
 ${agent.group.userMd.trim()}`
     );
+  } else {
+    parts.push(
+      `# About the User
+
+This group's USER.md is empty. As you learn STABLE facts about the human (preferences, role, working hours, how they like to be addressed), populate it via \`user_md_get\` then \`user_md_write\` (always get first \u2014 you need the etag). Reserve this for things you're confident are durable \u2014 project knowledge belongs in \`memory_write\`, personal notes about yourself in \`home_write\` on IDENTITY.md. **Do NOT send peer messages announcing USER.md changes \u2014 every agent in the group sees the new content in their system prompt on their next turn automatically.**`
+    );
   }
   parts.push(
-    "# Memory\n\nYou share a persistent memory backend with every other agent in this group. Use `memory_write` to remember things across sessions, and `memory_search` / `memory_read` / `memory_list` to recall them. This memory is for project knowledge \u2014 codebase notes, decisions, things the user told you about the work. For personal notes about yourself (preferences, persona quirks), use `home_write` on IDENTITY.md instead. Always check memory at the start of a session: another agent in the group may have already learned something useful."
+    "# Memory\n\nYou share a persistent memory backend with every other agent in this group. Use `memory_write` to remember things across sessions, and `memory_search` / `memory_read` / `memory_list` to recall them. This memory is for project knowledge \u2014 codebase notes, decisions, things the user told you about the work. For personal notes about yourself (preferences, persona quirks), use `home_write` on IDENTITY.md instead. Always check memory at the start of a session: another agent in the group may have already learned something useful. **Do NOT send peer messages announcing memory writes \u2014 every agent has access to the same store via `memory_search` and will find your note when they need it.**"
   );
   return parts.join("\n\n---\n\n");
 }
@@ -699,7 +862,7 @@ ${agent.group.userMd.trim()}`
 import { Type as Type2 } from "typebox";
 
 // ../daemon/src/runtime/tools/bootstrap.ts
-import { existsSync as existsSync10, rmSync as rmSync5 } from "fs";
+import { existsSync as existsSync10, rmSync as rmSync7 } from "fs";
 import { join as join11 } from "path";
 function bootstrapTool(agentDir) {
   return {
@@ -711,7 +874,7 @@ function bootstrapTool(agentDir) {
     async invoke() {
       const path = join11(agentDir, "BOOTSTRAP.md");
       if (existsSync10(path)) {
-        rmSync5(path);
+        rmSync7(path);
         return "BOOTSTRAP.md removed. Bootstrap is complete.";
       }
       return "BOOTSTRAP.md was already removed.";
@@ -720,7 +883,7 @@ function bootstrapTool(agentDir) {
 }
 
 // ../daemon/src/runtime/tools/home.ts
-import { readdirSync as readdirSync5, readFileSync as readFileSync8, statSync as statSync4, writeFileSync as writeFileSync5 } from "fs";
+import { readdirSync as readdirSync6, readFileSync as readFileSync8, statSync as statSync4, writeFileSync as writeFileSync5 } from "fs";
 import { join as join12 } from "path";
 var HOME_FILES_READABLE = [
   "IDENTITY.md",
@@ -812,7 +975,7 @@ function homeTools(agentDir) {
         if (entries.length === 0) {
           const dirEntries = (() => {
             try {
-              return readdirSync5(agentDir);
+              return readdirSync6(agentDir);
             } catch {
               return [];
             }
@@ -831,7 +994,7 @@ function memoryTools(memory) {
     {
       def: {
         name: "memory_write",
-        description: 'Write or update a memory note in the GROUP-SHARED memory. All agents in this group can read what you write. Use it for project knowledge, codebase notes, decisions, and findings \u2014 anything other agents in the group should benefit from. For personal notes about yourself (preferences, persona), use `home_write` on IDENTITY.md instead. Key is a path-like string with a markdown extension, e.g. "auth-flow.md" or "people/alice.md".',
+        description: 'Write or update a memory note in the GROUP-SHARED memory. All agents in this group can read what you write. Use it for project knowledge, codebase notes, decisions, and findings \u2014 anything other agents in the group should benefit from. For personal notes about yourself (preferences, persona) use `home_write` on IDENTITY.md. For STABLE facts about the human you\'re working with (their preferences, role, working hours, how they like to be addressed) use `user_md_get` then `user_md_write` \u2014 those land in every agent\'s system prompt directly. Key is a path-like string with a markdown extension, e.g. "auth-flow.md" or "decisions/2026-05-migration.md".',
         parameters: {
           type: "object",
           properties: {
@@ -917,7 +1080,7 @@ function messagingTools(host, fromAgentId) {
     {
       def: {
         name: "send_message",
-        description: "Send a message to another agent. Use the recipient's agent id (UUID).",
+        description: `Send a message to another agent. Use the recipient's agent id (UUID). Use this ONLY for things the recipient needs to ACT on: delegating a task, asking a peer for information you cannot get yourself, escalating a decision. Do NOT use it for status updates or to announce changes to group-shared resources \u2014 USER.md and the group memory backend both propagate to every agent in the group automatically on their next turn, so messages like "I updated USER.md" or "I wrote a new memory note" are pure noise and will trigger an inbox-wake loop on the recipient.`,
         parameters: {
           type: "object",
           properties: {
@@ -1021,6 +1184,64 @@ function messagingTools(host, fromAgentId) {
   ];
 }
 
+// ../daemon/src/runtime/tools/user-md.ts
+function userMdTools(host, groupId) {
+  return [
+    {
+      def: {
+        name: "user_md_get",
+        description: "Read the group-shared USER.md (facts every agent in the group knows about the human). Returns the current content followed by an `etag:` line \u2014 you MUST pass that etag back as `if_match` on the next `user_md_write` so the daemon can detect concurrent edits by other agents in the group. Always call this immediately before any `user_md_write`.",
+        parameters: {
+          type: "object",
+          properties: {}
+        }
+      },
+      async invoke() {
+        const { content, etag } = await host.get(groupId);
+        const body = content.length > 0 ? content : "(USER.md is empty)";
+        return `${body}
+
+---
+etag: ${etag}`;
+      }
+    },
+    {
+      def: {
+        name: "user_md_write",
+        description: "Replace the group-shared USER.md with new content. Use this for STABLE user-specific facts (preferences, role, working hours, how the human likes to be addressed). MANDATORY workflow: (1) call `user_md_get` first, (2) integrate your change into the full content preserving everything unrelated, (3) call `user_md_write` with the merged content and the etag you got from `user_md_get`. If another agent in the group wrote to USER.md between your get and write, this returns an etag-mismatch error \u2014 just call `user_md_get` again and retry the merge. The full result must fit under 12 KB. For project knowledge use `memory_write`; for notes about yourself use `home_write` on IDENTITY.md.",
+        parameters: {
+          type: "object",
+          properties: {
+            content: {
+              type: "string",
+              description: "Full new contents of USER.md (this is a complete replacement, NOT an append). Include everything you want to keep."
+            },
+            if_match: {
+              type: "string",
+              description: "The etag returned by your most recent `user_md_get`. The write fails if USER.md changed in the meantime."
+            }
+          },
+          required: ["content", "if_match"]
+        }
+      },
+      async invoke(args) {
+        const content = String(args.content ?? "");
+        const ifMatch = String(args.if_match ?? "");
+        if (!ifMatch) {
+          throw new Error(
+            'user_md_write: "if_match" is required \u2014 call user_md_get first to obtain the current etag.'
+          );
+        }
+        const { etag, totalBytes } = await host.write(groupId, content, ifMatch);
+        return `wrote USER.md (${totalBytes} bytes, new etag: ${etag})`;
+      }
+    }
+  ];
+}
+
+// ../daemon/src/runtime/tools/web.ts
+import { fetch as undiciFetch2 } from "undici";
+
 // ../daemon/src/runtime/tools/web-extract.ts
 import { Readability } from "@mozilla/readability";
 import { parseHTML } from "linkedom";
@@ -1101,7 +1322,7 @@ function extractReadable(html, url, mode) {
 // ../daemon/src/runtime/tools/web-ssrf.ts
 import { lookup as dnsLookupCb } from "dns";
 import { lookup as dnsLookup } from "dns/promises";
-import { Agent } from "undici";
+import { Agent, fetch as undiciFetch } from "undici";
 var SsrFBlockedError = class extends Error {
   constructor(message) {
     super(message);
@@ -1210,7 +1431,7 @@ async function closeDispatcher(d) {
   }
 }
 async function guardedFetch(opts) {
-  const fetcher = opts.fetchImpl ?? globalThis.fetch;
+  const fetcher = opts.fetchImpl ?? undiciFetch;
   const maxRedirects = opts.maxRedirects ?? 3;
   const abortController = new AbortController();
   const timeoutId = opts.timeoutMs ? setTimeout(() => abortController.abort(new Error("timeout")), opts.timeoutMs) : null;
@@ -1291,7 +1512,10 @@ var DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWe
 var DEFAULT_CACHE_TTL_MS = 15 * 6e4;
 var DEFAULT_CACHE_MAX = 100;
 var DEFAULT_MAX_LENGTH = 2e4;
-var DEFAULT_TIMEOUT_MS = 2e4;
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_MAX_BODY_BYTES = 3 * 1024 * 1024;
+var FIRECRAWL_FALLBACK_THRESHOLD = 200;
+var FIRECRAWL_DEFAULT_URL = "https://api.firecrawl.dev";
 function stripHtml(s) {
   return s.replace(/<[^>]*>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").trim();
 }
@@ -1325,6 +1549,92 @@ async function searxngSearch(query, limit, baseURL, fetchFn) {
     snippet: r.content ?? ""
   }));
 }
+function describeError(err) {
+  if (!(err instanceof Error)) return String(err);
+  const parts = [];
+  const seen = /* @__PURE__ */ new Set();
+  let cur = err;
+  while (cur instanceof Error && !seen.has(cur)) {
+    seen.add(cur);
+    const code = cur.code;
+    parts.push(code ? `[${code}] ${cur.message}` : cur.message);
+    cur = cur.cause;
+  }
+  return parts.join(" \u2014 cause: ");
+}
+async function readBodyCapped(res, maxBytes) {
+  const ct = res.headers.get("content-type") ?? "";
+  const charset = /charset=([^;]+)/i.exec(ct)?.[1]?.trim().toLowerCase() || "utf-8";
+  const decoder = new TextDecoder(charset, { fatal: false });
+  const reader = res.body?.getReader();
+  if (!reader) {
+    const text = await res.text();
+    if (text.length > maxBytes) return { text: text.slice(0, maxBytes), truncated: true };
+    return { text, truncated: false };
+  }
+  const chunks = [];
+  let total = 0;
+  let truncated = false;
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    if (!value) continue;
+    if (total + value.byteLength > maxBytes) {
+      const keep = maxBytes - total;
+      if (keep > 0) chunks.push(value.subarray(0, keep));
+      truncated = true;
+      try {
+        await reader.cancel();
+      } catch {
+      }
+      break;
+    }
+    chunks.push(value);
+    total += value.byteLength;
+  }
+  const sum = chunks.reduce((s, c) => s + c.byteLength, 0);
+  const merged = new Uint8Array(sum);
+  let off = 0;
+  for (const c of chunks) {
+    merged.set(c, off);
+    off += c.byteLength;
+  }
+  return { text: decoder.decode(merged), truncated };
+}
+async function firecrawlScrape(url, mode, env, fetchFn, timeoutMs) {
+  const apiKey = env.FIRECRAWL_API_KEY;
+  if (!apiKey) return null;
+  const base = (env.FIRECRAWL_URL ?? FIRECRAWL_DEFAULT_URL).replace(/\/$/, "");
+  const ac = new AbortController();
+  const t = setTimeout(() => ac.abort(new Error("firecrawl timeout")), timeoutMs);
+  try {
+    const res = await fetchFn(`${base}/v1/scrape`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: `Bearer ${apiKey}`,
+        accept: "application/json"
+      },
+      body: JSON.stringify({
+        url,
+        formats: ["markdown"],
+        onlyMainContent: true
+      }),
+      signal: ac.signal
+    });
+    if (!res.ok) return null;
+    const body = await res.json();
+    if (!body.success || !body.data?.markdown) return null;
+    const md = body.data.markdown;
+    const title = body.data.metadata?.title;
+    const text = mode === "text" ? markdownToPlain(md) : md;
+    return title ? { text, title } : { text };
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(t);
+  }
+}
 function cacheGet(cache, key) {
   const entry = cache.get(key);
   if (!entry) return null;
@@ -1345,12 +1655,14 @@ function cacheSet(cache, key, value, ttlMs, maxEntries) {
   }
 }
 function webTools(opts) {
-  const fetchFn = opts?.fetchImpl ?? fetch;
+  const fetchFn = opts?.fetchImpl ?? undiciFetch2;
   const env = opts?.env ?? process.env;
   const allowPrivate = opts?.allowPrivate ?? false;
   const cacheTtlMs = opts?.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
   const cacheMax = opts?.cacheMax ?? DEFAULT_CACHE_MAX;
   const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBodyBytes = opts?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
+  const firecrawlDisabled = opts?.firecrawlDisabled ?? false;
   const cache = /* @__PURE__ */ new Map();
   return [
     {
@@ -1388,7 +1700,7 @@ function webTools(opts) {
     {
       def: {
         name: "web_fetch",
-        description: "Fetch a URL and return its readable content. HTML is extracted via Readability and converted to markdown. Results are cached for 15 minutes.",
+        description: "Fetch a URL and return its readable content. HTML is extracted via Readability and converted to markdown. When the primary extraction returns near-empty content from a 2xx HTML page (typical of JS-only shells), the tool automatically retries via Firecrawl if FIRECRAWL_API_KEY is configured. Results are cached for 15 minutes.",
         parameters: {
           type: "object",
           properties: {
@@ -1414,7 +1726,7 @@ function webTools(opts) {
         const cacheKey = `${mode}|${url}`;
         const cached = cacheGet(cache, cacheKey);
         if (cached) return formatOutput(cached, maxLen);
-        let result;
+        let result = null;
         try {
           result = await guardedFetch({
             url,
@@ -1429,18 +1741,14 @@ function webTools(opts) {
               }
             }
           });
-        } catch (err) {
-          if (err instanceof SsrFBlockedError) throw new Error(`web_fetch: ${err.message}`);
-          throw err;
-        }
-        try {
           if (!result.response.ok) {
-            throw new Error(`web_fetch: ${result.response.status} ${result.response.statusText}`);
+            throw new Error(`${result.response.status} ${result.response.statusText}`);
           }
           const ct = result.response.headers.get("content-type") ?? "";
-          const body = await result.response.text();
+          const isHtml = ct.includes("text/html") || ct.includes("xhtml");
+          const { text: body, truncated } = await readBodyCapped(result.response, maxBodyBytes);
           let extracted;
-          if (ct.includes("text/html") || ct.includes("xhtml")) {
+          if (isHtml) {
             extracted = extractReadable(body, result.finalUrl, mode);
           } else if (ct.includes("application/json")) {
             try {
@@ -1451,10 +1759,32 @@ function webTools(opts) {
           } else {
             extracted = { text: body };
           }
+          if (isHtml && !firecrawlDisabled && extracted.text.length < FIRECRAWL_FALLBACK_THRESHOLD) {
+            const rescued = await firecrawlScrape(result.finalUrl, mode, env, fetchFn, timeoutMs);
+            if (rescued) {
+              extracted = {
+                ...rescued,
+                text: `${rescued.text}
+
+[content rendered via Firecrawl fallback \u2014 primary extraction returned ${extracted.text.length} chars]`
+              };
+            }
+          }
+          if (truncated) {
+            extracted = {
+              ...extracted,
+              text: `${extracted.text}
+
+[raw body truncated at ${maxBodyBytes} bytes before extraction \u2014 page exceeded the size cap]`
+            };
+          }
           cacheSet(cache, cacheKey, extracted, cacheTtlMs, cacheMax);
           return formatOutput(extracted, maxLen);
+        } catch (err) {
+          if (err instanceof SsrFBlockedError) throw new Error(`web_fetch: ${err.message}`);
+          throw new Error(`web_fetch: ${describeError(err)}`);
         } finally {
-          await result.release();
+          if (result) await result.release();
         }
       }
     }
@@ -1501,13 +1831,16 @@ function createBazilionCustomTools(opts) {
   if (opts.messagingHost) {
     handlers.push(...messagingTools(opts.messagingHost, opts.agent.agent.id));
   }
+  if (opts.userMdHost) {
+    handlers.push(...userMdTools(opts.userMdHost, opts.agent.group.id));
+  }
   return handlers.map(ourToolToPiTool);
 }
 
 // ../daemon/src/runtime/pi/session.ts
 var BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
 async function createBazilionSession(opts) {
-  const { agent, paths, env, memory, enabledProviders, messagingHost, refreshApiKey } = opts;
+  const { agent, paths, env, memory, enabledProviders, messagingHost, userMdHost, refreshApiKey } = opts;
   const { providerName, modelId } = splitModelString(agent.model);
   if (enabledProviders.size > 0 && !enabledProviders.has(providerName)) {
     throw new Error(`${providerName} provider is disabled \u2014 enable it on the /config page`);
@@ -1544,12 +1877,17 @@ async function createBazilionSession(opts) {
   const sessionManager = existing ? SessionManager.open(existing, sessionDir, cwd) : SessionManager.create(cwd, sessionDir);
   const settingsManager = SettingsManager.inMemory({
     compaction: { enabled: false },
-    retry: { enabled: true, maxRetries: 2, baseDelayMs: 500, maxDelayMs: 8e3 }
+    retry: {
+      enabled: true,
+      maxRetries: 2,
+      baseDelayMs: 500,
+      provider: { maxRetryDelayMs: 8e3 }
+    }
   });
   const bazilionPrompt = buildSystemPrompt(agent);
   const resourceLoader = createBazilionResourceLoader(bazilionPrompt);
   await resourceLoader.reload();
-  const customTools = createBazilionCustomTools({ agent, memory, messagingHost, env });
+  const customTools = createBazilionCustomTools({ agent, memory, messagingHost, userMdHost, env });
   const allowedTools = [...BUILTIN_TOOL_NAMES, ...customTools.map((t) => t.name)];
   const { session } = await createAgentSession({
     cwd,
@@ -1681,7 +2019,7 @@ function createBazilionResourceLoader(appendSystemPrompt) {
 function findMostRecent(sessionDir) {
   if (!existsSync11(sessionDir)) return null;
   let newest = null;
-  for (const entry of readdirSync6(sessionDir)) {
+  for (const entry of readdirSync7(sessionDir)) {
     if (!entry.endsWith(".jsonl")) continue;
     const path = join13(sessionDir, entry);
     try {
@@ -1709,7 +2047,7 @@ async function readInput() {
   }
   return parsed;
 }
-function createIpcMessagingHost() {
+function createIpcClient() {
   const pending = /* @__PURE__ */ new Map();
   process.on("message", (msg) => {
     if (!isIpcReply(msg)) return;
@@ -1723,17 +2061,14 @@ function createIpcMessagingHost() {
     for (const [, entry] of pending) entry.reject(new Error("worker IPC channel disconnected"));
     pending.clear();
   });
-  function call(method, args) {
+  return (method, args) => {
     if (!process.send) {
       return Promise.reject(new Error("worker: no IPC channel \u2014 daemon must spawn with stdio:ipc"));
     }
     const id = randomUUID5();
     const message = { type: "rpc", id, method, args };
     return new Promise((resolve3, reject) => {
-      pending.set(id, {
-        resolve: (v) => resolve3(v),
-        reject
-      });
+      pending.set(id, { resolve: (v) => resolve3(v), reject });
       process.send?.(message, void 0, void 0, (err) => {
         if (err) {
           pending.delete(id);
@@ -1741,7 +2076,9 @@ function createIpcMessagingHost() {
         }
       });
     });
-  }
+  };
+}
+function createIpcMessagingHost(call) {
   return {
     agentExists: (agentId) => call("agentExists", { agentId }),
     sendMessage: (input) => call("sendMessage", input),
@@ -1752,6 +2089,12 @@ function createIpcMessagingHost() {
     findReplies: (agentId, replyTo) => call("findReplies", { agentId, replyTo })
   };
 }
+function createIpcUserMdHost(call) {
+  return {
+    get: (groupId) => call("userMdGet", { groupId }),
+    write: (groupId, content, ifMatch) => call("userMdWrite", { groupId, content, ifMatch })
+  };
+}
 function isIpcReply(msg) {
   if (!msg || typeof msg !== "object") return false;
   const m = msg;
@@ -1778,7 +2121,9 @@ async function main() {
   const paths = resolvePaths();
   const memory = qmdBackend(join14(agent.group.path, "memory"));
   await memory.init();
-  const messagingHost = createIpcMessagingHost();
+  const ipcCall = createIpcClient();
+  const messagingHost = createIpcMessagingHost(ipcCall);
+  const userMdHost = createIpcUserMdHost(ipcCall);
   const { session, dispose } = await createBazilionSession({
     agent,
     paths,
@@ -1786,6 +2131,7 @@ async function main() {
     memory,
     enabledProviders: new Set(enabledProviders),
     messagingHost,
+    userMdHost,
     apiKey
   });
   abortSession = () => {