@audiofool/moondog 0.3.0-beta.10 → 0.3.0-beta.12

package/src/runtime/pi/agent-runtime.mjs

@@ -14,6 +14,7 @@ import { spotifyErrorReason } from "../../integrations/spotify/web-api-client.mj
 import { appleMusicStorefront } from "../../integrations/apple-music/storefront.mjs";
 import { formatWebSources } from "../../surfaces/cli/web-command.mjs";
 import { chooseConversationName } from "./conversation-name.mjs";
+import { prepareImageInput } from "./image-input.mjs";
 import { withArgumentRepair } from "./tool-arguments.mjs";
 import { restateInLanguage } from "./reply-language.mjs";
 import { chooseRecordName, RECORD_NAME_RULES } from "./record-name.mjs";
@@ -4957,11 +4958,11 @@ function createToolFactories(
         name: descriptor.tool_name,
         label: descriptor.label,
         description: kind === "search"
-          ? "Search public reviews, music news, interviews and concert information. Send only a concise public query under 128 characters, never private history or profile data; site:, quoted phrases and -exclusions narrow it. Results come with source links; content_kind says whether they are search-engine snippets or a generated summary."
-          : "Read a public HTTP(S) page. Send only its URL. The result is the page's own text (cut short when long) or a generated summary, as content_kind says. Inaccessible pages are reported; this does not log in or operate an interactive browser.",
+          ? "Search public reviews, music news, interviews and concert information. Send only a concise public query under 128 characters, never private history or profile data; site:, quoted phrases and -exclusions narrow it. Returns 5 sources by default; ask for up to 10 with limit when the first ones aren't enough. Results come with source links; content_kind says whether they are search-engine snippets or a generated summary."
+          : "Read a public HTTP(S) page. Send only its URL. The result is the page's own text or a generated summary, as content_kind says. A long page comes about 15,000 characters at a time: when next_offset is set, call again with that offset to read on. Inaccessible pages are reported; this does not log in or operate an interactive browser.",
         parameters: Type.Object(kind === "search"
-          ? { query: Type.String({ minLength: 1, maxLength: 500 }) }
-          : { url: Type.String({ minLength: 1, maxLength: 2048 }) }, { additionalProperties: false }),
+          ? { query: Type.String({ minLength: 1, maxLength: 500 }), limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })) }
+          : { url: Type.String({ minLength: 1, maxLength: 2048 }), offset: Type.Optional(Type.Integer({ minimum: 0, maximum: 200_000 })) }, { additionalProperties: false }),
         executionMode: "sequential",
         execute: async (_toolCallId, parameters, signal) => executeDomain(
           () => kind === "search"
@@ -6614,11 +6615,12 @@ function createToolFactories(
         name: descriptor.tool_name,
         label: descriptor.label,
         description:
-          "Find playlists in the listener's own library on this Mac by words of their name, or list the newest ones without words. Saved is the built-in playlist of single songs and made tracks they kept. Each playlist says how many entries it holds and how many of them Moondog made. With playlist_id, it reads that playlist's entries in order.",
+          "Find playlists in the listener's own library on this Mac by words of their name, or list the newest ones without words. Saved is the built-in playlist of single songs and made tracks they kept. Each playlist says how many entries it holds and how many of them Moondog made. With playlist_id, it reads that playlist's entries in order, 200 at a time; offset reads on.",
         parameters: Type.Object(
           {
             query: Type.Optional(Type.String({ maxLength: 120 })),
             playlist_id: Type.Optional(Type.String({ minLength: 1, maxLength: 128 })),
+            offset: Type.Optional(Type.Integer({ minimum: 0 })),
           },
           { additionalProperties: false },
         ),
@@ -6628,8 +6630,8 @@ function createToolFactories(
               const playlist = await application.libraryPlaylist(parameters.playlist_id);
               if (!playlist) throw Object.assign(new Error("That playlist is not in the library."), { code: "library_playlist_not_found" });
               return { playlist: { id: playlist.id, name: playlist.saved ? "Saved" : playlist.name, origin: playlist.origin,
-                entries: playlist.entries.slice(0, 60).map((entry) => ({ title: entry.title, artist: entry.artist_credit, ...(entry.kind === "made" ? { made_by_moondog: true } : {}) })),
-                total: playlist.entries.length } };
+                entries: playlist.entries.slice(parameters.offset ?? 0, (parameters.offset ?? 0) + 200).map((entry) => ({ title: entry.title, artist: entry.artist_credit, ...(entry.kind === "made" ? { made_by_moondog: true } : {}) })),
+                offset: parameters.offset ?? 0, total: playlist.entries.length } };
             }
             return { playlists: application.findLibraryPlaylists(parameters.query ?? "") };
           },
@@ -6674,6 +6676,31 @@ function createToolFactories(
         ),
       }),
     ],
+    [
+      "collection.edit",
+      (descriptor) => ({
+        name: descriptor.tool_name,
+        label: descriptor.label,
+        description:
+          "Change one of the listener's own playlists on this Mac, only when they ask: name renames it, remove takes songs out, move puts songs before another song (before), or at the end without it, in the order given. To reorder a whole playlist, move every song in the new order. Songs go by title as listed; give the artist when two share a title. To add songs use moondog_collection_save. Say what changed; not_found names songs it did not find.",
+        parameters: Type.Object(
+          {
+            playlist_id: Type.String({ minLength: 1, maxLength: 128 }),
+            name: Type.Optional(Type.String({ minLength: 1, maxLength: 120 })),
+            remove: Type.Optional(Type.Array(LIBRARY_SONG, { minItems: 1, maxItems: 200 })),
+            move: Type.Optional(Type.Array(LIBRARY_SONG, { minItems: 1, maxItems: 200 })),
+            before: Type.Optional(LIBRARY_SONG),
+          },
+          { additionalProperties: false },
+        ),
+        executionMode: "sequential",
+        execute: executeDomain(
+          async (_toolCallId, parameters) => application.editLibraryPlaylist({ playlistId: parameters.playlist_id, name: parameters.name,
+            remove: parameters.remove, move: parameters.move, before: parameters.before }),
+          (value) => structuredClone(value),
+        ),
+      }),
+    ],
     [
       "collection.import",
       (descriptor) => ({
@@ -6890,6 +6917,11 @@ const isOnAirRequest = (text) => {
 };
 const ON_AIR_NOTE = "The listener pressed On Air. Answer this message by starting Moondog's stream with moondog_start_stream: instrumental tracks made on this Mac from their taste and the moment, one after another, with a broad set of descriptors as that tool asks. Do not search, play or queue catalog music for it. If making music isn't available here, say so in one sentence.";
 
+const LIBRARY_SONG = Type.Object({
+  title: Type.String({ minLength: 1, maxLength: 200 }),
+  artist: Type.Optional(Type.String({ maxLength: 200 })),
+}, { additionalProperties: false });
+
 function sessionSection(application) {
   return sessionAvailability({
     spotifyReady: application.spotifyReady?.() ?? false,
@@ -6941,7 +6973,7 @@ Public web research rules:
 
 Grounding and evidence rules:
 - Before naming or selecting tracks as present in the user's library, call moondog_library_search.
-- The listener's collection in Moondog (Saved and their own playlists, which can hold songs and tracks Moondog made) is separate from their imported library and from Spotify. When they ask to save, keep, collect or like music (收藏, 存下来) without naming Spotify, use moondog_collection_save; use moondog_spotify_library_save only when they name Spotify or Liked Songs. To play one of their playlists, find it with moondog_collection_list and play it with moondog_collection_play. When they paste a playlist link and ask to import or add it, use moondog_collection_import.
+- The listener's collection in Moondog (Saved and their own playlists, which can hold songs and tracks Moondog made) is separate from their imported library and from Spotify. When they ask to save, keep, collect or like music (收藏, 存下来) without naming Spotify, use moondog_collection_save; use moondog_spotify_library_save only when they name Spotify or Liked Songs. To play one of their playlists, find it with moondog_collection_list and play it with moondog_collection_play. To rename one, take songs out or reorder it, use moondog_collection_edit. When they paste a playlist link and ask to import or add it, use moondog_collection_import.
 - When personalization matters, call moondog_profile_summary instead of assuming a profile.
 - Before saying how often, when, or whether the listener played something, check the listening coverage. Plays inside a gap, outside every source's span, or after recent_plays_checked_at are unknown, not zero. When a gap could hold the plays in question, lead with that boundary, for example "I have no records from August 30 to October 21", and do not conclude that the listener rarely plays it or only just found it.
 - The saved reading is enduring taste up to its listening date. Recent attention is the present: stored plays from the last four weeks, and Spotify's four-week top artists, which count listening on every device, including plays Moondog never stored and songs Moondog itself played or queued; stored plays count those apart as moondog_started_plays. When they disagree about what the listener plays lately, trust recent attention, and say which one you read.
@@ -7671,8 +7703,11 @@ export class PiAgentRuntime {
   };
 
   constructor({ application, models, model, provider, modelId, effort, effortLevels, thinkingLevel, modelFetch, modelRetryDelay, toolTrace = null,
-    nameConversations = false, clock = () => new Date() }) {
+    nameConversations = false, clock = () => new Date(), visionChoice = null }) {
     this.application = application;
+    // The vision model chosen in Settings, read at each message with images, so a change there
+    // needs no new agent; null leaves images to the chat model.
+    this.visionChoice = typeof visionChoice === "function" ? visionChoice : null;
     // The command passes a local trace of tool calls; tests and other hosts leave it out.
     this.toolTrace = toolTrace;
     this.clock = clock;
@@ -8114,7 +8149,7 @@ export class PiAgentRuntime {
   // A finished reply keeps only its text, so the transcript, not this agent, says what was declared.
   #sessionUpdate(messages) {
     // An On Air reply carries the host's note in its session section; the next reply drops it again.
-    const session = [sessionSection(this.application), this.onAirTurn ? ON_AIR_NOTE : null].filter(Boolean).join("\n\n") || null;
+    const session = [sessionSection(this.application), this.onAirTurn ? ON_AIR_NOTE : null, this.libraryNote].filter(Boolean).join("\n\n") || null;
     if ((getCurrentSystemMessage(messages)?.sections?.session ?? null) === session) return [];
     return [{ role: "system", content: "", sections: { session }, timestamp: Date.now() }];
   }
@@ -8129,6 +8164,28 @@ export class PiAgentRuntime {
       throw Object.assign(new Error("Moondog is answering in this conversation in another window or terminal."), { code: "conversation_busy" });
     }
     if ((this.application?.latestConversationEntryId?.() ?? null) !== this.syncedEntryId) this.restoreSession();
+    // Attached images become text the conversation keeps, and, for a chat model that sees, the
+    // images themselves. A vision model's description can take a few seconds, so the prompt counts
+    // as running from here; Ctrl+C stops it.
+    let nativeImages = [];
+    if (callbacks.images?.length) {
+      this.promptInFlight = true;
+      this.imageAbort = new AbortController();
+      try {
+        const visionChoice = await Promise.resolve(this.visionChoice?.()).catch(() => null);
+        callbacks.onImagesStart?.({ count: callbacks.images.length, vision: Boolean(visionChoice) });
+        const prepared = await prepareImageInput({ text, images: callbacks.images, models: this.models, model: this.model,
+          visionChoice, signal: this.imageAbort.signal });
+        text = prepared.text;
+        nativeImages = prepared.images;
+        callbacks.onImagesReady?.({ mode: prepared.mode, text, ...(prepared.vision ? { vision: prepared.vision } : {}) });
+      } catch (error) {
+        this.promptInFlight = false;
+        throw error;
+      } finally {
+        this.imageAbort = null;
+      }
+    }
     // What the listener sees of the reply is saved as it grows, so the other one shows it too.
     const host = callbacks;
     let shownText = "";
@@ -8153,6 +8210,8 @@ export class PiAgentRuntime {
     this.#refreshSurface();
     this.loopSurfaceVersion = this.surfaceVersion;
     this.onAirTurn = isOnAirRequest(text);
+    // What this conversation's messages name with @, read fresh for this reply.
+    this.libraryNote = this.application?.libraryMentionNote?.([...this.agent.state.messages.filter((message) => message.role === "user").map(extractMessageText), text]) ?? null;
     const sessionUpdate = this.#sessionUpdate(this.agent.state.messages);
     if (sessionUpdate.length) this.agent.state.messages = [...this.agent.state.messages, ...sessionUpdate];
     const historyStartIndex = this.agent.state.messages.length;
@@ -8544,7 +8603,8 @@ export class PiAgentRuntime {
         }
       });
 
-      await this.agent.prompt(text);
+      if (nativeImages.length) await this.agent.prompt(text, nativeImages);
+      else await this.agent.prompt(text);
       // One bounded continuation keeps an explicit queue request from stopping
       // at a plan/empty adjacency result or a redundant confirmation. It cannot
       // grant new authority, retry a write, or outlive cancellation.
@@ -8797,6 +8857,8 @@ To confirm, reply: ${preview.confirmation}`].join("\n\n");
   }
 
   abort() {
+    // A vision model may still be describing the message's images.
+    this.imageAbort?.abort();
     if (this.activePromptState) {
       this.activePromptState.abortRequested = true;
       this.activePromptState.restatement?.abort();

package/src/runtime/pi/configured-runtime.mjs

@@ -2,6 +2,7 @@ import { getSupportedThinkingLevels } from "@earendil-works/pi-ai";
 
 import { PiAgentRuntime } from "./agent-runtime.mjs";
 import { createPiAuthContext } from "./authentication.mjs";
+import { readModelRoles } from "./model-roles.mjs";
 import { getPublicPiModelCatalog } from "./public-model-catalog.mjs";
 import { createPersistentCredentialStore } from "./persistent-credential-store.mjs";
 import { clampEffort, EFFORT_LEVELS, readPiRuntimeSelection } from "./runtime-settings.mjs";
@@ -110,5 +111,7 @@ export async function createConfiguredRuntime(
     effortLevels: getSupportedThinkingLevels(model),
     thinkingLevel: effort ? clampEffort(model, effort) : undefined,
     ...(toolTrace ? { toolTrace } : {}),
+    // Read at each message with images: the vision model from Settings, or null for the chat model.
+    visionChoice: async () => (await readModelRoles(environment).catch(() => ({}))).vision ?? null,
   });
 }

package/src/runtime/pi/image-input.mjs

@@ -0,0 +1,148 @@
+import { readFile as readFileFromDisk } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { clampThinkingLevel } from "@earendil-works/pi-ai";
+
+// Images the listener attaches to a message, after Hermes Agent's image routing
+// (agent/image_routing.py, image_input_mode "auto"):
+//
+// - A vision model chosen in Settings describes each image first ("text"): its words go into the
+//   message, and the chat model reads them. A chosen auxiliary model wins over the chat model, as in
+//   Hermes, so the listener can keep a cheap chat model and a good eye.
+// - Otherwise a chat model that takes images gets them as they are ("native").
+// - Otherwise nothing can see them, and the message is refused with what to change.
+//
+// Either way the conversation keeps only text: a short stand-in for a native image, or the vision
+// model's description, so a conversation picked up later still knows what was shown.
+
+export const IMAGE_LIMITS = Object.freeze({
+  count: 4,
+  // Base64 characters of one image, about 6 MB of image; the window scales larger ones down first.
+  dataChars: 8_000_000,
+  mimeTypes: Object.freeze(["image/png", "image/jpeg", "image/webp", "image/gif"]),
+  nameChars: 120,
+  // Conversation entries hold at most 8,000 characters, the listener's own words first.
+  entryChars: 8_000,
+  descriptionChars: 1_600,
+});
+
+const VISION_PROMPT = [
+  "You describe an image for another model, which answers a music listener in Moondog and cannot see it.",
+  "Read any text exactly as written: track lists, artist and album names, playlists, setlists, posters,",
+  "streaming app screenshots, chat messages. Name album covers, artists, instruments or places you can recognise,",
+  "and say how sure you are when you guess. Describe mood and setting briefly.",
+  "Write plain text without a preamble, under 180 words.",
+].join(" ");
+
+function failure(code, message) {
+  return Object.assign(new Error(message), { code });
+}
+
+const takesImages = (model) => Array.isArray(model?.input) && model.input.includes("image");
+
+/** The attached images as Pi takes them, checked; an empty list when there are none. */
+export function normalizeImages(images) {
+  if (images === undefined || images === null) return [];
+  if (!Array.isArray(images)) throw failure("image_input_invalid", "Images must come as a list.");
+  if (images.length > IMAGE_LIMITS.count) throw failure("image_input_too_many", `Attach at most ${IMAGE_LIMITS.count} images to one message.`);
+  return images.map((image, index) => {
+    const mimeType = String(image?.mimeType ?? "").toLowerCase();
+    const data = typeof image?.data === "string" ? image.data.replace(/^data:[^,]*,/u, "") : "";
+    if (!IMAGE_LIMITS.mimeTypes.includes(mimeType)) throw failure("image_input_unsupported", "Moondog takes PNG, JPEG, WebP and GIF images.");
+    if (!data || data.length > IMAGE_LIMITS.dataChars || !/^[A-Za-z0-9+/]+={0,2}$/u.test(data)) {
+      throw failure("image_input_too_large", "That image is too large or unreadable. Try a smaller one.");
+    }
+    const name = typeof image?.name === "string" && image.name.trim()
+      ? image.name.trim().replace(/[\p{C}[\]]/gu, "").slice(0, IMAGE_LIMITS.nameChars) : `image ${index + 1}`;
+    return { type: "image", data, mimeType, name };
+  });
+}
+
+// Local image paths in a terminal message, as Hermes Agent's gateway finds them: an absolute or ~/
+// path ending in an image extension, quoted when it has spaces, or with the "\ " a Finder drag writes.
+const EXTENSION_TYPES = Object.freeze({ png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", webp: "image/webp", gif: "image/gif" });
+const QUOTED_PATH = /(["'])((?:~\/|\/)[^"'\n]+?\.(?:png|jpe?g|webp|gif))\1/giu;
+const BARE_PATH = /(?<![\w./:])((?:~\/|\/)(?:\\ |[^\s"'])+?\.(?:png|jpe?g|webp|gif))(?=[\s,.;:!?)]*(?:\s|$))/giu;
+
+/** The image files a message names, read from disk; paths that aren't readable images are left as words. */
+export async function imagesFromText(text, { readFile = readFileFromDisk, home = os.homedir() } = {}) {
+  const found = [];
+  const quoted = String(text ?? "").replace(QUOTED_PATH, (_, _quote, file) => { found.push(file); return " "; });
+  for (const match of quoted.matchAll(BARE_PATH)) found.push(match[1].replace(/\\ /gu, " "));
+  const images = [];
+  for (const written of [...new Set(found)].slice(0, IMAGE_LIMITS.count)) {
+    const file = written.startsWith("~/") ? path.join(home, written.slice(2)) : written;
+    let bytes;
+    try { bytes = await readFile(file); } catch { continue; }
+    const mimeType = EXTENSION_TYPES[path.extname(file).slice(1).toLowerCase()];
+    const data = Buffer.from(bytes).toString("base64");
+    if (data.length > IMAGE_LIMITS.dataChars) {
+      throw failure("image_input_too_large", `${path.basename(file)} is too large to send. Try a smaller copy or a screenshot of part of it.`);
+    }
+    images.push({ data, mimeType, name: path.basename(file) });
+  }
+  return images;
+}
+
+const modelLabel = (model) => model?.name ?? model?.id ?? "the vision model";
+
+/** Native, text through a vision model, or a refusal; the vision model is resolved here. */
+export async function chooseImageRoute({ models, model, visionChoice }) {
+  if (visionChoice?.provider && visionChoice?.model) {
+    const vision = models.getModel(visionChoice.provider, visionChoice.model);
+    if (!vision || !takesImages(vision)) {
+      throw failure("vision_model_unavailable", "The vision model chosen in Settings can't see images any more. Choose another under Model > Auxiliary models > Vision.");
+    }
+    if (!(await models.checkAuth?.(visionChoice.provider))) {
+      throw failure("vision_model_signed_out", `${modelLabel(vision)} needs a sign-in before it can see images. Add its key in Settings > Model.`);
+    }
+    return { mode: "text", vision };
+  }
+  if (takesImages(model)) return { mode: "native" };
+  throw failure("vision_model_missing",
+    `${modelLabel(model)} can't see images. Choose a model that can under Settings > Model > Auxiliary models > Vision, or switch to one that sees.`);
+}
+
+async function describe({ models, vision, image, text, signal }) {
+  const lightest = clampThinkingLevel(vision, "off");
+  const message = await models.completeSimple(vision, {
+    systemPrompt: VISION_PROMPT,
+    messages: [{ role: "user", timestamp: Date.now(), content: [
+      { type: "text", text: text ? `The listener's message, for context: ${text.slice(0, 1_000)}` : "The listener sent this image without words." },
+      { type: "image", data: image.data, mimeType: image.mimeType },
+    ] }],
+  }, { maxTokens: 900, maxRetries: 1, signal, ...(lightest !== "off" ? { reasoning: lightest } : {}) });
+  if (!message || message.stopReason === "aborted") throw failure("vision_cancelled", "Stopped describing the image.");
+  if (message.stopReason === "error") throw failure("vision_failed", `${modelLabel(vision)} couldn't describe the image. Try again in a moment.`);
+  return message.content.filter((block) => block.type === "text").map((block) => block.text).join("").replace(/\s+/gu, " ").trim();
+}
+
+const clip = (value, limit) => (value.length > limit ? `${value.slice(0, limit - 1).trimEnd()}…` : value);
+
+/**
+ * The message as the conversation keeps it, and the images the chat model gets with it (only in
+ * native mode). Without images it returns the text unchanged.
+ */
+export async function prepareImageInput({ text, images, models, model, visionChoice, signal }) {
+  const attached = normalizeImages(images);
+  if (!attached.length) return { text, images: [], mode: null };
+  const route = await chooseImageRoute({ models, model, visionChoice });
+  const words = String(text ?? "").trim();
+  const room = Math.max(400, IMAGE_LIMITS.entryChars - words.length - 40);
+  const each = Math.min(IMAGE_LIMITS.descriptionChars, Math.floor(room / attached.length) - 60);
+  const notes = [];
+  for (const [index, image] of attached.entries()) {
+    const label = attached.length > 1 ? `Image ${index + 1} (${image.name})` : `Image (${image.name})`;
+    if (route.mode === "native") { notes.push(`[${label} attached]`); continue; }
+    const seen = await describe({ models, vision: route.vision, image, text: words, signal });
+    notes.push(`[${label}, as ${modelLabel(route.vision)} describes it: ${clip(seen || "nothing it could make out", Math.max(80, each))}]`);
+  }
+  const kept = [words, notes.join("\n")].filter(Boolean).join("\n\n");
+  return {
+    text: kept.slice(0, IMAGE_LIMITS.entryChars),
+    images: route.mode === "native" ? attached.map(({ type, data, mimeType }) => ({ type, data, mimeType })) : [],
+    mode: route.mode,
+    ...(route.vision ? { vision: { provider: route.vision.provider, model: route.vision.id, name: modelLabel(route.vision) } } : {}),
+  };
+}

package/src/runtime/pi/model-roles.mjs

@@ -0,0 +1,121 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+
+import { resolveMoondogAuthFile } from "./persistent-credential-store.mjs";
+
+// What each model is for, after Hermes Agent's main model and auxiliary tasks: one chat model answers,
+// and a side task can have a model of its own. A role without a saved choice is "auto": vision uses the
+// chat model when it can see images, and web search takes the first service that is set up.
+// The chat model stays in settings.json, which older Moondogs read with a fixed shape; the other
+// roles live in model-roles.json beside it.
+//
+// kind "model" is a provider and model from the catalog; kind "service" is a web service by id.
+// A "planned" role holds its place in the settings and in this file, but nothing can choose it yet.
+export const MODEL_ROLES = Object.freeze([
+  Object.freeze({ id: "chat", kind: "model", state: "enabled", stored: "settings" }),
+  Object.freeze({ id: "vision", kind: "model", state: "enabled", input: "image" }),
+  Object.freeze({ id: "web_search", kind: "service", state: "enabled" }),
+  Object.freeze({ id: "audio", kind: "model", state: "planned", input: "audio" }),
+]);
+
+export const WEB_SERVICES = Object.freeze(["deepseek", "brave", "codex"]);
+
+const ROLES_VERSION = 1;
+const printable = (value, max) => typeof value === "string" && value.length > 0 && value.length <= max && !/[\p{C}\s]/u.test(value);
+
+function rolesError(message, cause) {
+  const error = new Error(message, cause ? { cause } : undefined);
+  error.name = "MoondogRuntimeSettingsError";
+  error.code = "model_roles_invalid";
+  return error;
+}
+
+export const modelRole = (id) => MODEL_ROLES.find((role) => role.id === id) ?? null;
+
+/** One role's choice in its saved shape, or null for auto. Throws on a shape the role can't take. */
+export function normalizeRoleChoice(roleId, value) {
+  const role = modelRole(roleId);
+  if (!role || role.stored) throw rolesError(`Moondog has no model role called ${roleId}.`);
+  if (value === null || value === undefined || value === "auto") return null;
+  if (role.kind === "service") {
+    const service = typeof value?.service === "string" ? value.service.trim().toLowerCase() : "";
+    if (!WEB_SERVICES.includes(service)) throw rolesError(`Web search can use ${WEB_SERVICES.join(", ")} or auto.`);
+    return { service };
+  }
+  const provider = typeof value?.provider === "string" ? value.provider.trim().toLowerCase() : "";
+  const model = typeof value?.model === "string" ? value.model.trim() : "";
+  if (!printable(provider, 80) || !printable(model, 200)) throw rolesError(`The ${roleId} role needs a provider and a model.`);
+  return { provider, model, ...(typeof value.effort === "string" && printable(value.effort, 20) ? { effort: value.effort } : {}) };
+}
+
+export function resolveModelRolesFile(environment = process.env) {
+  return path.join(path.dirname(resolveMoondogAuthFile(environment)), "model-roles.json");
+}
+
+// Every saved role as it is in the file, including roles a newer Moondog wrote, so a write here
+// never drops them.
+async function readDocument(environment) {
+  let source;
+  try {
+    source = await readFile(resolveModelRolesFile(environment), "utf8");
+  } catch (error) {
+    if (error?.code === "ENOENT") return {};
+    throw rolesError("Moondog could not read its model roles.", error);
+  }
+  let document;
+  try {
+    document = JSON.parse(source);
+  } catch (error) {
+    throw rolesError("Moondog's model roles are not valid JSON.", error);
+  }
+  if (!document || typeof document !== "object" || document.version !== ROLES_VERSION ||
+      !document.roles || typeof document.roles !== "object" || Array.isArray(document.roles)) {
+    throw rolesError("Moondog's model roles use an unsupported shape.");
+  }
+  return document.roles;
+}
+
+/** The saved choice for each role this Moondog knows, by role id; a role left on auto is absent. */
+export async function readModelRoles(environment = process.env) {
+  const saved = await readDocument(environment);
+  const roles = {};
+  for (const role of MODEL_ROLES) {
+    if (role.stored || role.state !== "enabled" || saved[role.id] === undefined) continue;
+    // One bad entry leaves that role on auto instead of failing every role.
+    try {
+      const choice = normalizeRoleChoice(role.id, saved[role.id]);
+      if (choice) roles[role.id] = choice;
+    } catch {}
+  }
+  return roles;
+}
+
+// One change at a time, so two quick choices never lose either.
+let writing = Promise.resolve();
+/** Saves one role's choice; null puts it back on auto. */
+export function writeModelRole(roleId, value, environment = process.env) {
+  const role = modelRole(roleId);
+  if (!role || role.stored) return Promise.reject(rolesError(`Moondog has no model role called ${roleId}.`));
+  if (role.state !== "enabled") return Promise.reject(rolesError(`The ${roleId} role isn't available yet.`));
+  let choice;
+  try { choice = normalizeRoleChoice(roleId, value); } catch (error) { return Promise.reject(error); }
+  const next = writing.then(async () => {
+    const roles = { ...(await readDocument(environment).catch(() => ({}))) };
+    if (choice) roles[roleId] = choice;
+    else delete roles[roleId];
+    const file = resolveModelRolesFile(environment);
+    await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+    // Another Moondog may read the roles at any moment, so it never sees half a file.
+    const temporary = `${file}.${randomUUID()}.tmp`;
+    try {
+      await writeFile(temporary, `${JSON.stringify({ version: ROLES_VERSION, roles }, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
+      await rename(temporary, file);
+    } finally {
+      await rm(temporary, { force: true });
+    }
+    return choice;
+  });
+  writing = next.catch(() => {});
+  return next;
+}

package/src/runtime/pi/model-settings.mjs

@@ -9,6 +9,7 @@ import {
 import { isEndpointProvider, removeEndpoint, saveEndpoint } from "./custom-endpoints.mjs";
 import { LEGACY_MODEL_ALIASES } from "./model-catalog.mjs";
 import { readConversationModels, readModelList, sameModel, writeConversationModel, writeModelList } from "./model-list.mjs";
+import { MODEL_ROLES, modelRole, readModelRoles, writeModelRole } from "./model-roles.mjs";
 import { chatgptProviderId } from "./provider-registry.mjs";
 import { createPersistentCredentialStore } from "./persistent-credential-store.mjs";
 import { getPublicPiModelCatalog } from "./public-model-catalog.mjs";
@@ -22,6 +23,8 @@ import { clampEffort, readPiRuntimeSelection } from "./runtime-settings.mjs";
 // key, then Anthropic, Google and xAI. Any other service is reached as an endpoint by address; Pi's
 // other providers show only once a key for one is saved or in the environment.
 const FEATURED = ["openai", "anthropic", "google", "xai"];
+// Whether a catalog model takes an input besides text, such as "image".
+const takes = (model, input) => Boolean(input) && Array.isArray(model?.input) && model.input.includes(input);
 // Moondog's own errors carry fixed text. Pi wraps them, and its own text may not be safe.
 const moondogErrorNames = new Set(["MoondogCredentialStoreError", "MoondogRuntimeSettingsError"]);
 
@@ -183,13 +186,71 @@ export function createModelSettings({
           const model = models.getModel(entry.provider, entry.model);
           return { provider: entry.provider, model: entry.model, name: model?.name ?? entry.model,
             providerName: names.get(entry.provider) ?? entry.provider, ready: Boolean(model) && ready.get(entry.provider) === true,
-            reasoning: thinks(model) };
+            reasoning: thinks(model), vision: takes(model, "image") };
         }),
         default: selection ? { provider: selection.provider, model: selection.model, effort: selection.effort ?? null,
           reasoning: thinks(defaultModel), thinking: selection.effort && defaultModel ? clampEffort(defaultModel, selection.effort) : null } : null,
       };
     },
 
+    // What each model is for: the chat model from settings.json, then each other role's choice from
+    // model-roles.json, null for auto. Vision on auto uses the chat model when it can see images.
+    // web is the web research's services(), which knows the web services; Settings passes it in.
+    async roles({ web = null } = {}) {
+      const selection = await current();
+      let saved;
+      try {
+        saved = await readModelRoles(environment);
+      } catch (error) {
+        throw moondogError(error) ?? failure("model_roles_invalid", "Moondog could not read its model roles.");
+      }
+      const source = await catalog();
+      let models;
+      try {
+        models = withAuth(source);
+      } catch (error) {
+        throw moondogError(error) ?? failure("auth_status_failed", "Moondog could not read its saved sign-ins.");
+      }
+      const names = new Map(source.providers().map(provider => [provider.id, provider.name]));
+      const describe = (choice) => {
+        const model = models.getModel(choice.provider, choice.model);
+        return { provider: choice.provider, model: choice.model, name: model?.name ?? choice.model,
+          providerName: names.get(choice.provider) ?? choice.provider, known: Boolean(model), vision: takes(model, "image") };
+      };
+      const chat = selection ? describe(selection) : null;
+      return MODEL_ROLES.map((role) => {
+        const base = { id: role.id, kind: role.kind, state: role.state, ...(role.input ? { input: role.input } : {}) };
+        if (role.id === "chat") return { ...base, choice: chat };
+        if (role.id === "web_search") {
+          return { ...base, choice: saved.web_search ?? null,
+            ...(web ? { services: web.services, search: web.search?.provider ?? null, read: web.read?.provider ?? null } : {}) };
+        }
+        if (role.state !== "enabled") return { ...base, choice: null };
+        // Hermes Agent's auxiliary vision on auto: the main model when it takes images, else none.
+        return { ...base, choice: saved[role.id] ? describe(saved[role.id]) : null,
+          auto: chat && takes(models.getModel(selection.provider, selection.model), role.input) ? chat : null };
+      });
+    },
+
+    // Saves one role's choice; null puts it on auto. A model must be in the catalog and take the
+    // input its role is for.
+    async setRole(roleId, choice) {
+      const role = modelRole(String(roleId ?? ""));
+      if (!role || role.stored) throw failure("model_role_unknown", "Moondog has no such model role.");
+      if (role.state !== "enabled") throw failure("model_role_unavailable", "That model role isn't available yet.");
+      if (role.kind === "model" && choice) {
+        const provider = String(choice.provider ?? "").trim().toLowerCase();
+        const model = (await catalog()).createModels().getModel(provider, String(choice.model ?? "").trim());
+        if (!model) throw failure("model_unknown", "Moondog doesn't know that model.");
+        if (!takes(model, role.input)) throw failure("model_role_unsuitable", `That model can't take ${role.input} input.`);
+      }
+      try {
+        return await writeModelRole(role.id, choice ?? null, environment);
+      } catch (error) {
+        throw moondogError(error) ?? failure("model_roles_invalid", "Moondog could not save its model roles.");
+      }
+    },
+
     // The model each conversation was given in the window's menu, and a change to one: null gives a
     // conversation back to the default.
     conversationModels: () => readConversationModels(environment),
@@ -262,7 +323,7 @@ export function createModelSettings({
         refresh,
         models: source.models(providerId).filter(({ id }) => !LEGACY_MODEL_ALIASES.includes(id)).map(({ id, name, contextWindow, reasoning, catalogNote }) => {
           const model = full.getModel(providerId, id);
-          return { id, name, ...(contextWindow ? { contextWindow } : {}), reasoning,
+          return { id, name, ...(contextWindow ? { contextWindow } : {}), reasoning, vision: takes(model, "image"),
             effortLevels: model ? getSupportedThinkingLevels(model) : [],
             ...(catalogNote ? { retained: true } : {}) };
         }),

package/src/runtime/pi/runtime-settings.mjs

@@ -109,11 +109,13 @@ export async function writePiRuntimeSelection(
 // saved selection, or undefined when none is saved; the surface decides whether to rebuild, with
 // runtimeBehindSelection.
 // The second argument says which file changed: "settings" (the default), "auth" (a sign-in),
-// "endpoints" (an endpoint added, edited or removed) or "models" (the window's menu list).
+// "endpoints" (an endpoint added, edited or removed), "models" (the window's menu list) or "roles"
+// (a vision model or web service chosen in model-roles.json).
 export function watchRuntimeSettings({ environment = process.env, interval = 2000, onChange } = {}) {
   const directory = path.dirname(resolveMoondogAuthFile(environment));
   const files = { settings: resolveMoondogSettingsFile(environment), auth: resolveMoondogAuthFile(environment),
-    endpoints: path.join(directory, "endpoints.json"), models: path.join(directory, "models.json") };
+    endpoints: path.join(directory, "endpoints.json"), models: path.join(directory, "models.json"),
+    roles: path.join(directory, "model-roles.json") };
   const listeners = Object.entries(files).map(([changed, file]) => {
     const listener = (now, before) => {
       if (now.mtimeMs === before.mtimeMs) return;

package/src/surfaces/cli/brand-components.mjs

@@ -564,5 +564,5 @@ function wrapDescription(text, width) {
 }
 
 export function listeningHelp() {
-  return tr("# Your listening room\n\n## 01 / Listen and discover\n\nJust type to talk about music, ask for a playlist, or go exploring.\nTo talk, Moondog needs a model: `/auth` signs you in, then `/model` picks one.\nWhile an answer is coming, each step shows in the conversation, and the footer names the one in progress.\n\n- `/taste` - your profile: open any song or artist to see why it's there\n- `/taste report` - the whole picture on one page\n- `/import` - bring in your history, your library, or a playlist\n- `/lyrics` - the lyrics on the sleeve; `/lyrics sync` finds more\n- `/service` - choose where I play music: Spotify, Apple Music, or NetEase\n- `/spotify` - play, queue, and make playlists in Spotify\n- Ask to move playback onto your iPhone, computer, or another Spotify device by name\n- `/web search <query>` or `/web read <url>` - look things up on public music sites\n\n## 02 / Make it yours\n\n- `/profile` - the same profile as `/taste`\n- `/profile corrections` - everything you've told me\n- `/profile retract <id>` - undo one of those\n- `/profile help` - the full set of profile commands\n- `/remember <text>`, `/memory`, `/forget <id>` - what I remember between conversations\n\n## 03 / Around the room\n\n- `/home` - back to the record sleeve; the conversation stays\n- `/theme paper|charcoal|terminal|auto` - change the look\n- `/art braille|ascii|off` - change the artwork\n- `/motion on|off` - turn the animation on or off\n- `/commands` or Ctrl+P - search every command; your message stays put\n- Page Up and Page Down scroll the conversation. The header and your draft stay put\n- Up and Down in the draft recall what you typed in this conversation\n- `/resume` - pick up a saved conversation (type to filter, Enter opens)\n- `/new` - start fresh; this conversation stays saved\n- `/status`, `/sources`, `/tools`, `/doctor` - see what's connected and working\n- `/reload` - reload model settings and sign-ins\n- `/update [--check] [--channel latest|beta]` - check for a release, or close this room and install it\n- `/help all` - every command, including the ones for the shell\n- `/language` - choose the language Moondog uses\n- `/quit` - leave\n\nTab moves to the menu on the home screen. Esc goes back to typing.\nEnter sends, and Shift+Enter starts a new line.\nTab or Enter accepts a suggestion.\nCtrl+C stops a model or web request, or leaves when nothing is running.\nIn the profile, type to filter, Tab switches lists, and Enter shows what you can do.\nCtrl+R refreshes it and Ctrl+O opens the full report.\nYour profile works without a model.\nPut quotes around paths, titles, and names that have spaces.");
+  return tr("# Your listening room\n\n## 01 / Listen and discover\n\nJust type to talk about music, ask for a playlist, or go exploring.\nTo talk, Moondog needs a model: `/auth` signs you in, then `/model` picks one.\nWhile an answer is coming, each step shows in the conversation, and the footer names the one in progress.\n\n- `/taste` - your profile: open any song or artist to see why it's there\n- `/taste report` - the whole picture on one page\n- `/import` - bring in your history, your library, or a playlist\n- `/library` - your saved songs and playlists\n- `/generation on|off` - let Moondog make music on this Mac, or stop\n- `/onair` - go on air, or come off it\n- `/lyrics` - the lyrics on the sleeve; `/lyrics sync` finds more\n- `/service` - choose where I play music: Spotify, Apple Music, or NetEase\n- `/spotify` - play, queue, and make playlists in Spotify\n- Ask to move playback onto your iPhone, computer, or another Spotify device by name\n- `/web search <query>` or `/web read <url>` - look things up on public music sites\n\n## 02 / Make it yours\n\n- `/profile` - the same profile as `/taste`\n- `/profile corrections` - everything you've told me\n- `/profile retract <id>` - undo one of those\n- `/profile help` - the full set of profile commands\n- `/remember <text>`, `/memory`, `/forget <id>` - what I remember between conversations\n\n## 03 / Around the room\n\n- `/home` - back to the record sleeve; the conversation stays\n- `/theme paper|charcoal|terminal|auto` - change the look\n- `/art braille|ascii|off` - change the artwork\n- `/motion on|off` - turn the animation on or off\n- `/commands` or Ctrl+P - search every command; your message stays put\n- Page Up and Page Down scroll the conversation. The header and your draft stay put\n- Up and Down in the draft recall what you typed in this conversation\n- `/resume` - pick up a saved conversation (type to filter, Enter opens)\n- `/new` - start fresh; this conversation stays saved\n- `/rewind` or Alt+R - edit an earlier message in place; the messages after it are replaced\n- `/branch` - go on from here in a new conversation; this one stays saved\n- `/status`, `/sources`, `/tools`, `/doctor` - see what's connected and working\n- `/reload` - reload model settings and sign-ins\n- `/model roles` - see what side tasks use: the model that reads images and the web search service\n- `/effort` - choose how hard the model thinks\n- `/endpoint add` or `/endpoint remove <id>` - add a model service by its address, or remove it\n- `/update [--check] [--channel latest|beta]` - check for a release, or close this room and install it\n- `/help all` - every command, including the ones for the shell\n- `/language` - choose the language Moondog uses\n- `/quit` - leave\n\nTab moves to the menu on the home screen. Esc goes back to typing.\nEnter sends, and Shift+Enter starts a new line.\nTab or Enter accepts a suggestion.\nCtrl+C stops a model or web request, or leaves when nothing is running.\nIn the profile, type to filter, Tab switches lists, and Enter shows what you can do.\nCtrl+R refreshes it and Ctrl+O opens the full report.\nYour profile works without a model.\nPut quotes around paths, titles, and names that have spaces.");
 }