@audiofool/moondog 0.3.0-beta.10 → 0.3.0-beta.11

package/src/profile/listening-history-store.mjs

@@ -844,6 +844,7 @@ export class ListeningHistoryStore {
   #coverageSources = null;
   #evidence = null;
   #ledgerChanges = 0;
+  #listeningDays = null;
 
   constructor(database) {
     this.#database = database;
@@ -2058,6 +2059,59 @@ export class ListeningHistoryStore {
       ...(started.length ? { moondog_started_plays: started.length } : {}), top_artists: artists });
   }
 
+  // The listener's own plays a local day at a time, for the window's taste record: every day's plays, the
+  // same for the most played artists, and the plays Moondog started, kept apart as recentAttention keeps
+  // them. Counts and artist names only. Kept until the stored plays change, since a long history is many rows.
+  listeningDays({ subjectId, artists = 8 } = {}) {
+    if (this.#closed) throw new Error("Listening history store is closed");
+    if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
+    if (!Number.isSafeInteger(artists) || artists < 0 || artists > 20) throw new TypeError("Artist count is invalid");
+    const subject = subjectId.toLowerCase();
+    const version = `${this.listeningDataVersion()}:${subject}:${artists}`;
+    if (this.#listeningDays?.version === version) return structuredClone(this.#listeningDays.value);
+    const effective = `e.subject_id = ?
+      AND NOT EXISTS (SELECT 1 FROM listening_event_supersessions s WHERE s.superseded_event_id = e.listening_event_id)
+      AND NOT EXISTS (SELECT 1 FROM listening_event_mirrors m WHERE m.mirror_event_id = e.listening_event_id)`;
+    const suggestions = this.#suggestionsForMatching();
+    const since = suggestions.length ? new Date(Date.parse(suggestions[0].requested_at) - SUGGESTION_WINDOW.beforeMs).toISOString() : null;
+    const started = !since ? [] : markMoondogStartedPlays(this.#database.prepare(`SELECT
+      e.listening_event_id, e.record_json AS event_json, t.record_json AS track_json FROM listening_events e
+      JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+      WHERE ${effective} AND e.occurred_at >= ?`)
+      .all(subject, since).map((row) => ({ id: row.listening_event_id, event: JSON.parse(row.event_json), track: JSON.parse(row.track_json) })),
+    suggestions).rows.filter((row) => row.moondog_suggestion_id).map((row) => row.id);
+    const own = `${effective} AND e.listening_event_id NOT IN (SELECT value FROM json_each(?))`;
+    const left = JSON.stringify(started);
+    const pairs = (rows) => rows.map((row) => [row.day, row.plays]);
+    const days = this.#database.prepare(`SELECT date(e.occurred_at, 'localtime') AS day, COUNT(*) AS plays
+      FROM listening_events e WHERE ${own} GROUP BY day ORDER BY day`).all(subject, left);
+    let value = null;
+    if (days.length) {
+      const credited = `FROM listening_events e
+        JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
+        JOIN json_each(t.record_json, '$.artist_credits') a
+        WHERE ${own} AND json_extract(a.value, '$.name') IS NOT NULL`;
+      const top = artists === 0 ? [] : this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name, COUNT(*) AS plays
+        ${credited} GROUP BY name ORDER BY plays DESC, name LIMIT ?`).all(subject, left, artists);
+      const byArtist = new Map(top.map((row) => [row.name, []]));
+      if (top.length) {
+        for (const row of this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name,
+          date(e.occurred_at, 'localtime') AS day, COUNT(*) AS plays
+          ${credited} AND json_extract(a.value, '$.name') IN (SELECT value FROM json_each(?))
+          GROUP BY name, day ORDER BY day`).iterate(subject, left, JSON.stringify(top.map((artist) => artist.name)))) {
+          byArtist.get(row.name)?.push([row.day, row.plays]);
+        }
+      }
+      const moondog = started.length ? this.#database.prepare(`SELECT date(occurred_at, 'localtime') AS day, COUNT(*) AS plays
+        FROM listening_events WHERE listening_event_id IN (SELECT value FROM json_each(?)) GROUP BY day ORDER BY day`).all(left) : [];
+      value = { first_day: days[0].day, last_day: days.at(-1).day, days: pairs(days),
+        artists: top.map((artist) => ({ name: artist.name, plays: artist.plays, days: byArtist.get(artist.name) })),
+        moondog_days: pairs(moondog) };
+    }
+    this.#listeningDays = { version, value };
+    return structuredClone(value);
+  }
+
   // Which spans of time the stored plays cover, and where plays may be missing.
   listeningCoverage({ subjectId } = {}) {
     if (this.#closed) throw new Error("Listening history store is closed");

package/src/runtime/pi/agent-runtime.mjs

@@ -14,6 +14,7 @@ import { spotifyErrorReason } from "../../integrations/spotify/web-api-client.mj
 import { appleMusicStorefront } from "../../integrations/apple-music/storefront.mjs";
 import { formatWebSources } from "../../surfaces/cli/web-command.mjs";
 import { chooseConversationName } from "./conversation-name.mjs";
+import { prepareImageInput } from "./image-input.mjs";
 import { withArgumentRepair } from "./tool-arguments.mjs";
 import { restateInLanguage } from "./reply-language.mjs";
 import { chooseRecordName, RECORD_NAME_RULES } from "./record-name.mjs";
@@ -4957,11 +4958,11 @@ function createToolFactories(
         name: descriptor.tool_name,
         label: descriptor.label,
         description: kind === "search"
-          ? "Search public reviews, music news, interviews and concert information. Send only a concise public query under 128 characters, never private history or profile data; site:, quoted phrases and -exclusions narrow it. Results come with source links; content_kind says whether they are search-engine snippets or a generated summary."
-          : "Read a public HTTP(S) page. Send only its URL. The result is the page's own text (cut short when long) or a generated summary, as content_kind says. Inaccessible pages are reported; this does not log in or operate an interactive browser.",
+          ? "Search public reviews, music news, interviews and concert information. Send only a concise public query under 128 characters, never private history or profile data; site:, quoted phrases and -exclusions narrow it. Returns 5 sources by default; ask for up to 10 with limit when the first ones aren't enough. Results come with source links; content_kind says whether they are search-engine snippets or a generated summary."
+          : "Read a public HTTP(S) page. Send only its URL. The result is the page's own text or a generated summary, as content_kind says. A long page comes about 15,000 characters at a time: when next_offset is set, call again with that offset to read on. Inaccessible pages are reported; this does not log in or operate an interactive browser.",
         parameters: Type.Object(kind === "search"
-          ? { query: Type.String({ minLength: 1, maxLength: 500 }) }
-          : { url: Type.String({ minLength: 1, maxLength: 2048 }) }, { additionalProperties: false }),
+          ? { query: Type.String({ minLength: 1, maxLength: 500 }), limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })) }
+          : { url: Type.String({ minLength: 1, maxLength: 2048 }), offset: Type.Optional(Type.Integer({ minimum: 0, maximum: 200_000 })) }, { additionalProperties: false }),
         executionMode: "sequential",
         execute: async (_toolCallId, parameters, signal) => executeDomain(
           () => kind === "search"
@@ -7671,8 +7672,11 @@ export class PiAgentRuntime {
   };
 
   constructor({ application, models, model, provider, modelId, effort, effortLevels, thinkingLevel, modelFetch, modelRetryDelay, toolTrace = null,
-    nameConversations = false, clock = () => new Date() }) {
+    nameConversations = false, clock = () => new Date(), visionChoice = null }) {
     this.application = application;
+    // The vision model chosen in Settings, read at each message with images, so a change there
+    // needs no new agent; null leaves images to the chat model.
+    this.visionChoice = typeof visionChoice === "function" ? visionChoice : null;
     // The command passes a local trace of tool calls; tests and other hosts leave it out.
     this.toolTrace = toolTrace;
     this.clock = clock;
@@ -8129,6 +8133,28 @@ export class PiAgentRuntime {
       throw Object.assign(new Error("Moondog is answering in this conversation in another window or terminal."), { code: "conversation_busy" });
     }
     if ((this.application?.latestConversationEntryId?.() ?? null) !== this.syncedEntryId) this.restoreSession();
+    // Attached images become text the conversation keeps, and, for a chat model that sees, the
+    // images themselves. A vision model's description can take a few seconds, so the prompt counts
+    // as running from here; Ctrl+C stops it.
+    let nativeImages = [];
+    if (callbacks.images?.length) {
+      this.promptInFlight = true;
+      this.imageAbort = new AbortController();
+      try {
+        const visionChoice = await Promise.resolve(this.visionChoice?.()).catch(() => null);
+        callbacks.onImagesStart?.({ count: callbacks.images.length, vision: Boolean(visionChoice) });
+        const prepared = await prepareImageInput({ text, images: callbacks.images, models: this.models, model: this.model,
+          visionChoice, signal: this.imageAbort.signal });
+        text = prepared.text;
+        nativeImages = prepared.images;
+        callbacks.onImagesReady?.({ mode: prepared.mode, text, ...(prepared.vision ? { vision: prepared.vision } : {}) });
+      } catch (error) {
+        this.promptInFlight = false;
+        throw error;
+      } finally {
+        this.imageAbort = null;
+      }
+    }
     // What the listener sees of the reply is saved as it grows, so the other one shows it too.
     const host = callbacks;
     let shownText = "";
@@ -8544,7 +8570,8 @@ export class PiAgentRuntime {
         }
       });
 
-      await this.agent.prompt(text);
+      if (nativeImages.length) await this.agent.prompt(text, nativeImages);
+      else await this.agent.prompt(text);
       // One bounded continuation keeps an explicit queue request from stopping
       // at a plan/empty adjacency result or a redundant confirmation. It cannot
       // grant new authority, retry a write, or outlive cancellation.
@@ -8797,6 +8824,8 @@ To confirm, reply: ${preview.confirmation}`].join("\n\n");
   }
 
   abort() {
+    // A vision model may still be describing the message's images.
+    this.imageAbort?.abort();
     if (this.activePromptState) {
       this.activePromptState.abortRequested = true;
       this.activePromptState.restatement?.abort();

package/src/runtime/pi/configured-runtime.mjs

@@ -2,6 +2,7 @@ import { getSupportedThinkingLevels } from "@earendil-works/pi-ai";
 
 import { PiAgentRuntime } from "./agent-runtime.mjs";
 import { createPiAuthContext } from "./authentication.mjs";
+import { readModelRoles } from "./model-roles.mjs";
 import { getPublicPiModelCatalog } from "./public-model-catalog.mjs";
 import { createPersistentCredentialStore } from "./persistent-credential-store.mjs";
 import { clampEffort, EFFORT_LEVELS, readPiRuntimeSelection } from "./runtime-settings.mjs";
@@ -110,5 +111,7 @@ export async function createConfiguredRuntime(
     effortLevels: getSupportedThinkingLevels(model),
     thinkingLevel: effort ? clampEffort(model, effort) : undefined,
     ...(toolTrace ? { toolTrace } : {}),
+    // Read at each message with images: the vision model from Settings, or null for the chat model.
+    visionChoice: async () => (await readModelRoles(environment).catch(() => ({}))).vision ?? null,
   });
 }

package/src/runtime/pi/image-input.mjs

@@ -0,0 +1,148 @@
+import { readFile as readFileFromDisk } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { clampThinkingLevel } from "@earendil-works/pi-ai";
+
+// Images the listener attaches to a message, after Hermes Agent's image routing
+// (agent/image_routing.py, image_input_mode "auto"):
+//
+// - A vision model chosen in Settings describes each image first ("text"): its words go into the
+//   message, and the chat model reads them. A chosen auxiliary model wins over the chat model, as in
+//   Hermes, so the listener can keep a cheap chat model and a good eye.
+// - Otherwise a chat model that takes images gets them as they are ("native").
+// - Otherwise nothing can see them, and the message is refused with what to change.
+//
+// Either way the conversation keeps only text: a short stand-in for a native image, or the vision
+// model's description, so a conversation picked up later still knows what was shown.
+
+export const IMAGE_LIMITS = Object.freeze({
+  count: 4,
+  // Base64 characters of one image, about 6 MB of image; the window scales larger ones down first.
+  dataChars: 8_000_000,
+  mimeTypes: Object.freeze(["image/png", "image/jpeg", "image/webp", "image/gif"]),
+  nameChars: 120,
+  // Conversation entries hold at most 8,000 characters, the listener's own words first.
+  entryChars: 8_000,
+  descriptionChars: 1_600,
+});
+
+const VISION_PROMPT = [
+  "You describe an image for another model, which answers a music listener in Moondog and cannot see it.",
+  "Read any text exactly as written: track lists, artist and album names, playlists, setlists, posters,",
+  "streaming app screenshots, chat messages. Name album covers, artists, instruments or places you can recognise,",
+  "and say how sure you are when you guess. Describe mood and setting briefly.",
+  "Write plain text without a preamble, under 180 words.",
+].join(" ");
+
+function failure(code, message) {
+  return Object.assign(new Error(message), { code });
+}
+
+const takesImages = (model) => Array.isArray(model?.input) && model.input.includes("image");
+
+/** The attached images as Pi takes them, checked; an empty list when there are none. */
+export function normalizeImages(images) {
+  if (images === undefined || images === null) return [];
+  if (!Array.isArray(images)) throw failure("image_input_invalid", "Images must come as a list.");
+  if (images.length > IMAGE_LIMITS.count) throw failure("image_input_too_many", `Attach at most ${IMAGE_LIMITS.count} images to one message.`);
+  return images.map((image, index) => {
+    const mimeType = String(image?.mimeType ?? "").toLowerCase();
+    const data = typeof image?.data === "string" ? image.data.replace(/^data:[^,]*,/u, "") : "";
+    if (!IMAGE_LIMITS.mimeTypes.includes(mimeType)) throw failure("image_input_unsupported", "Moondog takes PNG, JPEG, WebP and GIF images.");
+    if (!data || data.length > IMAGE_LIMITS.dataChars || !/^[A-Za-z0-9+/]+={0,2}$/u.test(data)) {
+      throw failure("image_input_too_large", "That image is too large or unreadable. Try a smaller one.");
+    }
+    const name = typeof image?.name === "string" && image.name.trim()
+      ? image.name.trim().replace(/[\p{C}[\]]/gu, "").slice(0, IMAGE_LIMITS.nameChars) : `image ${index + 1}`;
+    return { type: "image", data, mimeType, name };
+  });
+}
+
+// Local image paths in a terminal message, as Hermes Agent's gateway finds them: an absolute or ~/
+// path ending in an image extension, quoted when it has spaces, or with the "\ " a Finder drag writes.
+const EXTENSION_TYPES = Object.freeze({ png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", webp: "image/webp", gif: "image/gif" });
+const QUOTED_PATH = /(["'])((?:~\/|\/)[^"'\n]+?\.(?:png|jpe?g|webp|gif))\1/giu;
+const BARE_PATH = /(?<![\w./:])((?:~\/|\/)(?:\\ |[^\s"'])+?\.(?:png|jpe?g|webp|gif))(?=[\s,.;:!?)]*(?:\s|$))/giu;
+
+/** The image files a message names, read from disk; paths that aren't readable images are left as words. */
+export async function imagesFromText(text, { readFile = readFileFromDisk, home = os.homedir() } = {}) {
+  const found = [];
+  const quoted = String(text ?? "").replace(QUOTED_PATH, (_, _quote, file) => { found.push(file); return " "; });
+  for (const match of quoted.matchAll(BARE_PATH)) found.push(match[1].replace(/\\ /gu, " "));
+  const images = [];
+  for (const written of [...new Set(found)].slice(0, IMAGE_LIMITS.count)) {
+    const file = written.startsWith("~/") ? path.join(home, written.slice(2)) : written;
+    let bytes;
+    try { bytes = await readFile(file); } catch { continue; }
+    const mimeType = EXTENSION_TYPES[path.extname(file).slice(1).toLowerCase()];
+    const data = Buffer.from(bytes).toString("base64");
+    if (data.length > IMAGE_LIMITS.dataChars) {
+      throw failure("image_input_too_large", `${path.basename(file)} is too large to send. Try a smaller copy or a screenshot of part of it.`);
+    }
+    images.push({ data, mimeType, name: path.basename(file) });
+  }
+  return images;
+}
+
+const modelLabel = (model) => model?.name ?? model?.id ?? "the vision model";
+
+/** Native, text through a vision model, or a refusal; the vision model is resolved here. */
+export async function chooseImageRoute({ models, model, visionChoice }) {
+  if (visionChoice?.provider && visionChoice?.model) {
+    const vision = models.getModel(visionChoice.provider, visionChoice.model);
+    if (!vision || !takesImages(vision)) {
+      throw failure("vision_model_unavailable", "The vision model chosen in Settings can't see images any more. Choose another under Model > Auxiliary models > Vision.");
+    }
+    if (!(await models.checkAuth?.(visionChoice.provider))) {
+      throw failure("vision_model_signed_out", `${modelLabel(vision)} needs a sign-in before it can see images. Add its key in Settings > Model.`);
+    }
+    return { mode: "text", vision };
+  }
+  if (takesImages(model)) return { mode: "native" };
+  throw failure("vision_model_missing",
+    `${modelLabel(model)} can't see images. Choose a model that can under Settings > Model > Auxiliary models > Vision, or switch to one that sees.`);
+}
+
+async function describe({ models, vision, image, text, signal }) {
+  const lightest = clampThinkingLevel(vision, "off");
+  const message = await models.completeSimple(vision, {
+    systemPrompt: VISION_PROMPT,
+    messages: [{ role: "user", timestamp: Date.now(), content: [
+      { type: "text", text: text ? `The listener's message, for context: ${text.slice(0, 1_000)}` : "The listener sent this image without words." },
+      { type: "image", data: image.data, mimeType: image.mimeType },
+    ] }],
+  }, { maxTokens: 900, maxRetries: 1, signal, ...(lightest !== "off" ? { reasoning: lightest } : {}) });
+  if (!message || message.stopReason === "aborted") throw failure("vision_cancelled", "Stopped describing the image.");
+  if (message.stopReason === "error") throw failure("vision_failed", `${modelLabel(vision)} couldn't describe the image. Try again in a moment.`);
+  return message.content.filter((block) => block.type === "text").map((block) => block.text).join("").replace(/\s+/gu, " ").trim();
+}
+
+const clip = (value, limit) => (value.length > limit ? `${value.slice(0, limit - 1).trimEnd()}…` : value);
+
+/**
+ * The message as the conversation keeps it, and the images the chat model gets with it (only in
+ * native mode). Without images it returns the text unchanged.
+ */
+export async function prepareImageInput({ text, images, models, model, visionChoice, signal }) {
+  const attached = normalizeImages(images);
+  if (!attached.length) return { text, images: [], mode: null };
+  const route = await chooseImageRoute({ models, model, visionChoice });
+  const words = String(text ?? "").trim();
+  const room = Math.max(400, IMAGE_LIMITS.entryChars - words.length - 40);
+  const each = Math.min(IMAGE_LIMITS.descriptionChars, Math.floor(room / attached.length) - 60);
+  const notes = [];
+  for (const [index, image] of attached.entries()) {
+    const label = attached.length > 1 ? `Image ${index + 1} (${image.name})` : `Image (${image.name})`;
+    if (route.mode === "native") { notes.push(`[${label} attached]`); continue; }
+    const seen = await describe({ models, vision: route.vision, image, text: words, signal });
+    notes.push(`[${label}, as ${modelLabel(route.vision)} describes it: ${clip(seen || "nothing it could make out", Math.max(80, each))}]`);
+  }
+  const kept = [words, notes.join("\n")].filter(Boolean).join("\n\n");
+  return {
+    text: kept.slice(0, IMAGE_LIMITS.entryChars),
+    images: route.mode === "native" ? attached.map(({ type, data, mimeType }) => ({ type, data, mimeType })) : [],
+    mode: route.mode,
+    ...(route.vision ? { vision: { provider: route.vision.provider, model: route.vision.id, name: modelLabel(route.vision) } } : {}),
+  };
+}

package/src/runtime/pi/model-roles.mjs

@@ -0,0 +1,121 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+
+import { resolveMoondogAuthFile } from "./persistent-credential-store.mjs";
+
+// What each model is for, after Hermes Agent's main model and auxiliary tasks: one chat model answers,
+// and a side task can have a model of its own. A role without a saved choice is "auto": vision uses the
+// chat model when it can see images, and web search takes the first service that is set up.
+// The chat model stays in settings.json, which older Moondogs read with a fixed shape; the other
+// roles live in model-roles.json beside it.
+//
+// kind "model" is a provider and model from the catalog; kind "service" is a web service by id.
+// A "planned" role holds its place in the settings and in this file, but nothing can choose it yet.
+export const MODEL_ROLES = Object.freeze([
+  Object.freeze({ id: "chat", kind: "model", state: "enabled", stored: "settings" }),
+  Object.freeze({ id: "vision", kind: "model", state: "enabled", input: "image" }),
+  Object.freeze({ id: "web_search", kind: "service", state: "enabled" }),
+  Object.freeze({ id: "audio", kind: "model", state: "planned", input: "audio" }),
+]);
+
+export const WEB_SERVICES = Object.freeze(["deepseek", "brave", "codex"]);
+
+const ROLES_VERSION = 1;
+const printable = (value, max) => typeof value === "string" && value.length > 0 && value.length <= max && !/[\p{C}\s]/u.test(value);
+
+function rolesError(message, cause) {
+  const error = new Error(message, cause ? { cause } : undefined);
+  error.name = "MoondogRuntimeSettingsError";
+  error.code = "model_roles_invalid";
+  return error;
+}
+
+export const modelRole = (id) => MODEL_ROLES.find((role) => role.id === id) ?? null;
+
+/** One role's choice in its saved shape, or null for auto. Throws on a shape the role can't take. */
+export function normalizeRoleChoice(roleId, value) {
+  const role = modelRole(roleId);
+  if (!role || role.stored) throw rolesError(`Moondog has no model role called ${roleId}.`);
+  if (value === null || value === undefined || value === "auto") return null;
+  if (role.kind === "service") {
+    const service = typeof value?.service === "string" ? value.service.trim().toLowerCase() : "";
+    if (!WEB_SERVICES.includes(service)) throw rolesError(`Web search can use ${WEB_SERVICES.join(", ")} or auto.`);
+    return { service };
+  }
+  const provider = typeof value?.provider === "string" ? value.provider.trim().toLowerCase() : "";
+  const model = typeof value?.model === "string" ? value.model.trim() : "";
+  if (!printable(provider, 80) || !printable(model, 200)) throw rolesError(`The ${roleId} role needs a provider and a model.`);
+  return { provider, model, ...(typeof value.effort === "string" && printable(value.effort, 20) ? { effort: value.effort } : {}) };
+}
+
+export function resolveModelRolesFile(environment = process.env) {
+  return path.join(path.dirname(resolveMoondogAuthFile(environment)), "model-roles.json");
+}
+
+// Every saved role as it is in the file, including roles a newer Moondog wrote, so a write here
+// never drops them.
+async function readDocument(environment) {
+  let source;
+  try {
+    source = await readFile(resolveModelRolesFile(environment), "utf8");
+  } catch (error) {
+    if (error?.code === "ENOENT") return {};
+    throw rolesError("Moondog could not read its model roles.", error);
+  }
+  let document;
+  try {
+    document = JSON.parse(source);
+  } catch (error) {
+    throw rolesError("Moondog's model roles are not valid JSON.", error);
+  }
+  if (!document || typeof document !== "object" || document.version !== ROLES_VERSION ||
+      !document.roles || typeof document.roles !== "object" || Array.isArray(document.roles)) {
+    throw rolesError("Moondog's model roles use an unsupported shape.");
+  }
+  return document.roles;
+}
+
+/** The saved choice for each role this Moondog knows, by role id; a role left on auto is absent. */
+export async function readModelRoles(environment = process.env) {
+  const saved = await readDocument(environment);
+  const roles = {};
+  for (const role of MODEL_ROLES) {
+    if (role.stored || role.state !== "enabled" || saved[role.id] === undefined) continue;
+    // One bad entry leaves that role on auto instead of failing every role.
+    try {
+      const choice = normalizeRoleChoice(role.id, saved[role.id]);
+      if (choice) roles[role.id] = choice;
+    } catch {}
+  }
+  return roles;
+}
+
+// One change at a time, so two quick choices never lose either.
+let writing = Promise.resolve();
+/** Saves one role's choice; null puts it back on auto. */
+export function writeModelRole(roleId, value, environment = process.env) {
+  const role = modelRole(roleId);
+  if (!role || role.stored) return Promise.reject(rolesError(`Moondog has no model role called ${roleId}.`));
+  if (role.state !== "enabled") return Promise.reject(rolesError(`The ${roleId} role isn't available yet.`));
+  let choice;
+  try { choice = normalizeRoleChoice(roleId, value); } catch (error) { return Promise.reject(error); }
+  const next = writing.then(async () => {
+    const roles = { ...(await readDocument(environment).catch(() => ({}))) };
+    if (choice) roles[roleId] = choice;
+    else delete roles[roleId];
+    const file = resolveModelRolesFile(environment);
+    await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+    // Another Moondog may read the roles at any moment, so it never sees half a file.
+    const temporary = `${file}.${randomUUID()}.tmp`;
+    try {
+      await writeFile(temporary, `${JSON.stringify({ version: ROLES_VERSION, roles }, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
+      await rename(temporary, file);
+    } finally {
+      await rm(temporary, { force: true });
+    }
+    return choice;
+  });
+  writing = next.catch(() => {});
+  return next;
+}

package/src/runtime/pi/model-settings.mjs

@@ -9,6 +9,7 @@ import {
 import { isEndpointProvider, removeEndpoint, saveEndpoint } from "./custom-endpoints.mjs";
 import { LEGACY_MODEL_ALIASES } from "./model-catalog.mjs";
 import { readConversationModels, readModelList, sameModel, writeConversationModel, writeModelList } from "./model-list.mjs";
+import { MODEL_ROLES, modelRole, readModelRoles, writeModelRole } from "./model-roles.mjs";
 import { chatgptProviderId } from "./provider-registry.mjs";
 import { createPersistentCredentialStore } from "./persistent-credential-store.mjs";
 import { getPublicPiModelCatalog } from "./public-model-catalog.mjs";
@@ -22,6 +23,8 @@ import { clampEffort, readPiRuntimeSelection } from "./runtime-settings.mjs";
 // key, then Anthropic, Google and xAI. Any other service is reached as an endpoint by address; Pi's
 // other providers show only once a key for one is saved or in the environment.
 const FEATURED = ["openai", "anthropic", "google", "xai"];
+// Whether a catalog model takes an input besides text, such as "image".
+const takes = (model, input) => Boolean(input) && Array.isArray(model?.input) && model.input.includes(input);
 // Moondog's own errors carry fixed text. Pi wraps them, and its own text may not be safe.
 const moondogErrorNames = new Set(["MoondogCredentialStoreError", "MoondogRuntimeSettingsError"]);
 
@@ -183,13 +186,71 @@ export function createModelSettings({
           const model = models.getModel(entry.provider, entry.model);
           return { provider: entry.provider, model: entry.model, name: model?.name ?? entry.model,
             providerName: names.get(entry.provider) ?? entry.provider, ready: Boolean(model) && ready.get(entry.provider) === true,
-            reasoning: thinks(model) };
+            reasoning: thinks(model), vision: takes(model, "image") };
         }),
         default: selection ? { provider: selection.provider, model: selection.model, effort: selection.effort ?? null,
           reasoning: thinks(defaultModel), thinking: selection.effort && defaultModel ? clampEffort(defaultModel, selection.effort) : null } : null,
       };
     },
 
+    // What each model is for: the chat model from settings.json, then each other role's choice from
+    // model-roles.json, null for auto. Vision on auto uses the chat model when it can see images.
+    // web is the web research's services(), which knows the web services; Settings passes it in.
+    async roles({ web = null } = {}) {
+      const selection = await current();
+      let saved;
+      try {
+        saved = await readModelRoles(environment);
+      } catch (error) {
+        throw moondogError(error) ?? failure("model_roles_invalid", "Moondog could not read its model roles.");
+      }
+      const source = await catalog();
+      let models;
+      try {
+        models = withAuth(source);
+      } catch (error) {
+        throw moondogError(error) ?? failure("auth_status_failed", "Moondog could not read its saved sign-ins.");
+      }
+      const names = new Map(source.providers().map(provider => [provider.id, provider.name]));
+      const describe = (choice) => {
+        const model = models.getModel(choice.provider, choice.model);
+        return { provider: choice.provider, model: choice.model, name: model?.name ?? choice.model,
+          providerName: names.get(choice.provider) ?? choice.provider, known: Boolean(model), vision: takes(model, "image") };
+      };
+      const chat = selection ? describe(selection) : null;
+      return MODEL_ROLES.map((role) => {
+        const base = { id: role.id, kind: role.kind, state: role.state, ...(role.input ? { input: role.input } : {}) };
+        if (role.id === "chat") return { ...base, choice: chat };
+        if (role.id === "web_search") {
+          return { ...base, choice: saved.web_search ?? null,
+            ...(web ? { services: web.services, search: web.search?.provider ?? null, read: web.read?.provider ?? null } : {}) };
+        }
+        if (role.state !== "enabled") return { ...base, choice: null };
+        // Hermes Agent's auxiliary vision on auto: the main model when it takes images, else none.
+        return { ...base, choice: saved[role.id] ? describe(saved[role.id]) : null,
+          auto: chat && takes(models.getModel(selection.provider, selection.model), role.input) ? chat : null };
+      });
+    },
+
+    // Saves one role's choice; null puts it on auto. A model must be in the catalog and take the
+    // input its role is for.
+    async setRole(roleId, choice) {
+      const role = modelRole(String(roleId ?? ""));
+      if (!role || role.stored) throw failure("model_role_unknown", "Moondog has no such model role.");
+      if (role.state !== "enabled") throw failure("model_role_unavailable", "That model role isn't available yet.");
+      if (role.kind === "model" && choice) {
+        const provider = String(choice.provider ?? "").trim().toLowerCase();
+        const model = (await catalog()).createModels().getModel(provider, String(choice.model ?? "").trim());
+        if (!model) throw failure("model_unknown", "Moondog doesn't know that model.");
+        if (!takes(model, role.input)) throw failure("model_role_unsuitable", `That model can't take ${role.input} input.`);
+      }
+      try {
+        return await writeModelRole(role.id, choice ?? null, environment);
+      } catch (error) {
+        throw moondogError(error) ?? failure("model_roles_invalid", "Moondog could not save its model roles.");
+      }
+    },
+
     // The model each conversation was given in the window's menu, and a change to one: null gives a
     // conversation back to the default.
     conversationModels: () => readConversationModels(environment),
@@ -262,7 +323,7 @@ export function createModelSettings({
         refresh,
         models: source.models(providerId).filter(({ id }) => !LEGACY_MODEL_ALIASES.includes(id)).map(({ id, name, contextWindow, reasoning, catalogNote }) => {
           const model = full.getModel(providerId, id);
-          return { id, name, ...(contextWindow ? { contextWindow } : {}), reasoning,
+          return { id, name, ...(contextWindow ? { contextWindow } : {}), reasoning, vision: takes(model, "image"),
             effortLevels: model ? getSupportedThinkingLevels(model) : [],
             ...(catalogNote ? { retained: true } : {}) };
         }),

package/src/runtime/pi/runtime-settings.mjs

@@ -109,11 +109,13 @@ export async function writePiRuntimeSelection(
 // saved selection, or undefined when none is saved; the surface decides whether to rebuild, with
 // runtimeBehindSelection.
 // The second argument says which file changed: "settings" (the default), "auth" (a sign-in),
-// "endpoints" (an endpoint added, edited or removed) or "models" (the window's menu list).
+// "endpoints" (an endpoint added, edited or removed), "models" (the window's menu list) or "roles"
+// (a vision model or web service chosen in model-roles.json).
 export function watchRuntimeSettings({ environment = process.env, interval = 2000, onChange } = {}) {
   const directory = path.dirname(resolveMoondogAuthFile(environment));
   const files = { settings: resolveMoondogSettingsFile(environment), auth: resolveMoondogAuthFile(environment),
-    endpoints: path.join(directory, "endpoints.json"), models: path.join(directory, "models.json") };
+    endpoints: path.join(directory, "endpoints.json"), models: path.join(directory, "models.json"),
+    roles: path.join(directory, "model-roles.json") };
   const listeners = Object.entries(files).map(([changed, file]) => {
     const listener = (now, before) => {
       if (now.mtimeMs === before.mtimeMs) return;

package/src/surfaces/cli/model-roles-command.mjs

@@ -0,0 +1,60 @@
+import { MODEL_ROLES, WEB_SERVICES } from "../../runtime/pi/model-roles.mjs";
+
+// /model roles, /model vision and /model web: the terminal's side of Settings' Auxiliary models.
+// The chat model stays /model <provider> <model>.
+export const MODEL_ROLE_WORDS = Object.freeze({ roles: null, vision: "vision", web: "web_search", audio: "audio" });
+
+const titles = { chat: "Chat", vision: "Vision", web_search: "Web search", audio: "Audio" };
+const usage = "Usage: /model roles, /model vision auto|<provider> <model>, or /model web auto|deepseek|brave|codex.";
+
+const modelName = (choice) => `${choice.name ?? choice.model} (${choice.providerName ?? choice.provider})`;
+
+function describe(role) {
+  if (role.state !== "enabled") return "not available yet; its place is kept for a later Moondog";
+  if (role.id === "chat") return role.choice ? modelName(role.choice) : "none chosen; /model picks one";
+  if (role.kind === "service") {
+    const service = (id) => role.services?.find((entry) => entry.id === id || entry.provider === id);
+    if (role.choice) {
+      const chosen = service(role.choice.service);
+      return `${chosen?.name ?? role.choice.service}${chosen && !chosen.ready ? " (not set up; falls back to the next service)" : ""}`;
+    }
+    const used = service(role.search);
+    return used?.ready ? `auto, now ${used.name}` : "auto, but no service is set up; /web status says how";
+  }
+  if (role.choice) return modelName(role.choice);
+  return role.auto ? `auto, now ${modelName(role.auto)}` : "auto, but the chat model can't see images";
+}
+
+export function formatModelRoles(roles) {
+  const lines = roles.map((role) => `- ${titles[role.id] ?? role.id}: ${describe(role)}`);
+  return `# Chat and auxiliary models\n\n${lines.join("\n")}\n\n` +
+    "`/model vision <provider> <model>` gives images a model of their own, and `/model vision auto` gives them back to the chat model when it can see.\n" +
+    "`/model web deepseek|brave|codex` picks the web search service, and `/model web auto` takes the first one set up.";
+}
+
+export async function runModelRolesCommand({ args = [], modelSettings, webResearch } = {}) {
+  if (!modelSettings?.roles) throw new Error("Model roles aren't available when Moondog is started this way.");
+  const [word, ...rest] = args;
+  const roleId = MODEL_ROLE_WORDS[String(word ?? "").toLowerCase()];
+  const show = async () => formatModelRoles(await modelSettings.roles({ web: await webResearch?.services?.().catch(() => null) ?? null }));
+  if (roleId === null) {
+    if (rest.length) throw new Error(usage);
+    return show();
+  }
+  if (roleId === undefined) throw new Error(usage);
+  const role = MODEL_ROLES.find((entry) => entry.id === roleId);
+  if (role.state !== "enabled") return `${titles[roleId]} isn't available yet. Its place is kept, so a later Moondog can add it.`;
+  if (!rest.length) return show();
+  let choice;
+  if (rest.length === 1 && rest[0].toLowerCase() === "auto") choice = null;
+  else if (role.kind === "service") {
+    if (rest.length !== 1 || !WEB_SERVICES.includes(rest[0].toLowerCase())) throw new Error(usage);
+    choice = { service: rest[0].toLowerCase() };
+  } else {
+    if (rest.length !== 2) throw new Error(usage);
+    choice = { provider: rest[0], model: rest[1] };
+  }
+  await modelSettings.setRole(roleId, choice);
+  if (roleId === "web_search") await webResearch?.refresh?.();
+  return show();
+}