@isaacthoman/pulpo 0.57.0 → 0.59.0

package/README.md

@@ -11,6 +11,7 @@ pulpo auth 2fa setup
 pulpo auth 2fa confirm
 pulpo settings export --output pulpo-settings.json
 pulpo icon upload ./acme.svg --name Acme --mode monochrome
+pulpo model test acme-model "Reply with a short greeting" --no-agent --preset reasoning=off
 ```
 
 Use `pulpo help` or `pulpo <command> --help` for the complete command reference.
@@ -21,6 +22,29 @@ canonical [Lucide](https://lucide.dev/icons/) name. Discover available names wit
 `pulpo model icons camera`. Invalid names are rejected before the model request is
 sent. Add `--json` for stable `{ "name": "..." }` rows in scripts.
 
+Test a newly configured model with `pulpo model test <model-id> [prompt...]`.
+The command requires exactly one of `--agent` or `--no-agent`, plus one explicit
+`--preset <preset-id>=<choice-id>` for every preset exposed by the model. It
+does not silently apply preset defaults. For example:
+
+```bash
+pulpo model test acme-model "Investigate the failing build" \
+  --agent \
+  --preset reasoning=high \
+  --preset web-search=enabled
+
+git diff | pulpo model test acme-model \
+  --no-agent \
+  --preset reasoning=off \
+  --preset web-search=disabled
+```
+
+Model tests stream assistant text to stdout and use temporary chats by default.
+Pass `--keep` to retain the test in normal chat history, `--no-stream` to wait
+for the completed text, `--json` for one final result, or `--jsonl` for the
+replayable response event stream. Model testing requires a user session created
+by `pulpo auth login`; management tokens cannot access user chat endpoints.
+
 The major command groups are `context`, `auth`, `token`, `instance`, `settings`,
 `provider`, `lab`, `icon`, `model`, `user`, `usage`, `audit`, `workspace`, `banner`,
 `job`, `export`, and `backup`. There is intentionally no restore command.

package/dist/index.js

@@ -11,7 +11,7 @@ import { mkdtemp, readFile as readFile2, rm as rm2, writeFile as writeFile2 } fr
 import { tmpdir } from "node:os";
 import { basename, join as join2 } from "node:path";
 import { pathToFileURL } from "node:url";
-import { Command, CommanderError } from "commander";
+import { Command, CommanderError, Option } from "commander";
 
 // ../../node_modules/zod/v4/classic/external.js
 var external_exports = {};
@@ -1359,8 +1359,8 @@ function formatError(error51, mapper = (issue2) => issue2.message) {
           let i = 0;
           while (i < fullpath.length) {
             const el = fullpath[i];
-            const terminal = i === fullpath.length - 1;
-            if (!terminal) {
+            const terminal2 = i === fullpath.length - 1;
+            if (!terminal2) {
               curr[el] = curr[el] || { _errors: [] };
             } else {
               curr[el] = curr[el] || { _errors: [] };
@@ -1397,7 +1397,7 @@ function treeifyError(error51, mapper = (issue2) => issue2.message) {
         let i = 0;
         while (i < fullpath.length) {
           const el = fullpath[i];
-          const terminal = i === fullpath.length - 1;
+          const terminal2 = i === fullpath.length - 1;
           if (typeof el === "string") {
             curr.properties ?? (curr.properties = {});
             (_a3 = curr.properties)[el] ?? (_a3[el] = { errors: [] });
@@ -1407,7 +1407,7 @@ function treeifyError(error51, mapper = (issue2) => issue2.message) {
             (_b = curr.items)[el] ?? (_b[el] = { errors: [] });
             curr = curr.items[el];
           }
-          if (terminal) {
+          if (terminal2) {
             curr.errors.push(mapper(issue2));
           }
           i++;
@@ -16462,6 +16462,167 @@ var responseSnapshotSchema = external_exports.object({
   updatedAt: isoDateSchema
 });
 var embeddedResponseSnapshotSchema = responseSnapshotSchema.omit({ output: true });
+function targetItemIndex(output, payload, type) {
+  const itemId = typeof payload.item_id === "string" ? payload.item_id : typeof payload.itemId === "string" ? payload.itemId : void 0;
+  if (itemId) {
+    const byId = output.findIndex((item) => item.id === itemId);
+    return byId;
+  }
+  const outputIndex = typeof payload.output_index === "number" ? payload.output_index : typeof payload.outputIndex === "number" ? payload.outputIndex : void 0;
+  if (outputIndex !== void 0 && output[outputIndex]?.type === type)
+    return outputIndex;
+  const agentTurn = typeof payload.agent_turn === "number" ? payload.agent_turn : void 0;
+  const contentIndex = typeof payload.content_index === "number" ? payload.content_index : typeof payload.contentIndex === "number" ? payload.contentIndex : void 0;
+  if (agentTurn !== void 0 && contentIndex !== void 0) {
+    const byAgentPart = output.findIndex((item) => item.type === type && item.agent_turn === agentTurn && item.agent_content_index === contentIndex);
+    if (byAgentPart >= 0)
+      return byAgentPart;
+  }
+  for (let index = output.length - 1; index >= 0; index -= 1) {
+    if (output[index]?.type === type && output[index]?.status === "in_progress")
+      return index;
+  }
+  for (let index = output.length - 1; index >= 0; index -= 1) {
+    if (output[index]?.type === type)
+      return index;
+  }
+  return -1;
+}
+function appendOutputText(output, delta, payload) {
+  const copy = output.slice();
+  const index = targetItemIndex(copy, payload, "message");
+  let message = index >= 0 ? { ...copy[index] } : void 0;
+  if (!message) {
+    const itemId = typeof payload.item_id === "string" ? payload.item_id : typeof payload.itemId === "string" ? payload.itemId : void 0;
+    message = { ...itemId ? { id: itemId } : {}, type: "message", role: "assistant", status: "in_progress", content: [] };
+    copy.push(message);
+  } else {
+    copy[index] = message;
+  }
+  const content = Array.isArray(message.content) ? message.content.slice() : [];
+  const contentIndex = typeof payload.content_index === "number" ? payload.content_index : typeof payload.contentIndex === "number" ? payload.contentIndex : void 0;
+  const partIndex = contentIndex !== void 0 && content[contentIndex]?.type === "output_text" ? contentIndex : content.findIndex((item) => item.type === "output_text");
+  let part = partIndex >= 0 ? { ...content[partIndex] } : void 0;
+  if (!part) {
+    part = { type: "output_text", text: "" };
+    content.push(part);
+  } else {
+    content[partIndex] = part;
+  }
+  part.text = `${typeof part.text === "string" ? part.text : ""}${delta}`;
+  message.content = content;
+  return copy;
+}
+function appendReasoning(output, delta, payload) {
+  const copy = output.slice();
+  const index = targetItemIndex(copy, payload, "reasoning");
+  let reasoning = index >= 0 ? { ...copy[index] } : void 0;
+  if (!reasoning) {
+    const itemId = typeof payload.item_id === "string" ? payload.item_id : typeof payload.itemId === "string" ? payload.itemId : void 0;
+    reasoning = { ...itemId ? { id: itemId } : {}, type: "reasoning", status: "in_progress", summary: [] };
+    copy.push(reasoning);
+  } else {
+    copy[index] = reasoning;
+  }
+  const summary = Array.isArray(reasoning.summary) ? reasoning.summary.slice() : [];
+  const partIndex = summary.findIndex((item) => item.type === "summary_text");
+  let part = partIndex >= 0 ? { ...summary[partIndex] } : void 0;
+  if (!part) {
+    part = { type: "summary_text", text: "" };
+    summary.push(part);
+  } else {
+    summary[partIndex] = part;
+  }
+  part.text = `${typeof part.text === "string" ? part.text : ""}${delta}`;
+  reasoning.summary = summary;
+  return copy;
+}
+function upsertOutputItem(output, match, value) {
+  const copy = output.slice();
+  const index = copy.findIndex((item) => Boolean(item) && typeof item === "object" && match(item));
+  if (index < 0)
+    copy.push(value);
+  else
+    copy[index] = { ...copy[index], ...value };
+  return copy;
+}
+function applyAgentEventOutput(output, event) {
+  const payload = event.payload;
+  if (event.type.startsWith("pulpo.agent.workspace.")) {
+    return upsertOutputItem(output, (item) => item.type === "pulpo_workspace", payload);
+  }
+  if (event.type === "pulpo.compaction.updated" && typeof payload.id === "string") {
+    return upsertOutputItem(output, (item) => item.id === payload.id, payload);
+  }
+  if (event.type === "pulpo.agent.attachment.created" && typeof payload.attachment_id === "string") {
+    return upsertOutputItem(output, (item) => item.type === "pulpo_attachment" && item.attachment_id === payload.attachment_id, payload);
+  }
+  if (!event.type.startsWith("pulpo.agent.tool.") || typeof payload.id !== "string")
+    return output;
+  if (event.type === "pulpo.agent.tool.delta") {
+    return upsertOutputItem(output, (item) => item.id === payload.id, {
+      id: payload.id,
+      type: "pulpo_tool",
+      output: typeof payload.delta === "string" ? payload.delta : "",
+      status: "running"
+    });
+  }
+  if (event.type === "pulpo.agent.tool.completed") {
+    return upsertOutputItem(output, (item) => item.id === payload.id, {
+      ...payload,
+      type: "pulpo_tool",
+      status: payload.isError ? "failed" : "completed"
+    });
+  }
+  return upsertOutputItem(output, (item) => item.id === payload.id, payload);
+}
+function applyResponseEventToSnapshot(snapshot, event) {
+  if (event.sequence <= snapshot.sequence)
+    return snapshot;
+  const payload = event.payload;
+  const delta = typeof payload.delta === "string" ? payload.delta : "";
+  let output = snapshot.output;
+  if (delta && event.type === "response.output_text.delta")
+    output = appendOutputText(output, delta, payload);
+  if (delta && event.type === "response.reasoning_summary_text.delta")
+    output = appendReasoning(output, delta, payload);
+  output = applyAgentEventOutput(output, event);
+  return {
+    ...snapshot,
+    status: snapshot.status === "queued" ? "in_progress" : snapshot.status,
+    sequence: event.sequence,
+    output,
+    updatedAt: event.emittedAt
+  };
+}
+function mergeResponseSnapshots(current, incoming) {
+  if (incoming.sequence < current.sequence)
+    return current;
+  if (incoming.sequence === current.sequence) {
+    const currentTerminal = current.status !== "queued" && current.status !== "in_progress";
+    const incomingTerminal = incoming.status !== "queued" && incoming.status !== "in_progress";
+    if (currentTerminal && !incomingTerminal)
+      return current;
+    if (incomingTerminal && !currentTerminal) {
+      return incoming.output.length === 0 && current.output.length > 0 ? { ...incoming, output: current.output } : incoming;
+    }
+    if (incoming.updatedAt < current.updatedAt)
+      return current;
+    if (incoming.updatedAt === current.updatedAt) {
+      if (current.output.length === 0 && incoming.output.length > 0)
+        return incoming;
+      return current;
+    }
+    if (incoming.output.length === 0 && current.output.length > 0) {
+      return { ...incoming, output: current.output };
+    }
+  }
+  const incomingIsActive = incoming.status === "queued" || incoming.status === "in_progress";
+  if (incomingIsActive && incoming.output.length === 0 && current.output.length > 0) {
+    return { ...incoming, output: current.output };
+  }
+  return incoming;
+}
 var catalogIconModeSchema = external_exports.enum(["original", "monochrome"]);
 var catalogIconReferenceSchema = external_exports.object({
   id: idSchema,
@@ -17416,11 +17577,222 @@ async function confirmExact(io, expected, yes, json2) {
   }
 }
 
+// src/model-test.ts
+import { randomUUID } from "node:crypto";
+import { io as createSocket } from "socket.io-client";
+function explicitAgentMode(options) {
+  const enabled = options.agentEnabled === true;
+  const disabled = options.agentDisabled === false;
+  if (enabled === disabled) {
+    throw selectionError(["choose --agent or --no-agent"]);
+  }
+  return enabled;
+}
+function terminal(snapshot) {
+  return snapshot.status !== "queued" && snapshot.status !== "in_progress";
+}
+async function followResponse(input) {
+  if (terminal(input.snapshot)) return input.snapshot;
+  return new Promise((resolve, reject) => {
+    let current = input.snapshot;
+    const pending = /* @__PURE__ */ new Map();
+    const socket = createSocket(input.baseUrl, {
+      path: "/socket.io",
+      auth: { sessionToken: input.token },
+      autoConnect: false,
+      reconnection: true
+    });
+    let settled = false;
+    const cleanup = () => {
+      socket.removeAllListeners();
+      socket.disconnect();
+    };
+    const fail = (error51) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(error51);
+    };
+    const finishIfTerminal = () => {
+      if (settled || !terminal(current)) return;
+      settled = true;
+      cleanup();
+      resolve(current);
+    };
+    const flush = () => {
+      for (; ; ) {
+        const event = pending.get(current.sequence + 1);
+        if (!event) break;
+        pending.delete(event.sequence);
+        current = applyResponseEventToSnapshot(current, event);
+        input.onEvent?.(event);
+      }
+    };
+    socket.on("connect", () => {
+      socket.emit("response.subscribe", {
+        responseId: current.responseId,
+        afterSequence: current.sequence
+      });
+    });
+    socket.on("connect_error", (error51) => fail(new Error(`Realtime connection failed: ${error51.message}`)));
+    socket.on("response.event", (event) => {
+      if (event.responseId !== current.responseId || event.sequence <= current.sequence) return;
+      pending.set(event.sequence, event);
+      flush();
+    });
+    socket.on("response.snapshot", (snapshot) => {
+      if (snapshot.responseId !== current.responseId) return;
+      current = mergeResponseSnapshots(current, snapshot);
+      for (const sequence of pending.keys()) if (sequence <= current.sequence) pending.delete(sequence);
+      flush();
+      input.onSnapshot?.(current);
+      finishIfTerminal();
+    });
+    socket.connect();
+  });
+}
+function selectionError(lines) {
+  return new Error(`Explicit selections are required:
+${lines.map((line) => `  ${line}`).join("\n")}`);
+}
+function resolvePresetSelections(model, values) {
+  const parsed = /* @__PURE__ */ new Map();
+  for (const value of values) {
+    const separator = value.indexOf("=");
+    if (separator <= 0 || separator === value.length - 1) {
+      throw new Error(`Invalid preset selection ${JSON.stringify(value)}; expected <preset-id>=<choice-id>`);
+    }
+    const presetId = value.slice(0, separator);
+    const choiceId = value.slice(separator + 1);
+    if (parsed.has(presetId)) throw new Error(`Preset ${presetId} was selected more than once`);
+    parsed.set(presetId, choiceId);
+  }
+  const exposed = new Map(model.presets.map((preset) => [preset.id, preset]));
+  const unknown2 = [...parsed.keys()].filter((presetId) => !exposed.has(presetId));
+  if (unknown2.length) throw new Error(`Unknown preset${unknown2.length === 1 ? "" : "s"} for ${model.id}: ${unknown2.join(", ")}`);
+  const missing = model.presets.filter((preset) => !parsed.has(preset.id));
+  if (missing.length) {
+    throw selectionError(missing.map((preset) => `--preset ${preset.id}=<${preset.choices.map((choice) => choice.id).join("|")}>`));
+  }
+  for (const preset of model.presets) {
+    const choiceId = parsed.get(preset.id);
+    if (!preset.choices.some((choice) => choice.id === choiceId)) {
+      throw new Error(
+        `Unknown choice ${choiceId} for preset ${preset.id}; choose ${preset.choices.map((choice) => choice.id).join(", ")}`
+      );
+    }
+  }
+  return Object.fromEntries(parsed);
+}
+async function readModelTestPrompt(io, promptParts) {
+  const argument = promptParts.join(" ").trim();
+  if (argument) return argument;
+  if (io.stdin.isTTY) throw new Error("Prompt is required as an argument or on stdin");
+  const chunks = [];
+  for await (const chunk of io.stdin) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  const prompt = Buffer.concat(chunks).toString("utf8").trim();
+  if (!prompt) throw new Error("Prompt is required as an argument or on stdin");
+  return prompt;
+}
+function responseText(snapshot) {
+  const parts = [];
+  for (const item of snapshot.output) {
+    if (!item || typeof item !== "object") continue;
+    const content = item.content;
+    if (!Array.isArray(content)) continue;
+    for (const part of content) {
+      if (!part || typeof part !== "object") continue;
+      const candidate = part;
+      if (candidate.type === "output_text" && typeof candidate.text === "string") parts.push(candidate.text);
+    }
+  }
+  return parts.join("");
+}
+function writeJsonLine(output, value) {
+  output.write(`${JSON.stringify(value)}
+`);
+}
+async function runModelTest(input) {
+  const chatId = randomUUID();
+  const responseId = randomUUID();
+  const temporary = !input.keep;
+  const created = await input.client.request("/api/chats/start", {
+    method: "POST",
+    headers: { "idempotency-key": responseId },
+    body: {
+      chat: {
+        clientId: chatId,
+        modelId: input.model.id,
+        title: `CLI test: ${input.model.name}`.slice(0, 200),
+        temporary
+      },
+      response: {
+        clientId: responseId,
+        parentResponseId: null,
+        input: input.prompt,
+        modelId: input.model.id,
+        presetSelections: input.presetSelections,
+        attachmentIds: [],
+        agentMode: input.agentMode
+      }
+    }
+  });
+  let writtenText = "";
+  const writeProgress = (snapshot) => {
+    if (!input.streamText) return;
+    const text = responseText(snapshot);
+    if (text.startsWith(writtenText)) {
+      input.io.stdout.write(text.slice(writtenText.length));
+      writtenText = text;
+    }
+  };
+  if (input.jsonl) writeJsonLine(input.io.stdout, { type: "response.snapshot", chatId: created.chat.id, snapshot: created.response });
+  const finalSnapshot = await (input.follow ?? followResponse)({
+    baseUrl: input.baseUrl,
+    token: input.token,
+    snapshot: created.response,
+    onEvent: (event) => {
+      if (input.jsonl) writeJsonLine(input.io.stdout, { type: "response.event", chatId: created.chat.id, event });
+      if (input.streamText && event.type === "response.output_text.delta") {
+        const delta = event.payload.delta;
+        if (typeof delta === "string") {
+          input.io.stdout.write(delta);
+          writtenText += delta;
+        }
+      }
+    },
+    onSnapshot: (snapshot) => {
+      if (input.jsonl) writeJsonLine(input.io.stdout, { type: "response.snapshot", chatId: created.chat.id, snapshot });
+      writeProgress(snapshot);
+    }
+  });
+  writeProgress(finalSnapshot);
+  if (input.streamText) input.io.stdout.write("\n");
+  return {
+    chatId: created.chat.id,
+    responseId: finalSnapshot.responseId,
+    modelId: input.model.id,
+    agentMode: input.agentMode,
+    presetSelections: input.presetSelections,
+    temporary,
+    snapshot: finalSnapshot
+  };
+}
+
 // src/index.ts
-var CLI_VERSION = true ? "0.57.0" : "0.1.0";
+var CLI_VERSION = true ? "0.59.0" : "0.1.0";
 var CLI_BUNDLED = true;
 var commandIo = /* @__PURE__ */ new WeakMap();
 var commandClientFactory = /* @__PURE__ */ new WeakMap();
+var NamedOption = class extends Option {
+  constructor(flags, optionAttribute, description) {
+    super(flags, description);
+    this.optionAttribute = optionAttribute;
+  }
+  attributeName() {
+    return this.optionAttribute;
+  }
+};
 function catalogIconContentType(filename) {
   const extension = /\.([^.]+)$/.exec(filename)?.[1]?.toLowerCase();
   return extension === "png" ? "image/png" : extension === "jpg" || extension === "jpeg" ? "image/jpeg" : extension === "webp" ? "image/webp" : extension === "svg" ? "image/svg+xml" : null;
@@ -17542,7 +17914,14 @@ async function clientFor(command, authenticated = true) {
   if (capability && !info.capabilities.includes(capability)) {
     throw new Error(`The selected Pulpo instance does not advertise the ${capability} capability`);
   }
-  return { client, contextName: connection.contextName, context: connection.context, info };
+  return {
+    client,
+    contextName: connection.contextName,
+    context: connection.context,
+    info,
+    token: connection.token ?? "",
+    url: connection.url
+  };
 }
 async function twoFactorSecret(io, json2, prompt = "Authenticator or recovery code: ") {
   const value = process.env.PULPO_2FA_CODE;
@@ -17902,6 +18281,46 @@ function createProgram(io = processIo, dependencies = {}) {
     const icons = CHAT_PRESET_ICON_NAMES.filter((name) => !normalized || name.includes(normalized)).map((name) => ({ name }));
     emit(io, command, icons);
   });
+  model.command("test <id> [prompt...]").description("Send a model smoke-test message with explicit user-facing options").addOption(new NamedOption("--agent", "agentEnabled", "run with agent mode enabled").conflicts("agentDisabled")).addOption(new NamedOption("--no-agent", "agentDisabled", "run with agent mode disabled").default(void 0).conflicts("agentEnabled")).option("--preset <preset=choice>", "explicit preset choice; repeat for every exposed preset", (value, previous) => [...previous, value], []).option("--keep", "keep the test in normal chat history").option("--no-stream", "wait and print only the completed response").option("--jsonl", "stream response events as JSON Lines").action(async (id, promptParts, options, command) => {
+    const globals = globalOptions(command);
+    if (globals.json && options.jsonl) throw new Error("--json and --jsonl cannot be used together");
+    if (options.jsonl && options.stream === false) throw new Error("--jsonl and --no-stream cannot be used together");
+    const agentMode = explicitAgentMode(options);
+    const prompt = await readModelTestPrompt(io, promptParts ?? []);
+    const { client, token: token2, url: url2 } = await clientFor(command);
+    if (!token2) throw new Error("Model tests require a logged-in session. Run `pulpo auth login`.");
+    const catalog = await client.request("/api/models");
+    const selectedModel = catalog.data.find((candidate) => candidate.id === id);
+    if (!selectedModel) throw new Error(`Exposed model not found: ${id}`);
+    if (agentMode && !selectedModel.agentEnabled) throw new Error(`Model ${id} does not expose agent mode`);
+    if (agentMode && !catalog.agentAvailable) throw new Error("Agent mode is not available on the selected Pulpo instance");
+    const presetSelections = resolvePresetSelections(selectedModel, options.preset);
+    const result = await runModelTest({
+      client,
+      baseUrl: url2,
+      token: token2,
+      model: selectedModel,
+      prompt,
+      agentMode,
+      presetSelections,
+      keep: Boolean(options.keep),
+      streamText: !globals.json && !options.jsonl && options.stream !== false,
+      jsonl: Boolean(options.jsonl),
+      io,
+      follow: dependencies.followResponse ?? followResponse
+    });
+    if (globals.json) emit(io, command, result);
+    else if (!options.jsonl && options.stream === false) io.stdout.write(`${responseText(result.snapshot)}
+`);
+    if (!globals.json && !options.jsonl) {
+      io.stderr.write(`Chat ${result.chatId} \xB7 response ${result.responseId}${result.temporary ? " \xB7 temporary" : ""}
+`);
+    }
+    if (!["completed", "incomplete"].includes(result.snapshot.status)) {
+      const detail = result.snapshot.error && typeof result.snapshot.error === "object" ? result.snapshot.error.message : result.snapshot.error;
+      throw new Error(typeof detail === "string" ? detail : `Response ended with status ${result.snapshot.status}`);
+    }
+  });
   const user = registerFileCrud(program, io, { name: "user", pluralPath: "/api/management/v1/users" });
   for (const [action, patch] of [["approve", { role: "user" }], ["block", { blocked: true }], ["unblock", { blocked: false }]]) {
     user.command(`${action} <id>`).action(async (id, _options, command) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isaacthoman/pulpo",
-  "version": "0.57.0",
+  "version": "0.59.0",
   "description": "Operator-first command-line client for Pulpo",
   "type": "module",
   "bin": {
@@ -31,7 +31,8 @@
     "pack:check": "npm pack --dry-run"
   },
   "dependencies": {
-    "commander": "^14.0.2"
+    "commander": "^14.0.2",
+    "socket.io-client": "^4.8.3"
   },
   "devDependencies": {
     "@pulpo/client-core": "0.1.0",