@chatpanel/events 0.69.0 → 0.70.0

package/context-attachments.js

@@ -0,0 +1,148 @@
+// ATTACHED CONTEXT, AS THE MODEL SEES IT — the same on every client.
+//
+// A message carries `attachments`: pages, selections, fetched links, files, images, and the
+// user's own records. This is how they reach a model: text sources folded into the message
+// as <context> blocks, images as image blocks in the provider's wire shape, and — when the
+// turn has tools and the sources are big — a MANIFEST plus a `source` tool instead of the
+// full text, so "hi" on a long page does not pay for the page.
+//
+// Moved out of the extension's providers.js so the desktop's turn assembles attachments
+// exactly the way the panel does. Pure: nothing here fetches or renders.
+
+import { makeSourceStore, manifestText, readSource, approxTokens } from './sources-retrieval.js';
+
+// Flatten a stored message (with attachments) into the text the model sees.
+// Image attachments are excluded here — they go to the model as image blocks
+// (see toMultimodalMessages), not as text.
+export function renderContent(m) {
+  let text = m.content || '';
+  const ctx = (m.attachments || []).filter((a) => a.kind !== 'image');
+  if (ctx.length) {
+    const blocks = ctx
+      .map((a) => {
+        const head = `[${a.kind || 'context'}] ${a.title || a.url || ''}`.trim();
+        return `<context source="${(a.url || a.title || '').replace(/"/g, '')}">\n# ${head}\n${a.text || ''}\n</context>`;
+      })
+      .join('\n\n');
+    text = text ? `${text}\n\n${blocks}` : blocks;
+  }
+  return text;
+}
+
+export function toChatMessages(messages) {
+  return messages
+    .filter((m) => m.role === 'user' || m.role === 'assistant')
+    .map((m) => ({ role: m.role, content: renderContent(m) }));
+}
+
+// Image attachments on a message: { dataUrl: 'data:<media>;base64,<...>' }.
+export function imageAttachmentsOf(m) {
+  return (m.attachments || []).filter((a) => a.kind === 'image' && a.dataUrl);
+}
+
+// Like toChatMessages, but emits multimodal content (text + image blocks) for
+// user messages that carry images, in the given provider's wire format. Falls
+// back to plain string content when there are no images. `provider` is
+// 'openai' | 'anthropic'.
+export function toMultimodalMessages(messages, provider) {
+  return messages
+    .filter((m) => m.role === 'user' || m.role === 'assistant')
+    .map((m) => {
+      const text = renderContent(m);
+      const imgs = m.role === 'user' ? imageAttachmentsOf(m) : [];
+      if (imgs.length === 0) return { role: m.role, content: text };
+      if (provider === 'anthropic') {
+        const content = [];
+        for (const a of imgs) {
+          const match = /^data:([^;]+);base64,(.+)$/s.exec(a.dataUrl);
+          if (match) content.push({ type: 'image', source: { type: 'base64', media_type: match[1], data: match[2] } });
+        }
+        if (text) content.push({ type: 'text', text });
+        return { role: 'user', content: content.length ? content : text };
+      }
+      // openai (and OpenAI-compatible vision endpoints)
+      const content = [];
+      if (text) content.push({ type: 'text', text });
+      for (const a of imgs) content.push({ type: 'image_url', image_url: { url: a.dataUrl } });
+      return { role: 'user', content: content.length ? content : text };
+    });
+}
+
+/**
+ * Hand the model a MANIFEST of what is attached, and a tool to read it.
+ *
+ * Attachments used to be flattened into the first message — every attached tab, in full,
+ * before the model had said anything. "hi" on a long page paid for the whole page, and five
+ * attached tabs put five documents in the prompt to answer a question about one paragraph of
+ * one of them.
+ *
+ * Two conditions, both necessary:
+ *   - THE TURN MUST CARRY TOOLS. Deferring content a model cannot then fetch does not save
+ *     tokens, it deletes the context — the worst possible outcome, and silently.
+ *   - IT MUST BE WORTH A ROUND TRIP. Below the threshold the extra call costs more than the
+ *     text it avoids, so small attachments still travel inline.
+ *
+ * Returns the rewritten messages plus a store, or null when nothing was deferred.
+ */
+export function deferAttachedSources(messages, tools, { minTokens = 700 } = {}) {
+  if (!tools?.specs?.length) return null;
+  const carried = [];
+  for (const m of messages || []) {
+    for (const a of m?.attachments || []) {
+      if (a?.kind === 'image' || !a?.text) continue;
+      carried.push(a);
+    }
+  }
+  if (!carried.length) return null;
+  const store = makeSourceStore(carried.map((a) => ({
+    kind: a.kind || 'context', title: a.title, url: a.url, text: a.text,
+  })));
+  if (store.tokens < minTokens) return null;
+  // Same index, same id: the manifest and the store must agree or the model asks for
+  // something real and is told it does not exist.
+  const idFor = new Map(carried.map((a, i) => [a, store.entries[i]?.id]));
+  const out = (messages || []).map((m) => {
+    if (!m?.attachments?.some((a) => idFor.get(a))) return m;
+    return {
+      ...m,
+      attachments: m.attachments.map((a) => {
+        const id = idFor.get(a);
+        // The stub keeps the title and url — knowing WHAT is attached is what lets the model
+        // decide whether to read it, and that part is cheap.
+        return id ? { ...a, text: `(not included — read with \`source\`: id ${id}, ~${approxTokens(a.text)} tokens)` } : a;
+      }),
+    };
+  });
+  return { messages: out, store };
+}
+
+export const SOURCE_TOOL_SPEC = {
+  name: 'source',
+  description: 'Read an attached source (a page, tab, selection or file the user attached). Their content is NOT in the conversation — read what you need from here.',
+  parameters: {
+    type: 'object',
+    properties: {
+      id: { type: 'string', description: 'The source id from the manifest, e.g. page-1.' },
+      query: { type: 'string', description: 'What you are looking for. A large source returns the matching sections rather than its first page.' },
+    },
+    required: ['id'],
+  },
+};
+
+/** Add `source` to an existing toolset without disturbing what is already there. */
+export function withSourceTool(tools, store) {
+  const spec = { ...SOURCE_TOOL_SPEC };
+  const system = [
+    tools?.system,
+    `${manifestText(store)}\n\nTheir content is NOT in this conversation. Call \`source\` with an id — and a query when the source is large — to read what you need.`,
+  ].filter(Boolean).join('\n\n');
+  return {
+    ...tools,
+    specs: [...(tools?.specs || []), spec],
+    system,
+    systemParts: { ...(tools?.systemParts || {}), source: approxTokens(manifestText(store)) },
+    execute: async (name, input, meta) => (name === 'source'
+      ? JSON.stringify(readSource(store, typeof input === 'string' ? JSON.parse(input || '{}') : (input || {})))
+      : tools.execute(name, input, meta)),
+  };
+}

package/find-tool.js

@@ -34,11 +34,18 @@ export const FIND_DESCRIPTION =
 // meeting history" — while `find` was sitting in its toolset. The old line named the tool
 // and left the capability to be inferred, and inference is what small models are worst at.
 export const FIND_RESIDENT =
-  "You HAVE access to the user's own ChatPanel data — their past chats, notes, and "
-  + 'meeting transcripts and summaries — through the `find` tool, plus the web. When the '
-  + 'question is about past meetings, notes, people, decisions, or anything the user '
-  + 'discussed or wrote, call `find` FIRST and answer from what it returns. Never tell '
-  + 'the user you cannot access their meetings, notes or history: you can.';
+  "You HAVE access to the user's own ChatPanel data — past chats, notes, meeting "
+  + 'transcripts and summaries — through `find`, plus the web. For anything they discussed or '
+  + 'wrote, call `find` FIRST and answer from it; never say you cannot access their meetings, '
+  + 'notes or history. '
+  // Named here, resident, because a relayed agent with a web search of its own otherwise
+  // reaches for that: the desktop asked Codex about the weather, Codex searched on its own,
+  // read pages that were scripts and no temperature, and answered that it could not tell.
+  // The `weather` action answers in one request. And a `find` call is shown to the user as
+  // a step — an agent's own search is not. The whole block stays under the extension's
+  // 120-token resident cap (test-data-dispatch.mjs): the manual travels with `describe`.
+  + 'For anything current — weather, prices, news — use `find` (actions `weather`, '
+  + '`web_search`), not a search tool of your own; the user sees `find` calls as steps.';
 
 /**
  * Wrap the real search/read tools (history, web search, weather…) as the one `find` tool.

package/index.js

@@ -283,3 +283,6 @@ export { McpClient, mcpProvider } from './mcp-client.js';
 export { mcpSharedSystem, mcpInventorySystem, sourceCitationSystem, combineSystemPrompt, toolStatus, widgetAuthoringSystem, vaultWidgetSystem, wantsVaultGuidance } from './tool-hints.js';
 export { adaptiveToolRetryHint, createAdaptiveToolPolicy, isInvalidToolParametersResult } from './adaptive-tool-policy.js';
 export { getMcpProviders, testMcpServer, resetMcp } from './mcp-manager.js';
+export { WEATHER_TOOL_NAME, WEATHER_TOOL_SYSTEM, weatherToolProvider } from './weather-tool.js';
+// Attached context as the model sees it — <context> blocks, image blocks, deferred sources.
+export { renderContent, toChatMessages, toMultimodalMessages, imageAttachmentsOf, deferAttachedSources, withSourceTool, SOURCE_TOOL_SPEC } from './context-attachments.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/events",
-  "version": "0.69.0",
+  "version": "0.70.0",
   "description": "The canonical ChatPanel event-log and capability contracts \u2014 typed durable facts, clock-free deterministic linearization, schema upcasting, and the invariants the replay harness asserts. Pure, dependency-free ESM shared by the ChatPanel extension, gateway and bridge.",
   "type": "module",
   "main": "index.js",
@@ -105,7 +105,9 @@
     "./tool-hints.js": "./tool-hints.js",
     "./adaptive-tool-policy.js": "./adaptive-tool-policy.js",
     "./mcp-client.js": "./mcp-client.js",
-    "./mcp-manager.js": "./mcp-manager.js"
+    "./mcp-manager.js": "./mcp-manager.js",
+    "./weather-tool.js": "./weather-tool.js",
+    "./context-attachments.js": "./context-attachments.js"
   },
   "files": [
     "LICENSE",
@@ -116,6 +118,7 @@
     "capability.js",
     "citations.js",
     "client-prefs.js",
+    "context-attachments.js",
     "cowriter-router.js",
     "cowriter.js",
     "curate.js",
@@ -203,6 +206,7 @@
     "vault.js",
     "view.js",
     "voice-intents.js",
+    "weather-tool.js",
     "weather.js",
     "web-search-tool.js",
     "web-search.js",

package/weather-tool.js

@@ -0,0 +1,65 @@
+// `weather` as a TOOL — the spec, the guidance, and the answer shape, without the fetch.
+//
+// The extension answers "how is the weather in X" with this rather than with a web search,
+// because weather sites are rendered by scripts and a fetched page is navigation and no
+// temperature — which is exactly what a desktop turn without this tool produced: "I couldn't
+// retrieve reliable live weather". The engine (`weather.js`: the wttr.in query, the parse,
+// the ambiguity check) is already shared; this is the tool every client arms in front of it.
+//
+// `fetchJson(url, { timeoutMs })` is injected: each host owns its network guard and applies
+// it to a URL a MODEL supplied — attacker-influenced by construction.
+
+import { getWeather } from './weather.js';
+
+export const WEATHER_TOOL_NAME = 'weather';
+
+export const WEATHER_TOOL_SYSTEM =
+  'For weather, call `weather` FIRST — it answers the whole question in one request. Only '
+  + 'fall back to web_search if it tells you to. Report the location it says it resolved, '
+  + 'because a bare town name can geocode to the wrong place.';
+
+export function weatherToolProvider({ fetchJson } = {}) {
+  if (typeof fetchJson !== 'function') throw new Error('weatherToolProvider: fetchJson required');
+  return {
+    specs: [
+      {
+        name: 'weather',
+        // A read, declared: the round runner overlaps reads and serialises everything it
+        // cannot classify, and "weather" is not a verb its name heuristic knows.
+        annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: true },
+        description:
+          'Current conditions and a short forecast for one place, in a single request. Use this '
+          + 'for any weather question instead of searching. Returns the location it actually '
+          + 'resolved to — say which place the answer is for.',
+        parameters: {
+          type: 'object',
+          properties: {
+            location: {
+              type: 'string',
+              description: 'A place, as the user said it. Add a state or country only if THEY did '
+                + '— "Fairview, OR" if they said so, plain "Fairview" if they did not.',
+            },
+          },
+          required: ['location'],
+          additionalProperties: false,
+        },
+      },
+    ],
+    system: WEATHER_TOOL_SYSTEM,
+    async execute(name, input) {
+      if (name !== 'weather') return JSON.stringify({ error: `Unknown tool: ${name}` });
+      const location = String(input?.location || '').trim();
+      if (!location) return 'No location provided to weather.';
+      const got = await getWeather(location, { fetchJson });
+      // THE FALLBACK IS AN INSTRUCTION, not an error string. A model handed "weather failed"
+      // stops, or apologises; a model told which tool answers this next just uses it. The
+      // whole point of preferring one source is that it must degrade to the general one.
+      if (!got.ok) {
+        return `The weather service could not answer for "${location}" (${got.reason}). `
+          + `Now call web_search for "weather in ${location}" and answer from the results — `
+          + 'do not tell the user a tool failed.';
+      }
+      return { text: got.text, note: 'ChatPanel · wttr.in' };
+    },
+  };
+}