codecaine-cli 0.1.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/cli.mjs +3212 -185
- package/dist/resources/bb-plugin/package.json +22 -0
- package/dist/resources/bb-plugin/src/server.ts +1084 -0
- package/dist/resources/bb-plugin/src/snapshot.ts +74 -0
- package/dist/resources/bb-plugin/src/tools.ts +926 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1084 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bb-plugin-canvas-editor — the canvas editor's layer tools, inside bb.
|
|
3
|
+
*
|
|
4
|
+
* WHY a plugin at all: bb has no user-configurable MCP servers, so there is no
|
|
5
|
+
* per-run `--mcp-config` to hand a coding agent. `bb.agents.registerTool` is the
|
|
6
|
+
* sanctioned way to put tools in front of whichever provider CLI the user has
|
|
7
|
+
* authenticated, and a plugin is the only thing allowed to call it.
|
|
8
|
+
*
|
|
9
|
+
* WHY the HTTP hop: this file is loaded inside the *bb server process*. That
|
|
10
|
+
* process has no handle on the editor's renderer, where the document actually
|
|
11
|
+
* lives — so every execute() posts to the loopback endpoint the editor opened
|
|
12
|
+
* for us (`electron/bb/tool-server.ts`), which fans the call out to whichever
|
|
13
|
+
* open project answers for that workspace:
|
|
14
|
+
*
|
|
15
|
+
* provider ─► bb ─► execute() ─POST /tool─► editor main
|
|
16
|
+
* └─► renderer relay ─► canvas API
|
|
17
|
+
*
|
|
18
|
+
* The editor hands us that endpoint's URL and bearer token by calling our own
|
|
19
|
+
* `POST /config` route once per session. Nothing is persisted: the token is
|
|
20
|
+
* per-session, and a bb server that outlives the editor must fail loudly rather
|
|
21
|
+
* than keep a stale secret around.
|
|
22
|
+
*
|
|
23
|
+
* Typed against `@get-bb/plugin-sdk` (verified against bb-app 0.42.1). The
|
|
24
|
+
* import is types-only and erased before bb loads this file, so the package
|
|
25
|
+
* does not have to be installed for the plugin to run — which is the point:
|
|
26
|
+
* this directory is registered with bb by path and has no node_modules of its
|
|
27
|
+
* own.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
31
|
+
import { tmpdir } from "node:os";
|
|
32
|
+
import { join } from "node:path";
|
|
33
|
+
import type { BbPluginApi, PluginAgentToolResult } from "@get-bb/plugin-sdk";
|
|
34
|
+
import { frameOfLiveId, layerSnapshot, SNAPSHOT_MAX_LAYERS } from "./snapshot.js";
|
|
35
|
+
import { CANVAS_TOOLS, FRAMES_TOOLS, SCREENSHOT_TOOL } from "./tools.js";
|
|
36
|
+
|
|
37
|
+
/** Where the editor's tool server lives, as handed over by POST /config. */
|
|
38
|
+
interface CallbackTarget {
|
|
39
|
+
/** Base URL with no trailing slash, e.g. "http://127.0.0.1:51234". */
|
|
40
|
+
url: string;
|
|
41
|
+
token: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Screenshots are spilled to the OS temp dir rather than into an app-owned
|
|
46
|
+
* folder: the editor has no directory it prunes on launch (unlike ui-editor's
|
|
47
|
+
* voice temp), and inventing one would mean inventing its retention policy too.
|
|
48
|
+
* The OS already sweeps this one.
|
|
49
|
+
*/
|
|
50
|
+
const SHOT_DIR = join(tmpdir(), "canvas-agent-shots");
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A tool call answered by the wrong document is worse than one that fails: the
|
|
54
|
+
* model silently edits somebody else's design. A thread's workspace path is
|
|
55
|
+
* learned from `bb.agents.configure` — the per-call tool context carries
|
|
56
|
+
* threadId and projectId but no path — so that callback doubles as the place
|
|
57
|
+
* the mapping is recorded. It runs when a turn starts, so after a restart the
|
|
58
|
+
* project's folder, as the editor's pushes name it, stands in (`pathByProject`).
|
|
59
|
+
*/
|
|
60
|
+
const MAX_TRACKED_THREADS = 512;
|
|
61
|
+
|
|
62
|
+
/** Waits between attempts at the editor after a refused connection — about
|
|
63
|
+
* four seconds in all, long enough for an editor that is restarting its
|
|
64
|
+
* endpoint to push the new one, short enough not to stall a turn. */
|
|
65
|
+
const EDITOR_RETRY_DELAYS_MS = [250, 750, 1500, 2000];
|
|
66
|
+
/** How long a call waits for an editor to push its address when there is none
|
|
67
|
+
* yet, and how often it looks. */
|
|
68
|
+
const CALLBACK_WAIT_MS = 4_000;
|
|
69
|
+
const CALLBACK_POLL_MS = 200;
|
|
70
|
+
|
|
71
|
+
/** Sleep, or resolve false at once if the turn is cancelled. */
|
|
72
|
+
function pause(ms: number, signal?: AbortSignal): Promise<boolean> {
|
|
73
|
+
if (signal?.aborted) return Promise.resolve(false);
|
|
74
|
+
return new Promise((resolve) => {
|
|
75
|
+
const timer = setTimeout(() => {
|
|
76
|
+
signal?.removeEventListener("abort", onAbort);
|
|
77
|
+
resolve(true);
|
|
78
|
+
}, ms);
|
|
79
|
+
const onAbort = () => {
|
|
80
|
+
clearTimeout(timer);
|
|
81
|
+
resolve(false);
|
|
82
|
+
};
|
|
83
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Standing instructions, per bb project — the editor's "system prompt".
|
|
89
|
+
*
|
|
90
|
+
* Per project rather than one global string because the editor keeps them in
|
|
91
|
+
* the project's own settings, so two projects open against one bb server must
|
|
92
|
+
* not overwrite each other's. Bounded for the same reason as the thread map: a
|
|
93
|
+
* long-lived server sees unboundedly many projects.
|
|
94
|
+
*/
|
|
95
|
+
const MAX_TRACKED_PROJECTS = 256;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* How much of the two per-turn contributions can be sent.
|
|
99
|
+
*
|
|
100
|
+
* bb truncates a plugin's contributed instructions past 4096 characters, and a
|
|
101
|
+
* prompt that silently loses its last paragraph is worse than one that was
|
|
102
|
+
* never that long. The project's own instructions are kept whole and the
|
|
103
|
+
* selection block is what gives way — it is regenerable, since the assistant
|
|
104
|
+
* can call `getSelection` for the same answer.
|
|
105
|
+
*/
|
|
106
|
+
const MAX_INSTRUCTIONS = 4096;
|
|
107
|
+
/** Layers named in the per-turn selection block before it summarises. */
|
|
108
|
+
const MAX_SELECTION_LAYERS = 12;
|
|
109
|
+
|
|
110
|
+
/** The per-turn line about the board's selection. Names for a person reading
|
|
111
|
+
* the transcript, ids for the model to call `getLayer` and `updateLayer`
|
|
112
|
+
* with. */
|
|
113
|
+
/**
|
|
114
|
+
* Where an element inside a running project actually is.
|
|
115
|
+
*
|
|
116
|
+
* A layer of the design is a node id and that is the whole answer. An element
|
|
117
|
+
* of somebody's running application is not: its id is a handle this session
|
|
118
|
+
* issued, meaningless in a codebase, and the editor's tools can only write a
|
|
119
|
+
* stylesheet held *over* the page — which the next build throws away. So this
|
|
120
|
+
* carries the other vocabulary, and says what to do with it.
|
|
121
|
+
*/
|
|
122
|
+
interface LiveElement {
|
|
123
|
+
address?: unknown;
|
|
124
|
+
selector?: unknown;
|
|
125
|
+
path?: unknown;
|
|
126
|
+
tag?: unknown;
|
|
127
|
+
component?: unknown;
|
|
128
|
+
band?: unknown;
|
|
129
|
+
maxWidth?: unknown;
|
|
130
|
+
state?: unknown;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function liveLine(layer: { name?: unknown; live?: LiveElement }): string | null {
|
|
134
|
+
const live = layer.live;
|
|
135
|
+
if (!live || typeof live.selector !== "string") return null;
|
|
136
|
+
const named =
|
|
137
|
+
(typeof layer.name === "string" && layer.name) ||
|
|
138
|
+
(typeof live.component === "string" && live.component) ||
|
|
139
|
+
(typeof live.tag === "string" && live.tag) ||
|
|
140
|
+
"an element";
|
|
141
|
+
let line =
|
|
142
|
+
`${named} — inside the page running at ${typeof live.address === "string" ? live.address : "the project"}, ` +
|
|
143
|
+
`found by the CSS selector \`${live.selector}\``;
|
|
144
|
+
if (typeof live.path === "string") line += ` (path: ${live.path})`;
|
|
145
|
+
if (typeof live.component === "string") line += `, rendered by the React component \`${live.component}\``;
|
|
146
|
+
if (typeof live.band === "string" && typeof live.maxWidth === "number") {
|
|
147
|
+
line += `, being designed at ${live.band} — @media (max-width: ${live.maxWidth}px) and narrower`;
|
|
148
|
+
}
|
|
149
|
+
if (typeof live.state === "string" && live.state) {
|
|
150
|
+
line += `, in its \`:${live.state}\` state`;
|
|
151
|
+
}
|
|
152
|
+
return line + ".";
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** What every live element in a selection has in common, said once. */
|
|
156
|
+
const LIVE_RULE =
|
|
157
|
+
"Those are elements of a running application, not layers of this design. The layer tools reach them, and what they write is a stylesheet the editor holds OVER the page: it looks right and it is discarded the moment the page is rebuilt from its source. A change the person means to keep is a change to the code — find the element by its selector and edit whatever renders it. Use the layer tools there only to look, or to show them what a change would look like before you write it. Check it in light and dark mode.";
|
|
158
|
+
|
|
159
|
+
/** LIVE_RULE in the room the default instructions leave — about 800
|
|
160
|
+
* characters, which one live element's line plus the full rule overran. */
|
|
161
|
+
const LIVE_RULE_SHORT =
|
|
162
|
+
"Those are elements of a running app: a layer-tool edit to them is discarded when the page rebuilds, so a lasting change goes in its source, checked in light and dark. Layers of this design are edited with the layer tools.";
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The selection cut to `room` characters, keeping what matters most.
|
|
166
|
+
*
|
|
167
|
+
* Over bb's cap the whole block used to be dropped, and with the default
|
|
168
|
+
* instructions any selection holding a live element was over it — so the
|
|
169
|
+
* one turn that most needed the live rule never got it. Shorter rule first,
|
|
170
|
+
* then no per-element lines, then the head alone.
|
|
171
|
+
*/
|
|
172
|
+
export function fitSelection(selection: string, room: number): string | null {
|
|
173
|
+
if (selection.length <= room) return selection;
|
|
174
|
+
const short = selection.replace(LIVE_RULE, LIVE_RULE_SHORT);
|
|
175
|
+
if (short.length <= room) return short;
|
|
176
|
+
const head = selection.split("\n")[0] ?? "";
|
|
177
|
+
const headAndRule = selection.includes(LIVE_RULE) ? `${head}\n\n${LIVE_RULE_SHORT}` : head;
|
|
178
|
+
if (headAndRule.length <= room) return headAndRule;
|
|
179
|
+
return head.length <= room ? head : null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function selectionBlock(layers: readonly unknown[]): string {
|
|
183
|
+
const live: string[] = [];
|
|
184
|
+
const named = layers
|
|
185
|
+
.slice(0, MAX_SELECTION_LAYERS)
|
|
186
|
+
.map((entry) => {
|
|
187
|
+
const layer = (entry ?? {}) as {
|
|
188
|
+
id?: unknown;
|
|
189
|
+
name?: unknown;
|
|
190
|
+
type?: unknown;
|
|
191
|
+
live?: LiveElement;
|
|
192
|
+
};
|
|
193
|
+
const id = typeof layer.id === "string" ? layer.id : "?";
|
|
194
|
+
const name =
|
|
195
|
+
(typeof layer.name === "string" && layer.name) ||
|
|
196
|
+
(typeof layer.type === "string" && layer.type) ||
|
|
197
|
+
"layer";
|
|
198
|
+
const line = liveLine(layer);
|
|
199
|
+
if (line) live.push(line);
|
|
200
|
+
return `${name} (${id})`;
|
|
201
|
+
})
|
|
202
|
+
.join(", ");
|
|
203
|
+
const rest = layers.length - MAX_SELECTION_LAYERS;
|
|
204
|
+
const tail = rest > 0 ? `, and ${rest} more — call getSelection for all of them` : "";
|
|
205
|
+
const head =
|
|
206
|
+
`The person currently has ${layers.length} layer${layers.length === 1 ? "" : "s"} selected on the board: ` +
|
|
207
|
+
`${named}${tail}. When they say "this", "these" or "the selection", that is what they mean. ` +
|
|
208
|
+
`The selection changes as they work — call getSelection if you need it again later in the conversation.`;
|
|
209
|
+
if (live.length === 0) return head;
|
|
210
|
+
return [head, "", ...live.map((line) => `- ${line}`), "", LIVE_RULE].join("\n");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* How to reach this design — the plugin's own word on it, not the project's.
|
|
215
|
+
*
|
|
216
|
+
* Returned from `configure`, which bb pushes into the thread's instructions as
|
|
217
|
+
* a section of its own. Deliberately NOT part of the project's editable system
|
|
218
|
+
* prompt: that budget belongs to the person, this is the plugin saying what its
|
|
219
|
+
* tools are, and the two must not compete for the same 4096 characters.
|
|
220
|
+
*
|
|
221
|
+
* It exists because of what a coding agent does when it has a machine's whole
|
|
222
|
+
* tool surface in front of it. Somebody asked "rewrite the text in this frame"
|
|
223
|
+
* and watched the assistant reach for `mcp__pencil__get_app_state` — another
|
|
224
|
+
* design application's MCP server, which cannot see this board — and then
|
|
225
|
+
* explain the failure with it, inventing a connection that had to be restarted.
|
|
226
|
+
*
|
|
227
|
+
* **The prefix is the trap, and it cuts both ways.** For an ACP provider bb
|
|
228
|
+
* hands a plugin's tools over as a session MCP server of its own, named
|
|
229
|
+
* `bb-bridge` (`ACP_BRIDGE_MCP_SERVER_NAME`, @get-bb/plugin-sdk) — so these
|
|
230
|
+
* tools arrive as `mcp__bb-bridge__listLayers` and the like. A rule saying
|
|
231
|
+
* "never call an mcp__ tool" therefore tells the model to refuse the very tools
|
|
232
|
+
* this plugin exists to give it. What separates ours from somebody else's is
|
|
233
|
+
* the *name*, never the prefix.
|
|
234
|
+
*/
|
|
235
|
+
export const CANVAS_TOOL_GUIDE = [
|
|
236
|
+
"THE DESIGN ON THE BOARD IS REACHED THROUGH THESE TOOLS, BY THESE NAMES:",
|
|
237
|
+
CANVAS_TOOLS.map((tool) => tool.name).join(", ") + ".",
|
|
238
|
+
"Your provider may show them prefixed — `mcp__bb-bridge__listLayers` — as bb hands them " +
|
|
239
|
+
"over as a session MCP server named `bb-bridge`. Prefixed or not, they are the editor's own, " +
|
|
240
|
+
"already connected: nothing to start, connect, configure or open first.",
|
|
241
|
+
"It is the NAME that makes a tool this design's, never the prefix. A tool not listed above " +
|
|
242
|
+
"does not touch this design, however it sounds. " +
|
|
243
|
+
"Never use one for work on this board, or to explain a failure on it.",
|
|
244
|
+
"THE BOARD SHOWS WHAT YOU ARE DOING. Every write you make already puts a band of colour round " +
|
|
245
|
+
"the layer for a few seconds. holdLayer is the longer claim: it keeps a layer for a " +
|
|
246
|
+
"whole job, says what you are doing to it, and refuses the person's edits meanwhile. Take one " +
|
|
247
|
+
"before your first change when you are REWRITING it — redesigning, restyling, replacing its " +
|
|
248
|
+
"insides — so nobody edits under a half-applied change. Reading takes nothing; " +
|
|
249
|
+
"nor do variants BESIDE one, which are new layers.",
|
|
250
|
+
"Release before your turn ends even " +
|
|
251
|
+
"when you are giving up, a tool failed, or you are handing the question back — say so. " +
|
|
252
|
+
"askAboutLayer keeps the hold and puts your question on the board; holdLayer picks " +
|
|
253
|
+
"it back up. The person can take a layer back with the button beside its name, and a hold whose " +
|
|
254
|
+
"assistant goes quiet frees itself; getLayer reports the run on a layer, and an edit to one no longer held still lands.",
|
|
255
|
+
"If a tool reports it cannot reach the editor, the tool has already retried: report its " +
|
|
256
|
+
"message as it stands — do not assume more than it says — and stop. If the tools are missing from this session altogether, say " +
|
|
257
|
+
"THAT, plainly, first. In neither case substitute another " +
|
|
258
|
+
"application's tools or edit the design's files: the editor overwrites them.",
|
|
259
|
+
"WRITE ORDINARY WEB MARKUP. replaceLayer and insertHtml take any HTML — sections, tables, " +
|
|
260
|
+
"svg, canvas, forms with real inputs. Lay it out with flexbox and grid, which the inspector " +
|
|
261
|
+
"edits; absolute positioning only on the outermost element the board places. Linked stylesheets are ignored. " +
|
|
262
|
+
"Behaviour goes on a layer in a webpage (updateLayer's `script`, or script tags in markup) " +
|
|
263
|
+
"and runs in Interact and Preview with `element` as that layer, never on the canvas.",
|
|
264
|
+
"TO SEE AN EFFECT WORK: setPageView (in-view, motion playing), then observePage to read it or " +
|
|
265
|
+
"canvasPointer to use it with a real mouse. Restructure like a drag with moveLayer, " +
|
|
266
|
+
"groupLayers and duplicateLayers.",
|
|
267
|
+
"A LIVE PROJECT ON THIS MACHINE (localhost, LAN): a change asked for there is done only once " +
|
|
268
|
+
"SAVED in its source — a layer-tool edit is discarded at the next rebuild. Write it as the " +
|
|
269
|
+
"project's own code: the component or stylesheet that renders it, its Tailwind classes and " +
|
|
270
|
+
"theme variables, no inline styles or hard-coded colours. Then check LIGHT AND DARK MODE: " +
|
|
271
|
+
"every colour from a token with a value for each (listVariables; a new one gets both), and " +
|
|
272
|
+
"look at the page in each (setPageView theme light, then dark, then system) before saying done. A page " +
|
|
273
|
+
"served from the web has no source here: say so.",
|
|
274
|
+
"COPYING BETWEEN LIVE PROJECTS: a copy is the original, never a redesign. Read it in its " +
|
|
275
|
+
"source project first; keep its markup, words, data and look. Each colour, size, radius, " +
|
|
276
|
+
"shadow and font becomes var(--name) holding that same value: the target's own " +
|
|
277
|
+
"(listVariables) only if equal, else a new one (setVariable).",
|
|
278
|
+
"Start by looking: getSelection is what the person is pointing at when they say \"this\"; " +
|
|
279
|
+
"listLayers gives bearings; listVariables gives the design " +
|
|
280
|
+
"system behind it — change a shared colour there, not on every layer.",
|
|
281
|
+
].join("\n\n");
|
|
282
|
+
|
|
283
|
+
function toolResult(text: string, isError = false): PluginAgentToolResult {
|
|
284
|
+
return { content: [{ type: "text", text }], isError };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** Only ever point the plugin at the local machine. The /config route is
|
|
288
|
+
* `auth: "local"` already, but this endpoint receives a bearer token and would
|
|
289
|
+
* otherwise be a one-request exfiltration relay for anything that reached the
|
|
290
|
+
* route by some other path. */
|
|
291
|
+
function isLoopbackUrl(raw: string): boolean {
|
|
292
|
+
let parsed: URL;
|
|
293
|
+
try {
|
|
294
|
+
parsed = new URL(raw);
|
|
295
|
+
} catch {
|
|
296
|
+
return false;
|
|
297
|
+
}
|
|
298
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return false;
|
|
299
|
+
const host = parsed.hostname.replace(/^\[|\]$/g, "");
|
|
300
|
+
return host === "127.0.0.1" || host === "localhost" || host === "::1";
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export default function plugin(bb: BbPluginApi) {
|
|
304
|
+
let callback: CallbackTarget | null = null;
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Take the editor's callback from any push that carries one.
|
|
308
|
+
*
|
|
309
|
+
* Every route below accepts `{url, token}`, not just /config, and that is the
|
|
310
|
+
* whole repair strategy for this plugin's one real failure: everything it
|
|
311
|
+
* knows lives in memory, and bb restarts a plugin host whenever it reloads,
|
|
312
|
+
* updates or re-installs. Before this, a restart left the tools registered
|
|
313
|
+
* and permanently unreachable — the agent would be told the editor was not
|
|
314
|
+
* connected, and (having a shell) would go and edit the design's files
|
|
315
|
+
* instead. Now the next selection change or settings save puts it back.
|
|
316
|
+
*/
|
|
317
|
+
function adoptCallback(record: { url?: unknown; token?: unknown }): boolean {
|
|
318
|
+
const url =
|
|
319
|
+
typeof record.url === "string" ? record.url.trim().replace(/\/+$/, "") : "";
|
|
320
|
+
const token = typeof record.token === "string" ? record.token : "";
|
|
321
|
+
if (!url || !token || !isLoopbackUrl(url)) return false;
|
|
322
|
+
if (callback?.url !== url || callback.token !== token) {
|
|
323
|
+
bb.log.info(`canvas layer tools connected to the editor at ${url}`);
|
|
324
|
+
}
|
|
325
|
+
callback = { url, token };
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
/** threadId → the environment path bb is running that thread in. */
|
|
329
|
+
const workspaceByThread = new Map<string, string>();
|
|
330
|
+
/**
|
|
331
|
+
* bb projectId → the folder the editor has that project open from, as the
|
|
332
|
+
* editor itself says on every push.
|
|
333
|
+
*
|
|
334
|
+
* The fallback for `workspaceByThread`, which only `configure()` fills — and
|
|
335
|
+
* that runs when a turn starts, so a plugin host bb restarted mid-turn came
|
|
336
|
+
* back with it empty and every call after carried no path. A push is what
|
|
337
|
+
* repairs a restarted plugin (see /state), so it brings this back too. Only
|
|
338
|
+
* a fallback: a thread in a worktree runs somewhere else, and says so.
|
|
339
|
+
*/
|
|
340
|
+
const pathByProject = new Map<string, string>();
|
|
341
|
+
/** bb projectId → what the editor wants the assistant told first. */
|
|
342
|
+
const instructionsByProject = new Map<string, string>();
|
|
343
|
+
/** bb projectId → what is selected on that project's board right now. */
|
|
344
|
+
const selectionByProject = new Map<string, string>();
|
|
345
|
+
let shotSeq = 0;
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Keep one project's entry, evicting the oldest once the server has seen
|
|
349
|
+
* more projects than it will hold.
|
|
350
|
+
*
|
|
351
|
+
* An empty value deletes rather than keeps the last one: somebody who
|
|
352
|
+
* cleared the instructions box wants the assistant told nothing, and
|
|
353
|
+
* somebody who deselected is pointing at nothing — in both cases the stale
|
|
354
|
+
* answer is worse than no answer.
|
|
355
|
+
*/
|
|
356
|
+
function remember(
|
|
357
|
+
map: Map<string, string>,
|
|
358
|
+
projectId: string,
|
|
359
|
+
value: string,
|
|
360
|
+
): void {
|
|
361
|
+
if (!value) {
|
|
362
|
+
map.delete(projectId);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
if (map.size >= MAX_TRACKED_PROJECTS && !map.has(projectId)) {
|
|
366
|
+
const oldest = map.keys().next();
|
|
367
|
+
if (!oldest.done) map.delete(oldest.value);
|
|
368
|
+
}
|
|
369
|
+
map.set(projectId, value);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Record the folder a push names for its project. Never cleared by a push
|
|
373
|
+
* that leaves it out: most of them are about something else. */
|
|
374
|
+
function adoptPath(projectId: string, record: { path?: unknown }): void {
|
|
375
|
+
const path = typeof record.path === "string" ? record.path.trim() : "";
|
|
376
|
+
if (projectId && path) remember(pathByProject, projectId, path);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* POST /api/v1/plugins/canvas-editor/http/config { url, token }
|
|
381
|
+
*
|
|
382
|
+
* "local" auth checks Origin/Host against bb's own app origins and, for a
|
|
383
|
+
* mutation, requires a JSON content-type. A Node `fetch` sends no Origin
|
|
384
|
+
* header at all, so the editor's call passes; the content-type requirement is
|
|
385
|
+
* what stops a browser from forging this one.
|
|
386
|
+
*/
|
|
387
|
+
bb.http.route(
|
|
388
|
+
"POST",
|
|
389
|
+
"/config",
|
|
390
|
+
async (context) => {
|
|
391
|
+
let body: unknown;
|
|
392
|
+
try {
|
|
393
|
+
body = await context.req.json();
|
|
394
|
+
} catch {
|
|
395
|
+
body = null;
|
|
396
|
+
}
|
|
397
|
+
const record = (body ?? {}) as { url?: unknown; token?: unknown };
|
|
398
|
+
const url =
|
|
399
|
+
typeof record.url === "string"
|
|
400
|
+
? record.url.trim().replace(/\/+$/, "")
|
|
401
|
+
: "";
|
|
402
|
+
const token = typeof record.token === "string" ? record.token : "";
|
|
403
|
+
if (!url || !token) {
|
|
404
|
+
return context.json(
|
|
405
|
+
{ ok: false, error: "body must be { url, token }" },
|
|
406
|
+
400,
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
if (!isLoopbackUrl(url)) {
|
|
410
|
+
return context.json(
|
|
411
|
+
{ ok: false, error: `url must be a loopback address, got "${url}"` },
|
|
412
|
+
400,
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
adoptCallback({ url, token });
|
|
416
|
+
return context.json({
|
|
417
|
+
ok: true,
|
|
418
|
+
tools: CANVAS_TOOLS.map((tool) => tool.name),
|
|
419
|
+
});
|
|
420
|
+
},
|
|
421
|
+
{ auth: "local" },
|
|
422
|
+
);
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* POST /api/v1/plugins/canvas-editor/http/instructions { projectId, instructions, path? }
|
|
426
|
+
*
|
|
427
|
+
* The editor's standing instructions for one project, pushed when the panel
|
|
428
|
+
* opens and again whenever somebody edits them in Project settings. Kept
|
|
429
|
+
* here rather than in bb's own per-thread instructions because they belong to
|
|
430
|
+
* the editor's project file, travel with it, and must apply to a conversation
|
|
431
|
+
* the user has not started yet.
|
|
432
|
+
*/
|
|
433
|
+
bb.http.route(
|
|
434
|
+
"POST",
|
|
435
|
+
"/instructions",
|
|
436
|
+
async (context) => {
|
|
437
|
+
let body: unknown;
|
|
438
|
+
try {
|
|
439
|
+
body = await context.req.json();
|
|
440
|
+
} catch {
|
|
441
|
+
body = null;
|
|
442
|
+
}
|
|
443
|
+
const record = (body ?? {}) as {
|
|
444
|
+
projectId?: unknown;
|
|
445
|
+
instructions?: unknown;
|
|
446
|
+
path?: unknown;
|
|
447
|
+
url?: unknown;
|
|
448
|
+
token?: unknown;
|
|
449
|
+
};
|
|
450
|
+
adoptCallback(record);
|
|
451
|
+
const projectId =
|
|
452
|
+
typeof record.projectId === "string" ? record.projectId.trim() : "";
|
|
453
|
+
if (!projectId) {
|
|
454
|
+
return context.json(
|
|
455
|
+
{ ok: false, error: "body must be { projectId, instructions }" },
|
|
456
|
+
400,
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
adoptPath(projectId, record);
|
|
460
|
+
const instructions =
|
|
461
|
+
typeof record.instructions === "string" ? record.instructions.trim() : "";
|
|
462
|
+
remember(instructionsByProject, projectId, instructions);
|
|
463
|
+
if (!instructions) return context.json({ ok: true, cleared: true });
|
|
464
|
+
bb.log.info(`canvas instructions set for project ${projectId}`);
|
|
465
|
+
return context.json({ ok: true, length: instructions.length });
|
|
466
|
+
},
|
|
467
|
+
{ auth: "local" },
|
|
468
|
+
);
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* POST /api/v1/plugins/canvas-editor/http/selection { projectId, layers, path? }
|
|
472
|
+
*
|
|
473
|
+
* What the person is pointing at, pushed by the editor whenever the board's
|
|
474
|
+
* selection changes. It becomes part of the next turn's instructions, which
|
|
475
|
+
* is what makes "make this bigger" a sentence the assistant can act on
|
|
476
|
+
* without anybody typing an id.
|
|
477
|
+
*
|
|
478
|
+
* Pushed rather than pulled because `contributeInstructions` has to answer
|
|
479
|
+
* synchronously, on the thread-start path — there is no room there to ask the
|
|
480
|
+
* editor and wait.
|
|
481
|
+
*/
|
|
482
|
+
bb.http.route(
|
|
483
|
+
"POST",
|
|
484
|
+
"/selection",
|
|
485
|
+
async (context) => {
|
|
486
|
+
let body: unknown;
|
|
487
|
+
try {
|
|
488
|
+
body = await context.req.json();
|
|
489
|
+
} catch {
|
|
490
|
+
body = null;
|
|
491
|
+
}
|
|
492
|
+
const record = (body ?? {}) as {
|
|
493
|
+
projectId?: unknown;
|
|
494
|
+
layers?: unknown;
|
|
495
|
+
path?: unknown;
|
|
496
|
+
url?: unknown;
|
|
497
|
+
token?: unknown;
|
|
498
|
+
};
|
|
499
|
+
// A selection changes often, which is what makes it the best repair
|
|
500
|
+
// signal this plugin has: it is the push most likely to arrive first
|
|
501
|
+
// after bb restarted the plugin host.
|
|
502
|
+
adoptCallback(record);
|
|
503
|
+
const projectId =
|
|
504
|
+
typeof record.projectId === "string" ? record.projectId.trim() : "";
|
|
505
|
+
if (!projectId) {
|
|
506
|
+
return context.json(
|
|
507
|
+
{ ok: false, error: "body must be { projectId, layers }" },
|
|
508
|
+
400,
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
adoptPath(projectId, record);
|
|
512
|
+
const layers = Array.isArray(record.layers) ? record.layers : [];
|
|
513
|
+
remember(
|
|
514
|
+
selectionByProject,
|
|
515
|
+
projectId,
|
|
516
|
+
layers.length === 0 ? "" : selectionBlock(layers),
|
|
517
|
+
);
|
|
518
|
+
return context.json({ ok: true, count: layers.length });
|
|
519
|
+
},
|
|
520
|
+
{ auth: "local" },
|
|
521
|
+
);
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* POST /api/v1/plugins/canvas-editor/http/state
|
|
525
|
+
* { url?, token?, projectId?, instructions?, layers?, path? }
|
|
526
|
+
*
|
|
527
|
+
* Say what this plugin is holding — and take everything the caller knows on
|
|
528
|
+
* the way in.
|
|
529
|
+
*
|
|
530
|
+
* The reporting half is what lets the panel say "not connected to the board"
|
|
531
|
+
* instead of leaving somebody to find that out from a tool call that failed
|
|
532
|
+
* mid-conversation. The *writing* half is the fix for the failure that
|
|
533
|
+
* reporting alone only made visible: everything here lives in memory, bb
|
|
534
|
+
* restarts a plugin host whenever it reloads or updates one, and the pushes
|
|
535
|
+
* that fill these maps happen on *change* — a settings save, a new
|
|
536
|
+
* selection. So a restart left a project with no instructions and no
|
|
537
|
+
* selection until somebody happened to change one, and "make this bigger"
|
|
538
|
+
* quietly meant nothing in between.
|
|
539
|
+
*
|
|
540
|
+
* The editor therefore sends its whole picture with every check, and this
|
|
541
|
+
* route applies it. Only keys that are actually present are applied, so a
|
|
542
|
+
* bare `{}` — the diagnostic anybody would reach for — still just reports.
|
|
543
|
+
*/
|
|
544
|
+
bb.http.route(
|
|
545
|
+
"POST",
|
|
546
|
+
"/state",
|
|
547
|
+
async (context) => {
|
|
548
|
+
let body: unknown;
|
|
549
|
+
try {
|
|
550
|
+
body = await context.req.json();
|
|
551
|
+
} catch {
|
|
552
|
+
body = null;
|
|
553
|
+
}
|
|
554
|
+
const record = (body ?? {}) as {
|
|
555
|
+
projectId?: unknown;
|
|
556
|
+
instructions?: unknown;
|
|
557
|
+
layers?: unknown;
|
|
558
|
+
path?: unknown;
|
|
559
|
+
url?: unknown;
|
|
560
|
+
token?: unknown;
|
|
561
|
+
};
|
|
562
|
+
adoptCallback(record);
|
|
563
|
+
const projectId =
|
|
564
|
+
typeof record.projectId === "string" ? record.projectId.trim() : "";
|
|
565
|
+
if (projectId) {
|
|
566
|
+
adoptPath(projectId, record);
|
|
567
|
+
if (typeof record.instructions === "string") {
|
|
568
|
+
remember(instructionsByProject, projectId, record.instructions.trim());
|
|
569
|
+
}
|
|
570
|
+
if (Array.isArray(record.layers)) {
|
|
571
|
+
remember(
|
|
572
|
+
selectionByProject,
|
|
573
|
+
projectId,
|
|
574
|
+
record.layers.length === 0 ? "" : selectionBlock(record.layers),
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
return context.json({
|
|
579
|
+
ok: true,
|
|
580
|
+
connected: callback !== null,
|
|
581
|
+
projects: [...new Set([
|
|
582
|
+
...instructionsByProject.keys(),
|
|
583
|
+
...selectionByProject.keys(),
|
|
584
|
+
])].map((id) => ({
|
|
585
|
+
projectId: id,
|
|
586
|
+
instructions: (instructionsByProject.get(id) ?? "").length,
|
|
587
|
+
selection: (selectionByProject.get(id) ?? "").length,
|
|
588
|
+
})),
|
|
589
|
+
});
|
|
590
|
+
},
|
|
591
|
+
{ auth: "local" },
|
|
592
|
+
);
|
|
593
|
+
|
|
594
|
+
/** Resolve once the editor has pushed an address, or with null after
|
|
595
|
+
* CALLBACK_WAIT_MS or an abort. Polled: the address is a plain variable
|
|
596
|
+
* several routes write, and a call in this gap is rare. */
|
|
597
|
+
async function waitForCallback(
|
|
598
|
+
signal?: AbortSignal,
|
|
599
|
+
): Promise<CallbackTarget | null> {
|
|
600
|
+
const until = Date.now() + CALLBACK_WAIT_MS;
|
|
601
|
+
while (callback === null && Date.now() < until) {
|
|
602
|
+
if (!(await pause(CALLBACK_POLL_MS, signal))) return null;
|
|
603
|
+
}
|
|
604
|
+
return callback;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** One tool call, round-tripped through the editor. Transport faults come
|
|
608
|
+
* back as `{ error }` so the model sees a recoverable tool failure rather
|
|
609
|
+
* than a crashed turn. */
|
|
610
|
+
async function callEditor(
|
|
611
|
+
tool: string,
|
|
612
|
+
params: unknown,
|
|
613
|
+
ctx: { threadId?: string | null; projectId?: string | null; signal?: AbortSignal },
|
|
614
|
+
): Promise<{ result?: unknown; error?: string }> {
|
|
615
|
+
// A project just created or opened pushes its address a beat after bb
|
|
616
|
+
// starts the thread, and an editor that restarted its endpoint pushes a
|
|
617
|
+
// new one; a call landing in that gap used to fail on the first try and
|
|
618
|
+
// the model told the person to reopen a window that was open. So a
|
|
619
|
+
// missing address is waited for briefly, and a refused connection is
|
|
620
|
+
// retried against whatever address is current by then.
|
|
621
|
+
let response: Response | null = null;
|
|
622
|
+
let lastError: unknown = null;
|
|
623
|
+
for (let attempt = 0; attempt < EDITOR_RETRY_DELAYS_MS.length + 1; attempt++) {
|
|
624
|
+
if (attempt > 0) {
|
|
625
|
+
if (!(await pause(EDITOR_RETRY_DELAYS_MS[attempt - 1], ctx.signal))) break;
|
|
626
|
+
}
|
|
627
|
+
const target = callback ?? (await waitForCallback(ctx.signal));
|
|
628
|
+
if (target === null) break;
|
|
629
|
+
try {
|
|
630
|
+
response = await fetch(`${target.url}/tool`, {
|
|
631
|
+
method: "POST",
|
|
632
|
+
headers: {
|
|
633
|
+
"content-type": "application/json",
|
|
634
|
+
authorization: `Bearer ${target.token}`,
|
|
635
|
+
},
|
|
636
|
+
body: JSON.stringify({
|
|
637
|
+
tool,
|
|
638
|
+
params:
|
|
639
|
+
params && typeof params === "object" && !Array.isArray(params)
|
|
640
|
+
? params
|
|
641
|
+
: {},
|
|
642
|
+
// Routes the call to the project that owns this thread's workspace;
|
|
643
|
+
// with an empty string the editor can only guess (see relay.ts).
|
|
644
|
+
cwd:
|
|
645
|
+
(ctx.threadId ? workspaceByThread.get(ctx.threadId) : undefined) ??
|
|
646
|
+
(ctx.projectId ? pathByProject.get(ctx.projectId) : undefined) ??
|
|
647
|
+
"",
|
|
648
|
+
}),
|
|
649
|
+
...(ctx.signal ? { signal: ctx.signal } : {}),
|
|
650
|
+
});
|
|
651
|
+
} catch (error) {
|
|
652
|
+
lastError = error;
|
|
653
|
+
if (ctx.signal?.aborted) break;
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
// A stale token while a newer address has already arrived is the
|
|
657
|
+
// editor having restarted under this call: try the new one.
|
|
658
|
+
if (response.status === 401 && callback !== null && callback !== target) {
|
|
659
|
+
response = null;
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
if (response.status === 401 && callback === target) callback = null;
|
|
663
|
+
break;
|
|
664
|
+
}
|
|
665
|
+
if (response === null && lastError === null) {
|
|
666
|
+
return {
|
|
667
|
+
error:
|
|
668
|
+
"The canvas editor is not reachable: the window holding this design is " +
|
|
669
|
+
"not open — the project was closed, or the app restarted. Tell the " +
|
|
670
|
+
"person that, and stop. If it IS open, clicking into this chat hands " +
|
|
671
|
+
"the address back, so sending the message again is all it takes (the " +
|
|
672
|
+
"Agent panel also shows a banner with Reconnect). Do not guess at " +
|
|
673
|
+
"another cause and do not try another application's tools: this is not " +
|
|
674
|
+
"MCP, there is nothing to restart, and a new thread changes nothing. Do " +
|
|
675
|
+
"NOT edit the design's files instead — the editor holds the document in " +
|
|
676
|
+
"memory and overwrites them, so that work would be discarded.",
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
if (response === null) {
|
|
680
|
+
// Every retry failed. The editor moved or went away: its endpoint is
|
|
681
|
+
// per session, and a bb server outlives editors. Drop the address
|
|
682
|
+
// rather than keep failing against it — that is what makes the panel's
|
|
683
|
+
// own indicator say "can't reach this board", and the next push from a
|
|
684
|
+
// live editor puts a working one back.
|
|
685
|
+
callback = null;
|
|
686
|
+
return {
|
|
687
|
+
error:
|
|
688
|
+
`could not reach the canvas editor (${lastError instanceof Error ? lastError.message : String(lastError)}). ` +
|
|
689
|
+
"The usual cause is the editor window for this design being closed. " +
|
|
690
|
+
"Report that message as it stands rather than guessing, and ask the " +
|
|
691
|
+
"person to send the message again — clicking into this chat, or the " +
|
|
692
|
+
"Reconnect button on the Agent panel's banner, puts the address back. " +
|
|
693
|
+
"Do not reach for another application's tools, and do NOT edit the " +
|
|
694
|
+
"design's files instead: anything written there is discarded.",
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
if (response.status === 401) {
|
|
698
|
+
// The editor restarted and minted a fresh token and no new address came
|
|
699
|
+
// while retrying; ours was dropped above, so the next call reports "not
|
|
700
|
+
// connected" instead of retrying a dead secret.
|
|
701
|
+
return {
|
|
702
|
+
error:
|
|
703
|
+
"the canvas editor rejected this connection (stale token) — the editor " +
|
|
704
|
+
"restarted since this plugin was configured. Ask the person to send the " +
|
|
705
|
+
"message again, or to press Reconnect on the Agent panel's banner. Do " +
|
|
706
|
+
"not guess at another cause, and do not edit the design's files instead.",
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
let payload: unknown;
|
|
710
|
+
try {
|
|
711
|
+
payload = await response.json();
|
|
712
|
+
} catch {
|
|
713
|
+
return {
|
|
714
|
+
error: `the canvas editor returned a non-JSON response (HTTP ${response.status})`,
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
const record = (payload ?? {}) as { result?: unknown; error?: unknown };
|
|
718
|
+
if (typeof record.error === "string") return { error: record.error };
|
|
719
|
+
if (!response.ok) {
|
|
720
|
+
return { error: `the canvas editor returned HTTP ${response.status}` };
|
|
721
|
+
}
|
|
722
|
+
return { result: record.result };
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Spill the PNG bytes to disk and report the path.
|
|
727
|
+
*
|
|
728
|
+
* bb's provider bridge decodes a tool response by keeping only text parts and
|
|
729
|
+
* dropping everything else, which is the shared path for claude-code and every
|
|
730
|
+
* ACP bridge — an inline image reaches the model on one provider alone. So the
|
|
731
|
+
* path is the contract and the image part is a bonus for that provider.
|
|
732
|
+
*/
|
|
733
|
+
function saveShot(base64: string, label: string): string | null {
|
|
734
|
+
const safe = label.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 32) || "shot";
|
|
735
|
+
const file = join(SHOT_DIR, `canvas-${safe}-${Date.now()}-${++shotSeq}.png`);
|
|
736
|
+
try {
|
|
737
|
+
mkdirSync(SHOT_DIR, { recursive: true });
|
|
738
|
+
writeFileSync(file, Buffer.from(base64, "base64"));
|
|
739
|
+
return file;
|
|
740
|
+
} catch {
|
|
741
|
+
return null;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
function spillScreenshot(
|
|
746
|
+
result: unknown,
|
|
747
|
+
threadId: string,
|
|
748
|
+
): PluginAgentToolResult {
|
|
749
|
+
const record = (result ?? {}) as {
|
|
750
|
+
png?: unknown;
|
|
751
|
+
width?: unknown;
|
|
752
|
+
height?: unknown;
|
|
753
|
+
};
|
|
754
|
+
if (typeof record.png !== "string" || record.png.length === 0) {
|
|
755
|
+
return toolResult(JSON.stringify(result));
|
|
756
|
+
}
|
|
757
|
+
const safeThread =
|
|
758
|
+
threadId.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 32) || "thread";
|
|
759
|
+
const file = join(
|
|
760
|
+
SHOT_DIR,
|
|
761
|
+
`canvas-${safeThread}-${Date.now()}-${++shotSeq}.png`,
|
|
762
|
+
);
|
|
763
|
+
try {
|
|
764
|
+
mkdirSync(SHOT_DIR, { recursive: true });
|
|
765
|
+
writeFileSync(file, Buffer.from(record.png, "base64"));
|
|
766
|
+
} catch (error) {
|
|
767
|
+
return toolResult(
|
|
768
|
+
`the screenshot was taken but could not be saved: ${error instanceof Error ? error.message : String(error)}`,
|
|
769
|
+
true,
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
return {
|
|
773
|
+
content: [
|
|
774
|
+
{
|
|
775
|
+
type: "text",
|
|
776
|
+
text: JSON.stringify({
|
|
777
|
+
path: file,
|
|
778
|
+
width: record.width,
|
|
779
|
+
height: record.height,
|
|
780
|
+
}),
|
|
781
|
+
},
|
|
782
|
+
{ type: "image", data: record.png, mimeType: "image/png" },
|
|
783
|
+
],
|
|
784
|
+
isError: false,
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* The same spill, for a result that carries several pictures.
|
|
790
|
+
*
|
|
791
|
+
* A frame that only exists as base64 in a tool result is a frame the model
|
|
792
|
+
* never sees — bb keeps text parts and drops the rest before most providers
|
|
793
|
+
* are handed the result, which is the whole reason `spillScreenshot` exists.
|
|
794
|
+
* A dozen of them is a dozen files and one JSON list of paths; the image
|
|
795
|
+
* parts ride along for the one provider that keeps them.
|
|
796
|
+
*
|
|
797
|
+
* Two things it is careful about that the screenshot's version does not have
|
|
798
|
+
* to be:
|
|
799
|
+
*
|
|
800
|
+
* - **The rest of the result survives.** An observed scene is its tracks
|
|
801
|
+
* and its animations, with the frames alongside; returning only the paths
|
|
802
|
+
* would throw the answer away and keep the illustrations.
|
|
803
|
+
* - **Either encoding.** Video frames come as JPEG because a dozen
|
|
804
|
+
* photographic stills as PNG is tens of megabytes; an element rebuilt
|
|
805
|
+
* from markup comes as PNG because it is flat colour and type.
|
|
806
|
+
*
|
|
807
|
+
* Anything with no frames in it is passed through unchanged, so a refusal or
|
|
808
|
+
* an empty answer reads as itself rather than as a failed save.
|
|
809
|
+
*/
|
|
810
|
+
function spillFrames(
|
|
811
|
+
result: unknown,
|
|
812
|
+
threadId: string,
|
|
813
|
+
): PluginAgentToolResult {
|
|
814
|
+
const record = (result ?? {}) as { frames?: unknown };
|
|
815
|
+
if (!Array.isArray(record.frames) || record.frames.length === 0) {
|
|
816
|
+
return toolResult(JSON.stringify(result));
|
|
817
|
+
}
|
|
818
|
+
const safeThread =
|
|
819
|
+
threadId.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 32) || "thread";
|
|
820
|
+
const written: Array<Record<string, unknown>> = [];
|
|
821
|
+
const images: Array<{ type: "image"; data: string; mimeType: string }> = [];
|
|
822
|
+
try {
|
|
823
|
+
mkdirSync(SHOT_DIR, { recursive: true });
|
|
824
|
+
for (const entry of record.frames) {
|
|
825
|
+
const { jpeg, png, ...rest } = (entry ?? {}) as Record<string, unknown>;
|
|
826
|
+
const bytes = typeof jpeg === "string" ? jpeg : typeof png === "string" ? png : "";
|
|
827
|
+
if (!bytes) continue;
|
|
828
|
+
const jpg = typeof jpeg === "string";
|
|
829
|
+
const file = join(
|
|
830
|
+
SHOT_DIR,
|
|
831
|
+
`canvas-${safeThread}-${Date.now()}-${++shotSeq}.${jpg ? "jpg" : "png"}`,
|
|
832
|
+
);
|
|
833
|
+
writeFileSync(file, Buffer.from(bytes, "base64"));
|
|
834
|
+
// Everything the tool said about this frame, with the bytes swapped
|
|
835
|
+
// for the path — so a still keeps its step and its label, and a video
|
|
836
|
+
// frame keeps the time the decoder actually landed on.
|
|
837
|
+
written.push({ path: file, ...rest });
|
|
838
|
+
images.push({
|
|
839
|
+
type: "image",
|
|
840
|
+
data: bytes,
|
|
841
|
+
mimeType: jpg ? "image/jpeg" : "image/png",
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
} catch (error) {
|
|
845
|
+
return toolResult(
|
|
846
|
+
`the frames were taken but could not be saved: ${error instanceof Error ? error.message : String(error)}`,
|
|
847
|
+
true,
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
if (written.length === 0) return toolResult(JSON.stringify(result));
|
|
851
|
+
return {
|
|
852
|
+
content: [
|
|
853
|
+
{ type: "text", text: JSON.stringify({ ...record, frames: written }) },
|
|
854
|
+
...images,
|
|
855
|
+
],
|
|
856
|
+
isError: false,
|
|
857
|
+
};
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
for (const definition of CANVAS_TOOLS) {
|
|
861
|
+
bb.agents.registerTool({
|
|
862
|
+
name: definition.name,
|
|
863
|
+
description: definition.description,
|
|
864
|
+
// Raw JSON schema rather than zod: the renderer validates every target
|
|
865
|
+
// and patch it is handed anyway, and a zod round-trip would rewrite the
|
|
866
|
+
// schema the parity test pins. Params therefore arrive unvalidated.
|
|
867
|
+
parameters: definition.inputSchema as Record<string, unknown>,
|
|
868
|
+
async execute(params, ctx) {
|
|
869
|
+
const outcome = await callEditor(definition.name, params, ctx);
|
|
870
|
+
if (typeof outcome.error === "string") {
|
|
871
|
+
return toolResult(outcome.error, true);
|
|
872
|
+
}
|
|
873
|
+
if (definition.name === SCREENSHOT_TOOL) {
|
|
874
|
+
return spillScreenshot(outcome.result, ctx.threadId);
|
|
875
|
+
}
|
|
876
|
+
if (FRAMES_TOOLS.includes(definition.name)) {
|
|
877
|
+
return spillFrames(outcome.result, ctx.threadId);
|
|
878
|
+
}
|
|
879
|
+
return toolResult(JSON.stringify(outcome.result));
|
|
880
|
+
},
|
|
881
|
+
});
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* Mentioning a layer in the composer.
|
|
886
|
+
*
|
|
887
|
+
* bb's own `@` menu lists files, folders and sections — the things a coding
|
|
888
|
+
* agent works on. What somebody using a design editor wants to point at is a
|
|
889
|
+
* *layer*, and the whole reason this plugin exists is that the layer is
|
|
890
|
+
* reachable. So the same menu gets the board: type `@` (or `#`) and the rows
|
|
891
|
+
* are the layers of the open design, with "the current selection" first.
|
|
892
|
+
*
|
|
893
|
+
* The context a picked row resolves to is agent-visible and user-hidden, so
|
|
894
|
+
* a mention costs the person nothing in their own message and still tells the
|
|
895
|
+
* model exactly which id to call `getLayer` and `updateLayer` with. That is
|
|
896
|
+
* the difference between "make this bigger" working and not.
|
|
897
|
+
*
|
|
898
|
+
* `search` is time-boxed by bb (2s) and failure-isolated: if the editor is
|
|
899
|
+
* not answering, the menu simply has no layer rows. `resolve` throwing would
|
|
900
|
+
* block the send, so it degrades to a line of text saying the layer could not
|
|
901
|
+
* be read rather than to an exception.
|
|
902
|
+
*/
|
|
903
|
+
const SELECTION_ITEM = "selection";
|
|
904
|
+
|
|
905
|
+
/** The snapshot a mention carries (`layerSnapshot`), gathered at send time.
|
|
906
|
+
* Every read is best effort: a picture or markup that cannot be had leaves a
|
|
907
|
+
* shorter snapshot, never a mention that blocks the send. */
|
|
908
|
+
async function mentionSnapshot(id: string): Promise<string> {
|
|
909
|
+
const read = await callEditor("getLayer", { target: { id } }, {});
|
|
910
|
+
if (typeof read.error === "string") return `The layer "${id}" could not be read: ${read.error}`;
|
|
911
|
+
const layer = (read.result ?? {}) as Record<string, unknown>;
|
|
912
|
+
|
|
913
|
+
// An element of a running page is named by its page too: the element's own
|
|
914
|
+
// handle dies with the session, the page's address and node do not.
|
|
915
|
+
const frameId = frameOfLiveId(id);
|
|
916
|
+
let page: { id: string; address: string } | null = null;
|
|
917
|
+
if (frameId) {
|
|
918
|
+
const frame = await callEditor("getLayer", { target: { id: frameId } }, {});
|
|
919
|
+
const src = ((frame.result ?? {}) as { props?: { src?: unknown } }).props?.src;
|
|
920
|
+
if (typeof src === "string" && src) page = { id: frameId, address: src };
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
const [exported, shot] = await Promise.all([
|
|
924
|
+
callEditor("exportHtml", { target: { id } }, {}),
|
|
925
|
+
callEditor("canvasScreenshot", { target: { id }, width: 480 }, {}),
|
|
926
|
+
]);
|
|
927
|
+
const html = ((exported.result ?? {}) as { html?: unknown }).html;
|
|
928
|
+
const png = ((shot.result ?? {}) as { png?: unknown }).png;
|
|
929
|
+
return layerSnapshot({
|
|
930
|
+
layer,
|
|
931
|
+
html: typeof html === "string" ? html : null,
|
|
932
|
+
page,
|
|
933
|
+
shot: typeof png === "string" && png ? saveShot(png, "mention") : null,
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
// Guarded because it is the one registration this plugin can live without:
|
|
938
|
+
// a bb whose `ui` surface predates mention providers would otherwise throw
|
|
939
|
+
// here and take the tools down with it, which is a much worse trade than a
|
|
940
|
+
// menu that lists files but no layers.
|
|
941
|
+
try {
|
|
942
|
+
bb.ui.registerMentionProvider({
|
|
943
|
+
id: "layers",
|
|
944
|
+
label: "Canvas layers",
|
|
945
|
+
// `@` is where people already look; `#` is for anybody who wants the
|
|
946
|
+
// board without wading past the file list.
|
|
947
|
+
triggers: ["@", "#"],
|
|
948
|
+
async search(context) {
|
|
949
|
+
const query = context.query.trim();
|
|
950
|
+
const rows: Array<{ id: string; title: string; subtitle?: string }> = [];
|
|
951
|
+
|
|
952
|
+
// The selection first, and only when there is one: a row that says
|
|
953
|
+
// "0 selected" wastes the only line most menus show.
|
|
954
|
+
const selection = await callEditor("getSelection", {}, {
|
|
955
|
+
threadId: context.threadId,
|
|
956
|
+
});
|
|
957
|
+
const selected = (selection.result ?? {}) as { count?: number };
|
|
958
|
+
if (typeof selected.count === "number" && selected.count > 0) {
|
|
959
|
+
if (!query || "selection".startsWith(query.toLowerCase())) {
|
|
960
|
+
rows.push({
|
|
961
|
+
id: SELECTION_ITEM,
|
|
962
|
+
title: `Current selection (${selected.count} layer${selected.count === 1 ? "" : "s"})`,
|
|
963
|
+
subtitle: "What is picked on the board right now",
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
const listed = await callEditor(
|
|
969
|
+
"listLayers",
|
|
970
|
+
query ? { search: query, limit: 8 } : { limit: 8 },
|
|
971
|
+
{ threadId: context.threadId },
|
|
972
|
+
);
|
|
973
|
+
const layers = ((listed.result ?? {}) as { layers?: unknown }).layers;
|
|
974
|
+
if (Array.isArray(layers)) {
|
|
975
|
+
for (const entry of layers) {
|
|
976
|
+
const layer = (entry ?? {}) as {
|
|
977
|
+
id?: unknown;
|
|
978
|
+
name?: unknown;
|
|
979
|
+
type?: unknown;
|
|
980
|
+
text?: unknown;
|
|
981
|
+
};
|
|
982
|
+
if (typeof layer.id !== "string") continue;
|
|
983
|
+
const name =
|
|
984
|
+
typeof layer.name === "string" && layer.name
|
|
985
|
+
? layer.name
|
|
986
|
+
: typeof layer.type === "string"
|
|
987
|
+
? layer.type
|
|
988
|
+
: layer.id;
|
|
989
|
+
const subtitle = [layer.type, layer.text]
|
|
990
|
+
.filter((part): part is string => typeof part === "string" && part.length > 0)
|
|
991
|
+
.join(" · ");
|
|
992
|
+
rows.push({
|
|
993
|
+
id: `layer:${layer.id}`,
|
|
994
|
+
title: name,
|
|
995
|
+
...(subtitle ? { subtitle } : {}),
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
return rows;
|
|
1000
|
+
},
|
|
1001
|
+
async resolve(itemId) {
|
|
1002
|
+
if (itemId === SELECTION_ITEM) {
|
|
1003
|
+
const outcome = await callEditor("getSelection", {}, {});
|
|
1004
|
+
if (typeof outcome.error === "string") {
|
|
1005
|
+
return { context: `The canvas selection could not be read: ${outcome.error}` };
|
|
1006
|
+
}
|
|
1007
|
+
const layers = ((outcome.result ?? {}) as { layers?: unknown }).layers;
|
|
1008
|
+
const ids = Array.isArray(layers)
|
|
1009
|
+
? layers
|
|
1010
|
+
.map((entry) => (entry as { id?: unknown } | null)?.id)
|
|
1011
|
+
.filter((id): id is string => typeof id === "string")
|
|
1012
|
+
: [];
|
|
1013
|
+
const shots = await Promise.all(
|
|
1014
|
+
ids.slice(0, SNAPSHOT_MAX_LAYERS).map((id) => mentionSnapshot(id)),
|
|
1015
|
+
);
|
|
1016
|
+
const rest = ids.length - shots.length;
|
|
1017
|
+
return {
|
|
1018
|
+
context:
|
|
1019
|
+
"The layers selected on the board right now (address them by id with getLayer/updateLayer):\n" +
|
|
1020
|
+
shots.join("\n\n") +
|
|
1021
|
+
(rest > 0 ? `\n\n…and ${rest} more — call getSelection for them.` : ""),
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
const id = itemId.startsWith("layer:") ? itemId.slice("layer:".length) : itemId;
|
|
1025
|
+
return {
|
|
1026
|
+
context:
|
|
1027
|
+
"A layer of the open design (address it by id with getLayer/updateLayer):\n" +
|
|
1028
|
+
(await mentionSnapshot(id)),
|
|
1029
|
+
};
|
|
1030
|
+
},
|
|
1031
|
+
});
|
|
1032
|
+
} catch (error) {
|
|
1033
|
+
bb.log.info(
|
|
1034
|
+
`canvas layer mentions unavailable on this bb: ${error instanceof Error ? error.message : String(error)}`,
|
|
1035
|
+
);
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// Registering a configure() provider means bb stops auto-selecting all of our
|
|
1039
|
+
// tools, so this has to return the full set. We pay that cost to learn
|
|
1040
|
+
// `environment.path`, which is not available anywhere in the per-call context.
|
|
1041
|
+
bb.agents.configure((context) => {
|
|
1042
|
+
if (context.environment.path) {
|
|
1043
|
+
// Insertion-ordered eviction: a long-lived bb server sees unboundedly
|
|
1044
|
+
// many threads, and a stale entry only ever costs a mis-routed call.
|
|
1045
|
+
if (
|
|
1046
|
+
workspaceByThread.size >= MAX_TRACKED_THREADS &&
|
|
1047
|
+
!workspaceByThread.has(context.thread.id)
|
|
1048
|
+
) {
|
|
1049
|
+
const oldest = workspaceByThread.keys().next();
|
|
1050
|
+
if (!oldest.done) workspaceByThread.delete(oldest.value);
|
|
1051
|
+
}
|
|
1052
|
+
workspaceByThread.set(context.thread.id, context.environment.path);
|
|
1053
|
+
}
|
|
1054
|
+
return {
|
|
1055
|
+
tools: CANVAS_TOOLS.map((tool) => tool.name),
|
|
1056
|
+
skills: [],
|
|
1057
|
+
instructions: CANVAS_TOOL_GUIDE,
|
|
1058
|
+
};
|
|
1059
|
+
});
|
|
1060
|
+
|
|
1061
|
+
// What the assistant is told before the conversation starts. bb resolves this
|
|
1062
|
+
// when a thread starts and when a turn is submitted, so an edit in Project
|
|
1063
|
+
// settings reaches the next turn without reloading anything.
|
|
1064
|
+
bb.agents.contributeInstructions((context) => {
|
|
1065
|
+
const standing = instructionsByProject.get(context.projectId) ?? "";
|
|
1066
|
+
const selection = selectionByProject.get(context.projectId) ?? "";
|
|
1067
|
+
if (!standing) return selection || null;
|
|
1068
|
+
if (!selection) return standing;
|
|
1069
|
+
// Over bb's cap the project's own instructions are kept whole and the
|
|
1070
|
+
// selection gives way, because the assistant can ask for it
|
|
1071
|
+
// (getSelection) and cannot ask for the half of a prompt that went
|
|
1072
|
+
// missing — but it gives way a piece at a time, see `fitSelection`.
|
|
1073
|
+
const fitted = fitSelection(selection, MAX_INSTRUCTIONS - standing.length - 2);
|
|
1074
|
+
return fitted ? `${standing}\n\n${fitted}` : standing.slice(0, MAX_INSTRUCTIONS);
|
|
1075
|
+
});
|
|
1076
|
+
|
|
1077
|
+
bb.onDispose(() => {
|
|
1078
|
+
callback = null;
|
|
1079
|
+
workspaceByThread.clear();
|
|
1080
|
+
pathByProject.clear();
|
|
1081
|
+
instructionsByProject.clear();
|
|
1082
|
+
selectionByProject.clear();
|
|
1083
|
+
});
|
|
1084
|
+
}
|