dsh-comfyui-canvas 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +184 -0
- package/README.zh.md +210 -0
- package/comfyui-bridge/ComfyUI-DSH-Canvas/__init__.py +174 -0
- package/comfyui-bridge/ComfyUI-DSH-Canvas/entry/bridge.js +414 -0
- package/cordis.patch.yml +9 -0
- package/docs/NEXT-VERSION-PLAN.md +199 -0
- package/docs/PLAN-v0.1.2.md +307 -0
- package/docs/architecture-flow.html +210 -0
- package/docs/architecture.html +252 -0
- package/docs/comfyui-nav-icon.svg +5 -0
- package/docs/screenshots/01-startup-prompt.png +0 -0
- package/docs/screenshots/02-canvas-ready.png +0 -0
- package/docs/screenshots/03-workflow-output.png +0 -0
- package/lib/client.js +810 -0
- package/lib/index.js +726 -0
- package/package.json +57 -0
- package/screenshots.json +5 -0
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
// ComfyUI-DSH-Canvas bridge frontend.
|
|
2
|
+
// Injected into the ComfyUI page via app.registerExtension. It watches the
|
|
3
|
+
// LiteGraph canvas and posts graph changes to the bridge backend, so the DSH
|
|
4
|
+
// plugin can read what is on the canvas without touching the iframe DOM.
|
|
5
|
+
//
|
|
6
|
+
// M0: passive reporting (read path).
|
|
7
|
+
// M1: command listener (write path) — add_node / connect / set_param /
|
|
8
|
+
// remove_node executed directly on LiteGraph, results POSTed back.
|
|
9
|
+
// M2: load_workflow (rewrite loop) / run (queuePrompt) / highlight (debug).
|
|
10
|
+
// M3: validate (structural checks, no execution) + per-tab targeting.
|
|
11
|
+
|
|
12
|
+
import { app } from "../../scripts/app.js";
|
|
13
|
+
import { api } from "../../scripts/api.js";
|
|
14
|
+
|
|
15
|
+
const REPORT_URL = "/dsh-bridge/report";
|
|
16
|
+
const RESULT_URL = "/dsh-bridge/result";
|
|
17
|
+
const COMMAND_EVENT = "dsh-bridge-command";
|
|
18
|
+
const DEBOUNCE_MS = 300;
|
|
19
|
+
|
|
20
|
+
// Stable per-tab identity: lets the backend route a command to one frontend
|
|
21
|
+
// even when several tabs are open (DSH canvas tab + a native ComfyUI tab).
|
|
22
|
+
// The backend echoes the last-reported clientId back as `target`, and only
|
|
23
|
+
// that tab executes; the others skip the command.
|
|
24
|
+
const CLIENT_ID = (crypto?.randomUUID?.() ?? String(Math.random()).slice(2));
|
|
25
|
+
|
|
26
|
+
// ------- M0: passive reporting -------
|
|
27
|
+
|
|
28
|
+
function summarize(graph) {
|
|
29
|
+
const nodes = graph?.nodes ?? [];
|
|
30
|
+
return nodes.map((node) => ({
|
|
31
|
+
id: String(node.id),
|
|
32
|
+
type: node.type ?? node.comfyClass ?? "unknown",
|
|
33
|
+
title: node.title || node.type || "",
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function serializeGraph(graph) {
|
|
38
|
+
try {
|
|
39
|
+
return graph?.serialize?.() ?? null;
|
|
40
|
+
} catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function report() {
|
|
46
|
+
try {
|
|
47
|
+
const graph = app.graph;
|
|
48
|
+
if (!graph) return;
|
|
49
|
+
const body = {
|
|
50
|
+
clientId: CLIENT_ID,
|
|
51
|
+
nodes: summarize(graph),
|
|
52
|
+
workflow: serializeGraph(graph),
|
|
53
|
+
prompt: null, // API-format prompt lands with the run path (M2)
|
|
54
|
+
};
|
|
55
|
+
fetch(REPORT_URL, {
|
|
56
|
+
method: "POST",
|
|
57
|
+
headers: { "Content-Type": "application/json" },
|
|
58
|
+
body: JSON.stringify(body),
|
|
59
|
+
}).catch(() => {
|
|
60
|
+
// Backend may not be up yet during startup; next change retries.
|
|
61
|
+
});
|
|
62
|
+
} catch {
|
|
63
|
+
// Never let a reporting error break the canvas.
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
let timer = null;
|
|
68
|
+
function scheduleReport() {
|
|
69
|
+
clearTimeout(timer);
|
|
70
|
+
timer = setTimeout(report, DEBOUNCE_MS);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ------- M1: write path -------
|
|
74
|
+
|
|
75
|
+
async function resolveLiteGraph() {
|
|
76
|
+
if (window.LiteGraph) return window.LiteGraph;
|
|
77
|
+
try {
|
|
78
|
+
const mod = await import("../../scripts/litegraph.js");
|
|
79
|
+
return mod.LiteGraph ?? window.LiteGraph;
|
|
80
|
+
} catch {
|
|
81
|
+
return window.LiteGraph;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Apply a widget value with combo validation + callback, shared by set_param
|
|
86
|
+
// and inject_text. Throws on an unknown combo value so it fails loudly instead
|
|
87
|
+
// of silently rendering empty, then lets the widget's own callback react.
|
|
88
|
+
function applyWidgetValue(node, widget, value) {
|
|
89
|
+
widget.value = value;
|
|
90
|
+
if (widget.type === "combo" || Array.isArray(widget.options)) {
|
|
91
|
+
const labels = (Array.isArray(widget.options) ? widget.options : widget.options?.values ?? [])
|
|
92
|
+
.map((o) => (typeof o === "object" && o !== null ? (o.value ?? o[0]) : o));
|
|
93
|
+
if (labels.length > 0 && !labels.some((l) => String(l) === String(value))) {
|
|
94
|
+
throw new Error(`value ${JSON.stringify(value)} is not a valid option for ${widget.name} (${labels.join(", ")})`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
widget.callback?.(widget.value, widget);
|
|
98
|
+
node.onWidgetChanged?.(widget, widget.value, null, null);
|
|
99
|
+
node.setDirtyCanvas?.(true, true);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function executeCommand(cmd, payload) {
|
|
103
|
+
const graph = app.graph;
|
|
104
|
+
const LiteGraph = await resolveLiteGraph();
|
|
105
|
+
|
|
106
|
+
switch (cmd) {
|
|
107
|
+
case "add_node": {
|
|
108
|
+
const type = payload.class ?? payload.type;
|
|
109
|
+
if (!type) throw new Error("add_node requires payload.class");
|
|
110
|
+
const node = LiteGraph?.createNode?.(type) ?? null;
|
|
111
|
+
if (!node) throw new Error(`unknown node class: ${type}`);
|
|
112
|
+
if (Array.isArray(payload.pos)) node.pos = [payload.pos[0], payload.pos[1]];
|
|
113
|
+
graph.add(node);
|
|
114
|
+
scheduleReport();
|
|
115
|
+
return { nodeId: String(node.id), type: node.type };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
case "connect": {
|
|
119
|
+
const { srcId, srcSlot, dstId, dstSlot } = payload;
|
|
120
|
+
const src = graph.getNodeById(+srcId);
|
|
121
|
+
const dst = graph.getNodeById(+dstId);
|
|
122
|
+
if (!src) throw new Error(`source node not found: ${srcId}`);
|
|
123
|
+
if (!dst) throw new Error(`destination node not found: ${dstId}`);
|
|
124
|
+
const ok = src.connect(+srcSlot, dst, +dstSlot);
|
|
125
|
+
if (!ok) throw new Error("connection rejected by LiteGraph");
|
|
126
|
+
scheduleReport();
|
|
127
|
+
return { connected: `${srcId}:${srcSlot} -> ${dstId}:${dstSlot}` };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
case "set_param": {
|
|
131
|
+
const { nodeId, key, value } = payload;
|
|
132
|
+
const node = graph.getNodeById(+nodeId);
|
|
133
|
+
if (!node) throw new Error(`node not found: ${nodeId}`);
|
|
134
|
+
const widget = (node.widgets ?? []).find((w) => w.name === key);
|
|
135
|
+
if (!widget) throw new Error(`widget not found: ${key}`);
|
|
136
|
+
applyWidgetValue(node, widget, value);
|
|
137
|
+
scheduleReport();
|
|
138
|
+
return { key, value };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
case "remove_node": {
|
|
142
|
+
const { nodeId } = payload;
|
|
143
|
+
const node = graph.getNodeById(+nodeId);
|
|
144
|
+
if (!node) throw new Error(`node not found: ${nodeId}`);
|
|
145
|
+
graph.remove(node);
|
|
146
|
+
scheduleReport();
|
|
147
|
+
return { removed: String(nodeId) };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// M2: load a whole workflow back onto the canvas (rewrite loop).
|
|
151
|
+
case "load_workflow": {
|
|
152
|
+
const data = payload.workflow;
|
|
153
|
+
if (!data) throw new Error("load_workflow requires payload.workflow");
|
|
154
|
+
if (typeof data === "string") {
|
|
155
|
+
try { payload.workflow = JSON.parse(data); } catch {
|
|
156
|
+
throw new Error("payload.workflow is not valid JSON");
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
await app.loadGraphData(payload.workflow);
|
|
160
|
+
scheduleReport();
|
|
161
|
+
return { loaded: true };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// M2: run the current canvas graph (queuePrompt) and report the task id.
|
|
165
|
+
// `overrides` (optional) temporarily sets widget values before the graph
|
|
166
|
+
// is compiled, then restores them — the canvas is left untouched, so the
|
|
167
|
+
// same graph can be run repeatedly with different seeds/prompts.
|
|
168
|
+
case "run": {
|
|
169
|
+
if (typeof app.graphToPrompt !== "function") {
|
|
170
|
+
throw new Error("this ComfyUI frontend lacks app.graphToPrompt");
|
|
171
|
+
}
|
|
172
|
+
const overrides = Array.isArray(payload.overrides) ? payload.overrides : [];
|
|
173
|
+
// Snapshot the touched widgets so we can restore them afterwards.
|
|
174
|
+
const saved = [];
|
|
175
|
+
for (const o of overrides) {
|
|
176
|
+
const node = graph.getNodeById(+o.nodeId);
|
|
177
|
+
if (!node) throw new Error(`override node not found: ${o.nodeId}`);
|
|
178
|
+
const widget = (node.widgets ?? []).find((w) => w.name === o.key);
|
|
179
|
+
if (!widget) throw new Error(`override widget not found: ${o.nodeId}.${o.key}`);
|
|
180
|
+
saved.push({ node, widget, prev: widget.value });
|
|
181
|
+
widget.value = o.value;
|
|
182
|
+
}
|
|
183
|
+
try {
|
|
184
|
+
const { output } = await app.graphToPrompt();
|
|
185
|
+
const resp = await api.fetchApi("/prompt", {
|
|
186
|
+
method: "POST",
|
|
187
|
+
headers: { "Content-Type": "application/json" },
|
|
188
|
+
body: JSON.stringify({ prompt: output }),
|
|
189
|
+
});
|
|
190
|
+
const json = await resp.json().catch(() => null);
|
|
191
|
+
if (!resp.ok) throw new Error(json?.error?.message ?? `prompt HTTP ${resp.status}`);
|
|
192
|
+
return { promptId: json.prompt_id, nodeErrors: json.node_errors ?? null };
|
|
193
|
+
} finally {
|
|
194
|
+
// Restore regardless of outcome so the canvas is never left altered.
|
|
195
|
+
for (const s of saved) {
|
|
196
|
+
s.widget.value = s.prev;
|
|
197
|
+
s.node.setDirtyCanvas?.(true, true);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// M4: batch run — execute the graph once per override set (a parameter
|
|
203
|
+
// matrix), queueing each as its own prompt. Returns the prompt ids in the
|
|
204
|
+
// same order as the runs. ComfyUI executes the queue serially, so this is
|
|
205
|
+
// a convenient way to sweep seeds / prompts / strengths.
|
|
206
|
+
case "batch_run": {
|
|
207
|
+
if (typeof app.graphToPrompt !== "function") {
|
|
208
|
+
throw new Error("this ComfyUI frontend lacks app.graphToPrompt");
|
|
209
|
+
}
|
|
210
|
+
const runs = Array.isArray(payload.runs) ? payload.runs : [];
|
|
211
|
+
const results = [];
|
|
212
|
+
for (const run of runs ?? []) {
|
|
213
|
+
const overrides = Array.isArray(run.overrides) ? run.overrides : [];
|
|
214
|
+
const saved = [];
|
|
215
|
+
for (const o of overrides) {
|
|
216
|
+
const node = graph.getNodeById(+o.nodeId);
|
|
217
|
+
if (!node) throw new Error(`override node not found: ${o.nodeId}`);
|
|
218
|
+
const widget = (node.widgets ?? []).find((w) => w.name === o.key);
|
|
219
|
+
if (!widget) throw new Error(`override widget not found: ${o.nodeId}.${o.key}`);
|
|
220
|
+
saved.push({ node, widget, prev: widget.value });
|
|
221
|
+
widget.value = o.value;
|
|
222
|
+
}
|
|
223
|
+
try {
|
|
224
|
+
const { output } = await app.graphToPrompt();
|
|
225
|
+
const resp = await api.fetchApi("/prompt", {
|
|
226
|
+
method: "POST",
|
|
227
|
+
headers: { "Content-Type": "application/json" },
|
|
228
|
+
body: JSON.stringify({ prompt: output }),
|
|
229
|
+
});
|
|
230
|
+
const json = await resp.json().catch(() => null);
|
|
231
|
+
if (!resp.ok) throw new Error(json?.error?.message ?? `prompt HTTP ${resp.status}`);
|
|
232
|
+
results.push({ promptId: json.prompt_id, nodeErrors: json.node_errors ?? null });
|
|
233
|
+
} finally {
|
|
234
|
+
for (const s of saved) {
|
|
235
|
+
s.widget.value = s.prev;
|
|
236
|
+
s.node.setDirtyCanvas?.(true, true);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return { queued: results.length, runs: results };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// M2: flash/highlight nodes by id (debug output).
|
|
244
|
+
case "highlight": {
|
|
245
|
+
const ids = Array.isArray(payload.ids) ? payload.ids : [];
|
|
246
|
+
for (const id of ids) {
|
|
247
|
+
const node = graph.getNodeById(+id);
|
|
248
|
+
if (node && typeof node.flash === "function") node.flash();
|
|
249
|
+
}
|
|
250
|
+
return { highlighted: ids.map(String) };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// M3: validate — structural checks only, never submits /prompt, so a valid
|
|
254
|
+
// graph is NOT queued (unlike `run`). Flags nodes with unconnected required
|
|
255
|
+
// inputs so the agent can locate/fix errors without spending a generation.
|
|
256
|
+
// Dynamic-slot nodes (switch / index / selector, or numbered inputs like
|
|
257
|
+
// `image15`) intentionally leave trailing slots unconnected — those are
|
|
258
|
+
// reported as warnings, never as errors, so debug does not false-flag them.
|
|
259
|
+
case "validate": {
|
|
260
|
+
const nodeErrors = {};
|
|
261
|
+
const warnings = {};
|
|
262
|
+
for (const node of graph?.nodes ?? []) {
|
|
263
|
+
const missing = [];
|
|
264
|
+
const dynamic = [];
|
|
265
|
+
for (const input of node.inputs ?? []) {
|
|
266
|
+
if (input.link == null && !input.optional && !input.widget) {
|
|
267
|
+
const type = String(node.type ?? "");
|
|
268
|
+
const isDynamicSlot = /switch|index|selector|multi/i.test(type)
|
|
269
|
+
|| /^(image|input|value|mask|opt|optional)\d+$/i.test(input.name);
|
|
270
|
+
(isDynamicSlot ? dynamic : missing).push(
|
|
271
|
+
isDynamicSlot
|
|
272
|
+
? `dynamic slot "${input.name}" unconnected (switch/index node — expected)`
|
|
273
|
+
: `missing required input "${input.name}"`,
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (missing.length) nodeErrors[String(node.id)] = missing;
|
|
278
|
+
if (dynamic.length) warnings[String(node.id)] = dynamic;
|
|
279
|
+
}
|
|
280
|
+
const offendingIds = Object.keys(nodeErrors);
|
|
281
|
+
return { valid: offendingIds.length === 0, nodeErrors, warnings, offendingIds };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// M3: ask the frontend to re-report the current graph immediately, so
|
|
285
|
+
// comfyui_read_workflow does not serve a stale cache when the page is
|
|
286
|
+
// open but idle (no recent change to trigger a debounced report).
|
|
287
|
+
case "refresh_report": {
|
|
288
|
+
report();
|
|
289
|
+
return { reported: true };
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// v0.1.1: inject text into the canvas. Mode 2 (newClass): create a source
|
|
293
|
+
// node, set its widget, optionally connect to a target input — a one-step
|
|
294
|
+
// "conversation text → canvas node". Mode 1 (nodeId): write an existing
|
|
295
|
+
// widget directly. Both reuse applyWidgetValue so combo validation + the
|
|
296
|
+
// widget callback fire identically to set_param.
|
|
297
|
+
case "inject_text": {
|
|
298
|
+
const { text, nodeId, widgetKey = "text", newClass, targetId, targetSlot, sourceSlot = 0 } = payload;
|
|
299
|
+
if (newClass) {
|
|
300
|
+
const node = LiteGraph?.createNode?.(newClass) ?? null;
|
|
301
|
+
if (!node) throw new Error(`unknown node class: ${newClass}`);
|
|
302
|
+
graph.add(node);
|
|
303
|
+
const widget = (node.widgets ?? []).find((w) => w.name === widgetKey);
|
|
304
|
+
if (widget) applyWidgetValue(node, widget, text);
|
|
305
|
+
const connected = targetId != null && targetSlot != null;
|
|
306
|
+
if (connected) {
|
|
307
|
+
const dst = graph.getNodeById(+targetId);
|
|
308
|
+
if (!dst) throw new Error(`target node not found: ${targetId}`);
|
|
309
|
+
const ok = node.connect(+sourceSlot, dst, +targetSlot);
|
|
310
|
+
if (!ok) throw new Error("connection rejected by LiteGraph");
|
|
311
|
+
}
|
|
312
|
+
scheduleReport();
|
|
313
|
+
return { nodeId: String(node.id), created: true, type: node.type, connected };
|
|
314
|
+
}
|
|
315
|
+
if (nodeId == null) throw new Error("inject_text requires nodeId or newClass");
|
|
316
|
+
const node = graph.getNodeById(+nodeId);
|
|
317
|
+
if (!node) throw new Error(`node not found: ${nodeId}`);
|
|
318
|
+
const widget = (node.widgets ?? []).find((w) => w.name === widgetKey);
|
|
319
|
+
if (!widget) throw new Error(`widget not found: ${widgetKey}`);
|
|
320
|
+
applyWidgetValue(node, widget, text);
|
|
321
|
+
scheduleReport();
|
|
322
|
+
return { nodeId: String(nodeId), key: widgetKey, value: text };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// v0.1.1: export the current canvas as API-format workflow JSON — the
|
|
326
|
+
// format /prompt and comfy-cli run_workflow consume. Bridges the live
|
|
327
|
+
// canvas to headless/MCP batch runs.
|
|
328
|
+
case "export_api": {
|
|
329
|
+
if (typeof app.graphToPrompt !== "function") {
|
|
330
|
+
throw new Error("this ComfyUI frontend lacks app.graphToPrompt");
|
|
331
|
+
}
|
|
332
|
+
const { output } = await app.graphToPrompt();
|
|
333
|
+
return { workflow: output, nodeCount: Object.keys(output || {}).length };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
default:
|
|
337
|
+
throw new Error(`unknown command: ${cmd}`);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function postResult(cmdId, ok, raw) {
|
|
342
|
+
const body = { id: cmdId, ok };
|
|
343
|
+
if (ok) {
|
|
344
|
+
body.result = raw;
|
|
345
|
+
} else {
|
|
346
|
+
body.error = raw?.message ?? String(raw ?? "unknown error");
|
|
347
|
+
}
|
|
348
|
+
fetch(RESULT_URL, {
|
|
349
|
+
method: "POST",
|
|
350
|
+
headers: { "Content-Type": "application/json" },
|
|
351
|
+
body: JSON.stringify(body),
|
|
352
|
+
}).catch(() => {});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
app.registerExtension({
|
|
356
|
+
name: "ComfyUI-DSH-Canvas",
|
|
357
|
+
setup() {
|
|
358
|
+
const graph = app.graph;
|
|
359
|
+
if (!graph) return;
|
|
360
|
+
|
|
361
|
+
// M0: report on canvas changes, preserving any existing handler.
|
|
362
|
+
// The arrow keeps `this` from the setup scope; forward the graph itself so
|
|
363
|
+
// a handler that relies on `this` still sees the graph.
|
|
364
|
+
const prevOnChange = graph.onChange;
|
|
365
|
+
graph.onChange = (...args) => {
|
|
366
|
+
if (typeof prevOnChange === "function") {
|
|
367
|
+
try { prevOnChange.apply(graph, args); } catch { /* ignore */ }
|
|
368
|
+
}
|
|
369
|
+
scheduleReport();
|
|
370
|
+
};
|
|
371
|
+
const prevGraphConfigure = graph.configure ?? null;
|
|
372
|
+
if (typeof graph.configure === "function") {
|
|
373
|
+
graph.configure = function (...args) {
|
|
374
|
+
const out = prevGraphConfigure.apply(this, args);
|
|
375
|
+
scheduleReport();
|
|
376
|
+
return out;
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// M1: listen for write commands pushed over the WebSocket.
|
|
381
|
+
// Per-tab targeting (M3): when the backend names a target clientId (the
|
|
382
|
+
// last one that reported), only that tab executes — several open ComfyUI
|
|
383
|
+
// tabs must not each run the same command. No target = any tab may run.
|
|
384
|
+
api.addEventListener(COMMAND_EVENT, async (event) => {
|
|
385
|
+
const msg = event?.detail;
|
|
386
|
+
if (!msg || !msg.id || !msg.cmd) return;
|
|
387
|
+
if (msg.target && msg.target !== CLIENT_ID) return;
|
|
388
|
+
try {
|
|
389
|
+
const raw = await executeCommand(msg.cmd, msg.payload ?? {});
|
|
390
|
+
postResult(msg.id, true, raw);
|
|
391
|
+
} catch (err) {
|
|
392
|
+
console.error("[ComfyUI-DSH-Canvas] command failed", msg.cmd, err);
|
|
393
|
+
postResult(msg.id, false, err);
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
// v0.1.1 fix: node previews (SaveImage/PreviewImage thumbnails) sometimes
|
|
398
|
+
// do not appear after a manual run inside the DSH canvas iframe, even
|
|
399
|
+
// though the images land in output/. ComfyUI's own executed handler sets
|
|
400
|
+
// node.imgs, but inside the nested iframe the canvas redraw (driven by
|
|
401
|
+
// requestAnimationFrame) may not fire afterwards, so the preview stays
|
|
402
|
+
// empty. Forcing a redraw here on every executed event closes that gap —
|
|
403
|
+
// harmless when ComfyUI already redrew, and it is exactly the event the
|
|
404
|
+
// README previously noted the bridge did not touch.
|
|
405
|
+
api.addEventListener("executed", () => {
|
|
406
|
+
try {
|
|
407
|
+
app.canvas?.setDirty?.(true, true);
|
|
408
|
+
} catch { /* never break the canvas over a redraw */ }
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
// Initial report once the canvas is ready.
|
|
412
|
+
setTimeout(report, 1500);
|
|
413
|
+
},
|
|
414
|
+
});
|
package/cordis.patch.yml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# dsh-comfyui-canvas bundle layer: applied automatically when the package is
|
|
2
|
+
# installed (package.json declares dsh.bundle.patch), i.e. a profile that runs
|
|
3
|
+
# `dsh plugin add github:<you>/dsh-comfyui-canvas` gets this wiring without any
|
|
4
|
+
# manual cordis.patch.yml edits. Later layers override these rows by id.
|
|
5
|
+
|
|
6
|
+
# 挂载插件行:注册 host 端画布工具 + client 端画布标签页/设置页。
|
|
7
|
+
- insert:
|
|
8
|
+
- id: comfyui-canvas
|
|
9
|
+
name: dsh-comfyui-canvas
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# dsh-comfyui-canvas — 下个版本开发规划(v0.1.1)
|
|
2
|
+
|
|
3
|
+
> **本文档是「新会话接手说明书」**:下个版本开发请新开一个会话,先读本文件 + `docs/architecture.html`(分层架构)+ `docs/architecture-flow.html`(流程图),即可接续全部开发意图。
|
|
4
|
+
> 最后更新:2026-09-01
|
|
5
|
+
|
|
6
|
+
> **最近一次更新(2026-09-01)**:
|
|
7
|
+
> - 设置面板导航图标改为**大写「C」字母**(此前为不美观的 C 形填充样式),并**移除 label 前的 🎨 emoji**——左侧导航只显示干净的「C + ComfyUI 画布」
|
|
8
|
+
> - **修复 bug「画布运行后节点预览不显示」**(README 已知问题):手动在 DSH 画布 iframe 内运行工作流后,SaveImage/PreviewImage 节点预览缩略图不出现
|
|
9
|
+
> - 根因(已诊断确认):DSH 画布 iframe 设置了 `referrerpolicy="no-referrer"`,与原生 ComfyUI 标签页环境不一致;且 bridge 不触碰 `executed` 事件流,iframe 内画布重绘(rAF 驱动)在 executed 后未触发
|
|
10
|
+
> - 修复:① `lib/client.js` 移除 iframe 的 `referrerpolicy`(iframe 内请求 Referer 本来就是 ComfyUI 自己的 URL,不会泄漏 DSH URL);② `comfyui-bridge/.../entry/bridge.js` 新增监听 ComfyUI `executed` 事件 → `app.canvas.setDirty(true, true)` 强制重绘
|
|
11
|
+
> - 三处已同步:源码 → node_modules 副本 → E 盘 custom_nodes;**需重启 ComfyUI 使 bridge 改动生效**
|
|
12
|
+
> - **修复 bug「画布启动按钮点不动 / 卡在正在启动…」**:在 DSH 画布直接点「启动 ComfyUI」,host 收到 `launchRequested` 后静默失败、界面永远停在"正在启动…"
|
|
13
|
+
> - 根因(已诊断确认):host 启动 watcher 调 `shell.start(shell.resolve({ command, workdir }))` 时**没传 `sandboxPolicy`**,Windows 上 `pwsh-sandbox` 按默认 `workspace-write`(root=F:\Deepseek-harness)执行启动器——而 `ComfyUI启动器.bat` 在 **E 盘**启动 python 并写 output/temp,受限 token 下被拒、进程立即退出;且错误被 `void proc.done.catch(() => {})` 吞掉,客户端无超时、永久卡死
|
|
14
|
+
> - 修复:① `lib/index.js` 启动时显式传 `sandboxPolicy: { mode: 'danger-full-access', workspaceRoot }`(外部服务启动不受工作区沙箱限制),并把启动失败写入新增的 `launchError` 配置字段;② `lib/client.js` 启动卡片读取并展示 `launchError`,加 45s 启动超时(超时未在线则复位"正在启动…"允许重试)
|
|
15
|
+
> - 已同步:源码 → node_modules 副本;**需重启 DSH(host 改动生效)+ 刷新页面(client 改动生效)**
|
|
16
|
+
> - 涉及文件:`lib/client.js` + `lib/index.js` + `comfyui-bridge/ComfyUI-DSH-Canvas/entry/bridge.js`(本仓库)+ DSH 核心 `packages/client/ui-settings-general`(navIcon 映射,不在本仓库 git 内)
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## 0. 如何接续(先读这里)
|
|
21
|
+
|
|
22
|
+
下个会话请按此顺序恢复上下文:
|
|
23
|
+
|
|
24
|
+
1. **读本文档** —— 理解下版本目标、技术边界、实施步骤
|
|
25
|
+
2. **读 `docs/architecture.html` / `docs/architecture-flow.html`** —— 在浏览器打开这两张图,理解三侧(浏览器 / DSH Host / ComfyUI)架构
|
|
26
|
+
3. **跑一次 `npm run check`** —— 确认当前代码语法可用
|
|
27
|
+
4. **确认环境三副本可访问**(见 §1.3)
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## 1. 项目现状快照
|
|
32
|
+
|
|
33
|
+
### 1.1 基本信息
|
|
34
|
+
- **包名/版本**:`dsh-comfyui-canvas` / `0.1.0`
|
|
35
|
+
- **仓库(已开源 Public)**:https://github.com/wbin0001/dsh-comfyui-canvas
|
|
36
|
+
- **许可证**:MIT;**语言**:host = JS(ESM),bridge 后端 = Python,bridge 前端 = JS
|
|
37
|
+
- **README**:`README.md`(en)+ `README.zh.md`(zh),功能表已按「画布操作 / 生产任务 / 环境维护」三面组织
|
|
38
|
+
|
|
39
|
+
### 1.2 当前 12 个 agent 工具(host `lib/index.js`)
|
|
40
|
+
```
|
|
41
|
+
comfyui_read_workflow 读画布(先 refresh_report 再读;无上报返回 ready:false 诊断)
|
|
42
|
+
comfyui_add_node 加节点(class + 可选 pos,立即显示在画布)
|
|
43
|
+
comfyui_connect 连线 srcId:srcSlot → dstId:dstSlot
|
|
44
|
+
comfyui_set_param 改 widget 值(支持 combo 校验 + callback)
|
|
45
|
+
comfyui_remove_node 删节点
|
|
46
|
+
comfyui_load_workflow 整图替换(兼容 read 结果整对象回传的 unwrap)
|
|
47
|
+
comfyui_run 运行(可选 overrides 临时覆盖 widget,跑完还原画布)
|
|
48
|
+
comfyui_get_outputs 取回出图(读 /history/:id + /view,可选 downloadDir 落盘)
|
|
49
|
+
comfyui_batch_run 参数矩阵批量(runs=[{overrides:[...]}],逐组入队)
|
|
50
|
+
comfyui_debug 纯校验(不触发执行;switch/index 动态槽归 warnings 不报错)
|
|
51
|
+
comfyui_config 会话隔离的画布感知 + 连接配置
|
|
52
|
+
comfyui_upgrade git pull 核心 + 全部自定义节点
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### 1.3 三副本位置(改代码必须三处同步)
|
|
56
|
+
| 角色 | 路径 | 说明 |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| 源码主仓(权威源) | `F:\Deepseek-harness\projects\dsh-comfyui-canvas` | git 管理,这是改代码的地方 |
|
|
59
|
+
| DSH 运行时副本 | `F:\Deepseek-harness\.dsh\profiles\web\node_modules\dsh-comfyui-canvas` | DSH 启动实际加载这份 —— 改完 host/client 要复制过来 |
|
|
60
|
+
| ComfyUI 桥接节点 | `E:\AI-ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-DSH-Canvas` | 只同步 `comfyui-bridge/` 下的 `__init__.py` + `entry/bridge.js` —— 改 bridge 要复制过来并**重启 ComfyUI** |
|
|
61
|
+
|
|
62
|
+
> 同步约定:host/client 改动 → 同步到 node_modules 副本;bridge 改动 → 同步到 node_modules 副本 + E 盘 custom_nodes。改完跑 `node --check` + `python -m py_compile`。
|
|
63
|
+
|
|
64
|
+
### 1.4 关键机制速记
|
|
65
|
+
- **bridge 三件套**:`__init__.py`(Python 路由 + 可选 `DSH_BRIDGE_TOKEN` 鉴权 + report 8MB 上限 + clientId 定向)+ `entry/bridge.js`(前端注入,执行白名单命令)
|
|
66
|
+
- **命令白名单**(bridge.js `executeCommand`):`add_node / connect / set_param / remove_node / load_workflow / run / batch_run / validate / highlight / refresh_report`(+ 下版本将加 `inject_text`)
|
|
67
|
+
- **画布数据流**:host 工具 → HTTP `/dsh-bridge/command` → bridge 后端 → WebSocket `send_sync` → 前端执行 → POST `/dsh-bridge/result` → host 轮询
|
|
68
|
+
- **出图取回**:`get_outputs` 直连 ComfyUI 原生 `/history/:id` + `/view`(不经过 bridge),区分 `output/`(SaveImage 永久)与 `temp/`(PreviewImage 临时)
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## 2. 下版本目标:v0.1.1 ——「对话产物进画布」
|
|
73
|
+
|
|
74
|
+
### 2.1 一句话目标
|
|
75
|
+
让 **agent 在对话中生成的内容(文本 / 图片)直接作为 ComfyUI 工作流的节点输入**,形成「对话创意 → 画布产出」的闭环,无需手动切工具。
|
|
76
|
+
|
|
77
|
+
### 2.2 核心新增项(本版本必做)
|
|
78
|
+
|
|
79
|
+
#### A. 新增工具 `comfyui_attach_image` —— 把图片送进画布
|
|
80
|
+
- **职责**:把 agent 本机的一张图片上传进 ComfyUI 的 `input/` 目录,并(可选)更新指定 `LoadImage` 节点的文件名,使其立刻生效
|
|
81
|
+
- **实现位置**:**host 端(`lib/index.js`)** —— 走 ComfyUI **原生 `POST /upload/image`** 端点,**不经过 bridge**(源文件在 agent 本机,bridge 无本机路径概念)
|
|
82
|
+
- **为什么在 host**:云端 ComfyUI 场景下「DSH 机器」与「ComfyUI 机器」不是同一台,文件读取与上传必须由 host 从本机发起;这正是 ComfyUI 官方 web UI 传图的同一条路
|
|
83
|
+
- **接口草案**:
|
|
84
|
+
```js
|
|
85
|
+
comfyui_attach_image({
|
|
86
|
+
image: string // 本机图片绝对路径(必填)
|
|
87
|
+
nodeId?: number // 可选:指定 LoadImage 节点,上传后自动 set_param 指向新文件名
|
|
88
|
+
})
|
|
89
|
+
// 返回 { filename, subfolder, nodeId?, updated? }
|
|
90
|
+
```
|
|
91
|
+
- **要点**:上传后返回的文件名要与 `LoadImage` 节点 widget `image` 期望的值一致(ComfyUI `/upload/image` 返回的 `name` + `subfolder` 拼进 filename);可组合 `set_param` 复用现有路径
|
|
92
|
+
- **验收**:对话里生成/出现一张图 → `attach_image` → 画布上 LoadImage 显示该图 → `run` 能正常走通
|
|
93
|
+
|
|
94
|
+
#### B. 新增工具 `comfyui_inject_text` —— 把文本注入节点(含建源连线)
|
|
95
|
+
- **职责**:把一段文本写入某节点 widget(如 CLIPTextEncode 的 `text`);或先 `add_node` 一个 text/primitive 源节点、填值、再 `connect` 到目标节点的输入——即「对话文本作为独立可连线源」
|
|
96
|
+
- **实现位置**:**host 注册工具 + bridge 加 command 分支**(画布操作必须由前端 LiteGraph 执行)
|
|
97
|
+
- host `lib/index.js`:定义 `comfyui_inject_text`,拼 command 下发
|
|
98
|
+
- bridge.js `executeCommand` 新增 `inject_text` 分支:`add_node`(若给 class)→ `set_param` → `connect`
|
|
99
|
+
- **本质**:现有 `add_node + set_param + connect` 的组合封装,提供「一步到位」的语义
|
|
100
|
+
- **日常替代**:若只改已有节点 widget(多数文本场景),`set_param` 已够;`inject_text` 是「要新建源并连线」时的便捷入口
|
|
101
|
+
- **接口草案**:
|
|
102
|
+
```js
|
|
103
|
+
comfyui_inject_text({
|
|
104
|
+
text: string // 要注入的文本
|
|
105
|
+
nodeId?: number // 可选:目标节点,直接写其 widget(如 text/prompt)
|
|
106
|
+
widgetKey?:string // 可选:目标 widget 名,默认 "text"
|
|
107
|
+
newClass?: string // 可选:先新建此类型节点(如 "StringConcatenate")作源
|
|
108
|
+
targetId?: number // 可选:把新源节点 connect 到目标节点输入
|
|
109
|
+
targetSlot?:number // 可选:目标输入槽位
|
|
110
|
+
})
|
|
111
|
+
```
|
|
112
|
+
- **验收**:对话生成一段提示词 → `inject_text` → 画布出现新文本节点并接到 CLIP 类节点 → `run` 生效
|
|
113
|
+
|
|
114
|
+
### 2.3 架构边界原则(重要,写死)
|
|
115
|
+
| 操作类型 | 归属 | 执行路径 |
|
|
116
|
+
|---|---|---|
|
|
117
|
+
| 文件传输(上传图片到 input/) | **host** | 直连 ComfyUI 原生 `/upload/image`,不经过 bridge |
|
|
118
|
+
| 画布节点操作(增删连改) | **host 注册 + bridge 分支** | `/dsh-bridge/command` → 前端 LiteGraph 执行 |
|
|
119
|
+
| 读取执行结果 / 出图 | **host** | 直连 `/history/:id` + `/view` |
|
|
120
|
+
|
|
121
|
+
> 一句话:**凡是「本机文件 ↔ ComfyUI 文件」走 host + 原生 API;凡是「动画布 / LiteGraph」走 bridge command;读结果走原生 API。** 不要跨层。
|
|
122
|
+
|
|
123
|
+
### 2.4 其他候选(按优先级,时间允许再做)
|
|
124
|
+
1. `comfyui_export_api` —— 把当前画布导出为 **API 格式 workflow JSON**(供 MCP comfy-cli `run_workflow` 无人值守批量用)—— 打通「画布 ↔ MCP」衔接
|
|
125
|
+
2. `batch_run` 增强 —— 支持 zip 式参数矩阵(多字段同步扫)+ 返回每组的 `get_outputs` 摘要
|
|
126
|
+
3. 授权审批增强 —— 越权高的命令(`load_workflow`/`run`/`upgrade`)走 pending 审批流(现有授权遮罩已就位,补命令级钩子)
|
|
127
|
+
4. README 补「对话产物进画布」用法章节 + 录一段演示 GIF
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 3. 实施步骤(新会话照此执行)
|
|
132
|
+
|
|
133
|
+
```text
|
|
134
|
+
step 1 读本文档 + 两张架构图 → 跑 npm run check 确认基线
|
|
135
|
+
step 2 实现 A:comfyui_attach_image(host lib/index.js)
|
|
136
|
+
- 复用 bridgeToken/bridgeHeaders 风格;用 fetch 打 /upload/image
|
|
137
|
+
- multipart 构造(FormData/Blob,Node 22 可用)
|
|
138
|
+
- 注意 ComfyUI /upload/image 的字段:image=file, type=input, overwrite=false
|
|
139
|
+
- 可选:返回后调 set_param 更新 LoadImage
|
|
140
|
+
step 3 实现 B:comfyui_inject_text
|
|
141
|
+
- bridge.js executeCommand 加 inject_text 分支
|
|
142
|
+
- host 加 comfyui_inject_text 工具(含参数 schema)
|
|
143
|
+
step 4 三处同步(源码 → node_modules → E 盘 custom_nodes)
|
|
144
|
+
- node --check(lib 两个 JS + bridge.js)
|
|
145
|
+
- python -m py_compile(__init__.py)
|
|
146
|
+
step 5 真机验证(ComfyUI 运行中)
|
|
147
|
+
- attach_image:本机放一张图 → 上传 → 画布 LoadImage 可见 → run 走通
|
|
148
|
+
- inject_text:生成一段提示词 → 注入新节点并连线 → run 生效
|
|
149
|
+
step 6 README(中/英):
|
|
150
|
+
- What you get 表加新工具
|
|
151
|
+
- 新增「对话产物进画布」小节(含 attach_image / inject_text 用法)
|
|
152
|
+
- 12 → 14 工具计数
|
|
153
|
+
step 7 npm run check + python -m py_compile → git commit → 三处同步
|
|
154
|
+
step 8 发布(见 §4)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
**新增/修改文件清单(预计)**
|
|
158
|
+
- `lib/index.js`:+`comfyui_attach_image`,+`comfyui_inject_text`
|
|
159
|
+
- `comfyui-bridge/ComfyUI-DSH-Canvas/entry/bridge.js`:+`inject_text` 分支
|
|
160
|
+
- `README.md` / `README.zh.md`:工具表与用法
|
|
161
|
+
- `docs/architecture-flow.html`:可选补「对话产物 → 画布」一段
|
|
162
|
+
|
|
163
|
+
---
|
|
164
|
+
|
|
165
|
+
## 4. 发布与生态(上版本遗留待办,可一并收尾)
|
|
166
|
+
|
|
167
|
+
- **npm publish**(让 `dsh plugin add dsh-comfyui-canvas` 可装)
|
|
168
|
+
```powershell
|
|
169
|
+
cd F:\Deepseek-harness\projects\dsh-comfyui-canvas # 或新仓库本地路径
|
|
170
|
+
npm config set registry https://registry.npmjs.org # 当前指向 npmmirror,必须切回官方
|
|
171
|
+
npm adduser # 浏览器授权登录
|
|
172
|
+
npm whoami # 确认用户名
|
|
173
|
+
npm publish # 发布(tarball 已验证干净:无 pyc/缓存)
|
|
174
|
+
```
|
|
175
|
+
- 已确认:包名 `dsh-comfyui-canvas` 在 npm **未被占用**;package.json `files` 已排除 `__pycache__`/`*.pyc`
|
|
176
|
+
- **awesome-dsh-plugin 收录**(让 `dshmarket` 市场可见)
|
|
177
|
+
- 前置:仓库已加 `dsh-plugin` topic ✅;`dsh.bundle` manifest ✅;commit ≥10 ✅;需**仓库创建满 1 天**(CI 硬性检查,2026-09-01 后可提)
|
|
178
|
+
- 提交 PR:在 [awesome-dsh-plugin](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin) 的 `data/plugins/` 加一个文件 `wbin0001__dsh-comfyui-canvas.yml`
|
|
179
|
+
- 条目草稿(description 必须属实,会被对照代码检查):
|
|
180
|
+
```yaml
|
|
181
|
+
url: https://github.com/wbin0001/dsh-comfyui-canvas
|
|
182
|
+
name: wbin0001/dsh-comfyui-canvas
|
|
183
|
+
category: tools
|
|
184
|
+
description:
|
|
185
|
+
en: 'Embed your ComfyUI as a split-screen canvas in DSH Web and let the agent operate it live: read/write workflows, edit nodes, wire links, run, fetch output images, and batch-sweep parameters straight from the chat.'
|
|
186
|
+
zh: '把 ComfyUI 作为分屏画布嵌入 DSH,agent 可视化操作:读写工作流、改节点、连线、运行、取回出图、批量扫参,全程不离开对话。'
|
|
187
|
+
```
|
|
188
|
+
> 注意:description 含 `: `(冒号加空格)时必须加引号,详见该仓库 `contributing.md`
|
|
189
|
+
|
|
190
|
+
---
|
|
191
|
+
|
|
192
|
+
## 5. 已知事项 / 坑(写死,避免重踩)
|
|
193
|
+
|
|
194
|
+
- **运行时副本版本号滞后**:node_modules 副本是 0.2.0、源码是 0.1.0——功能一致,仅版本号不同;`npm publish` 后如需统一可重装官方版。当前不处理
|
|
195
|
+
- **DSH 侧工具热加载**:host 工具的 `name`/`description` 改动需**重启 DSH** 才注入 agent 工具列表;bridge 改动需**重启 ComfyUI/重开画布页**
|
|
196
|
+
- **was-ns 依赖**:ComfyUI venv 的 NumPy 已固定 `<2.4`(启动器会自动校验降级),不要随意升级 numpy,否则 was-ns/numba 报错
|
|
197
|
+
- **Cloud ComfyUI**:`networkMode` 支持 cloud-selfhosted/saas;云端需自行部署桥接节点 + DSH 可达 + token 两端一致
|
|
198
|
+
- **安全基线**:`/dsh-bridge/*` 保护 host→bridge 通道(token);前端上报(/report、result)不 gate;report ≤8MB;多标签 clientId 定向;命令无 client_id 时拒绝
|
|
199
|
+
- **版本号策略**:v0.1.0 已发布;下版升 **0.1.1**(用户指定——新增工具作为小版本增量推进,不跨次版本号)
|