@openbone/mcp 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ import { type ScriptHost } from "@openbone/runtime";
3
+ import type { ProjectSession } from "./session.js";
4
+ export declare function registerExtraTools(server: McpServer, session: ProjectSession, scriptHost: () => Promise<ScriptHost>): void;
@@ -0,0 +1,355 @@
1
+ import { writeFile } from "node:fs/promises";
2
+ import { decodeImage } from "./image-load.js";
3
+ import path from "node:path";
4
+ import { createCanvas } from "@napi-rs/canvas";
5
+ import { guessMouthMapping, lipSync, meshUvAffine, slotAttachments, traceMesh } from "@openbone/core";
6
+ import { OpenBoneInstance, Skeleton, applyAnimation, poseChainFrom, solvePose } from "@openbone/runtime";
7
+ import { z } from "zod";
8
+ import { atlasSettingsSchema } from "./atlas-schema.js";
9
+ import { EXPORT_FORMATS, assetBytes, exportBaseName, exportFile } from "./export.js";
10
+ /**
11
+ * MCP tools for what the editor does beyond plain ops: every export format, automatic mesh tracing, lip sync, the
12
+ * Pose tool (IK drag) and skeleton stats.
13
+ */
14
+ const text = (v) => ({ content: [{ type: "text", text: typeof v === "string" ? v : JSON.stringify(v, null, 1) }] });
15
+ const fail = (e) => ({ isError: true, content: [{ type: "text", text: `Error: ${e.message}` }] });
16
+ /** Mono samples of a PCM WAV (8/16/24/32-bit int or 32-bit float). */
17
+ function decodeWav(b) {
18
+ if (b.toString("ascii", 0, 4) !== "RIFF" || b.toString("ascii", 8, 12) !== "WAVE")
19
+ throw new Error("Lip sync reads WAV files (convert MP3/OGG to WAV first)");
20
+ let p = 12;
21
+ let fmt = { format: 1, channels: 1, rate: 44100, bits: 16 };
22
+ while (p + 8 <= b.length) {
23
+ const id = b.toString("ascii", p, p + 4);
24
+ const size = b.readUInt32LE(p + 4);
25
+ if (id === "fmt ")
26
+ fmt = { format: b.readUInt16LE(p + 8), channels: b.readUInt16LE(p + 10), rate: b.readUInt32LE(p + 12), bits: b.readUInt16LE(p + 22) };
27
+ else if (id === "data") {
28
+ const bytes = fmt.bits / 8;
29
+ const n = Math.floor(size / bytes / fmt.channels);
30
+ const samples = new Float32Array(n);
31
+ for (let i = 0; i < n; i++) {
32
+ let v = 0;
33
+ for (let c = 0; c < fmt.channels; c++) {
34
+ const o = p + 8 + (i * fmt.channels + c) * bytes;
35
+ v +=
36
+ fmt.format === 3 ? b.readFloatLE(o) : fmt.bits === 8 ? (b[o] - 128) / 128 : fmt.bits === 16 ? b.readInt16LE(o) / 32768 : fmt.bits === 24 ? b.readIntLE(o, 3) / 8388608 : b.readInt32LE(o) / 2147483648;
37
+ }
38
+ samples[i] = v / fmt.channels;
39
+ }
40
+ return { samples, rate: fmt.rate };
41
+ }
42
+ p += 8 + size + (size % 2);
43
+ }
44
+ throw new Error("WAV file has no audio data");
45
+ }
46
+ const stepSchema = z.object({
47
+ advance: z.number().min(0).optional(),
48
+ setInputs: z.record(z.string(), z.union([z.boolean(), z.number(), z.string()])).optional(),
49
+ fire: z.array(z.string()).optional(),
50
+ pointer: z.object({ type: z.enum(["down", "up", "move"]), x: z.number(), y: z.number() }).optional(),
51
+ });
52
+ export function registerExtraTools(server, session, scriptHost) {
53
+ const commit = async () => session.maybeAutosave();
54
+ // ---------------- export ----------------
55
+ server.registerTool("export_file", {
56
+ description: "Export the project like the editor's Export dialog (same code as `openbone export`). Formats: obone (whole project; embed: true makes it self-contained), json / obonb (runtime data: picked animations, state machines; obonb is binary; pack: true embeds a texture atlas), atlas (ZIP: runtime JSON + atlas pages as PNG/JPEG files; atlas.scales gives one data file + page set per scale), png / jpeg (one frame, or a ZIP of every frame with sequence: true), apng, gif, psd (one layer per frame), avi / mov (Motion-JPEG video with the run's sound), mp4 (H.264 + AAC) / webm (VP9 + Opus) through ffmpeg on the PATH, spritesheet (engine packs: engine unity | godot | flutter | generic; one clip per animation), web (the <openbone-player> library + animation + demo page as a ZIP, or one self-contained HTML file with singleFile). Atlas settings (`atlas`, same as pack_atlas): padding, maxSize / maxWidth / maxHeight, minWidth / minHeight, powerOfTwo, square, multipleOf4, rotate, scale or scales, stripWhitespace, polygons, premultipliedAlpha, alias, ignoreBlank, alphaThreshold, edgePadding, duplicatePadding, bleed, filterMin / filterMag, wrapX / wrapY, pixelFormat, pageFormat (png | jpeg) + jpegQuality, resample, debug. Frames come from `animation` (default the first) or a state machine run (`stateMachine` + `steps`). Writes to `out` (default next to the project) and returns the path and size.",
57
+ inputSchema: {
58
+ format: z.enum(EXPORT_FORMATS),
59
+ out: z.string().optional().describe("Output path (default <project>.<ext> next to the project)"),
60
+ animation: z.string().optional(),
61
+ stateMachine: z.string().optional(),
62
+ steps: z.array(stepSchema).optional(),
63
+ skin: z.string().optional(),
64
+ animations: z.array(z.string()).optional().describe("json/obonb/atlas: animations to keep (names; default all marked for export). spritesheet: clips (default all exported animations)"),
65
+ stateMachines: z.array(z.string()).optional().describe("json/obonb/atlas: state machines to keep (default all)"),
66
+ duration: z.number().positive().max(60).optional().describe("Seconds (default the animation's length, or 2 for state machines)"),
67
+ fps: z.number().int().min(1).max(60).default(30),
68
+ scale: z.number().positive().max(4).default(1),
69
+ transparent: z.boolean().default(false).describe("Transparent background (png, apng, gif, psd, spritesheet)"),
70
+ sequence: z.boolean().default(false).describe("png/jpeg: every frame in a ZIP instead of one frame"),
71
+ time: z.number().min(0).default(0).describe("png/jpeg single frame: time in the animation"),
72
+ engine: z.enum(["unity", "godot", "flutter", "generic"]).default("generic"),
73
+ singleFile: z.boolean().default(false).describe("web: one HTML file (works from file://)"),
74
+ minify: z.boolean().default(true).describe("web: minified player"),
75
+ binary: z.boolean().optional().describe("web: animation as .obonb (default true); atlas: runtime data as .obonb instead of .json (default false)"),
76
+ pack: z.boolean().default(false).describe("json/obonb/web: pack images into an embedded texture atlas"),
77
+ audio: z.boolean().default(true).describe("avi/mov/mp4/webm: include the sound the run plays (WAV decodes always; MP3/OGG/FLAC and the mp4/webm encoders need ffmpeg on the PATH)"),
78
+ quality: z.number().min(0).max(1).optional().describe("mp4/webm: quality 0..1 (default 0.85)"),
79
+ embed: z
80
+ .boolean()
81
+ .default(false)
82
+ .describe("obone/json/obonb: embed every image and audio file in the file, so it opens on its own anywhere (e.g. a copy to open in the web editor, which can't read files next to it)"),
83
+ atlas: atlasSettingsSchema
84
+ .optional()
85
+ .describe("Texture atlas settings for atlas / pack"),
86
+ },
87
+ }, async (a) => {
88
+ try {
89
+ const r = await exportFile(session, a, scriptHost);
90
+ const out = path.resolve(session.baseDir, a.out ?? `${exportBaseName(session)}${r.suffix}.${r.ext}`);
91
+ await writeFile(out, r.bytes);
92
+ return text(`Saved ${out} (${(r.bytes.length / 1024).toFixed(1)} KB)${r.note}`);
93
+ }
94
+ catch (e) {
95
+ return fail(e);
96
+ }
97
+ });
98
+ // ---------------- trace mesh ----------------
99
+ server.registerTool("trace_mesh", {
100
+ description: "Trace a mesh from an image's visible pixels (like the editor's Trace mesh / Re-trace outline): less transparent area is drawn and the mesh can bend. On a region: replaces it with a mesh in the same place, size, rotation and tint. On a mesh: re-traces it where it sits (same image mapping) and binds it again to the same bones with automatic weights (or to `bones`), with as many influences per vertex as before; its deform keys no longer apply. maxHull: outline detail cap (16 simple, 48 balanced, 96+ detailed); tolerance: how closely the outline follows the edge in image pixels (default 1% of the image's longer side; 1–2 for detailed edges like hair or feet); interior: about N×N inner vertices for bending (0 = outline only).",
101
+ inputSchema: {
102
+ slot: z.string(),
103
+ attachment: z.string().optional().describe("Region or mesh attachment name (default the slot's current one)"),
104
+ skin: z.string().optional().describe("Skin holding the attachment (default: where it is found)"),
105
+ maxHull: z.number().int().min(3).max(200).default(48),
106
+ interior: z.number().int().min(0).max(32).default(4),
107
+ padding: z.number().min(0).max(32).default(2),
108
+ threshold: z.number().int().min(0).max(254).default(8).describe("Alpha above this counts as visible"),
109
+ tolerance: z
110
+ .number()
111
+ .positive()
112
+ .max(64)
113
+ .optional()
114
+ .describe("Outline simplification in image pixels (smaller follows the edge more closely; default 1% of the image's longer side). maxHull only caps the count."),
115
+ bones: z.array(z.string()).min(1).optional().describe("Re-tracing a mesh: bind to these bones instead of the ones it used"),
116
+ falloff: z.number().positive().optional().describe("Re-tracing a weighted mesh: automatic weights falloff (default 2; higher = sharper)"),
117
+ },
118
+ }, async (a) => {
119
+ try {
120
+ const doc = session.doc;
121
+ const slot = doc.skeleton.slots.find((s) => s.name === a.slot || s.id === a.slot);
122
+ if (!slot)
123
+ throw new Error(`Slot "${a.slot}" not found`);
124
+ const name = a.attachment ?? slot.attachment;
125
+ const skinId = a.skin ? doc.skeleton.skins.find((k) => k.id === a.skin || k.name === a.skin)?.id : undefined;
126
+ if (a.skin && !skinId)
127
+ throw new Error(`Skin "${a.skin}" not found`);
128
+ const found = slotAttachments(doc, slot.id).find((x) => x.name === name && (!skinId || x.skin.id === skinId));
129
+ const att = found?.attachment;
130
+ if (!att || (att.type !== "region" && att.type !== "mesh"))
131
+ throw new Error(`"${name}" on slot "${slot.name}" is not a region or mesh attachment`);
132
+ if (att.type === "mesh" && att.linked)
133
+ throw new Error(`"${att.name}" is a linked mesh: it uses its parent's shape, so re-trace the parent ("${att.linked.mesh}")`);
134
+ if (att.type === "mesh") {
135
+ const users = doc.skeleton.skins.flatMap((k) => Object.values(k.attachments[slot.id] ?? {}).filter((x) => x.type === "mesh" && x.linked?.mesh === att.name && (x.linked.skin ?? found.skin.id) === found.skin.id));
136
+ if (users.length)
137
+ throw new Error(`Linked meshes use this mesh's shape (${users.map((u) => u.name).join(", ")}): re-tracing would break them`);
138
+ }
139
+ const img = doc.images.find((i) => i.id === att.image);
140
+ if (!img?.src)
141
+ throw new Error("The image has no file of its own (packed in an atlas)");
142
+ const el = await decodeImage(await session.readImageSrc(img.src));
143
+ const c = createCanvas(el.width, el.height);
144
+ const g = c.getContext("2d");
145
+ g.drawImage(el, 0, 0);
146
+ const px = g.getImageData(0, 0, el.width, el.height).data;
147
+ const alpha = new Uint8ClampedArray(el.width * el.height);
148
+ for (let i = 0; i < alpha.length; i++)
149
+ alpha[i] = px[i * 4 + 3];
150
+ const spacing = a.interior ? Math.max(6, Math.min(el.width, el.height) / a.interior) : 0;
151
+ const opts = { maxHull: a.maxHull, padding: a.padding, threshold: a.threshold, interior: spacing, tolerance: a.tolerance };
152
+ let placement;
153
+ let rebind = null;
154
+ if (att.type === "region")
155
+ placement = { x: att.x, y: att.y, width: att.width, height: att.height, scaleX: att.scaleX, scaleY: att.scaleY, rotation: att.rotation };
156
+ else {
157
+ const affine = meshUvAffine(att.vertices, att.uvs);
158
+ if (!affine)
159
+ throw new Error("Could not work out where this mesh's image sits");
160
+ placement = { x: 0, y: 0, width: 1, height: 1, uvAffine: affine };
161
+ const used = att.weights?.length ? [...new Set(att.weights.flatMap((inf) => inf.map((w) => w.bone)))] : [];
162
+ const bones = a.bones ?? used;
163
+ if (bones.length)
164
+ rebind = { bones, maxInfluences: att.weights?.length ? Math.min(4, Math.max(1, ...att.weights.map((inf) => inf.length))) : 2 };
165
+ }
166
+ const mesh = traceMesh(alpha, el.width, el.height, placement, opts);
167
+ if (!mesh)
168
+ throw new Error("Nothing visible to trace in this image");
169
+ session.store.transaction(att.type === "mesh" ? "re-trace mesh" : "trace mesh", (s) => {
170
+ s.apply("deleteAttachment", { slot: slot.id, attachment: att.name, skin: found.skin.id });
171
+ s.apply("addMesh", { slot: slot.id, skin: found.skin.id, name: att.name, image: att.image, ...mesh, color: att.color });
172
+ if (rebind)
173
+ s.apply("autoWeights", { slot: slot.id, attachment: att.name, skin: found.skin.id, ...rebind, ...(a.falloff ? { falloff: a.falloff } : {}) });
174
+ });
175
+ await commit();
176
+ const boundTo = rebind?.bones.map((id) => doc.skeleton.bones.find((b) => b.id === id || b.name === id)?.name ?? id);
177
+ return text({ ok: true, retraced: att.type === "mesh", outline: mesh.hull, inner: mesh.vertices.length / 2 - mesh.hull, triangles: mesh.triangles.length / 3, ...(boundTo ? { boundTo } : {}) });
178
+ }
179
+ catch (e) {
180
+ return fail(e);
181
+ }
182
+ });
183
+ // ---------------- lip sync ----------------
184
+ server.registerTool("lip_sync", {
185
+ description: "Lip sync: analyse a sound (WAV) and key a mouth slot's attachments to it, stepped, starting at `start` seconds in `animation` (created if missing); the sound is added to the animation's audio track. Six shapes: rest, closed (M/B/P), small, open, wide, round; the slot's attachment names are guessed per shape unless `mapping` is given.",
186
+ inputSchema: {
187
+ sound: z.string().describe("Sound name or id"),
188
+ slot: z.string().describe("Mouth slot with one attachment per mouth shape"),
189
+ animation: z.string().optional().describe("Animation name (default: a new 'lipsync <sound>' animation)"),
190
+ start: z.number().min(0).default(0),
191
+ mapping: z.record(z.enum(["rest", "closed", "small", "open", "wide", "round"]), z.string()).optional().describe("Shape → attachment name"),
192
+ },
193
+ }, async (a) => {
194
+ try {
195
+ const doc = session.doc;
196
+ const sound = doc.audio?.sounds.find((s) => s.name === a.sound || s.id === a.sound);
197
+ if (!sound)
198
+ throw new Error(`Sound "${a.sound}" not found`);
199
+ const asset = doc.audio?.assets.find((x) => x.id === sound.asset);
200
+ if (!asset)
201
+ throw new Error("The sound has no audio file");
202
+ const slot = doc.skeleton.slots.find((s) => s.name === a.slot || s.id === a.slot);
203
+ if (!slot)
204
+ throw new Error(`Slot "${a.slot}" not found`);
205
+ const names = slotAttachments(doc, slot.id).map((x) => x.name);
206
+ if (names.length < 2)
207
+ throw new Error("The mouth slot needs two or more attachments (one per mouth shape)");
208
+ const map = { ...guessMouthMapping(names), ...(a.mapping ?? {}) };
209
+ const { samples, rate } = decodeWav(await assetBytes(session, asset.src));
210
+ const keys = lipSync(samples, rate, { fps: doc.fps });
211
+ const speed = sound.rate ?? 1;
212
+ const fps = doc.fps;
213
+ const snap = (t) => Math.round(t * fps) / fps;
214
+ const end = a.start + samples.length / rate / speed;
215
+ let animName = a.animation ?? `lipsync ${sound.name}`;
216
+ session.store.transaction("lip sync", (s) => {
217
+ let anim = s.doc.animations.find((x) => x.name === animName || x.id === animName);
218
+ if (!anim) {
219
+ s.apply("createAnimation", { name: animName });
220
+ anim = s.doc.animations.find((x) => x.name === animName);
221
+ }
222
+ animName = anim.name;
223
+ const tl = anim.timelines.find((t) => t.target === "slot" && t.id === slot.id && t.property === "attachment");
224
+ for (const k of [...(tl?.keys ?? [])])
225
+ if (k.time >= a.start - 1e-6 && k.time <= end + 1e-6)
226
+ s.apply("deleteKey", { animation: anim.id, target: "slot", id: slot.id, property: "attachment", time: k.time });
227
+ s.apply("setKeys", { animation: anim.id, target: "slot", id: slot.id, property: "attachment", keys: keys.map((k) => ({ time: snap(a.start + k.time / speed), value: map[k.shape], curve: "stepped" })) });
228
+ s.apply("addAudioKey", { animation: anim.id, sound: sound.id, time: a.start });
229
+ if (anim.duration > 0 && end > anim.duration)
230
+ s.apply("updateAnimation", { animation: anim.id, duration: snap(end) });
231
+ });
232
+ await commit();
233
+ const used = [...new Set(keys.map((k) => k.shape))].map((sh) => `${sh}→${map[sh]}`).join(", ");
234
+ return text({ ok: true, animation: animName, keys: keys.length, seconds: Math.round((end - a.start) * 100) / 100, shapes: used });
235
+ }
236
+ catch (e) {
237
+ return fail(e);
238
+ }
239
+ });
240
+ // ---------------- pose tool ----------------
241
+ server.registerTool("pose_bones", {
242
+ description: "Pose tool (inverse kinematics without a constraint): bend `bone` and up to `chain`-1 of its parents so the bone's tip reaches a target point, given in ARTBOARD pixels (y-down, top-left origin; the same space as render_frame). Without `animation` it changes the setup pose; with `animation` (+ `time`) it sets rotation keys there. The root bone never moves.",
243
+ inputSchema: {
244
+ bone: z.string(),
245
+ x: z.number(),
246
+ y: z.number(),
247
+ chain: z.number().int().min(1).max(32).default(3),
248
+ animation: z.string().optional(),
249
+ time: z.number().min(0).default(0),
250
+ },
251
+ }, async (a) => {
252
+ try {
253
+ const doc = session.doc;
254
+ const sk = new Skeleton(doc);
255
+ const anim = a.animation ? doc.animations.find((x) => x.name === a.animation || x.id === a.animation) : undefined;
256
+ if (a.animation && !anim)
257
+ throw new Error(`Animation "${a.animation}" not found`);
258
+ sk.setToSetupPose();
259
+ if (anim)
260
+ applyAnimation(sk, anim, a.time);
261
+ sk.updateWorldTransform();
262
+ const bone = sk.findBone(a.bone) ?? sk.bones.find((b) => b.data.id === a.bone);
263
+ if (!bone)
264
+ throw new Error(`Bone "${a.bone}" not found`);
265
+ const [tx, ty] = new OpenBoneInstance(doc).artboardToSkeleton(a.x, a.y);
266
+ const chain = poseChainFrom(bone, a.chain);
267
+ const before = new Map(chain.map((b) => [b, b.rotation]));
268
+ const miss = solvePose({ bones: chain, local: [Math.max(bone.data.length, 0), 0] }, tx, ty);
269
+ const changed = chain.filter((b) => Math.abs(b.rotation - before.get(b)) > 1e-4);
270
+ session.store.transaction("pose bones", (s) => {
271
+ for (const b of changed) {
272
+ const rotation = Math.round(b.rotation * 100) / 100;
273
+ if (anim)
274
+ s.apply("setKeys", { animation: anim.id, target: "bone", id: b.data.id, property: "rotation", keys: [{ time: a.time, value: rotation }] });
275
+ else
276
+ s.apply("updateBone", { bone: b.data.id, rotation });
277
+ }
278
+ });
279
+ await commit();
280
+ return text({ ok: true, bones: changed.map((b) => `${b.data.name} ${Math.round(b.rotation * 10) / 10}°`), reached: miss < 0.5, distanceLeft: Math.round(miss * 10) / 10 });
281
+ }
282
+ catch (e) {
283
+ return fail(e);
284
+ }
285
+ });
286
+ // ---------------- stats ----------------
287
+ server.registerTool("get_stats", {
288
+ description: "Skeleton statistics like the editor's Stats panel: bones, slots, constraints, visible/total attachments, vertices, weighted vertex transforms, triangles, clipping masks, estimated draw calls, images and image area.",
289
+ inputSchema: { skin: z.string().optional() },
290
+ }, async (a) => {
291
+ try {
292
+ const d = session.doc;
293
+ const sk = d.skeleton;
294
+ const s = new Skeleton(d);
295
+ if (a.skin)
296
+ s.setSkin(a.skin);
297
+ s.updateWorldTransform();
298
+ let totalAtt = 0;
299
+ for (const skin of sk.skins)
300
+ for (const atts of Object.values(skin.attachments))
301
+ totalAtt += Object.keys(atts).length;
302
+ let vertices = 0;
303
+ let transforms = 0;
304
+ let triangles = 0;
305
+ let clipping = 0;
306
+ let visible = 0;
307
+ for (const slot of s.slots) {
308
+ const att = s.getAttachment(slot);
309
+ if (!att)
310
+ continue;
311
+ visible++;
312
+ if (att.type === "clipping")
313
+ clipping++;
314
+ if (att.type === "mesh") {
315
+ const n = att.vertices.length / 2;
316
+ vertices += n;
317
+ transforms += att.weights ? att.weights.reduce((t, w) => t + w.length, 0) : n;
318
+ triangles += att.triangles.length / 3;
319
+ }
320
+ else if (att.type === "region" || att.type === "sequence") {
321
+ vertices += 4;
322
+ transforms += 4;
323
+ triangles += 2;
324
+ }
325
+ }
326
+ let calls = 0;
327
+ let last = "";
328
+ for (const it of s.drawList()) {
329
+ const key = `${it.texture?.id ?? it.image?.id ?? "solid"}|${it.blend}|${it.clip ? "clip" : ""}`;
330
+ if (key !== last)
331
+ calls++;
332
+ last = key;
333
+ }
334
+ return text({
335
+ bones: sk.bones.length,
336
+ slots: sk.slots.length,
337
+ constraints: { ik: sk.ik.length, transform: sk.transform.length, path: sk.path.length, physics: sk.physics.length, sliders: sk.sliders?.length ?? 0 },
338
+ attachments: { visible, total: totalAtt },
339
+ vertices,
340
+ vertexTransforms: transforms,
341
+ triangles,
342
+ clippingMasks: clipping,
343
+ drawCallsEstimate: calls,
344
+ images: d.images.length,
345
+ atlasPages: d.atlas?.pages.length ?? 0,
346
+ imageMegapixels: Math.round((d.images.reduce((t, i) => t + i.width * i.height, 0) / 1e6) * 100) / 100,
347
+ animations: d.animations.length,
348
+ stateMachines: d.stateMachines.length,
349
+ });
350
+ }
351
+ catch (e) {
352
+ return fail(e);
353
+ }
354
+ });
355
+ }
@@ -0,0 +1,3 @@
1
+ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ import type { ProjectSession } from "./session.js";
3
+ export declare function registerImageTools(server: McpServer, session: ProjectSession): void;
@@ -0,0 +1,176 @@
1
+ import { readFile, writeFile } from "node:fs/promises";
2
+ import { decodeImage } from "./image-load.js";
3
+ import path from "node:path";
4
+ import { createCanvas } from "@napi-rs/canvas";
5
+ import { z } from "zod";
6
+ import { alphaBounds, crop, differenceMatte, featherEdges, flip, keepLargest, keyColor } from "./image-ops.js";
7
+ /**
8
+ * process_image: prepare art for rigging without leaving the MCP — key out a flat background (generated art usually
9
+ * comes on plain grey), cut a layer out by comparing a picture with a cleaned copy of it, trim to the visible pixels,
10
+ * crop, resize, flip, feather edges for seamless tiling — then save the PNG and import it (or replace an image in place).
11
+ */
12
+ const text = (v) => ({ content: [{ type: "text", text: typeof v === "string" ? v : JSON.stringify(v, null, 1) }] });
13
+ const fail = (e) => ({ isError: true, content: [{ type: "text", text: `Error: ${e.message}` }] });
14
+ const boxSchema = z.object({ x: z.number(), y: z.number(), width: z.number().positive(), height: z.number().positive() });
15
+ const opSchema = z.discriminatedUnion("op", [
16
+ z.object({
17
+ op: z.literal("keyColor"),
18
+ color: z.string().optional().describe("Background colour #rrggbb (default: the median of the border)"),
19
+ sampleFrom: z.enum(["all", "top"]).default("all").describe("Border to sample the default colour from; 'top' when the art touches the other edges"),
20
+ tolerance: z.number().min(0).max(441).default(18),
21
+ softness: z.number().min(1).max(441).default(34).describe("Colour distance over which pixels fade in; large values (200+) turn brightness into transparency, e.g. mist or glow keyed off black"),
22
+ keepBrighterThan: z.number().min(0).max(255).optional().describe("Keep pixels at least this bright opaque (white details on a light background)"),
23
+ despill: z.boolean().optional().describe("Remove the key colour's cast from edges (default: on for strongly coloured keys like #00ff00)"),
24
+ }),
25
+ z.object({
26
+ op: z.literal("differenceMatte"),
27
+ other: z.string().describe("The same picture with the object removed (path, or a project image name/id); resized to match when its size differs"),
28
+ threshold: z.number().min(0).max(1).default(0.06),
29
+ softness: z.number().min(0.001).max(1).default(0.09),
30
+ region: boxSchema.optional().describe("Only cut out inside this box (input pixels)"),
31
+ fadeRows: z.object({ from: z.number(), to: z.number() }).optional().describe("Fade the cut-out to nothing from row `from` to row `to` (where the object meets repainted ground)"),
32
+ keepLargest: z.boolean().default(true),
33
+ }),
34
+ z.object({ op: z.literal("trim"), threshold: z.number().int().min(0).max(254).default(8), padding: z.number().int().min(0).default(0) }),
35
+ z.object({ op: z.literal("crop"), ...boxSchema.shape }),
36
+ z.object({
37
+ op: z.literal("resize"),
38
+ width: z.number().int().positive().optional(),
39
+ height: z.number().int().positive().optional(),
40
+ scale: z.number().positive().optional(),
41
+ }),
42
+ z.object({ op: z.literal("flip"), horizontal: z.boolean().default(true), vertical: z.boolean().default(false) }),
43
+ z.object({
44
+ op: z.literal("featherEdges"),
45
+ left: z.number().min(0).optional(),
46
+ right: z.number().min(0).optional(),
47
+ top: z.number().min(0).optional(),
48
+ bottom: z.number().min(0).optional(),
49
+ }),
50
+ z.object({ op: z.literal("keepLargest"), threshold: z.number().int().min(0).max(254).default(12) }),
51
+ ]);
52
+ export function registerImageTools(server, session) {
53
+ const resolveInput = async (ref) => {
54
+ const img = session.doc.images.find((i) => i.id === ref || i.name === ref);
55
+ if (img) {
56
+ if (!img.src)
57
+ throw new Error(`Image "${ref}" has no file of its own (packed in an atlas)`);
58
+ const src = await session.readImageSrc(img.src);
59
+ return typeof src === "string" ? Buffer.from(src.replace(/^data:[^,]*,/, ""), "base64") : src;
60
+ }
61
+ return readFile(path.resolve(session.baseDir, ref));
62
+ };
63
+ const decode = async (buf, w, h) => {
64
+ const el = await decodeImage(buf);
65
+ const width = w ?? el.width;
66
+ const height = h ?? el.height;
67
+ const c = createCanvas(width, height);
68
+ const g = c.getContext("2d");
69
+ g.drawImage(el, 0, 0, width, height);
70
+ return { data: new Uint8ClampedArray(g.getImageData(0, 0, width, height).data), width, height };
71
+ };
72
+ const toCanvas = (p) => {
73
+ const c = createCanvas(p.width, p.height);
74
+ const g = c.getContext("2d");
75
+ const im = g.createImageData(p.width, p.height);
76
+ im.data.set(p.data);
77
+ g.putImageData(im, 0, 0);
78
+ return c;
79
+ };
80
+ server.registerTool("process_image", {
81
+ description: "Prepare art for rigging: run `ops` in order on an image (a file path or a project image name/id), save the result as a PNG at `out` and optionally import it (`importAs`: a new image of that name) or replace a project image in place (`replace`: regions using it keep their pixel density). Ops: keyColor (remove a flat background colour everywhere, soft edges, no colour fringe; colour defaults to the border median, sampleFrom:'top' for strips like grass that touch the bottom), differenceMatte (cut out what differs from `other`, the same picture with the object painted out — e.g. a tree, to animate it on its own layer; fadeRows fades it where it meets the ground), trim (crop to the visible pixels), crop, resize, flip, featherEdges (fade the sides so tiled copies overlap without seams), keepLargest (drop stray specks). Returns the size and `box`: where the result sits in the input picture (input pixels) — use it to place the layer exactly where it was in the full picture.",
82
+ inputSchema: {
83
+ input: z.string().describe("Image file path (relative to the project folder or absolute) or a project image name/id"),
84
+ out: z.string().describe("Output PNG path (relative to the project folder or absolute)"),
85
+ ops: z.array(opSchema).min(1),
86
+ importAs: z.string().optional().describe("Import the result as a new image with this name"),
87
+ replace: z.string().optional().describe("Replace this project image (name or id) with the result, keeping its id"),
88
+ },
89
+ }, async (a) => {
90
+ try {
91
+ let p = await decode(await resolveInput(a.input));
92
+ const inW = p.width;
93
+ const inH = p.height;
94
+ // The result's place in the input picture (scale tracks resizes).
95
+ let box = { x: 0, y: 0, width: inW, height: inH };
96
+ let sx = 1;
97
+ let sy = 1;
98
+ const notes = [];
99
+ for (const o of a.ops) {
100
+ switch (o.op) {
101
+ case "keyColor": {
102
+ const { color } = keyColor(p, o);
103
+ notes.push(`keyed out #${color.map((c) => c.toString(16).padStart(2, "0")).join("")}`);
104
+ break;
105
+ }
106
+ case "differenceMatte": {
107
+ const raw = await resolveInput(o.other);
108
+ differenceMatte(p, await decode(raw, p.width, p.height), o);
109
+ break;
110
+ }
111
+ case "trim": {
112
+ const b = alphaBounds(p, o.threshold, o.padding);
113
+ if (!b)
114
+ throw new Error("trim: nothing visible is left in the image");
115
+ p = crop(p, b);
116
+ box = { x: box.x + b.x / sx, y: box.y + b.y / sy, width: b.width / sx, height: b.height / sy };
117
+ break;
118
+ }
119
+ case "crop": {
120
+ p = crop(p, o);
121
+ box = { x: box.x + o.x / sx, y: box.y + o.y / sy, width: p.width / sx, height: p.height / sy };
122
+ break;
123
+ }
124
+ case "resize": {
125
+ const w = o.width ?? (o.scale ? Math.round(p.width * o.scale) : o.height ? Math.round((p.width * o.height) / p.height) : p.width);
126
+ const h = o.height ?? (o.scale ? Math.round(p.height * o.scale) : Math.round((p.height * w) / p.width));
127
+ const c = createCanvas(w, h);
128
+ c.getContext("2d").drawImage(toCanvas(p), 0, 0, w, h);
129
+ sx *= w / p.width;
130
+ sy *= h / p.height;
131
+ p = { data: new Uint8ClampedArray(c.getContext("2d").getImageData(0, 0, w, h).data), width: w, height: h };
132
+ break;
133
+ }
134
+ case "flip":
135
+ p = flip(p, o.horizontal, o.vertical);
136
+ break;
137
+ case "featherEdges":
138
+ featherEdges(p, o);
139
+ break;
140
+ case "keepLargest":
141
+ keepLargest(p, o.threshold);
142
+ break;
143
+ }
144
+ }
145
+ const out = path.resolve(session.baseDir, a.out);
146
+ const png = toCanvas(p).toBuffer("image/png");
147
+ await writeFile(out, png);
148
+ const result = {
149
+ ok: true,
150
+ out,
151
+ width: p.width,
152
+ height: p.height,
153
+ box: { x: round(box.x), y: round(box.y), width: round(box.width), height: round(box.height) },
154
+ ...(notes.length ? { notes } : {}),
155
+ };
156
+ if (a.importAs || a.replace) {
157
+ const src = session.filePath ? path.relative(session.baseDir, out).split(path.sep).join("/") : `data:image/png;base64,${png.toString("base64")}`;
158
+ if (a.replace) {
159
+ const img = session.doc.images.find((i) => i.id === a.replace || i.name === a.replace);
160
+ if (!img)
161
+ throw new Error(`Image "${a.replace}" not found`);
162
+ result.replaced = img.id;
163
+ Object.assign(result, session.store.apply("updateImage", { image: img.id, src, width: p.width, height: p.height }));
164
+ }
165
+ else
166
+ result.image = session.store.apply("addImage", { name: a.importAs, src, width: p.width, height: p.height }).id;
167
+ await session.maybeAutosave();
168
+ }
169
+ return text(result);
170
+ }
171
+ catch (e) {
172
+ return fail(e);
173
+ }
174
+ });
175
+ }
176
+ const round = (v) => Math.round(v * 100) / 100;
package/package.json CHANGED
@@ -1,6 +1,70 @@
1
1
  {
2
2
  "name": "@openbone/mcp",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
3
+ "version": "0.3.0",
4
+ "description": "MCP server for OpenBone: let AI agents rig, animate and wire up interactive skeletal animations",
5
+ "keywords": [
6
+ "openbone",
7
+ "skeletal-animation",
8
+ "2d-animation",
9
+ "animation",
10
+ "mcp",
11
+ "model-context-protocol",
12
+ "ai",
13
+ "agent"
14
+ ],
15
+ "homepage": "https://github.com/UXCentury/open-bone/tree/main/packages/mcp#readme",
16
+ "bugs": {
17
+ "url": "https://github.com/UXCentury/open-bone/issues"
18
+ },
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "git+https://github.com/UXCentury/open-bone.git",
22
+ "directory": "packages/mcp"
23
+ },
24
+ "license": "MIT",
25
+ "type": "module",
26
+ "exports": {
27
+ ".": {
28
+ "types": "./dist/index.d.ts",
29
+ "default": "./dist/index.js"
30
+ },
31
+ "./bin": {
32
+ "types": "./dist/bin.d.ts",
33
+ "default": "./dist/bin.js"
34
+ }
35
+ },
36
+ "bin": {
37
+ "openbone-mcp": "./dist/bin.js"
38
+ },
39
+ "files": [
40
+ "dist"
41
+ ],
42
+ "engines": {
43
+ "node": ">=20"
44
+ },
45
+ "publishConfig": {
46
+ "access": "public"
47
+ },
48
+ "dependencies": {
49
+ "@modelcontextprotocol/sdk": "^1.32.2",
50
+ "@napi-rs/canvas": "^0.1.80",
51
+ "esbuild": "^0.28.2",
52
+ "gifenc": "^1.0.3",
53
+ "ws": "^8.18.0",
54
+ "zod": "^4.1.0",
55
+ "@openbone/player": "0.3.0",
56
+ "@openbone/export": "0.3.0",
57
+ "@openbone/core": "0.3.0",
58
+ "@openbone/render-canvas": "0.3.0",
59
+ "@openbone/scripting": "0.3.0",
60
+ "@openbone/runtime": "0.3.0"
61
+ },
62
+ "devDependencies": {
63
+ "@types/ws": "^8.18.1"
64
+ },
65
+ "scripts": {
66
+ "build": "node ../build.mjs"
67
+ },
68
+ "main": "./dist/index.js",
69
+ "types": "./dist/index.d.ts"
6
70
  }