@map-audio/pam-mcp-server 1.7.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.js CHANGED
@@ -3,6 +3,7 @@ import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mc
3
3
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4
4
  import { z } from "zod";
5
5
  import { readFile } from "node:fs/promises";
6
+ import { inflateSync } from "node:zlib";
6
7
  import { basename, dirname, resolve } from "node:path";
7
8
  import { fileURLToPath } from "node:url";
8
9
  import { loadManifest, getParameterIndex, getParametersByCell, getGlobalParameters, findPresetDirs, resolvePresetPath, walkPresets, } from "./manifest.js";
@@ -76,23 +77,49 @@ function parsePresetFile(raw) {
76
77
  const dataStr = raw.slice(prefixLen + metadataSize);
77
78
  return { metadata: JSON.parse(metadataStr), data: JSON.parse(dataStr) };
78
79
  }
80
+ // PAM ≥ 1.4.61 stores variations ONLY in the data block, zlib+base64-compressed with a
81
+ // "PAMZ1:" marker (see helpers/VariationCodec.h). Decompress one entry to plain JSON;
82
+ // plain (older-format) entries pass through unchanged.
83
+ const VARIATION_MARKER = "PAMZ1:";
84
+ function decompressVariationEntry(v) {
85
+ if (typeof v !== "string" || !v.startsWith(VARIATION_MARKER))
86
+ return v;
87
+ try {
88
+ return inflateSync(Buffer.from(v.slice(VARIATION_MARKER.length), "base64")).toString("utf8");
89
+ }
90
+ catch {
91
+ return v;
92
+ }
93
+ }
94
+ /**
95
+ * Canonical variations array for a parsed preset, as plain-JSON strings (or nulls).
96
+ * The data block is authoritative (current format); metadata.variations is the fallback
97
+ * for legacy presets that still carried the duplicate copy there.
98
+ */
99
+ function variationsFromParsedPreset(parsed) {
100
+ const fromData = parsed.data?.variations;
101
+ const fromMeta = parsed.metadata?.variations;
102
+ const src = Array.isArray(fromData) && fromData.length === PRESET_VARIATION_SLOTS ? fromData
103
+ : Array.isArray(fromMeta) && fromMeta.length === PRESET_VARIATION_SLOTS ? fromMeta
104
+ : null;
105
+ if (!src)
106
+ return Array(PRESET_VARIATION_SLOTS).fill(null);
107
+ return src.map(decompressVariationEntry);
108
+ }
79
109
  /**
80
110
  * Read existing variations array from a preset file on disk.
81
- * Returns Array(8) of strings or nulls.
111
+ * Returns Array(8) of plain-JSON strings or nulls.
82
112
  */
83
113
  async function readExistingVariations(presetPath) {
84
114
  try {
85
115
  const fullPath = await resolvePresetPath(presetPath);
86
116
  const raw = await readFile(fullPath, "utf-8");
87
- const { metadata } = parsePresetFile(raw);
88
- if (metadata.variations && Array.isArray(metadata.variations) && metadata.variations.length === PRESET_VARIATION_SLOTS) {
89
- return [...metadata.variations];
90
- }
117
+ return variationsFromParsedPreset(parsePresetFile(raw));
91
118
  }
92
119
  catch {
93
120
  // Preset doesn't exist yet or can't be read — start fresh
121
+ return Array(PRESET_VARIATION_SLOTS).fill(null);
94
122
  }
95
- return Array(PRESET_VARIATION_SLOTS).fill(null);
96
123
  }
97
124
  // Mirrors interop.ts VARIATION_EXCLUDED_PARAMS (activePresetPath is excluded separately below, matching interop.ts)
98
125
  const VARIATION_EXCLUDED_PARAMS = [
@@ -152,19 +179,27 @@ function formatParam(p) {
152
179
  // --- Server ---
153
180
  const server = new McpServer({
154
181
  name: "pam",
155
- version: "1.7.0",
182
+ version: "1.8.0",
156
183
  }, {
157
184
  instructions: "PAM is an audio sampler plugin. When browsing samples, ALWAYS call get_default_locations " +
158
185
  "first to discover the user's actual library paths — never guess paths like ~/Music/PAM. " +
159
186
  "The sample library lives under ~/Library/Application Support/, NOT ~/Music/. " +
160
187
  "Use the returned paths with list_samples. When loading samples, use load_sample with " +
161
188
  "paths from list_samples. Transport must be playing for sequencer output. " +
162
- "To enable experimental features (drum synth, plugin host) call update_settings with both " +
163
- "experimentalMode:true AND the specific sub-flag (e.g. experimentalDrumSynth:true). " +
189
+ "To enable experimental features (plugin host, AI engines) call update_settings with both " +
190
+ "experimentalMode:true AND the specific sub-flag (e.g. experimentalPluginHost:true). " +
191
+ "The drum synth is a standard feature (PAM ≥ 1.5) — no settings flag needed. " +
164
192
  "Tools added in server version 1.7.0 — update_settings, configure_transport, " +
165
193
  "get_audio_device_setup, set_midi_io, set_midi_mapping, delete_preset, " +
166
194
  "set_preset_factory_status, get_sample_usage, repair_preset_samples, preview_sample, " +
167
195
  "set_plugin_params, skip_current_plugin_scan, remove_plugin_from_blocklist — require PAM ≥ 1.7.0. " +
196
+ "VX video tools (get_video_state, video_command, add_video_layer, update_video_layer, " +
197
+ "remove_video_layer, apply_video_layout, add_video_fx, set_video_fx, remove_video_fx, " +
198
+ "bind_video_param, set_video_settings) require PAM ≥ 1.8.0 AND PAM's window to be OPEN " +
199
+ "(the video engine renders in the WebView; there is no headless renderer). For " +
200
+ "audio-reactive control, note the visual audio-react params (visualWaveDisplace, etc.) " +
201
+ "are regular plugin params — drive them with set_parameters; layers/FX/bindings go through " +
202
+ "the video tools. When building a scene, call get_video_state first to learn layer/fx ids. " +
168
203
  "Earlier tool sets (looper, song mode, modulation edits, MIDI recording, sample analysis, " +
169
204
  "live resample, variation scheduling, AI generation) require PAM ≥ 1.4.50 / 1.5.0. " +
170
205
  "If a tool fails with \"PAM build does not support 'X'\", the connected plugin is older than " +
@@ -482,6 +517,77 @@ server.registerTool("load_sample", {
482
517
  await bridge.send("loadSampleToCell", filePath, cellId);
483
518
  return text(`Loaded ${basename(filePath)} into cell ${cellId}.`);
484
519
  });
520
+ // ---- Unified AI model management (notochord / aigen / magenta) ----
521
+ // All six map to the C++ MCPBridge model* actions (helpers/MCPBridge.cpp::handleModelOp).
522
+ // Each engine must have its experimental flags on (update_settings: experimentalMode
523
+ // + experimental<Engine>) — the bridge returns a clear error otherwise.
524
+ const MODEL_ENGINE = z
525
+ .enum(["notochord", "aigen", "magenta"])
526
+ .describe("Which AI engine. notochord = MIDI co-player (~18 MB blob); aigen = Stable Audio 3 sample generation (~3.2 GB ONNX); magenta = MagentaRT realtime audio (Apple Silicon, MLX).");
527
+ const MODEL_VARIANT = z
528
+ .enum(["base", "small"])
529
+ .optional()
530
+ .describe("Magenta only: which variant (base = 2.4B, needs ~M2 Max/M3 Pro+; small = 230M, default). Ignored for notochord/aigen (single model).");
531
+ server.registerTool("download_model", {
532
+ title: "Download AI Model",
533
+ description: "Start (or resume) downloading model weights for an AI engine. Async — returns immediately with download progress; poll get_model_status. aigen + notochord auto-load on completion; magenta needs a separate load_model. Requires the engine's experimental flags.",
534
+ inputSchema: { engine: MODEL_ENGINE, model: MODEL_VARIANT },
535
+ }, async ({ engine, model }) => {
536
+ const check = await requireBridge();
537
+ if (check !== true)
538
+ return check;
539
+ return json(await bridge.send("modelDownload", { engine, model }));
540
+ });
541
+ server.registerTool("get_model_status", {
542
+ title: "Get AI Model Status",
543
+ description: "Report downloaded / loaded / loading / download-in-flight + version + disk size + live download progress for an AI engine.",
544
+ inputSchema: { engine: MODEL_ENGINE },
545
+ }, async ({ engine }) => {
546
+ const check = await requireBridge();
547
+ if (check !== true)
548
+ return check;
549
+ return json(await bridge.send("modelStatus", { engine }));
550
+ });
551
+ server.registerTool("load_model", {
552
+ title: "Load AI Model",
553
+ description: "Load a downloaded model into memory (async — poll get_model_status for loaded:true). Use after unload, or for magenta after download. magenta accepts an optional variant; omit to load whichever is on disk.",
554
+ inputSchema: { engine: MODEL_ENGINE, model: MODEL_VARIANT },
555
+ }, async ({ engine, model }) => {
556
+ const check = await requireBridge();
557
+ if (check !== true)
558
+ return check;
559
+ return json(await bridge.send("modelLoad", { engine, model }));
560
+ });
561
+ server.registerTool("unload_model", {
562
+ title: "Unload AI Model",
563
+ description: "Free the loaded model from memory (weights stay on disk). May briefly stall the plugin while an in-flight load finishes (non-interruptible ORT/MLX load).",
564
+ inputSchema: { engine: MODEL_ENGINE },
565
+ }, async ({ engine }) => {
566
+ const check = await requireBridge();
567
+ if (check !== true)
568
+ return check;
569
+ return json(await bridge.send("modelUnload", { engine }));
570
+ });
571
+ server.registerTool("delete_model", {
572
+ title: "Delete AI Model",
573
+ description: "Delete downloaded model weights from disk. Unloads first automatically. DESTRUCTIVE — re-download to restore. magenta delete is variant-agnostic (removes both variants + shared resources).",
574
+ inputSchema: { engine: MODEL_ENGINE, model: MODEL_VARIANT },
575
+ }, async ({ engine, model }) => {
576
+ const check = await requireBridge();
577
+ if (check !== true)
578
+ return check;
579
+ return json(await bridge.send("modelDelete", { engine, model }));
580
+ });
581
+ server.registerTool("list_models", {
582
+ title: "List AI Models",
583
+ description: "List available model variants per engine with downloaded / loaded state. Omit engine to list all engines (notochord, aigen, magenta base + small).",
584
+ inputSchema: { engine: MODEL_ENGINE.optional() },
585
+ }, async ({ engine }) => {
586
+ const check = await requireBridge();
587
+ if (check !== true)
588
+ return check;
589
+ return json(await bridge.send("modelList", { engine }));
590
+ });
485
591
  server.registerTool("generate_sample", {
486
592
  title: "Generate Sample (AI)",
487
593
  description: "Generate a new audio sample via the on-device AI engine (Stable Audio 3 Small SFX, ~3.2 GB ONNX). The plugin must have the model downloaded first (Settings → Local models). Returns the on-disk path to the generated WAV and optionally loads it into a cell.",
@@ -563,6 +669,141 @@ server.registerTool("generate_kit", {
563
669
  return error(`Kit generation failed: ${e.message}`);
564
670
  }
565
671
  });
672
+ // --- AI Audio Workbench (Phase A, dev-gated) ---------------------------------
673
+ // Drives the Transform canvas headlessly: canvas state (mask regions / strength
674
+ // curve / prompt / seed / sigma), sidecar renders, and the take shelf. Requires
675
+ // experimentalMode + experimentalAIGen (update_settings) AND the SA3 Python
676
+ // sidecar running on 127.0.0.1:8093 (packages/sa3-workbench-sidecar/README.md).
677
+ const WORKBENCH_MASK_REGION = z.object({
678
+ startSec: z.number().min(0).describe("Region start in seconds (source time)."),
679
+ endSec: z.number().min(0).describe("Region end in seconds (must be > startSec)."),
680
+ lowHz: z.number().min(0).optional()
681
+ .describe("Optional band low edge in Hz (canvas v2 2D mask). Omit BOTH lowHz/highHz for a full-band region (1D back-compat)."),
682
+ highHz: z.number().min(0).optional()
683
+ .describe("Optional band high edge in Hz (must be > lowHz). lowHz<=20 && highHz>=source nyquist is treated as full band by the sidecar."),
684
+ });
685
+ const WORKBENCH_CURVE_POINT = z.object({
686
+ t: z.number().describe("Normalized time 0..1."),
687
+ v: z.number().describe("Value 0..1 (σ at this point)."),
688
+ tension: z.number().optional().describe("Curve tension (CurveCanvas semantics, default 0)."),
689
+ });
690
+ const WORKBENCH_STATE_FIELDS = {
691
+ sourceCellId: z.number().int().min(1).max(8).nullable().optional()
692
+ .describe("Cell whose sample is the transform source (null to clear)."),
693
+ prompt: z.string().optional().describe("Text prompt for the render."),
694
+ seed: z.number().int().min(0).optional().describe("Render seed."),
695
+ seedLocked: z.boolean().optional().describe("Lock seed across renders (iterate at fixed identity)."),
696
+ sigma: z.number().min(0.02).max(1).optional()
697
+ .describe("Deviation σ / init noise level. a2a: 0.25-0.35 preserves melody, 0.45-0.5 variation; inpaint regime ~0.85."),
698
+ steps: z.number().int().min(1).max(32).optional().describe("Denoise steps (8 is the validated recipe)."),
699
+ maskRegions: z.array(WORKBENCH_MASK_REGION).optional()
700
+ .describe("Transform-lane mask. Empty = pure a2a re-render of the whole source; regions = inpaint (regenerate inside, keep outside). Regions may carry an optional lowHz/highHz band."),
701
+ strengthCurve: z.array(WORKBENCH_CURVE_POINT).optional()
702
+ .describe("Strength-lane breakpoints (stored with full provenance; per-region σ application is experimental)."),
703
+ activeSourcePath: z.string().optional()
704
+ .describe("Render-chain head: absolute WAV path used as the transform source ('' = the source cell's sample). Managed automatically — each successful render advances it to the new take's wavPath; set explicitly only to point the chain at a prior take. Walk it back with workbench_undo / workbench_reset_chain."),
705
+ sketchPath: z.string().optional()
706
+ .describe("SKETCH mode: absolute path to a painted-gesture WAV ('' = no sketch). Normally written by the UI (workbenchSaveSketch saves into ai-cache/workbench/); settable directly for headless tests. When non-empty, renders feed the model source + sketch*sketchGain as init audio AND use that MIX as the preservation reference for masks/paste-back — the gesture survives outside masked regions."),
707
+ sketchGain: z.number().min(0).max(1).optional()
708
+ .describe("SKETCH mix gain 0..1 (default 0.5) applied to the sketch before summing with the source. If the mix peaks above 0.99 the sidecar scales the whole mix down to 0.99."),
709
+ };
710
+ server.registerTool("workbench_get_state", {
711
+ title: "Workbench: Get State",
712
+ description: "Read the AI Audio Workbench canvas state AND the take shelf: { state: { sourceCellId, prompt, seed, seedLocked, sigma, steps, maskRegions, strengthCurve, activeSourcePath, sketchPath, sketchGain }, takes: [...] }.",
713
+ inputSchema: {},
714
+ }, async () => {
715
+ const check = await requireBridge();
716
+ if (check !== true)
717
+ return check;
718
+ return json(await bridge.send("workbenchGetState"));
719
+ });
720
+ server.registerTool("workbench_set_state", {
721
+ title: "Workbench: Set State",
722
+ description: "Partial-merge the Workbench canvas state (only the keys you pass change). Also pushes the merged state into an open Workbench UI. Returns the full merged state.",
723
+ inputSchema: WORKBENCH_STATE_FIELDS,
724
+ }, async (args) => {
725
+ const check = await requireBridge();
726
+ if (check !== true)
727
+ return check;
728
+ return json(await bridge.send("workbenchSetState", args));
729
+ });
730
+ server.registerTool("workbench_render", {
731
+ title: "Workbench: Render",
732
+ description: "Render the Workbench transform via the SA3 sidecar and append the result to the take shelf. Renders the CURRENT canvas state merged with any overrides passed here (overrides do not persist). Blocks until done (up to ~3 min: pipeline rebuilds + multi-region inpaints render sequentially). Returns the take { id, wavPath, prompt, seed, sigma, maskRegions, createdAt, sourceCellId, durationSeconds, error? }. Source resolution: sourcePath override > activeSourcePath (chain head) > state.sourceCellId's sample file. A successful render ADVANCES the chain (activeSourcePath = new take's wavPath) — step back with workbench_undo or workbench_reset_chain.",
733
+ inputSchema: {
734
+ ...WORKBENCH_STATE_FIELDS,
735
+ sourcePath: z.string().optional()
736
+ .describe("Absolute path to a source WAV — bypasses sourceCellId resolution."),
737
+ },
738
+ }, async (args) => {
739
+ const check = await requireBridge();
740
+ if (check !== true)
741
+ return check;
742
+ try {
743
+ return json(await bridge.send("workbenchRender", args));
744
+ }
745
+ catch (e) {
746
+ return error(`Workbench render failed: ${e.message}`);
747
+ }
748
+ });
749
+ server.registerTool("workbench_list_takes", {
750
+ title: "Workbench: List Takes",
751
+ description: "List all Workbench takes (renders) with full provenance, oldest first.",
752
+ inputSchema: {},
753
+ }, async () => {
754
+ const check = await requireBridge();
755
+ if (check !== true)
756
+ return check;
757
+ return json(await bridge.send("workbenchListTakes"));
758
+ });
759
+ server.registerTool("workbench_load_take", {
760
+ title: "Workbench: Load Take",
761
+ description: "Load a rendered take's WAV into a cell (same native path as load_sample).",
762
+ inputSchema: {
763
+ takeId: z.string().describe("Take id from workbench_render / workbench_list_takes."),
764
+ cellId: z.number().int().min(1).max(8).describe("Target cell."),
765
+ },
766
+ }, async ({ takeId, cellId }) => {
767
+ const check = await requireBridge();
768
+ if (check !== true)
769
+ return check;
770
+ try {
771
+ return json(await bridge.send("workbenchLoadTake", { takeId, cellId }));
772
+ }
773
+ catch (e) {
774
+ return error(`Failed to load take: ${e.message}`);
775
+ }
776
+ });
777
+ server.registerTool("workbench_undo", {
778
+ title: "Workbench: Undo (pop chain)",
779
+ description: "Pop the Workbench render chain one step: restores the previous chain head (activeSourcePath) so the next render re-transforms the prior source. No-op at the chain root (cell sample). Takes are kept. Returns the full { state, takes }.",
780
+ inputSchema: {},
781
+ }, async () => {
782
+ const check = await requireBridge();
783
+ if (check !== true)
784
+ return check;
785
+ return json(await bridge.send("workbenchUndo"));
786
+ });
787
+ server.registerTool("workbench_reset_chain", {
788
+ title: "Workbench: Reset Chain",
789
+ description: "Clear the Workbench render chain: empties the undo stack and resets the source back to the cell sample (activeSourcePath = ''). Takes are kept. Returns the full { state, takes }.",
790
+ inputSchema: {},
791
+ }, async () => {
792
+ const check = await requireBridge();
793
+ if (check !== true)
794
+ return check;
795
+ return json(await bridge.send("workbenchResetChain"));
796
+ });
797
+ server.registerTool("workbench_sidecar_status", {
798
+ title: "Workbench: Sidecar Status",
799
+ description: "Ping the SA3 render sidecar (127.0.0.1:8093). Returns { connected, loaded } — loaded means a pipeline is resident (first render after a cold start also pays the model load).",
800
+ inputSchema: {},
801
+ }, async () => {
802
+ const check = await requireBridge();
803
+ if (check !== true)
804
+ return check;
805
+ return json(await bridge.send("workbenchSidecarStatus"));
806
+ });
566
807
  server.registerTool("trigger_midi", {
567
808
  title: "Trigger MIDI",
568
809
  description: "Send a MIDI note to a specific cell or globally. " +
@@ -1033,11 +1274,9 @@ server.registerTool("save_variation", {
1033
1274
  try {
1034
1275
  const fullPath = await resolvePresetPath(diskPath);
1035
1276
  const raw = await readFile(fullPath, "utf-8");
1036
- const { metadata: m } = parsePresetFile(raw);
1037
- existingMeta = m;
1038
- if (m.variations && Array.isArray(m.variations) && m.variations.length === PRESET_VARIATION_SLOTS) {
1039
- variations = [...m.variations];
1040
- }
1277
+ const parsed = parsePresetFile(raw);
1278
+ existingMeta = parsed.metadata;
1279
+ variations = variationsFromParsedPreset(parsed);
1041
1280
  }
1042
1281
  catch {
1043
1282
  // New preset or unreadable — start fresh
@@ -1426,6 +1665,473 @@ server.registerTool("import_visual_folder", {
1426
1665
  (result.operationId ? ` (operationId: ${result.operationId})` : ""));
1427
1666
  });
1428
1667
  // ============================================================
1668
+ // VX video subsystem — layers, FX, audio-reactive bindings, settings
1669
+ // ============================================================
1670
+ //
1671
+ // The video scene lives in PAM's WebView (R3F/WebGL). These tools forward semantic
1672
+ // commands into the live video stores and read back a full-scene snapshot. PAM's
1673
+ // window must be OPEN for commands to apply (no headless renderer). Audio-reactive
1674
+ // *visual params* (visualWaveDisplace, etc.) are regular plugin params — drive those
1675
+ // with set_parameters; everything else (layers/FX/bindings/output FX) goes here.
1676
+ // Per-layer FX types (FxRegistry). Some only apply to drawing/text (trails, beam).
1677
+ const VIDEO_LAYER_FX = "hueRotate, glow, audioReact, vibration, chromaticAberration, pixelate, rgbShift, " +
1678
+ "mirror, trails(drawing/text), beam(drawing/text), shake, rolls, barrelBlur, edge, " +
1679
+ "halftone, patches, brightnessContrast, lines, feedback, warp, datamosh, slicer, " +
1680
+ "polarSlitScan, cellVoronoi, spectralZones, paletteLut, asciiDither, pixelSort, " +
1681
+ "cameraShake, blur";
1682
+ // Output/global FX chain ids (videoStore, DEFAULT_VIDEO_FX_CONFIG).
1683
+ const VIDEO_OUTPUT_FX = "lines, barrelBlur, brightnessContrast, edge, glow, shake, rgbShift, halftone, mirror, " +
1684
+ "patches, pixelate, rolls, feedback, warp, datamosh, slicer, positionXY, speed, " +
1685
+ "cellVoronoi, polarSlitScan, spectralZones, audioReact";
1686
+ // Binding source vocabulary (sourceKey string form OR a ModSource object).
1687
+ const VIDEO_BIND_SOURCES = "Pass `source` as a sourceKey string: " +
1688
+ "`g:<id>` global (level, bandLow, bandMid, bandHigh, transient, transientLow, " +
1689
+ "transientMid, transientHigh, bpm, spectralCentroid, spectralFlux, stereoWidth, " +
1690
+ "varyActivity, phraseLevel, phraseBass, looperBus, masterOnsetLow/Mid/High, " +
1691
+ "masterStereoWidth, masterPanScalar, masterFullMix, masterCompressorGR, " +
1692
+ "sidechainLevel, sidechainEnv); " +
1693
+ "`t:<phase>` transport (beat, bar, phrase); " +
1694
+ "`c:<cell>:<field>` per-cell (level, envFollow, pulse, onset, spectrumPeak, " +
1695
+ "spectrumCentroid, pitch, postFaderL/R, samplerRMSL/R, synthRMS, beatPosition, " +
1696
+ "activeRetriggers, grainActivity, grainPos, grainSpread, grainPitch, " +
1697
+ "grainPitchSpread, grainReverseRatio); " +
1698
+ "`l:<track>:level` looper; `lf:<lfo>:rate|phase` LFO; " +
1699
+ "`p:<paramId>` any DAW param (e.g. p:macro_1). Or a structured object e.g. " +
1700
+ "{kind:'cell',cellIndex:1,field:'onset'}.";
1701
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
1702
+ async function getVideoStateRaw() {
1703
+ const r = await bridge.send("getVideoState");
1704
+ return (typeof r === "string" ? JSON.parse(r) : r) ?? {};
1705
+ }
1706
+ const sendVideoCommand = (command) => bridge.send("videoCommand", command);
1707
+ /** Create ops are fire-and-forget; resolve the minted id by diffing the snapshot. */
1708
+ async function resolveNewId(beforeIds, pick, timeoutMs = 2500) {
1709
+ const deadline = Date.now() + timeoutMs;
1710
+ while (Date.now() < deadline) {
1711
+ await sleep(120);
1712
+ const s = await getVideoStateRaw();
1713
+ const found = pick(s).find((id) => !beforeIds.has(id));
1714
+ if (found)
1715
+ return found;
1716
+ }
1717
+ return null;
1718
+ }
1719
+ server.registerTool("get_video_state", {
1720
+ title: "Get Video State",
1721
+ description: "Read the full VX video scene: layers (in render order, top-first) with their " +
1722
+ "per-layer FX chains, the output/global FX chain, audio-reactive bindings, " +
1723
+ "mod-matrix routes, the 3D model overlay, the active cell, and global render/sync " +
1724
+ "settings. Layer/FX/binding ids returned here are what the other video tools " +
1725
+ "address. If `available` is false, open PAM's window so the video UI mounts.",
1726
+ inputSchema: {},
1727
+ annotations: { readOnlyHint: true },
1728
+ }, async () => {
1729
+ const check = await requireBridge();
1730
+ if (check !== true)
1731
+ return check;
1732
+ const state = await getVideoStateRaw();
1733
+ return json(state);
1734
+ });
1735
+ server.registerTool("capture_ui", {
1736
+ title: "Capture UI Screenshot",
1737
+ description: "Render PAM's plugin UI (#app_root) to a PNG via html2canvas and return the saved " +
1738
+ "file path (read it to view the image). Optionally compose the view first: " +
1739
+ "gridSize (1|4|8), mainView (CellGrid|CtrlAll|Envelope|LFO|Vary|Mixer|SongMode|Video|" +
1740
+ "PresetBrowser; omit/null = cell grid), instrumentView ('synth'|'sampler', applied to " +
1741
+ "all cells). REQUIRES PAM's window OPEN — html2canvas needs the live DOM, there is no " +
1742
+ "headless renderer (same constraint as the video tools). Beats screen-capture: " +
1743
+ "deterministic, no cursor/screen control, exact UI bitmap.",
1744
+ inputSchema: {
1745
+ gridSize: z.number().optional().describe("Cell grid size to set before capture: 1, 4, or 8."),
1746
+ mainView: z
1747
+ .string()
1748
+ .optional()
1749
+ .describe("Main view to switch to before capture (CellGrid, Mixer, LFO, …). Omit for the cell grid."),
1750
+ instrumentView: z
1751
+ .enum(["synth", "sampler"])
1752
+ .optional()
1753
+ .describe("Switch all cells to the synth or sampler tab before capture."),
1754
+ settleMs: z.number().optional().describe("Delay (ms) after view changes before rasterizing. Default 350."),
1755
+ timeoutMs: z.number().optional().default(8000).describe("Max ms to wait for the WebView to return the image."),
1756
+ },
1757
+ annotations: { readOnlyHint: true },
1758
+ }, async ({ gridSize, mainView, instrumentView, settleMs, timeoutMs }) => {
1759
+ const check = await requireBridge();
1760
+ if (check !== true)
1761
+ return check;
1762
+ const requestId = `ui-${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
1763
+ const setup = { requestId };
1764
+ if (typeof gridSize === "number")
1765
+ setup.gridSize = gridSize;
1766
+ if (mainView !== undefined)
1767
+ setup.mainView = mainView;
1768
+ if (instrumentView !== undefined)
1769
+ setup.instrumentView = instrumentView;
1770
+ if (typeof settleMs === "number")
1771
+ setup.settleMs = settleMs;
1772
+ // Forward is fire-and-forget; poll the C++ cache until the WebView posts the PNG back.
1773
+ await bridge.send("captureUi", setup);
1774
+ const deadline = Date.now() + (timeoutMs ?? 8000);
1775
+ for (;;) {
1776
+ const res = (await bridge.send("getUiCapture", { requestId }));
1777
+ if (res.ready && res.requestId === requestId && res.path) {
1778
+ return text(`Saved UI screenshot: ${res.path}`);
1779
+ }
1780
+ if (Date.now() > deadline) {
1781
+ return error("capture_ui timed out waiting for the WebView to return the image. Is PAM's window " +
1782
+ "open and active? html2canvas needs the live DOM — a closed/suspended editor cannot render.");
1783
+ }
1784
+ await new Promise((r) => setTimeout(r, 150));
1785
+ }
1786
+ });
1787
+ server.registerTool("video_command", {
1788
+ title: "Video Command (advanced)",
1789
+ description: "Escape hatch to the full video command router — pass any { op, ...args } object. " +
1790
+ "Use the dedicated tools (add_video_layer, set_video_fx, bind_video_param, …) when " +
1791
+ "one fits; this reaches everything else. Ops: " +
1792
+ "addLayer{kind,name?,insertAt?}, removeLayer{layerId}, updateLayer{layerId,patch}, " +
1793
+ "moveLayer{layerId,toIndex}, duplicateLayer{layerId}, renameLayer{layerId,name}, " +
1794
+ "setLayerEnabled{layerId,enabled?}, toggleSolo{layerId}, selectLayer{layerId}, " +
1795
+ "resetLayer{layerId}, applyLayout{preset}, randomizeLayer{layerId}; " +
1796
+ "addLayerFx{layerId,fxType,params?,beforeFxId?}, removeLayerFx{layerId,fxId}, " +
1797
+ "removeAllLayerFx{layerId}, setLayerFxParam{layerId,fxId,param,value|params}, " +
1798
+ "setLayerFxEnabled{layerId,fxId,enabled?}, reorderLayerFx{layerId,fxIds}, " +
1799
+ "resetLayerFx{layerId,fxId}; bind{layerId,fxId?,paramKey,source,...}, " +
1800
+ "unbind{layerId,fxId?,paramKey}, setBindingField{layerId,fxId?,paramKey,field,value}, " +
1801
+ "fireImpulse{layerId,fxId?,paramKey}, removeBindingsForLayer{layerId}, clearBindings; " +
1802
+ "addOutputFx{fxId}, removeOutputFx{instanceId}, setOutputFxParam{instanceId|fxId,param,value}, " +
1803
+ "setOutputFxEnabled{instanceId,enabled}, setOutputFxStrength{instanceId,strength}; " +
1804
+ "addModRoute{route}, removeModRoute{id}, updateModRoute{id,updates}, clearModRoutes, " +
1805
+ "setModEnabled{enabled}; setModel{sourcePath?,visible?,blendMode?,audioReactive?,autoRotate?,transform?}, " +
1806
+ "resetModel; setSettings{...}, setActiveCell{cellId}, resetWorkspace. " +
1807
+ "Layer fx target a per-layer chain; output fx target the global chain (ids differ). " +
1808
+ VIDEO_BIND_SOURCES,
1809
+ inputSchema: {
1810
+ command: z
1811
+ .record(z.string(), z.any())
1812
+ .describe("The command object. Must include a string `op` field."),
1813
+ },
1814
+ }, async ({ command }) => {
1815
+ const check = await requireBridge();
1816
+ if (check !== true)
1817
+ return check;
1818
+ const result = await sendVideoCommand(command);
1819
+ return json(result);
1820
+ });
1821
+ server.registerTool("add_video_layer", {
1822
+ title: "Add Video Layer",
1823
+ description: "Add a layer to the VX scene and return its id. Kinds: video, image, camera, " +
1824
+ "text, drawing, model3d, adjustment (adjustment = full-canvas FX only; master is a " +
1825
+ "managed singleton and cannot be added). After adding, set its source/props with " +
1826
+ "update_video_layer and stack FX with add_video_fx.",
1827
+ inputSchema: {
1828
+ kind: z
1829
+ .enum(["video", "image", "camera", "text", "drawing", "model3d", "adjustment"])
1830
+ .describe("Layer kind."),
1831
+ name: z.string().optional().describe("Display name (auto-generated if omitted)."),
1832
+ insertAt: z
1833
+ .number()
1834
+ .int()
1835
+ .optional()
1836
+ .describe("Data-index to insert at (default: top of stack)."),
1837
+ },
1838
+ }, async ({ kind, name, insertAt }) => {
1839
+ const check = await requireBridge();
1840
+ if (check !== true)
1841
+ return check;
1842
+ const before = await getVideoStateRaw();
1843
+ const beforeIds = new Set((before.layers ?? []).map((l) => l.id));
1844
+ await sendVideoCommand({ op: "addLayer", kind, name, insertAt });
1845
+ const id = await resolveNewId(beforeIds, (s) => (s.layers ?? []).map((l) => l.id));
1846
+ return text(id
1847
+ ? `Added ${kind} layer: ${id}`
1848
+ : `Added ${kind} layer (window may be closed; call get_video_state to confirm).`);
1849
+ });
1850
+ server.registerTool("update_video_layer", {
1851
+ title: "Update Video Layer",
1852
+ description: "Update a layer's common props and/or kind-specific fields. transform is merged " +
1853
+ "(partial) against the current transform; other fields replace. Common: opacity " +
1854
+ "(0..1), blendMode (normal|add|multiply|screen), enabled, name, transform " +
1855
+ "{x,y(0..1),scale,rotation(rad),anchor}. Kind-specific via `fields`: text layers " +
1856
+ "{text,fontSize,fontWeight,color,align,lineHeight}; image/video/model3d {sourcePath,fit}; " +
1857
+ "camera {deviceId,mirror,fit}; drawing {paintColor,paintWidth}; model3d " +
1858
+ "{lightingMode,lightIntensity,rotateRateX/Y/Z,...}. Use get_video_state to see a " +
1859
+ "layer's current fields.",
1860
+ inputSchema: {
1861
+ layerId: z.string().describe("Target layer id (from get_video_state)."),
1862
+ opacity: z.number().min(0).max(1).optional(),
1863
+ blendMode: z.enum(["normal", "add", "multiply", "screen"]).optional(),
1864
+ enabled: z.boolean().optional(),
1865
+ name: z.string().optional(),
1866
+ transform: z
1867
+ .record(z.string(), z.any())
1868
+ .optional()
1869
+ .describe("Partial transform: {x?,y?,scale?,rotation?,anchor?}. Merged."),
1870
+ fields: z
1871
+ .record(z.string(), z.any())
1872
+ .optional()
1873
+ .describe("Kind-specific fields (text, sourcePath, fit, fontSize, color, …)."),
1874
+ },
1875
+ }, async ({ layerId, opacity, blendMode, enabled, name, transform, fields }) => {
1876
+ const check = await requireBridge();
1877
+ if (check !== true)
1878
+ return check;
1879
+ const patch = { ...(fields ?? {}) };
1880
+ if (opacity !== undefined)
1881
+ patch.opacity = opacity;
1882
+ if (blendMode !== undefined)
1883
+ patch.blendMode = blendMode;
1884
+ if (enabled !== undefined)
1885
+ patch.enabled = enabled;
1886
+ if (name !== undefined)
1887
+ patch.name = name;
1888
+ if (transform !== undefined)
1889
+ patch.transform = transform;
1890
+ const result = await sendVideoCommand({ op: "updateLayer", layerId, patch });
1891
+ return json(result);
1892
+ });
1893
+ server.registerTool("remove_video_layer", {
1894
+ title: "Remove Video Layer",
1895
+ description: "Remove a layer (and its FX + bindings). The master layer cannot be removed.",
1896
+ inputSchema: { layerId: z.string().describe("Layer id to remove.") },
1897
+ annotations: { destructiveHint: true },
1898
+ }, async ({ layerId }) => {
1899
+ const check = await requireBridge();
1900
+ if (check !== true)
1901
+ return check;
1902
+ const result = await sendVideoCommand({ op: "removeLayer", layerId });
1903
+ return json(result);
1904
+ });
1905
+ server.registerTool("apply_video_layout", {
1906
+ title: "Apply Video Layout",
1907
+ description: "Arrange the spatial layers with a layout preset: centered (stacked), grid, " +
1908
+ "columns, rows, or focus (one large + thumbnail strip). adjustment and master " +
1909
+ "layers are untouched.",
1910
+ inputSchema: {
1911
+ preset: z.enum(["centered", "grid", "columns", "rows", "focus"]).describe("Layout preset."),
1912
+ },
1913
+ }, async ({ preset }) => {
1914
+ const check = await requireBridge();
1915
+ if (check !== true)
1916
+ return check;
1917
+ const result = await sendVideoCommand({ op: "applyLayout", preset });
1918
+ return json(result);
1919
+ });
1920
+ server.registerTool("add_video_fx", {
1921
+ title: "Add Video FX",
1922
+ description: "Append an FX to a layer's chain (target='layer', needs layerId) or to the output/" +
1923
+ "global chain (target='output'), and return the new id. Per-layer FX: " +
1924
+ VIDEO_LAYER_FX +
1925
+ ". Output FX: " +
1926
+ VIDEO_OUTPUT_FX +
1927
+ ". FX params seed from registry defaults; override via `params`. Use set_video_fx " +
1928
+ "to tweak afterwards.",
1929
+ inputSchema: {
1930
+ target: z.enum(["layer", "output"]).default("layer").describe("Which chain."),
1931
+ layerId: z.string().optional().describe("Required when target='layer'."),
1932
+ fxType: z.string().describe("FX id (see the per-layer / output id lists)."),
1933
+ params: z
1934
+ .record(z.string(), z.number())
1935
+ .optional()
1936
+ .describe("Optional initial param overrides {paramKey: value}."),
1937
+ beforeFxId: z
1938
+ .string()
1939
+ .optional()
1940
+ .describe("Insert before this fx id instead of appending (layer target only)."),
1941
+ },
1942
+ }, async ({ target, layerId, fxType, params, beforeFxId }) => {
1943
+ const check = await requireBridge();
1944
+ if (check !== true)
1945
+ return check;
1946
+ if (target === "output") {
1947
+ const before = await getVideoStateRaw();
1948
+ const beforeIds = new Set((before.outputFx ?? []).map((f) => f.instanceId));
1949
+ await sendVideoCommand({ op: "addOutputFx", fxId: fxType });
1950
+ const id = await resolveNewId(beforeIds, (s) => (s.outputFx ?? []).map((f) => f.instanceId));
1951
+ // Apply param overrides now that the instance exists.
1952
+ if (params && id)
1953
+ for (const [param, value] of Object.entries(params))
1954
+ await sendVideoCommand({ op: "setOutputFxParam", instanceId: id, param, value });
1955
+ return text(id ? `Added output FX ${fxType}: ${id}` : `Added output FX ${fxType} (confirm via get_video_state).`);
1956
+ }
1957
+ if (!layerId)
1958
+ return error("add_video_fx: layerId is required when target='layer'.");
1959
+ const before = await getVideoStateRaw();
1960
+ const layer = (before.layers ?? []).find((l) => l.id === layerId);
1961
+ const beforeIds = new Set((layer?.fx ?? []).map((f) => f.id));
1962
+ await sendVideoCommand({ op: "addLayerFx", layerId, fxType, params, beforeFxId });
1963
+ const id = await resolveNewId(beforeIds, (s) => {
1964
+ const l = (s.layers ?? []).find((x) => x.id === layerId);
1965
+ return (l?.fx ?? []).map((f) => f.id);
1966
+ });
1967
+ return text(id ? `Added ${fxType} to layer ${layerId}: ${id}` : `Added ${fxType} (confirm via get_video_state).`);
1968
+ });
1969
+ server.registerTool("set_video_fx", {
1970
+ title: "Set Video FX",
1971
+ description: "Set FX params / enabled / strength on a per-layer FX (target='layer', needs " +
1972
+ "layerId + fxId) or an output FX (target='output', needs instanceId, or fxId to " +
1973
+ "auto-create). Pass a single {param,value}, multiple {params}, `enabled`, and/or " +
1974
+ "`strength` (output only). Get current ids/params from get_video_state.",
1975
+ inputSchema: {
1976
+ target: z.enum(["layer", "output"]).default("layer"),
1977
+ layerId: z.string().optional().describe("target='layer': owning layer id."),
1978
+ fxId: z.string().optional().describe("Layer fx id, or output fxId (auto-creates the output fx)."),
1979
+ instanceId: z.string().optional().describe("target='output': the output fx instance id."),
1980
+ param: z.string().optional().describe("Single param key to set."),
1981
+ value: z.number().optional().describe("Value for `param`."),
1982
+ params: z.record(z.string(), z.number()).optional().describe("Multiple params {key: value}."),
1983
+ enabled: z.boolean().optional().describe("Enable/bypass this FX."),
1984
+ strength: z.number().optional().describe("Output FX wet strength (output target only)."),
1985
+ },
1986
+ }, async ({ target, layerId, fxId, instanceId, param, value, params, enabled, strength }) => {
1987
+ const check = await requireBridge();
1988
+ if (check !== true)
1989
+ return check;
1990
+ const results = [];
1991
+ if (target === "layer") {
1992
+ if (!layerId || !fxId)
1993
+ return error("set_video_fx: layerId and fxId required for target='layer'.");
1994
+ if (param !== undefined && value !== undefined)
1995
+ results.push(await sendVideoCommand({ op: "setLayerFxParam", layerId, fxId, param, value }));
1996
+ if (params)
1997
+ results.push(await sendVideoCommand({ op: "setLayerFxParam", layerId, fxId, params }));
1998
+ if (enabled !== undefined)
1999
+ results.push(await sendVideoCommand({ op: "setLayerFxEnabled", layerId, fxId, enabled }));
2000
+ }
2001
+ else {
2002
+ if (param !== undefined && value !== undefined)
2003
+ results.push(await sendVideoCommand({ op: "setOutputFxParam", instanceId, fxId, param, value }));
2004
+ if (params)
2005
+ for (const [p, v] of Object.entries(params))
2006
+ results.push(await sendVideoCommand({ op: "setOutputFxParam", instanceId, fxId, param: p, value: v }));
2007
+ if (enabled !== undefined && instanceId)
2008
+ results.push(await sendVideoCommand({ op: "setOutputFxEnabled", instanceId, enabled }));
2009
+ if (strength !== undefined && instanceId)
2010
+ results.push(await sendVideoCommand({ op: "setOutputFxStrength", instanceId, strength }));
2011
+ }
2012
+ if (results.length === 0)
2013
+ return error("set_video_fx: nothing to set (provide param/value, params, enabled, or strength).");
2014
+ return json({ applied: results.length });
2015
+ });
2016
+ server.registerTool("remove_video_fx", {
2017
+ title: "Remove Video FX",
2018
+ description: "Remove an FX from a layer chain (target='layer' + layerId + fxId) or the output " +
2019
+ "chain (target='output' + instanceId).",
2020
+ inputSchema: {
2021
+ target: z.enum(["layer", "output"]).default("layer"),
2022
+ layerId: z.string().optional(),
2023
+ fxId: z.string().optional().describe("target='layer': fx id on the layer."),
2024
+ instanceId: z.string().optional().describe("target='output': output fx instance id."),
2025
+ },
2026
+ annotations: { destructiveHint: true },
2027
+ }, async ({ target, layerId, fxId, instanceId }) => {
2028
+ const check = await requireBridge();
2029
+ if (check !== true)
2030
+ return check;
2031
+ if (target === "layer") {
2032
+ if (!layerId || !fxId)
2033
+ return error("remove_video_fx: layerId and fxId required for target='layer'.");
2034
+ return json(await sendVideoCommand({ op: "removeLayerFx", layerId, fxId }));
2035
+ }
2036
+ if (!instanceId)
2037
+ return error("remove_video_fx: instanceId required for target='output'.");
2038
+ return json(await sendVideoCommand({ op: "removeOutputFx", instanceId }));
2039
+ });
2040
+ server.registerTool("bind_video_param", {
2041
+ title: "Bind Video Param (audio-reactive)",
2042
+ description: "Bind a per-layer FX param or layer field to an audio/transport source so it reacts " +
2043
+ "to the music. Omit fxId (or pass '') to target a layer field: transform.x, " +
2044
+ "transform.y, transform.scale, transform.rotation, opacity, hue, brightness, " +
2045
+ "contrast, saturation, playbackRate, fontSize, pulseAmount, or model3d fields. " +
2046
+ "Pass fxId to target an FX param (param key from that FX). " +
2047
+ VIDEO_BIND_SOURCES +
2048
+ " amount is -1..1; curve linear|exp|expInverse|bipolar; mode additive|replace|" +
2049
+ "multiply|centered; impulse off|pulse|latch (one-shot on threshold crossing). " +
2050
+ "attack/release default per-source if omitted.",
2051
+ inputSchema: {
2052
+ layerId: z.string().describe("Target layer id."),
2053
+ paramKey: z.string().describe("FX param key, or layer field (e.g. transform.scale, opacity)."),
2054
+ source: z
2055
+ .union([z.string(), z.record(z.string(), z.any())])
2056
+ .describe("sourceKey string (e.g. 'g:bandLow', 'c:1:onset', 't:beat') or ModSource object."),
2057
+ fxId: z.string().optional().describe("FX instance id; omit/'' for a layer-field target."),
2058
+ amount: z.number().min(-1).max(1).optional().describe("Modulation depth (-1..1), default 1."),
2059
+ curve: z.enum(["linear", "exp", "expInverse", "bipolar"]).optional(),
2060
+ mode: z.enum(["additive", "replace", "multiply", "centered"]).optional(),
2061
+ attackMs: z.number().min(0).optional(),
2062
+ releaseMs: z.number().min(0).optional(),
2063
+ threshold: z.number().min(0).max(1).optional().describe("Gate / impulse trigger level."),
2064
+ impulse: z.enum(["off", "pulse", "latch"]).optional(),
2065
+ },
2066
+ }, async ({ layerId, paramKey, source, fxId, amount, curve, mode, attackMs, releaseMs, threshold, impulse }) => {
2067
+ const check = await requireBridge();
2068
+ if (check !== true)
2069
+ return check;
2070
+ const result = await sendVideoCommand({
2071
+ op: "bind",
2072
+ layerId,
2073
+ fxId: fxId ?? "",
2074
+ paramKey,
2075
+ source,
2076
+ amount,
2077
+ curve,
2078
+ mode,
2079
+ attackMs,
2080
+ releaseMs,
2081
+ threshold,
2082
+ impulse,
2083
+ });
2084
+ return json(result);
2085
+ });
2086
+ server.registerTool("set_video_settings", {
2087
+ title: "Set Video Settings",
2088
+ description: "Set global VX render/sync settings: syncMode (off|beat-sync|beat-match), loopBars, " +
2089
+ "aspectRatio (auto|16:9|4:3|1:1|stretch), renderQuality (low|medium|high), " +
2090
+ "designAspect (w/h number or null), letterboxColor / canvasColor (#hex), the active " +
2091
+ "video cell (cellId), and the mod-matrix master enable. resetWorkspace clears the " +
2092
+ "scene (preserves the master layer).",
2093
+ inputSchema: {
2094
+ syncMode: z.enum(["off", "beat-sync", "beat-match"]).optional(),
2095
+ loopBars: z.number().optional(),
2096
+ aspectRatio: z.enum(["auto", "16:9", "4:3", "1:1", "stretch"]).optional(),
2097
+ renderQuality: z.enum(["low", "medium", "high"]).optional(),
2098
+ designAspect: z.number().nullable().optional(),
2099
+ letterboxColor: z.string().optional(),
2100
+ canvasColor: z.string().optional(),
2101
+ modEnabled: z.boolean().optional().describe("Mod-matrix master enable."),
2102
+ activeCell: z.number().int().optional().describe("Switch the active per-cell video slice."),
2103
+ resetWorkspace: z.boolean().optional().describe("If true, clear the whole scene first."),
2104
+ },
2105
+ }, async ({ syncMode, loopBars, aspectRatio, renderQuality, designAspect, letterboxColor, canvasColor, modEnabled, activeCell, resetWorkspace }) => {
2106
+ const check = await requireBridge();
2107
+ if (check !== true)
2108
+ return check;
2109
+ if (resetWorkspace)
2110
+ await sendVideoCommand({ op: "resetWorkspace" });
2111
+ if (activeCell !== undefined)
2112
+ await sendVideoCommand({ op: "setActiveCell", cellId: activeCell });
2113
+ if (modEnabled !== undefined)
2114
+ await sendVideoCommand({ op: "setModEnabled", enabled: modEnabled });
2115
+ const settings = {};
2116
+ if (syncMode !== undefined)
2117
+ settings.syncMode = syncMode;
2118
+ if (loopBars !== undefined)
2119
+ settings.loopBars = loopBars;
2120
+ if (aspectRatio !== undefined)
2121
+ settings.aspectRatio = aspectRatio;
2122
+ if (renderQuality !== undefined)
2123
+ settings.renderQuality = renderQuality;
2124
+ if (designAspect !== undefined)
2125
+ settings.designAspect = designAspect;
2126
+ if (letterboxColor !== undefined)
2127
+ settings.letterboxColor = letterboxColor;
2128
+ if (canvasColor !== undefined)
2129
+ settings.canvasColor = canvasColor;
2130
+ if (Object.keys(settings).length > 0)
2131
+ await sendVideoCommand({ op: "setSettings", ...settings });
2132
+ return text("Video settings updated.");
2133
+ });
2134
+ // ============================================================
1429
2135
  // Composite cell-operation tools
1430
2136
  // ============================================================
1431
2137
  const PATTERN_SUFFIXES = [
@@ -1813,7 +2519,7 @@ server.registerTool("set_drum_synth", {
1813
2519
  "sample for a pure synth voice, or layer over a sample. Each engine reinterprets the 8 generic params (p1..p8); pass " +
1814
2520
  "only the ones you want to nuance — unspecified params seed to the engine's musical default. Per-engine p-labels: " +
1815
2521
  DRUM_SYNTH_ENGINES.map((e) => `${e.name}=[${e.params.map((p) => p[0]).join(",")}]`).join("; ") + ". " +
1816
- "NOTE: requires a PAM build with drum synth (manifest.drumSynth.enabled) AND Settings → Experimental → Drum Synth ON. " +
2522
+ "NOTE: requires a PAM build with drum synth (manifest.drumSynth.enabled) — a standard PAM feature, no settings flag. " +
1817
2523
  "On builds without it, the cell{N}_synthEngine state key is ignored.",
1818
2524
  inputSchema: {
1819
2525
  cellId: z.number().min(1).max(8).describe("Cell number (1-8)."),
@@ -1984,56 +2690,110 @@ server.registerTool("gain_sample", {
1984
2690
  await bridge.send("gainSampleForCell", { cellId, gainDb });
1985
2691
  return text(`Applied ${gainDb > 0 ? "+" : ""}${gainDb} dB to the sample in cell ${cellId}.`);
1986
2692
  });
2693
+ // Poll the offline-render status flag until the export completes or times out.
2694
+ async function pollExportStatus(maxWaitMs = 180_000, pollMs = 500) {
2695
+ const t0 = Date.now();
2696
+ while (Date.now() - t0 < maxWaitMs) {
2697
+ await new Promise((r) => setTimeout(r, pollMs));
2698
+ const status = (await bridge.send("getExportStatus"));
2699
+ if (status.complete)
2700
+ return status.success ? "success" : "failed";
2701
+ }
2702
+ return "timeout";
2703
+ }
1987
2704
  server.registerTool("export_audio", {
1988
2705
  title: "Export Audio",
1989
- description: "Render and export audio from the current preset to a WAV file. " +
1990
- "Uses the current BPM to calculate duration. Waits for completion and returns the output file path. " +
1991
- "Use get_audio_info afterwards to analyze the exported file.",
2706
+ description: "Render and export audio from the current preset to a WAV file in PAM's Exports folder. " +
2707
+ "Waits for completion. Default mode renders the looping preset for `bars` bars at the current BPM. " +
2708
+ "Set `mode` to bounce stems or the whole Song Mode arrangement.\n\n" +
2709
+ "Modes: 'main' (full mix, default) · 'individual' (one cell solo, needs cellId) · " +
2710
+ "'all-cells' / 'selected-cells' (per-cell stems, the latter needs selectedCellIds) · " +
2711
+ "'per-bus' (per output bus) · 'song-mode' (the Song Mode timeline).\n\n" +
2712
+ "NOTE on 'song-mode': the OFFLINE bounce only renders the arrangement's FIRST section's live state — " +
2713
+ "variation/preset switches are not applied offline (the message thread is blocked during render). " +
2714
+ "For a true full track that captures every section change, use the `export_song` tool (live capture).",
1992
2715
  inputSchema: {
1993
- bars: z.number().min(1).max(128).default(4).describe("Number of bars to render."),
2716
+ bars: z.number().min(1).max(128).default(4).describe("Bars to render (ignored for 'song-mode', which derives length from the timeline)."),
1994
2717
  cellId: z
1995
2718
  .number()
1996
2719
  .min(1)
1997
2720
  .max(8)
1998
2721
  .optional()
1999
- .describe("Render only a specific cell in solo. Omit for full mix."),
2722
+ .describe("Render only a specific cell in solo (implies mode 'individual'). Omit for full mix."),
2723
+ mode: z
2724
+ .enum(["main", "individual", "all-cells", "selected-cells", "per-bus", "song-mode"])
2725
+ .optional()
2726
+ .describe("Export mode. Defaults to 'main' (or 'individual' when cellId is set)."),
2727
+ selectedCellIds: z
2728
+ .array(z.number().min(1).max(8))
2729
+ .optional()
2730
+ .describe("For mode 'selected-cells': cells to bounce as stems."),
2000
2731
  bitDepth: z
2001
2732
  .number()
2002
2733
  .default(24)
2003
2734
  .describe("Bit depth (16 or 24)."),
2004
2735
  },
2005
- }, async ({ bars, cellId, bitDepth }) => {
2736
+ }, async ({ bars, cellId, mode, selectedCellIds, bitDepth }) => {
2006
2737
  const check = await requireBridge();
2007
2738
  if (check !== true)
2008
2739
  return check;
2009
- const options = { bars, bitDepth };
2010
- if (cellId)
2011
- options.cellId = cellId;
2012
- const startResult = await bridge.send("exportAudio", options);
2013
- // If bridge returned old-style "ok", it was a pass-through with full options
2014
- if (typeof startResult === "string") {
2015
- return text(`Audio export started (${bars} bars, ${bitDepth}-bit).`);
2016
- }
2017
- const result = startResult;
2018
- const outputPath = result.outputPath;
2019
- const durationSecs = result.durationInSeconds;
2020
- const bpm = result.bpm;
2021
- // Poll for completion
2022
- const maxWaitMs = 120_000;
2023
- const pollMs = 500;
2024
- const t0 = Date.now();
2025
- while (Date.now() - t0 < maxWaitMs) {
2026
- await new Promise((r) => setTimeout(r, pollMs));
2027
- const status = (await bridge.send("getExportStatus"));
2028
- if (status.complete) {
2029
- if (status.success) {
2030
- return text(`Audio exported successfully.\nFile: ${outputPath}\n` +
2031
- `Duration: ${durationSecs.toFixed(1)}s (${bars} bars at ${bpm} BPM, ${bitDepth}-bit)`);
2032
- }
2033
- return error(`Audio export failed.`);
2740
+ // Resolve the effective mode. A bare cellId means "individual".
2741
+ const effectiveMode = mode ?? (cellId ? "individual" : "main");
2742
+ // Fast path: main / individual go through the MCP-friendly bridge branch,
2743
+ // which constructs options server-side and returns the output path + duration.
2744
+ if (effectiveMode === "main" || (effectiveMode === "individual" && cellId)) {
2745
+ const options = { bars, bitDepth };
2746
+ if (cellId)
2747
+ options.cellId = cellId;
2748
+ const startResult = await bridge.send("exportAudio", options);
2749
+ if (typeof startResult === "string") {
2750
+ return text(`Audio export started (${bars} bars, ${bitDepth}-bit).`);
2034
2751
  }
2752
+ const result = startResult;
2753
+ const outputPath = result.outputPath;
2754
+ const durationSecs = result.durationInSeconds;
2755
+ const bpm = result.bpm;
2756
+ const outcome = await pollExportStatus();
2757
+ if (outcome === "success")
2758
+ return text(`Audio exported successfully.\nFile: ${outputPath}\n` +
2759
+ `Duration: ${durationSecs.toFixed(1)}s (${bars} bars at ${bpm} BPM, ${bitDepth}-bit)`);
2760
+ if (outcome === "failed")
2761
+ return error("Audio export failed.");
2762
+ return text(`Export timed out. The render may still be running.\nExpected output: ${outputPath}`);
2035
2763
  }
2036
- return text(`Export timed out after ${maxWaitMs / 1000}s. The render may still be running.\nExpected output: ${outputPath}`);
2764
+ // Stem / song-mode modes: build full pass-through options (filename + mode +
2765
+ // durationInSeconds) so the C++ export handler runs the requested mode directly.
2766
+ if (effectiveMode === "selected-cells" && (!selectedCellIds || selectedCellIds.length === 0))
2767
+ return error("mode 'selected-cells' requires selectedCellIds.");
2768
+ if (effectiveMode === "individual" && !cellId)
2769
+ return error("mode 'individual' requires cellId.");
2770
+ const transport = (await bridge.send("getTransport"));
2771
+ const bpm = typeof transport.bpm === "number" && transport.bpm > 0 ? transport.bpm : 120;
2772
+ // 4/4 assumption for the placeholder duration; 'song-mode' recomputes its own length from the timeline.
2773
+ const durationInSeconds = (bars * 4 * 60) / bpm;
2774
+ const filename = `pam_export_${effectiveMode.replace(/[^a-z0-9]/gi, "_")}_${Date.now()}`;
2775
+ const options = {
2776
+ filename,
2777
+ mode: effectiveMode,
2778
+ durationInSeconds,
2779
+ bitDepth,
2780
+ };
2781
+ if (cellId)
2782
+ options.selectedCellId = cellId;
2783
+ if (selectedCellIds)
2784
+ options.selectedCellIds = selectedCellIds;
2785
+ await bridge.send("exportAudio", options);
2786
+ const outcome = await pollExportStatus();
2787
+ if (outcome === "success")
2788
+ return text(`Export complete (mode '${effectiveMode}').\n` +
2789
+ `File(s) written to PAM's Exports folder, base name: ${filename}.wav` +
2790
+ (effectiveMode === "song-mode"
2791
+ ? "\n\nReminder: the offline song-mode bounce captures only the first section's state. " +
2792
+ "Use export_song for a full track with section changes."
2793
+ : ""));
2794
+ if (outcome === "failed")
2795
+ return error(`Export failed (mode '${effectiveMode}').`);
2796
+ return text(`Export timed out (mode '${effectiveMode}'). The render may still be running.`);
2037
2797
  });
2038
2798
  // ============================================================
2039
2799
  // RESOURCES
@@ -3126,6 +3886,99 @@ server.registerTool("song_mode", {
3126
3886
  }
3127
3887
  return text("No-op");
3128
3888
  });
3889
+ server.registerTool("export_song", {
3890
+ title: "Export Song (Full Track)",
3891
+ description: "Bounce the full Song Mode arrangement to a single WAV — the 'render my whole track' button. " +
3892
+ "Set up the arrangement first with the `song_mode` tool (setSequence), then call this.\n\n" +
3893
+ "Two strategies:\n" +
3894
+ "• 'live' (DEFAULT, full fidelity): plays the arrangement in real time and captures the master mix, " +
3895
+ "so EVERY section/variation/preset change and all modulation is rendered. Needs a free `cellId` to " +
3896
+ "land the captured clip in (use an empty cell). Capture is capped at 64 bars; for longer songs bounce " +
3897
+ "in sections or raise the cap. The captured WAV also lands in PAM's Resamples folder.\n" +
3898
+ "• 'offline' (fast, LIMITED): renders via the offline song-mode bounce. Only the first section's state " +
3899
+ "is rendered (variation/preset switches are NOT applied offline). Use only for single-section sketches.\n\n" +
3900
+ "After a 'live' bounce, the captured file path is read back from the destination cell; run get_audio_info " +
3901
+ "(resolve against the Resamples location from get_default_locations) to verify it.",
3902
+ inputSchema: {
3903
+ strategy: z.enum(["live", "offline"]).default("live").describe("Capture strategy."),
3904
+ bars: z
3905
+ .number()
3906
+ .min(1)
3907
+ .max(64)
3908
+ .describe("Total arrangement length in bars (the full timeline). Required for 'live'."),
3909
+ cellId: z
3910
+ .number()
3911
+ .min(1)
3912
+ .max(8)
3913
+ .optional()
3914
+ .describe("For 'live': empty destination cell the bounced master clip is written into."),
3915
+ bitDepth: z.number().default(24).describe("Bit depth (16 or 24)."),
3916
+ tailBars: z
3917
+ .number()
3918
+ .min(0)
3919
+ .max(8)
3920
+ .default(1)
3921
+ .describe("For 'live': extra bars captured after the arrangement end for reverb/release tails."),
3922
+ },
3923
+ }, async ({ strategy, bars, cellId, bitDepth, tailBars }) => {
3924
+ const check = await requireBridge();
3925
+ if (check !== true)
3926
+ return check;
3927
+ // -------- Offline strategy: reuse the song-mode bounce (first-section only) --------
3928
+ if (strategy === "offline") {
3929
+ const filename = `pam_song_${Date.now()}`;
3930
+ const transport = (await bridge.send("getTransport"));
3931
+ const bpm = typeof transport.bpm === "number" && transport.bpm > 0 ? transport.bpm : 120;
3932
+ await bridge.send("exportAudio", {
3933
+ filename,
3934
+ mode: "song-mode",
3935
+ durationInSeconds: (bars * 4 * 60) / bpm,
3936
+ bitDepth,
3937
+ });
3938
+ const outcome = await pollExportStatus();
3939
+ if (outcome === "success")
3940
+ return text(`Offline song bounce complete: ${filename}.wav (in PAM's Exports folder).\n` +
3941
+ "Note: only the first section's state was rendered. Use strategy 'live' for the full track.");
3942
+ if (outcome === "failed")
3943
+ return error("Offline song bounce failed. Is Song Mode enabled with a non-empty sequence?");
3944
+ return text("Offline song bounce timed out. The render may still be running.");
3945
+ }
3946
+ // -------- Live strategy: real-time master capture of the whole arrangement --------
3947
+ if (typeof cellId !== "number")
3948
+ return error("'live' strategy requires an empty destination cellId.");
3949
+ const transport = (await bridge.send("getTransport"));
3950
+ const bpm = typeof transport.bpm === "number" && transport.bpm > 0 ? transport.bpm : 120;
3951
+ const totalBars = bars + (tailBars ?? 1);
3952
+ if (totalBars > 64)
3953
+ return error(`Total capture (${totalBars} bars incl. tail) exceeds the 64-bar live cap. Bounce in sections.`);
3954
+ const captureSeconds = (totalBars * 4 * 60) / bpm;
3955
+ // Reset playhead to the top so the arrangement starts at section 1, enable Song
3956
+ // Mode, then roll the transport and capture the full master mix.
3957
+ await bridge.send("fullStopTransport");
3958
+ await bridge.send("toggleSongMode", true);
3959
+ await bridge.send("setSongModePlayhead", 0);
3960
+ await bridge.send("startTransport");
3961
+ await bridge.send("resampleGraphToCell", JSON.stringify({ cellId, bars: totalBars, live: true, captureSource: 2, cellOnly: false }));
3962
+ // Wait out the arrangement (plus a small safety margin), then stop + finalize.
3963
+ await new Promise((r) => setTimeout(r, captureSeconds * 1000 + 1500));
3964
+ await bridge.send("stopLiveResample", true);
3965
+ await bridge.send("stopTransport");
3966
+ // Read the captured clip path back off the destination cell.
3967
+ let capturedPath = "(unknown — check PAM's Resamples folder)";
3968
+ try {
3969
+ const state = (await bridge.send("getState"));
3970
+ const cells = state.cells;
3971
+ const fp = cells?.[String(cellId)]?.filePath;
3972
+ if (typeof fp === "string" && fp.length > 0)
3973
+ capturedPath = fp;
3974
+ }
3975
+ catch {
3976
+ // fall through with the placeholder
3977
+ }
3978
+ return text(`Live song bounce complete (${bars} bars + ${tailBars ?? 1} tail @ ${bpm} BPM, ${bitDepth}-bit).\n` +
3979
+ `Captured into cell ${cellId}.\nClip: ${capturedPath}\n` +
3980
+ "Resolve the absolute path against the Resamples location from get_default_locations, then verify with get_audio_info.");
3981
+ });
3129
3982
  // ============================================================
3130
3983
  // TOOLS — Metronome
3131
3984
  // ============================================================
@@ -3290,8 +4143,12 @@ server.registerTool("export_slice", {
3290
4143
  // ============================================================
3291
4144
  server.registerTool("live_resample", {
3292
4145
  title: "Live Resample",
3293
- description: "Capture PAM's audio graph output into a cell as a new sample. Ops: 'start' / 'stop'. " +
3294
- "Transport must be playing for the capture to produce audio.",
4146
+ description: "Capture PAM's audio graph output into a cell as a new sample, in real time. Ops: 'start' / 'stop'. " +
4147
+ "Transport MUST be playing for the capture to produce audio. Because the capture runs live, " +
4148
+ "Song Mode section changes, modulation, and variation switches are all captured faithfully — " +
4149
+ "this is the building block for full-track / Song Mode bounces (see export_song).\n\n" +
4150
+ "`source` selects what to record: 'cell' (the destination cell's own output), 'input' (PAM's audio input), " +
4151
+ "or 'fullmix' (the whole master mix — use this to bounce the entire arrangement).",
3295
4152
  inputSchema: {
3296
4153
  op: z.enum(["start", "stop"]).describe("Operation."),
3297
4154
  cellId: z
@@ -3299,13 +4156,21 @@ server.registerTool("live_resample", {
3299
4156
  .min(1)
3300
4157
  .max(8)
3301
4158
  .optional()
3302
- .describe("For 'start': destination cell."),
4159
+ .describe("For 'start': destination cell the captured clip lands in. Use an empty/scratch cell for a full-mix bounce."),
3303
4160
  bars: z
3304
4161
  .number()
3305
4162
  .min(0.125)
3306
4163
  .max(64)
3307
4164
  .optional()
3308
- .describe("For 'start': capture length in bars."),
4165
+ .describe("For 'start': capture length in bars (max 64; the capture also stops on the 'stop' op)."),
4166
+ source: z
4167
+ .enum(["cell", "input", "fullmix"])
4168
+ .default("cell")
4169
+ .describe("For 'start': capture source. 'fullmix' records the entire master output (the whole song)."),
4170
+ normalize: z
4171
+ .boolean()
4172
+ .default(false)
4173
+ .describe("For 'start': peak-normalize the captured clip after recording."),
3309
4174
  finalize: z
3310
4175
  .boolean()
3311
4176
  .default(true)
@@ -3318,8 +4183,19 @@ server.registerTool("live_resample", {
3318
4183
  if (args.op === "start") {
3319
4184
  if (typeof args.cellId !== "number" || typeof args.bars !== "number")
3320
4185
  return error("'start' requires cellId + bars");
3321
- await bridge.send("resampleGraphToCell", JSON.stringify({ cellId: args.cellId, bars: args.bars }));
3322
- return text(`Started live resample → cell ${args.cellId} (${args.bars} bars).`);
4186
+ const source = args.source ?? "cell";
4187
+ const captureSource = source === "fullmix" ? 2 : source === "input" ? 1 : 0;
4188
+ await bridge.send("resampleGraphToCell", JSON.stringify({
4189
+ cellId: args.cellId,
4190
+ bars: args.bars,
4191
+ live: true,
4192
+ captureSource,
4193
+ // For a full-mix / input bounce the destination cell must NOT be soloed,
4194
+ // or we'd only capture that one (empty) cell.
4195
+ cellOnly: source === "cell",
4196
+ normalize: args.normalize ?? false,
4197
+ }));
4198
+ return text(`Started live resample → cell ${args.cellId} (${args.bars} bars, source '${source}').`);
3323
4199
  }
3324
4200
  await bridge.send("stopLiveResample", args.finalize ?? true);
3325
4201
  return text(`Stopped live resample (finalize=${args.finalize ?? true}).`);
@@ -3592,14 +4468,15 @@ server.registerTool("update_settings", {
3592
4468
  title: "Update Plugin Settings",
3593
4469
  description: "Merge a partial settings delta into PAM's plugin settings (the Settings panel state) and apply it live. " +
3594
4470
  "Returns the full merged settings object so you can discover available keys. This is the only way to flip " +
3595
- "experimental-feature gates from an agent. Common boolean keys: experimentalMode, experimentalDrumSynth, " +
3596
- "experimentalPluginHost, experimentalMcpBridge. To enable the drum synth or plugin host, set BOTH " +
3597
- "experimentalMode:true AND the specific sub-flag (e.g. experimentalDrumSynth:true) in one call. Other keys " +
4471
+ "experimental-feature gates from an agent. Common boolean keys: experimentalMode, " +
4472
+ "experimentalPluginHost, experimentalMcpBridge. To enable the plugin host, set BOTH " +
4473
+ "experimentalMode:true AND the specific sub-flag (e.g. experimentalPluginHost:true) in one call. " +
4474
+ "The drum synth needs no flag — it is a standard feature on PAM. Other keys " +
3598
4475
  "include UI/library preferences — call get_state or inspect the returned object for the current set.",
3599
4476
  inputSchema: {
3600
4477
  settings: z
3601
4478
  .record(z.string(), z.any())
3602
- .describe("Partial settings object to merge. Example: { experimentalMode: true, experimentalDrumSynth: true }"),
4479
+ .describe("Partial settings object to merge. Example: { experimentalMode: true, experimentalPluginHost: true }"),
3603
4480
  },
3604
4481
  }, async ({ settings }) => {
3605
4482
  const check = await requireBridge();