@williamzujkowski/live-coding-music-mcp 4.0.0 → 4.1.0

package/dist/server/tools/ai.js

@@ -8,7 +8,11 @@
  * pattern (tempo/key/style detection, layer detection, merge logic)
  * and that are used only here. They live in this module.
  */
+import { empty, err, withStashField } from './types.js';
+import { AiRateLimitError } from '../../services/ai/AiTransport.js';
 import { Logger } from '../../utils/Logger.js';
+import { lookup } from '../../utils/TableLookup.js';
+import { declaredBpm } from '../../utils/Tempo.js';
 const logger = new Logger();
 const SESSION_ID_PROP = {
     session_id: {
@@ -25,7 +29,8 @@ export const tools = [
             'task=jam generates a fresh layer (drums/bass/melody/pad/texture) and merges it into the current pattern, then auto-plays. ' +
             'All three share Gemini auth + rate limiting. ' +
             'Example: ai_assist({ task: "jam", layer: "bass" }). ' +
-            'Requires GEMINI_API_KEY env var. For non-AI pattern generation use generate_part; for full compositions use compose.',
+            'Needs GEMINI_API_KEY, or a logged-in claude/agy/codex CLI on PATH (#252). ' +
+            'For non-AI pattern generation use generate_part; for full compositions use compose.',
         inputSchema: {
             type: 'object',
             properties: {
@@ -78,7 +83,8 @@ async function getPatternFeedback(includeAudio, style, ctx, sid) {
     if (!ctx.geminiService.isAvailable()) {
         return {
             gemini_available: false,
-            error: 'Gemini API not configured. Set GEMINI_API_KEY environment variable to enable AI feedback.',
+            error: 'No AI transport available. Set GEMINI_API_KEY, or log in to one of the '
+                + 'claude/agy/codex CLIs on PATH (#252).',
         };
     }
     const pattern = await ctx.getCurrentPatternSafe(sid);
@@ -92,19 +98,40 @@ async function getPatternFeedback(includeAudio, style, ctx, sid) {
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         logger.error('Pattern feedback failed', { error: message });
-        if (message?.includes('rate limit') || message?.includes('Rate limit')) {
-            return { gemini_available: true, error: message };
+        // By type, not by phrase. `AiRateLimitError` is what the limiter
+        // throws; matching the words meant a reworded message would fall
+        // through to "Pattern analysis failed" and lose the one thing a
+        // caller can act on — that waiting works (#380, #392).
+        if (error instanceof AiRateLimitError) {
+            return err('transient', message, { isRetryable: true });
         }
         result.error = `Pattern analysis failed: ${message}`;
     }
     if (includeAudio && (sid || ctx.isInitialized())) {
         try {
-            const audioBlob = await captureAudioSampleForFeedback(ctx, sid);
-            if (audioBlob) {
-                result.audio_analysis = await ctx.geminiService.analyzeAudio(audioBlob, { style, duration: 5 });
+            const sample = await captureAudioSampleForFeedback(ctx, sid);
+            if (sample === null) {
+                logger.warn('Audio capture returned no data');
+                if (!result.error)
+                    result.error = 'Audio capture returned no data.';
+            }
+            else if (sample.silent) {
+                // Sending silence to Gemini does not fail — it returns a confident
+                // mood/style/energy for audio that is not there. Feedback silently
+                // decoupled from the audio is worse than no feedback, so refuse.
+                result.audio_analysis_skipped =
+                    'Captured audio was silent, so no feedback was requested. ' +
+                        'Is a pattern playing? Call playback({ action: "play" }) first.';
+                logger.warn('Skipping Gemini audio analysis: capture was silent', {
+                    peak: sample.peak,
+                });
             }
             else {
-                logger.warn('Audio capture returned no data');
+                // Measurements, not the waveform. Every installed CLI answers
+                // "CANNOT DECODE AUDIO", and agy will confabulate detailed
+                // analysis of audio it never examined rather than admit it.
+                result.audio_analysis = await ctx.geminiService.analyzeAudioMeasurements(sample.measurements, pattern, { style, duration: FEEDBACK_SAMPLE_MS / 1000 });
+                result.audio_levels = { peak: sample.peak, rms: sample.rms };
             }
         }
         catch (error) {
@@ -121,6 +148,8 @@ async function getPatternFeedback(includeAudio, style, ctx, sid) {
     }
     return result;
 }
+/** Seconds of audio sent to Gemini for feedback. */
+const FEEDBACK_SAMPLE_MS = 5000;
 /**
  * Captures a brief audio sample for Gemini analysis. Different from the
  * `capture_audio_sample` tool — this one connects directly to the
@@ -133,62 +162,31 @@ async function captureAudioSampleForFeedback(ctx, sid) {
         logger.warn('Cannot capture audio: controller page not available');
         return null;
     }
-    try {
-        const audioData = await page.evaluate(/* istanbul ignore next */ async () => {
-            return new Promise((resolve) => {
-                const analyzer = window.strudelAudioAnalyzer;
-                if (!analyzer || !analyzer.analyser) {
-                    resolve(null);
-                    return;
-                }
-                try {
-                    const audioCtx = analyzer.analyser.context;
-                    const destination = audioCtx.createMediaStreamDestination();
-                    analyzer.analyser.connect(destination);
-                    const mediaRecorder = new MediaRecorder(destination.stream, { mimeType: 'audio/webm;codecs=opus' });
-                    const chunks = [];
-                    mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0)
-                        chunks.push(e.data); };
-                    mediaRecorder.onstop = async () => {
-                        try {
-                            analyzer.analyser.disconnect(destination);
-                        }
-                        catch { /* already disconnected */ }
-                        if (chunks.length === 0) {
-                            resolve(null);
-                            return;
-                        }
-                        const blob = new Blob(chunks, { type: 'audio/webm' });
-                        const reader = new FileReader();
-                        reader.onloadend = () => {
-                            const base64 = reader.result;
-                            resolve(base64.split(',')[1] || null);
-                        };
-                        reader.onerror = () => resolve(null);
-                        reader.readAsDataURL(blob);
-                    };
-                    mediaRecorder.start();
-                    setTimeout(() => { if (mediaRecorder.state === 'recording')
-                        mediaRecorder.stop(); }, 5000);
-                }
-                catch {
-                    resolve(null);
-                }
-            });
-        });
-        if (!audioData)
-            return null;
-        const binaryString = atob(audioData);
-        const bytes = new Uint8Array(binaryString.length);
-        for (let i = 0; i < binaryString.length; i++)
-            bytes[i] = binaryString.charCodeAt(i);
-        return new Blob([bytes], { type: 'audio/webm' });
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        logger.error('Audio capture failed', { error: message });
+    // Ensures the recorder is injected for this session before exporting.
+    await ctx.getAudioCaptureService(sid);
+    const result = await ctx.audioExportService.exportAudio(page, {
+        duration: FEEDBACK_SAMPLE_MS,
+        format: 'wav',
+        output: 'base64',
+    });
+    if (!result.success || result.audio === undefined) {
+        logger.warn('Audio capture returned no data', { error: result.error });
         return null;
     }
+    // The bytes are deliberately discarded. No model in play can decode
+    // audio, so what gets sent is the measurement set, not the waveform.
+    return {
+        measurements: {
+            durationMs: result.duration,
+            peak: result.peak,
+            rms: result.rms,
+            sampleRate: result.sampleRate,
+            channels: result.channels,
+        },
+        peak: result.peak,
+        rms: result.rms,
+        silent: result.silent === true,
+    };
 }
 // ---------------------------------------------------------------------------
 // suggest_pattern_from_audio
@@ -198,7 +196,10 @@ async function suggestPatternFromAudio(style, role, ctx, sid) {
         return { error: 'Browser not initialized. Run init and play a pattern first.' };
     }
     if (!ctx.geminiService.isAvailable()) {
-        return { error: 'Gemini API not configured. Set GEMINI_API_KEY to enable AI features.' };
+        return {
+            error: 'No AI transport available. Set GEMINI_API_KEY, or log in to one of the '
+                + 'claude/agy/codex CLIs on PATH (#252).',
+        };
     }
     const controller = ctx.getController(sid);
     let bpm = 0, key = 'C', scale = 'major';
@@ -222,7 +223,7 @@ async function suggestPatternFromAudio(style, role, ctx, sid) {
         melody: 'a melodic line that harmonizes with the key',
         percussion: 'a percussion layer that adds rhythmic interest',
     };
-    const roleText = roleDesc[role] || roleDesc['complement'];
+    const roleText = lookup(roleDesc, role, roleDesc['complement']);
     const styleText = style ? ` in a ${style} style` : '';
     const tempoText = bpm > 0 ? `Detected tempo: ${bpm} BPM. ` : '';
     const keyText = `Detected key: ${key} ${scale}. `;
@@ -240,10 +241,17 @@ Example patterns:
     try {
         const geminiResponse = await ctx.geminiService.suggestVariations(prompt, style);
         if (!geminiResponse || geminiResponse.length === 0) {
-            return { error: 'Gemini returned no pattern suggestions.' };
+            // The model answered; it just had nothing to offer. Returning a
+            // bare `error` key made `isFailureShaped` convert this into an
+            // err() envelope, so a working call was reported as a failure —
+            // the same bug as #274, pointing the other way (#288).
+            return empty({
+                suggestions: [],
+                message: 'Gemini returned no pattern suggestions.',
+            });
         }
         const suggestedPattern = geminiResponse[0].code;
-        const validation = ctx.strudelEngine.validate(suggestedPattern);
+        const validation = await ctx.strudelEngine.validate(suggestedPattern);
         return {
             suggested_pattern: suggestedPattern,
             analysis: { bpm, key, scale },
@@ -293,7 +301,7 @@ async function jamWith(layer, styleHint, autoPlay = true, ctx, sid) {
     }
     let newLayer;
     try {
-        newLayer = generateComplementaryLayer(layer, key, tempo, detectedStyle, existingLayers, ctx);
+        newLayer = generateComplementaryLayer(layer, key, detectedStyle, existingLayers, ctx);
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -308,18 +316,18 @@ async function jamWith(layer, styleHint, autoPlay = true, ctx, sid) {
     }
     const mergedPattern = mergeLayerIntoPattern(currentPattern, newLayer, layer);
     try {
-        await ctx.writePatternSafe(mergedPattern, sid);
+        const written = await ctx.writePatternSafe(mergedPattern, sid);
         if (autoPlay && (sid || ctx.isInitialized())) {
             await ctx.getController(sid).play();
         }
-        return {
+        return withStashField({
             success: true,
             message: `Added ${layer} layer${styleHint ? ` (${styleHint} style)` : ''} to jam with your pattern`,
             layer,
             detected: { tempo, key, existingLayers },
             newLayer,
             pattern: mergedPattern.substring(0, 300) + (mergedPattern.length > 300 ? '...' : ''),
-        };
+        }, written);
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -333,20 +341,29 @@ async function jamWith(layer, styleHint, autoPlay = true, ctx, sid) {
         };
     }
 }
+/**
+ * The tempo a pattern's author meant, for generating a layer to sit on
+ * top of it.
+ *
+ * Uses the canonical parser rather than a regex of its own. The regex it
+ * had required a bare `setcpm(<n>)`, so it matched nothing the generator
+ * writes — every generated pattern is `setcpm(130/4)` — and fell through
+ * to guessing from words in the text, which is how `jam_with` came to
+ * read a tempo off a comment (#397).
+ *
+ * `declaredBpm`, not `impliedBpm`: this reads patterns written by someone
+ * else, and only the number they wrote is knowable. Working out what one
+ * will actually sound like takes an assumption about beats per cycle
+ * that holds for what this project generates and not for an arbitrary
+ * hand-written pattern.
+ */
 function detectTempoFromPattern(pattern) {
-    const cpmMatch = pattern.match(/setcpm\s*\(\s*(\d+(?:\.\d+)?)\s*\)/i);
-    if (cpmMatch)
-        return Math.round(parseFloat(cpmMatch[1]));
-    const bpmMatch = pattern.match(/setbpm\s*\(\s*(\d+(?:\.\d+)?)\s*\)/i);
-    if (bpmMatch)
-        return Math.round(parseFloat(bpmMatch[1]));
-    const cpsMatch = pattern.match(/setcps\s*\(\s*(\d+(?:\.\d+)?)\s*(?:\/\s*60)?\s*\)/i);
-    if (cpsMatch) {
-        const cps = parseFloat(cpsMatch[1]);
-        if (pattern.includes(`setcps(${cpsMatch[1]}/60`))
-            return Math.round(cps);
-        return Math.round(cps * 60);
-    }
+    const declared = declaredBpm(pattern);
+    if (declared !== undefined)
+        return Math.round(declared);
+    // No tempo call at all. Guessing from genre words is a poor answer,
+    // but it beats silently assuming 120 for a dnb pattern — and a real
+    // parse always wins over it now, which is the part that was broken.
     if (pattern.toLowerCase().includes('dnb'))
         return 174;
     if (pattern.toLowerCase().includes('techno'))
@@ -375,10 +392,13 @@ function detectKeyFromPattern(pattern) {
     }
     if (allNotes.length === 0)
         return 'C';
-    const noteCounts = {};
+    // Object.create(null): the keys come from pattern text, and both
+    // reading and writing 'constructor'/'__proto__' on a plain literal
+    // misbehave (#318).
+    const noteCounts = Object.create(null);
     for (const note of allNotes) {
         const normalizedNote = note.charAt(0).toUpperCase() + note.slice(1);
-        noteCounts[normalizedNote] = (noteCounts[normalizedNote] || 0) + 1;
+        noteCounts[normalizedNote] = lookup(noteCounts, normalizedNote, 0) + 1;
     }
     let mostCommonNote = 'C';
     let maxCount = 0;
@@ -430,7 +450,14 @@ function detectStyleFromPattern(pattern, styleHint) {
         return 'trap';
     return 'techno';
 }
-function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ctx) {
+function generateComplementaryLayer(layer, key, style, existingLayers, ctx) {
+    // No `tempo` parameter: it was declared, passed, and never read. The
+    // merged layer goes inside the host pattern's own stack and inherits
+    // its tempo call (see `mergeLayerIntoPattern`), so there is nothing
+    // here for a tempo to change. A cross-model reviewer read the dead
+    // parameter as evidence that this function retimes the layer, and
+    // concluded jam_with would desync by 4x; it cannot. Dead arguments
+    // are not free.
     switch (layer) {
         case 'drums':
             if (existingLayers.includes('drums')) {
@@ -443,7 +470,7 @@ function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ct
                     'jungle': 's("hh*32").gain(perlin.range(0.2, 0.4)).hpf(4000)',
                     'jazz': 's("~ ride ~ ride, ~ ~ ~ hh").gain(0.3).room(0.3)',
                 };
-                return percOptions[style] || percOptions['techno'];
+                return lookup(percOptions, style, percOptions['techno']);
             }
             return ctx.generator.generateDrumPattern(style, 0.6);
         case 'bass':
@@ -451,8 +478,14 @@ function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ct
         case 'melody': {
             let scaleName = 'minor';
             let octaveRange = [4, 5];
+            // `major`, not `dorian`. The convention names the mode of the ii
+            // CHORD; rooted on the key it is the wrong collection, and over
+            // the jazz progression this server generates (Dm7 G7 Cmaj7 in C)
+            // it puts Eb against Cmaj7's E (#449). `jam_with` cannot see the
+            // host pattern's chords, so it follows the generator's choice
+            // rather than disagreeing with it.
             if (style === 'jazz') {
-                scaleName = 'dorian';
+                scaleName = 'major';
                 octaveRange = [3, 5];
             }
             if (style === 'ambient') {
@@ -468,7 +501,7 @@ function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ct
                 'trap': '.gain(0.5).room(0.15)', 'jungle': '.delay(0.125).room(0.25).gain(0.55)',
                 'jazz': '.room(0.4).gain(0.5)',
             };
-            return ctx.generator.generateMelody(scale, 8, octaveRange) + (effects[style] || '.room(0.3).gain(0.5)');
+            return ctx.generator.generateMelody(scale, 8, octaveRange) + lookup(effects, style, '.room(0.3).gain(0.5)');
         }
         case 'pad': {
             const safeKey = key.toLowerCase();
@@ -483,7 +516,7 @@ function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ct
                 'jungle': `chord("<${safeKey}m9 ${fourth}m9>/8").dict('ireal').voicing().s("gm_epiano1").gain(0.25).room(0.4).delay(0.25)`,
                 'jazz': `chord("<${safeKey}m9 ${fourth}m9 ${fifth}7>/4").dict('ireal').voicing().s("gm_epiano1").gain(0.3).room(0.5)`,
             };
-            return padPatterns[style] || padPatterns['techno'];
+            return lookup(padPatterns, style, padPatterns['techno']);
         }
         case 'texture': {
             const texturePatterns = {
@@ -495,7 +528,7 @@ function generateComplementaryLayer(layer, key, tempo, style, existingLayers, ct
                 'jungle': `s("breaks125:8").fit().chop(32).gain(0.05).hpf(5000).room(0.4).pan(perlin.range(0.2, 0.8))`,
                 'jazz': `s("brush:1").struct("~ 1 ~ 1 ~ 1 ~ ~").gain(0.1).room(0.5)`,
             };
-            return texturePatterns[style] || texturePatterns['techno'];
+            return lookup(texturePatterns, style, texturePatterns['techno']);
         }
         default:
             throw new Error(`Unknown layer type: ${layer}`);