@manny-est/node-red-flowpilot 0.4.1 → 0.5.1

package/flowpilot.js

@@ -7,6 +7,8 @@ const documentSystemPrompt = require("./lib/document-system-prompt");
 const modifySystemPrompt = require("./lib/modify-system-prompt");
 const buildSystemPrompt = require("./lib/build-system-prompt");
 const personaPrompt = require("./lib/persona-prompt");
+const { buildCoreScript } = require("./lib/build-core-script");
+const { extractJsonObject } = require("./lib/envelope");
 
 module.exports = function flowPilotRuntime(RED) {
   const storage = createStorage(RED.settings.userDir);
@@ -554,13 +556,18 @@ module.exports = function flowPilotRuntime(RED) {
 
     let streamResult;
     try {
-      streamResult = await provider.chatStream(activeProvider, messages, function (delta) {
-        const visible = splitter.push(delta);
-        if (visible) {
-          visibleText += visible;
-          res.write("data: " + JSON.stringify({ delta: visible }) + "\n\n");
+      streamResult = await provider.chatStream(activeProvider, messages,
+        function (delta) {
+          const visible = splitter.push(delta);
+          if (visible) {
+            visibleText += visible;
+            res.write("data: " + JSON.stringify({ delta: visible }) + "\n\n");
+          }
+        },
+        function (reasoningDelta) {
+          res.write("data: " + JSON.stringify({ reasoningDelta: reasoningDelta }) + "\n\n");
         }
-      });
+      );
     } catch (err) {
       res.write("data: " + JSON.stringify({ error: err.message }) + "\n\n");
       res.end();
@@ -625,6 +632,11 @@ module.exports = function flowPilotRuntime(RED) {
   // static lib/debug/view.html that loads the SAME debug-utils.js the
   // sidebar uses).
   //
+  // Phase 9 refactor: the SOURCE is now split into lib/core/*.js fragments
+  // (see lib/build-core-script.js for why and how), but this route's
+  // behavior is unchanged — it still serves one complete script at this
+  // same URL, just assembled instead of read off disk verbatim.
+  //
   // INTENTIONALLY UNGATED (fixed in 0.4.1 — was needsPermission("settings.
   // read") in 0.4.0, which broke the editor on every adminAuth-enabled
   // instance): these are static client assets, fetched via plain <script
@@ -639,7 +651,7 @@ module.exports = function flowPilotRuntime(RED) {
   // routes (settings, chat, generate, modify, etc. below) stay gated.
 
   RED.httpAdmin.get("/flowpilot/core.js", function (req, res) {
-    res.sendFile(path.join(__dirname, "flowpilot-core.js"));
+    res.type("application/javascript").send(buildCoreScript());
   });
 
   RED.httpAdmin.get("/flowpilot/core.css", function (req, res) {
@@ -744,6 +756,10 @@ module.exports = function flowPilotRuntime(RED) {
       if (suggestedAction) { body.suggestedAction = suggestedAction; }
       const questionOptions = extractQuestionOptions(chatData);
       if (questionOptions) { body.questionOptions = questionOptions; }
+      // Pass reasoning_content through so the frontend can render a thinking
+      // block on the non-streaming (agent-loop) path too.
+      const rawMsg = result.raw && result.raw.choices && result.raw.choices[0] && result.raw.choices[0].message;
+      if (rawMsg && rawMsg.reasoning_content) { body.reasoningContent = rawMsg.reasoning_content; }
       res.json(body);
     } catch (err) {
       storage.appendAudit({ action: "chat_error", error: err.message });
@@ -926,29 +942,42 @@ module.exports = function flowPilotRuntime(RED) {
       // probe failure here just means "no tool support", not a /test failure.
       // Persist the result on the provider profile for the agentic tool-calling path.
       const probe = await provider.probeTools(activeProvider);
+      const reasoning = provider.detectReasoning(result.raw);
       storage.appendAudit({
         action: "capability_probe",
         providerName: activeProvider.providerName,
         baseUrl: activeProvider.baseUrl,
         model: activeProvider.model,
-        supportsTools: probe.supportsTools
+        supportsTools: probe.supportsTools,
+        isReasoningModel: reasoning.isReasoningModel
       });
 
       const updatedProviders = (settings.providers || []).map(function (p) {
         return p.id === activeProvider.id
-          ? Object.assign({}, p, { supportsTools: probe.supportsTools, toolsProbedAt: new Date().toISOString() })
+          ? Object.assign({}, p, {
+              supportsTools: probe.supportsTools,
+              toolsProbedAt: new Date().toISOString(),
+              isReasoningModel: reasoning.isReasoningModel,
+              reasoningProbedAt: new Date().toISOString(),
+              probedModel: activeProvider.model
+            })
           : p;
       });
       storage.saveSettings(Object.assign({}, settings, { providers: updatedProviders }));
 
+      const toolLabel = probe.supportsTools
+        ? "✓ Connected · ✓ Supports tools"
+        : "✓ Connected · ⚠ No tool support — compatibility mode";
+      const reasoningLabel = reasoning.isReasoningModel ? " · Reasoning model" : "";
+
       res.json({
         message: chatMessage || "[No assistant message returned by provider]",
         raw: result.raw ? "[raw response captured]" : null,
         capability: {
           supportsTools: probe.supportsTools,
-          label: probe.supportsTools
-            ? "✓ Connected · ✓ Supports tools"
-            : "✓ Connected · ⚠ No tool support — compatibility mode"
+          isReasoningModel: reasoning.isReasoningModel,
+          probedModel: activeProvider.model,
+          label: toolLabel + reasoningLabel
         }
       });
     } catch (err) {
@@ -957,130 +986,63 @@ module.exports = function flowPilotRuntime(RED) {
     }
   });
 
-  // ---- Generate: produce an importable flow fragment --------------------
-  // Uses the generation system prompt and expects the model to return a single
-  // JSON object { explanation, flow }. This first cut does NOT validate node
-  // types or wire integrity yet (that's the next chunk) — it returns the parsed
-  // envelope so the frontend can display it for review.
+  // ---- Probe: silent capability re-check after model change -----------
+  // Called by the frontend when it detects that the active provider's model
+  // changed since the last full pre-flight — stale supportsTools silently
+  // misroutes chat (agent-loop vs streaming/non-streaming). Runs probeTools
+  // + a minimal chat for reasoning detection, saves all results including
+  // probedModel, and returns { supportsTools, isReasoningModel, probedModel }.
 
-  // Given s[startIdx] === "{", scans forward with brace-depth counting that
-  // ignores braces inside string literals (so a value like "{{payload}}"
-  // can't be mistaken for structure) to find the index of the MATCHING
-  // closing "}". Returns -1 if the braces never balance before the string
-  // ends (truncated/malformed input).
-  function findMatchingBrace(s, startIdx) {
-    let depth = 0;
-    let inString = false;
-    let escaped = false;
-    for (let i = startIdx; i < s.length; i++) {
-      const ch = s[i];
-      if (inString) {
-        if (escaped) { escaped = false; }
-        else if (ch === "\\") { escaped = true; }
-        else if (ch === "\"") { inString = false; }
-        continue;
-      }
-      if (ch === "\"") { inString = true; }
-      else if (ch === "{") { depth++; }
-      else if (ch === "}") {
-        depth--;
-        if (depth === 0) { return i; }
-      }
-    }
-    return -1;
-  }
+  RED.httpAdmin.post("/flowpilot/probe", RED.auth.needsPermission("settings.write"), async function (req, res) {
+    try {
+      const settings = storage.getSettings();
+      const activeProvider = storage.getActiveProvider(settings);
 
-  function extractJsonObject(text) {
-    if (!text) { throw new Error("Empty response from provider."); }
-    let s = String(text).trim();
-    // Strip markdown code fences if the model wrapped the JSON.
-    s = s.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "").trim();
-
-    const firstBrace = s.indexOf("{");
-    const firstBracket = s.indexOf("[");
-
-    // The model occasionally returns a bare top-level array (e.g.
-    // `[ {...node...} ]`) instead of the {explanation, flow} envelope. If we
-    // fell through to the {...} extraction below, indexOf("{")/lastIndexOf("}")
-    // would grab just the first node object — which has no "flow" key and
-    // fails validation. Detect this case up front and wrap it as a minimal
-    // envelope instead.
-    if (firstBracket !== -1 && (firstBrace === -1 || firstBracket < firstBrace)) {
-      const lastBracket = s.lastIndexOf("]");
-      if (lastBracket !== -1 && lastBracket > firstBracket) {
-        try {
-          const arr = JSON.parse(s.slice(firstBracket, lastBracket + 1));
-          if (Array.isArray(arr)) {
-            return { explanation: "", flow: arr };
-          }
-        } catch (e) {
-          // Not a parseable array — fall through to the {...} extraction.
-        }
-      }
-    }
+      const probe = await provider.probeTools(activeProvider);
+      const chatResult = await provider.chat(activeProvider, [
+        { role: "system", content: "You are a helpful assistant." },
+        { role: "user", content: "Say hello." }
+      ]);
+      const reasoning = provider.detectReasoning(chatResult.raw);
 
-    const firstObjIdx = s.indexOf("{");
-    if (firstObjIdx === -1) {
-      // No JSON object found at all — flagged separately from a found-
-      // but-unparseable ({...} present, JSON.parse failed) "garbled" error,
-      // so callers can distinguish "model just answered in prose" (tolerate)
-      // from "model's JSON envelope is broken" (still an error).
-      const err = new Error("Provider did not return a JSON object.");
-      err.noJsonFound = true;
-      throw err;
+      const updatedProviders = (settings.providers || []).map(function (p) {
+        return p.id === activeProvider.id
+          ? Object.assign({}, p, {
+              supportsTools: probe.supportsTools,
+              toolsProbedAt: new Date().toISOString(),
+              isReasoningModel: reasoning.isReasoningModel,
+              reasoningProbedAt: new Date().toISOString(),
+              probedModel: activeProvider.model
+            })
+          : p;
+      });
+      storage.saveSettings(Object.assign({}, settings, { providers: updatedProviders }));
+
+      storage.appendAudit({
+        action: "auto_probe",
+        providerName: activeProvider.providerName,
+        baseUrl: activeProvider.baseUrl,
+        model: activeProvider.model,
+        supportsTools: probe.supportsTools,
+        isReasoningModel: reasoning.isReasoningModel
+      });
+
+      res.json({
+        supportsTools: probe.supportsTools,
+        isReasoningModel: reasoning.isReasoningModel,
+        probedModel: activeProvider.model
+      });
+    } catch (err) {
+      storage.appendAudit({ action: "auto_probe_error", error: err.message });
+      res.status(500).json({ error: err.message });
     }
+  });
 
-    // There may be more than one "{" before the real envelope — e.g. prose
-    // explaining a fix that mentions inline code like "{{payload}}" before
-    // the actual JSON (seen live: a review response started with "The
-    // template node is using `{{payload}}` with...", and slicing from THAT
-    // brace to the envelope's real closing "}" produced unparseable
-    // garbage). Try each candidate "{" in order with string-aware brace
-    // matching (findMatchingBrace, which ignores braces inside quoted
-    // strings) rather than just slicing from the first "{" to the last
-    // "}".
-    //
-    // A candidate must not just PARSE, it must also look like one of the
-    // known envelope shapes (have at least one recognized top-level key) —
-    // seen live: a pure-prose advice response that mentioned structured
-    // logging included the illustrative example
-    // `{"level":"info","event":"trivia_answer","user":"alex","correct":true}`,
-    // which IS valid standalone JSON, so the old "first candidate that
-    // parses wins" rule accepted it as "the envelope" and the caller threw
-    // "no recognizable modify fields" — when the right answer was to treat
-    // the whole reply as prose, since there was no real envelope at all.
-    const ENVELOPE_KEYS = ["explanation", "flow", "question", "changes", "newNodes", "newWires", "removeNodes", "newGroups", "prose"];
-    function looksLikeEnvelope(obj) {
-      if (!obj || typeof obj !== "object" || Array.isArray(obj)) { return false; }
-      return ENVELOPE_KEYS.some(function (k) { return k in obj; });
-    }
-
-    let lastError = null;
-    let searchFrom = firstObjIdx;
-    while (searchFrom !== -1 && searchFrom < s.length) {
-      const end = findMatchingBrace(s, searchFrom);
-      if (end !== -1) {
-        try {
-          const candidate = JSON.parse(s.slice(searchFrom, end + 1));
-          if (looksLikeEnvelope(candidate)) { return candidate; }
-          // Valid JSON, but not envelope-shaped (e.g. an illustrative
-          // example embedded in prose) — keep searching rather than
-          // accepting it.
-        } catch (e) {
-          lastError = e;
-        }
-      }
-      searchFrom = s.indexOf("{", searchFrom + 1);
-    }
-    // No candidate both parsed AND looked like a real envelope — equivalent
-    // to "the model just answered in prose," not "the envelope is broken."
-    // Let callers fall back to rendering this as a normal message instead
-    // of surfacing a parse error (same noJsonFound flag the "no { at all"
-    // branch above uses).
-    const err = lastError || new Error("Provider's JSON object could not be parsed.");
-    err.noJsonFound = true;
-    throw err;
-  }
+  // ---- Generate: produce an importable flow fragment --------------------
+  // Uses the generation system prompt and expects the model to return a single
+  // JSON object { explanation, flow }. This first cut does NOT validate node
+  // types or wire integrity yet (that's the next chunk) — it returns the parsed
+  // envelope so the frontend can display it for review.
 
   // ---------------------------------------------------------------------
   // Shared helper: resolve the active provider and assemble the messages
@@ -1538,29 +1500,42 @@ module.exports = function flowPilotRuntime(RED) {
     // Validate that changes contains no hallucinated ids, and that no id is
     // both patched and marked for removal. A group id from contextGroupIds
     // is allowed here too (see above) even though it's not in originalIds.
+    // Instead of rejecting the whole response when some ids are bad, drop
+    // only the offending patches and apply the rest — same philosophy as the
+    // applyInsertions partial-failure fix (Phase 8.5 #11). A skippedNote in
+    // the response body tells the user what was dropped and why.
     const extraIds = changeIds.filter(function (id) { return !originalIds.has(String(id)) && !contextGroupIds.has(String(id)); });
     const wronglyRemovedIds = changeIds.filter(function (id) { return removeSet.has(String(id)); });
 
-    const idProblems = [];
-    if (extraIds.length) { idProblems.push("unexpected id(s) in changes: " + extraIds.join(", ")); }
-    if (wronglyRemovedIds.length) { idProblems.push("id(s) in both changes and removeNodes: " + wronglyRemovedIds.join(", ")); }
-
-    if (idProblems.length > 0) {
-      storage.appendAudit({ action: "modify_id_mismatch", problems: idProblems });
-      return {
-        status: 422,
-        body: {
-          error: "The model returned inconsistent node ids (" + idProblems.join("; ") + "). Try again.",
-          raw: JSON.stringify(changes)
-        }
-      };
+    const skippedDescriptions = [];
+    if (extraIds.length) {
+      skippedDescriptions.push(
+        extraIds.length === 1
+          ? "Skipped 1 change — that node wasn't in your current selection; reselect it to include it"
+          : "Skipped " + extraIds.length + " changes — those nodes weren't in your current selection; reselect them to include them"
+      );
+    }
+    if (wronglyRemovedIds.length) {
+      skippedDescriptions.push(
+        wronglyRemovedIds.length === 1
+          ? "Skipped 1 change — that node was also marked for removal"
+          : "Skipped " + wronglyRemovedIds.length + " changes — those nodes were also marked for removal"
+      );
     }
+    if (skippedDescriptions.length > 0) {
+      storage.appendAudit({ action: "modify_id_mismatch_partial", skipped_extra: extraIds, skipped_conflict: wronglyRemovedIds });
+    }
+
+    const badIdSet = new Set(extraIds.map(String).concat(wronglyRemovedIds.map(String)));
+    const validChanges = changes.filter(function (c) {
+      return c && c.id !== undefined && c.id !== null && !badIdSet.has(String(c.id));
+    });
 
     // Each patch's "set" is shallow-merged onto a copy of the original node.
     // "id", "x", "y", "z" can never move via a patch — strip them
     // defensively even though the prompt already forbids them.
     const patchById = {};
-    changes.forEach(function (c) {
+    validChanges.forEach(function (c) {
       const set = (c.set && typeof c.set === "object") ? c.set : {};
       const clean = Object.assign({}, set);
       delete clean.id;
@@ -1587,7 +1562,8 @@ module.exports = function flowPilotRuntime(RED) {
     // is, and findLiveNode() already resolves a group id to the live
     // group object (Phase 8.5 C2 slice 1). This is how a group gets
     // renamed/restyled — pure property edit, no new apply-side code.
-    const groupPatchIds = changeIds.filter(function (id) {
+    const validChangeIds = validChanges.map(function (c) { return c.id; });
+    const groupPatchIds = validChangeIds.filter(function (id) {
       return contextGroupIds.has(String(id)) && !originalIds.has(String(id));
     });
     groupPatchIds.forEach(function (id) {
@@ -1698,6 +1674,7 @@ module.exports = function flowPilotRuntime(RED) {
       removeNodes: finalRemoveNodes,
       newGroups: newGroups
     };
+    if (skippedDescriptions.length > 0) { body.skippedNote = skippedDescriptions.join(". ") + "."; }
     if (result.suggestedAction) { body.suggestedAction = result.suggestedAction; }
 
     return { status: 200, body: body };

package/lib/build-core-script.js

@@ -0,0 +1,44 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+// flowpilot-core.js is loaded directly in the BROWSER via a single
+// <script src="flowpilot/core.js"> tag - one big IIFE, no module system,
+// every function/var sharing one closure by lexical scope (not via
+// require/import). Phase 9's refactor splits the SOURCE into focused files
+// under lib/core/ for maintainability, but the served SCRIPT must stay
+// exactly what it was: one concatenated text, so cross-fragment references
+// keep working unmodified and script-loading/timing behavior never changes.
+//
+// Order matters only for top-level code that runs immediately at load time
+// (e.g. a `var x = (function(){...})();` initializer) - plain function
+// declarations are hoisted within the shared closure and safe in any
+// order. Keep new fragments appended in the same relative order they held
+// in the original single file unless a specific dependency says otherwise.
+const FRAGMENT_ORDER = [
+  "redaction.js",
+  "history.js",
+  "markdown.js",
+  "selection-context.js",
+  "apply-review.js",
+  "modes.js",
+  "main.js",
+  "init.js"
+];
+
+const HEADER = "(function () {\n    \"use strict\";\n";
+const FOOTER = "\n})();\n";
+
+let cached = null;
+
+function buildCoreScript() {
+  if (cached) { return cached; }
+  const body = FRAGMENT_ORDER.map(function (name) {
+    return fs.readFileSync(path.join(__dirname, "core", name), "utf8");
+  }).join("\n");
+  cached = HEADER + body + FOOTER;
+  return cached;
+}
+
+module.exports = { buildCoreScript: buildCoreScript };

package/lib/build-system-prompt.js

@@ -25,6 +25,12 @@ Example "explanation" for a trivial goal:
 
 Deploy and trigger it; let me know what the debug output shows."
 
+ALWAYS include at least one debug node in your proposed flow so the test-and-review cycle has something to observe. This is REQUIRED, not optional. Specific rules:
+- HTTP endpoint flows (http in → function/change → http response): add a debug node tapping the function/change output BEFORE the http response node — wire the function/change output to BOTH the debug node AND the http response node.
+- Inject-triggered flows: the debug node at the end is fine.
+- Any other flow shape: add a debug node at the last meaningful output point.
+Never generate a build flow without a debug node. The loop cannot review what it cannot see.
+
 Everything below describes the envelope/rules for THIS step specifically — they work exactly as written, including the parts that say "Generate mode": for the purposes of this prompt, treat that phrase as describing this build step, not a separate mode. The "explanation" field's content rules below still apply — your "Plan:" block comes first, then that content follows immediately after it in the same field.
 
 ---