@manny-est/node-red-flowpilot 0.5.1 → 0.6.0-beta.1

package/lib/storage.js

@@ -1,10 +1,83 @@
 const fs = require("fs");
 const path = require("path");
 
+// Sentinel the client sends back for an apiKey field it never actually saw
+// (see maskProviderSecrets in flowpilot.js) to mean "leave the stored key
+// alone". Shared between the GET/POST /flowpilot/settings routes (masking)
+// and reconcileProviderSecrets below (unmasking on save) — both must agree
+// on the exact string or a real key could get silently overwritten.
+const API_KEY_UNCHANGED = "__FP_KEY_UNCHANGED__";
+
 function ensureDir(dir) {
   fs.mkdirSync(dir, { recursive: true });
 }
 
+// Strips control characters (CR/LF and friends) from a freshly-typed API
+// key before it's persisted — cheap insurance against a pasted value
+// corrupting a future header or a log line.
+function sanitizeApiKey(raw) {
+  return String(raw).replace(/[\x00-\x1F\x7F]/g, "");
+}
+
+// Reconciles an incoming providers list (as submitted by POST /flowpilot/
+// settings, where the client only ever sees the masked sentinel/"" for
+// apiKey — never a real key) against the previously-stored list, matched by
+// provider id. A sentinel or missing apiKey keeps the stored key; an empty
+// string clears it; anything else is a real retyped key. Also enforces the
+// server-writes-only discipline for confirmedBaseUrl/confirmedAt (the
+// provider-confirmation gate, B1): a client can never set or forge these,
+// and confirmation is dropped whenever baseUrl or apiKey actually changed
+// from what was last confirmed.
+// trustedConfirmation: true ONLY when called from saveSettings' own internal
+// callers (/flowpilot/test, /flowpilot/probe, right after a real passing
+// check) — never from the public POST /settings path. Those two routes
+// compute confirmedBaseUrl themselves (== the exact URL they just verified),
+// so it's server-computed data at that point, not client input; every OTHER
+// provider in the same save (anything not freshly (re)confirmed this call)
+// still only keeps its OWN prior confirmation, and only while baseUrl/apiKey
+// still match what was actually confirmed.
+function reconcileProviderSecrets(incoming, existing, trustedConfirmation) {
+  const existingById = {};
+  (Array.isArray(existing) ? existing : []).forEach(function (p) {
+    if (p && p.id) { existingById[p.id] = p; }
+  });
+
+  return (Array.isArray(incoming) ? incoming : []).map(function (p) {
+    const prior = existingById[p.id];
+    const next = Object.assign({}, p);
+    delete next.hasApiKey;
+
+    if (next.apiKey === API_KEY_UNCHANGED || next.apiKey === undefined) {
+      next.apiKey = prior ? (prior.apiKey || "") : "";
+    } else if (typeof next.apiKey === "string" && next.apiKey !== "") {
+      next.apiKey = sanitizeApiKey(next.apiKey);
+    }
+
+    // typeof, not truthiness: baseUrl "" is a real, documented, supported
+    // value (Anthropic's "leave blank for api.anthropic.com" convention),
+    // so confirmedBaseUrl can legitimately BE "" too — `"" && ...` would
+    // silently evaluate false and lock that configuration out of
+    // confirmation forever. Only an actually-absent field should fail.
+    if (trustedConfirmation && typeof p.confirmedBaseUrl === "string") {
+      next.confirmedBaseUrl = p.confirmedBaseUrl;
+      next.confirmedAt = p.confirmedAt;
+      return next;
+    }
+
+    // Untrusted path (public POST /settings): never trust what the client
+    // sent, start from the prior stored state, then drop it if baseUrl or
+    // the (now-reconciled) apiKey no longer matches what was confirmed.
+    if (prior && typeof prior.confirmedBaseUrl === "string" && next.baseUrl === prior.baseUrl && next.apiKey === prior.apiKey) {
+      next.confirmedBaseUrl = prior.confirmedBaseUrl;
+      next.confirmedAt = prior.confirmedAt;
+    } else {
+      delete next.confirmedBaseUrl;
+      delete next.confirmedAt;
+    }
+    return next;
+  });
+}
+
 function createStorage(userDir) {
   const baseDir = path.join(userDir, "flowpilot");
   const chatsDir = path.join(baseDir, "chats");
@@ -18,10 +91,15 @@ function createStorage(userDir) {
     return {
       id: "default",
       providerName: "LocalAI",
+      // "openai-compatible" (default) or "anthropic"
+      type: "openai-compatible",
       baseUrl: "http://localhost:8080",
       apiKey: "",
       model: "",
-      temperature: 0.2
+      temperature: 0.2,
+      // Configured context window size for this provider in tokens (0 = unknown).
+      // When set, FlowPilot warns when the assembled prompt approaches the limit.
+      numCtx: 0
     };
   }
 
@@ -33,7 +111,11 @@ function createStorage(userDir) {
     maxContextChars: 12000,
     defaultContextMode: "selected",
     allowConfigContext: false,
-    logFullContext: false,
+    // When true, provider turns (post-redaction messages, replies, and tool calls)
+    // are appended to debug.log
+    // (0600 perms). Auth headers/keys are never logged — only the content bytes
+    // that the provider actually received. Off by default (diagnostic tool).
+    debugLogging: false,
     streamingEnabled: true,
     // First-run welcome/warning shows until the user saves settings once.
     firstRunAcknowledged: false,
@@ -50,6 +132,9 @@ function createStorage(userDir) {
     // than cloud providers; users on that hardware raise this in Behavior
     // settings rather than living with a hardcoded ceiling.
     requestTimeoutMs: 180000,
+    // Hard output bound for each tool-capable agent turn. Classic completions
+    // remain uncapped so ordinary flow envelopes are never silently clipped.
+    agentTurnMaxTokens: 4096,
     // Max build->deploy->test->fix cycles the /build agentic loop will run
     // before stopping with an honest "couldn't fully verify" instead of
     // proposing another fix. Bounds against a non-converging loop burning
@@ -62,6 +147,14 @@ function createStorage(userDir) {
     // and shows a checkpoint question ("Continue with AI review, or stop?")
     // instead of auto-advancing. Default false = original auto-advance behavior.
     loopHoldStep: false,
+    // Routes Generate through the Phase 10 step-queue engine (graph read-back
+    // verification after import). The legacy path remains available when a
+    // user explicitly disables this setting.
+    enableStepQueue: true,
+    // W7 WRITE-tool loop. Default-off until the server/client Round 1
+    // plumbing has passed its integration and mandatory human live-test
+    // gates. This is separate from enableStepQueue (Generate checklist UI).
+    enableAgentWrite: false,
     // Lets the user silence the recurring secrets/size reminder bar after
     // typing an explicit acknowledgement in settings.
     suppressContextWarnings: false,
@@ -71,12 +164,15 @@ function createStorage(userDir) {
     // dedicated Node-RED credentials field is dropped by the frontend
     // regardless of this setting, via a different, always-on mechanism.
     redactionEnabled: true,
-    // Chat-only persona slider, 1-10: 1 is a plain Node-RED engineer, 10 is
-    // a comically over-the-top airline captain who happens to be a Node-RED
-    // expert. 3 matches the original "subtle co-pilot" voice this replaced.
+    // Chat-only persona slider, 1-5 (CLAUDE-032: was 1-10, collapsed to 5
+    // discrete levels after live testing found the old scale's
+    // interpolated-between-anchors design produced no discernible voice
+    // difference across most of its range): 1 is a plain Node-RED engineer,
+    // 5 is a comically over-the-top airline captain who happens to be a
+    // Node-RED expert. 2 ("subtle co-pilot") is the default.
     // See lib/persona-prompt.js — generated fresh per request, never baked
     // into the persisted systemPrompt below.
-    personaIntensity: 3,
+    personaIntensity: 2,
     // User-defined intent buttons: array of { label, text }.
     customIntents: [],
     systemPrompt: require("./default-system-prompt")
@@ -163,6 +259,7 @@ function createStorage(userDir) {
 
     if (!fs.existsSync(settingsFile)) {
       fs.writeFileSync(settingsFile, JSON.stringify(defaultSettings, null, 2), "utf8");
+      try { fs.chmodSync(settingsFile, 0o600); } catch (e) { /* best-effort */ }
     }
 
     if (!fs.existsSync(auditFile)) {
@@ -191,7 +288,13 @@ function createStorage(userDir) {
     }
   }
 
-  function saveSettings(settings) {
+  // options.trustConfirmation: pass true ONLY from /flowpilot/test or
+  // /flowpilot/probe's own internal saveSettings call, right after a real
+  // passing provider check — see reconcileProviderSecrets's own comment.
+  // Every other caller (in particular the public POST /settings route)
+  // omits this, so confirmedBaseUrl/confirmedAt stay strictly
+  // server-computed and can never be set via a settings save.
+  function saveSettings(settings, options) {
     init();
 
     let current = {};
@@ -205,16 +308,19 @@ function createStorage(userDir) {
     const merged = Object.assign({}, defaultSettings, current, settings || {});
     merged.systemPrompt = fixStaleSystemPrompt(merged.systemPrompt);
     delete merged._error;
-    // If the caller sent a providers list, it wins outright (Object.assign
-    // already did this, but be explicit for clarity/safety).
+    // If the caller sent a providers list, reconcile it against what's
+    // actually stored (real apiKey/confirmedBaseUrl never come from the
+    // client — see reconcileProviderSecrets above) rather than trusting it
+    // outright the way Object.assign would.
     if (settings && Array.isArray(settings.providers)) {
-      merged.providers = settings.providers;
+      merged.providers = reconcileProviderSecrets(settings.providers, current.providers, !!(options && options.trustConfirmation));
     }
     // Saving settings is an explicit user action; mark first-run complete so
     // the welcome/warning stops showing.
     merged.firstRunAcknowledged = true;
 
     fs.writeFileSync(settingsFile, JSON.stringify(merged, null, 2), "utf8");
+    try { fs.chmodSync(settingsFile, 0o600); } catch (e) { /* best-effort */ }
 
     return merged;
   }
@@ -240,7 +346,16 @@ function createStorage(userDir) {
 
   function appendTranscript(conversationId, entry) {
     init();
-    fs.appendFileSync(transcriptFile(conversationId), JSON.stringify(entry) + "\n", "utf8");
+    const file = transcriptFile(conversationId);
+    const isNewFile = !fs.existsSync(file);
+    // Same 0600-on-append pattern as appendDebugLog below — transcripts hold
+    // full conversation content, never world-readable even on first write.
+    const fd = fs.openSync(file, "a", 0o600);
+    fs.writeSync(fd, JSON.stringify(entry) + "\n");
+    fs.closeSync(fd);
+    if (isNewFile) {
+      try { fs.chmodSync(file, 0o600); } catch (e) { /* best-effort */ }
+    }
   }
 
   // Removes a conversation's transcript file (e.g. user deletes it from the
@@ -290,6 +405,32 @@ function createStorage(userDir) {
     return defaultSettings.systemPrompt;
   }
 
+  const debugLogFile = path.join(baseDir, "debug.log");
+
+  // Append one JSON-lines entry to debug.log.
+  // Written 0600 — diagnostic data, never world-readable.
+  // Auth keys are NOT included (only baseUrl + model from the provider
+  // profile, never apiKey). The bytes logged are post-redaction: the same
+  // content the provider actually received.
+  function appendDebugLog(entry) {
+    init();
+    const line = JSON.stringify({
+      timestamp: new Date().toISOString(),
+      ...entry
+    });
+    try {
+      // Open with O_APPEND | O_CREAT, mode 0600 so secrets never land
+      // in a world-readable file even on first write.
+      const fd = fs.openSync(debugLogFile, "a", 0o600);
+      fs.writeSync(fd, line + "\n");
+      fs.closeSync(fd);
+      // Ensure 0600 regardless of umask on subsequent opens.
+      fs.chmodSync(debugLogFile, 0o600);
+    } catch (err) {
+      console.error("[FlowPilot] debug log write failed:", err.message);
+    }
+  }
+
   init();
 
   return {
@@ -298,11 +439,13 @@ function createStorage(userDir) {
     backupsDir,
     settingsFile,
     auditFile,
+    debugLogFile,
     getSettings,
     saveSettings,
     getActiveProvider,
     getDefaultSystemPrompt,
     appendAudit,
+    appendDebugLog,
     appendTranscript,
     readTranscript,
     deleteTranscript,
@@ -310,4 +453,9 @@ function createStorage(userDir) {
   };
 }
 
-module.exports = createStorage;
+// Static, instance-independent — flowpilot.js's route handlers need the
+// exact same sentinel string that reconcileProviderSecrets checks against
+// above, without needing a storage instance to get it.
+createStorage.API_KEY_UNCHANGED = API_KEY_UNCHANGED;
+
+module.exports = createStorage;

package/lib/validator.js

@@ -0,0 +1,238 @@
+"use strict";
+
+// W2 — Class A validator / repair layer.
+//
+// Rules that are mechanically checkable or repairable belong here, not in
+// the model's attention budget. For each repair implemented, the
+// corresponding prompt rule is deleted or shortened in the same commit —
+// the win is measured in prompt shrinkage + corpus improvement, not in
+// code added.
+//
+// Server-side mirror of the client-side DIFF_SKIP in apply-review.js.
+// When a field appears here, it is stripped from changes[].set before
+// the diff reaches the client (rule A6). Keep this list in sync with
+// DIFF_SKIP in apply-review.js.
+const DIFF_SKIP_SERVER = {
+  // Appearance-tab metadata (all node types)
+  info: 1, inputLabels: 1, outputLabels: 1, icon: 1,
+  // debug node display flags
+  console: 1, tostatus: 1, targetType: 1, statusVal: 1, statusType: 1,
+  // function node internal flags
+  noerr: 1, initialize: 1, finalize: 1,
+  // mqtt in/out retain handling
+  rh: 1
+};
+
+// A1: missing wires on a non-comment node → inject [].
+// A2: comment node with non-empty wires → force empty.
+// A3: x/y/z present on a node → strip (editor handles placement).
+// A4: tab/subflow type → remove from array entirely.
+function repairFlowNodes(nodes, repairs) {
+  if (!Array.isArray(nodes)) { return nodes; }
+  const out = [];
+  nodes.forEach(function (n) {
+    if (!n || typeof n !== "object") { return; }
+
+    // A4: reject tab/subflow — editor types, not importable nodes
+    if (n.type === "tab" || n.type === "subflow") {
+      repairs.push({ rule: "A4", detail: "removed " + n.type + " node id=" + (n.id || "?") });
+      return;
+    }
+
+    const fixed = Object.assign({}, n);
+
+    // A3: strip x/y/z (editor assigns positions on import; if present
+    // they cause nodes to pile up at exact coordinates instead of
+    // being auto-arranged)
+    const stripped = [];
+    ["x", "y", "z"].forEach(function (k) {
+      if (k in fixed) { delete fixed[k]; stripped.push(k); }
+    });
+    if (stripped.length) {
+      repairs.push({ rule: "A3", detail: "stripped " + stripped.join(",") + " from id=" + (n.id || "?") });
+    }
+
+    if (n.type === "comment") {
+      // A2: comment nodes are passive annotations — wires must be empty
+      if (Array.isArray(n.wires) && n.wires.some(function (p) { return Array.isArray(p) && p.length > 0; })) {
+        fixed.wires = [];
+        repairs.push({ rule: "A2", detail: "forced wires:[] on comment id=" + (n.id || "?") });
+      } else if (!Array.isArray(n.wires)) {
+        fixed.wires = [];
+      }
+    } else {
+      // A1: every non-comment node must have a wires array
+      if (!Array.isArray(n.wires)) {
+        fixed.wires = [];
+        repairs.push({ rule: "A1", detail: "injected wires:[] on id=" + (n.id || "?") + " type=" + (n.type || "?") });
+      }
+    }
+
+    out.push(fixed);
+  });
+  return out;
+}
+
+// A5: http-request node headers in {key, value} shape → transform to
+// {keyType, keyValue, valueType, valueValue}. The flat shape is silently
+// ignored by Node-RED; the editor uses the keyed shape exclusively.
+function repairHttpHeaders(headers) {
+  if (!Array.isArray(headers)) { return headers; }
+  return headers.map(function (h) {
+    if (h && typeof h === "object" &&
+        "key" in h && "value" in h &&
+        !("keyType" in h)) {
+      return {
+        keyType: "other", keyValue: String(h.key || ""),
+        valueType: "other", valueValue: String(h.value || "")
+      };
+    }
+    return h;
+  });
+}
+
+// A6: forbidden fields in changes[].set → strip (server-side DIFF_SKIP).
+// A7: group inside set → strip (group membership changes via newGroups only).
+// A8: http-request headers {key,value} shape → transform.
+// A9: same id in changes AND removeNodes → drop from changes.
+//
+// Note: redaction-placeholder stripping (formerly A8 in planning notes)
+// was implemented as W0.2 (stripRedactionPlaceholders in flowpilot.js)
+// and runs after this function — no duplication needed here.
+function repairChanges(changes, removeNodes, repairs) {
+  if (!Array.isArray(changes)) { return changes; }
+  const removeSet = new Set(Array.isArray(removeNodes) ? removeNodes : []);
+
+  return changes.filter(function (entry) {
+    if (!entry || typeof entry !== "object") { return false; }
+    // A9: id appears in both changes and removeNodes — removeNodes wins
+    if (entry.id && removeSet.has(entry.id)) {
+      repairs.push({ rule: "A9", detail: "dropped id=" + entry.id + " from changes (also in removeNodes)" });
+      return false;
+    }
+    return true;
+  }).map(function (entry) {
+    if (!entry.set || typeof entry.set !== "object") { return entry; }
+    const cleanSet = {};
+    const droppedA6 = [];
+    let droppedGroup = false;
+
+    Object.keys(entry.set).forEach(function (k) {
+      // A6: forbidden internal fields
+      if (DIFF_SKIP_SERVER[k]) { droppedA6.push(k); return; }
+      // A7: group is informational context, not settable via changes
+      if (k === "group") { droppedGroup = true; return; }
+
+      let v = entry.set[k];
+      // A5: http-request node headers on the modify path
+      if (k === "headers") { v = repairHttpHeaders(v); }
+
+      cleanSet[k] = v;
+    });
+
+    if (droppedA6.length) {
+      repairs.push({ rule: "A6", detail: "stripped " + droppedA6.join(",") + " from set on id=" + entry.id });
+    }
+    if (droppedGroup) {
+      repairs.push({ rule: "A7", detail: "stripped group from set on id=" + entry.id });
+    }
+
+    return Object.assign({}, entry, { set: cleanSet });
+  });
+}
+
+// A10: newWires entries using fromId/toId → rename to from/to. The Modify
+// finalizer consumes only from/to, so leaving the common model-produced
+// aliases in place would make both endpoint references appear missing.
+function repairNewWires(newWires, repairs) {
+  if (!Array.isArray(newWires)) { return newWires; }
+  return newWires.map(function (wire, index) {
+    if (!wire || typeof wire !== "object") { return wire; }
+    const fixed = Object.assign({}, wire);
+    const renamed = [];
+    if (!("from" in fixed) && "fromId" in fixed) {
+      fixed.from = fixed.fromId;
+      delete fixed.fromId;
+      renamed.push("fromId->from");
+    }
+    if (!("to" in fixed) && "toId" in fixed) {
+      fixed.to = fixed.toId;
+      delete fixed.toId;
+      renamed.push("toId->to");
+    }
+    if (renamed.length) {
+      repairs.push({ rule: "A10", detail: "renamed " + renamed.join(",") + " on newWires[" + index + "]" });
+    }
+    return fixed;
+  });
+}
+
+// Switch rules/wires mismatch — detect, do not repair. The rule
+// alignment is a semantic contract (rules[i] corresponds to output port
+// i); auto-repairing the wrong choice would silently misroute messages.
+// Instead, surface as a targeted retry so the model sees exactly what's
+// wrong and can fix it in one shot.
+// Returns an array of { id, rulesLen, wiresLen } — empty when all clean.
+function detectSwitchMismatches(changes) {
+  const mismatches = [];
+  (Array.isArray(changes) ? changes : []).forEach(function (entry) {
+    if (!entry || !entry.set) { return; }
+    const rules = entry.set.rules;
+    const wires = entry.set.wires;
+    if (Array.isArray(rules) && Array.isArray(wires) && rules.length !== wires.length) {
+      mismatches.push({ id: entry.id, rulesLen: rules.length, wiresLen: wires.length });
+    }
+  });
+  return mismatches;
+}
+
+// Top-level entry point. Takes a parsed envelope object and returns:
+//   { envelope, repairs, switchMismatches }
+//
+// - envelope: repaired copy (original not mutated)
+// - repairs: array of { rule, detail } for each repair applied
+// - switchMismatches: array of { id, rulesLen, wiresLen } for switch
+//   nodes whose rules/wires arrays have different lengths (targeted retry
+//   candidates — callers surface these as skippedNotes or 422 bounces)
+function repairEnvelope(parsed) {
+  if (!parsed || typeof parsed !== "object") {
+    return { envelope: parsed, repairs: [], switchMismatches: [] };
+  }
+
+  const repairs = [];
+  const out = Object.assign({}, parsed);
+
+  // Flow array (Generate/Build): A1, A2, A3, A4
+  if (Array.isArray(out.flow)) {
+    out.flow = repairFlowNodes(out.flow, repairs);
+  }
+
+  // newNodes (Modify): A1, A2, A3, A4
+  if (Array.isArray(out.newNodes)) {
+    out.newNodes = repairFlowNodes(out.newNodes, repairs);
+  }
+
+  // changes (Modify): A5, A6, A7, A9
+  if (Array.isArray(out.changes)) {
+    out.changes = repairChanges(out.changes, out.removeNodes, repairs);
+  }
+
+  // newWires (Modify): A10
+  if (Array.isArray(out.newWires)) {
+    out.newWires = repairNewWires(out.newWires, repairs);
+  }
+
+  // Switch alignment check (detect only, not repair)
+  const switchMismatches = detectSwitchMismatches(out.changes);
+
+  return { envelope: out, repairs: repairs, switchMismatches: switchMismatches };
+}
+
+module.exports = {
+  repairEnvelope: repairEnvelope,
+  repairFlowNodes: repairFlowNodes,
+  repairChanges: repairChanges,
+  repairNewWires: repairNewWires,
+  detectSwitchMismatches: detectSwitchMismatches,
+  DIFF_SKIP_SERVER: DIFF_SKIP_SERVER
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@manny-est/node-red-flowpilot",
-  "version": "0.5.1",
+  "version": "0.6.0-beta.1",
   "description": "FlowPilot for Node-RED - an AI-powered development assistant sidebar",
   "main": "flowpilot.js",
   "keywords": [