@manny-est/node-red-flowpilot 0.2.1

package/lib/provider-openai-compatible.js

@@ -0,0 +1,377 @@
+const http = require("http");
+const https = require("https");
+
+function postJson(urlString, headers, body, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    let url;
+    try {
+      url = new URL(urlString);
+    } catch (err) {
+      reject(new Error(`Invalid provider URL: ${urlString}`));
+      return;
+    }
+
+    const payload = JSON.stringify(body);
+    const transport = url.protocol === "https:" ? https : http;
+    const options = {
+      method: "POST",
+      hostname: url.hostname,
+      port: url.port || (url.protocol === "https:" ? 443 : 80),
+      path: `${url.pathname}${url.search}`,
+      headers: Object.assign({
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(payload)
+      }, headers || {})
+    };
+
+    const req = transport.request(options, (res) => {
+      let data = "";
+      res.setEncoding("utf8");
+      res.on("data", chunk => { data += chunk; });
+      res.on("end", () => {
+        let parsed = null;
+        try {
+          parsed = data ? JSON.parse(data) : null;
+        } catch (err) {
+          reject(new Error(`Provider returned non-JSON response (${res.statusCode}): ${data.slice(0, 500)}`));
+          return;
+        }
+
+        if (res.statusCode < 200 || res.statusCode >= 300) {
+          const msg = parsed && parsed.error ? JSON.stringify(parsed.error) : data;
+          reject(new Error(`Provider request failed (${res.statusCode}): ${msg}`));
+          return;
+        }
+
+        resolve(parsed);
+      });
+    });
+
+    req.on("error", reject);
+    req.setTimeout(timeoutMs || 180000, () => {
+      req.destroy(new Error(`Provider request timed out after ${timeoutMs || 180000}ms`));
+    });
+
+    req.write(payload);
+    req.end();
+  });
+}
+
+function getJson(urlString, headers, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    let url;
+    try {
+      url = new URL(urlString);
+    } catch (err) {
+      reject(new Error(`Invalid provider URL: ${urlString}`));
+      return;
+    }
+
+    const transport = url.protocol === "https:" ? https : http;
+    const options = {
+      method: "GET",
+      hostname: url.hostname,
+      port: url.port || (url.protocol === "https:" ? 443 : 80),
+      path: `${url.pathname}${url.search}`,
+      headers: headers || {}
+    };
+
+    const req = transport.request(options, (res) => {
+      let data = "";
+      res.setEncoding("utf8");
+      res.on("data", chunk => { data += chunk; });
+      res.on("end", () => {
+        let parsed = null;
+        try {
+          parsed = data ? JSON.parse(data) : null;
+        } catch (err) {
+          reject(new Error(`Provider returned non-JSON response (${res.statusCode}): ${data.slice(0, 500)}`));
+          return;
+        }
+
+        if (res.statusCode < 200 || res.statusCode >= 300) {
+          const msg = parsed && parsed.error ? JSON.stringify(parsed.error) : data;
+          reject(new Error(`Provider request failed (${res.statusCode}): ${msg}`));
+          return;
+        }
+
+        resolve(parsed);
+      });
+    });
+
+    req.on("error", reject);
+    req.setTimeout(timeoutMs || 30000, () => {
+      req.destroy(new Error(`Provider request timed out after ${timeoutMs || 30000}ms`));
+    });
+
+    req.end();
+  });
+}
+
+// ---------------------------------------------------------------------
+// Models dropdown: lists models via the OpenAI-compatible GET /v1/models
+// endpoint, so the settings UI can suggest valid model names instead of
+// the user guessing and hitting a 404 from /v1/chat/completions. Unlike
+// chat()/probeTools(), settings.model is NOT required here — this is how
+// the user picks one. Never throws: a provider without /v1/models (or any
+// other failure) just means an empty list with an explanatory error, which
+// the UI shows as a hint while leaving the model field free-text.
+// ---------------------------------------------------------------------
+async function listModels(settings) {
+  const baseUrl = String(settings.baseUrl || "").replace(/\/+$/, "");
+  if (!baseUrl) throw new Error("Provider base URL is required.");
+
+  const headers = {};
+  if (settings.apiKey) {
+    headers.Authorization = `Bearer ${settings.apiKey}`;
+  }
+
+  try {
+    const response = await getJson(`${baseUrl}/v1/models`, headers, 30000);
+    const data = response && Array.isArray(response.data) ? response.data : [];
+    const models = data
+      .map(function (m) { return m && m.id; })
+      .filter(function (id) { return typeof id === "string" && id; });
+    return { models: models };
+  } catch (err) {
+    return { models: [], error: err.message };
+  }
+}
+
+// `options.tools` (OpenAI-style tool/function definitions) and
+// `options.toolChoice` are optional — when present, the request asks the
+// provider to call a tool instead of (or in addition to) replying with text.
+// `result.toolCalls` is the raw `message.tool_calls` array (or null), passed
+// through unparsed so the agent loop can validate/dispatch each call itself.
+// A tool-call-only response has no `content`, so the "[No assistant message
+// returned...]" fallback only applies when there are no tool calls either.
+async function chat(settings, messages, options) {
+  const baseUrl = String(settings.baseUrl || "").replace(/\/+$/, "");
+  if (!baseUrl) throw new Error("Provider base URL is required.");
+  if (!settings.model) throw new Error("Model is required.");
+
+  const headers = {};
+  if (settings.apiKey) {
+    headers.Authorization = `Bearer ${settings.apiKey}`;
+  }
+
+  const temperature = settings.temperature !== undefined ? Number(settings.temperature) : 0.2;
+
+  const body = {
+    model: settings.model,
+    messages,
+    temperature,
+    stream: false
+  };
+  if (options && Array.isArray(options.tools) && options.tools.length) {
+    body.tools = options.tools;
+    body.tool_choice = options.toolChoice || "auto";
+  }
+
+  const startedAt = Date.now();
+  const response = await postJson(`${baseUrl}/v1/chat/completions`, headers, body, 180000);
+  const totalMs = Date.now() - startedAt;
+
+  const message = response && response.choices && response.choices[0] && response.choices[0].message;
+  const content = message ? message.content : "";
+  const toolCalls = message && Array.isArray(message.tool_calls) && message.tool_calls.length
+    ? message.tool_calls
+    : null;
+
+  return {
+    raw: response,
+    content: content || (toolCalls ? "" : "[No assistant message returned by provider]"),
+    toolCalls: toolCalls,
+    timing: { totalMs },
+    usage: (response && response.usage) || null
+  };
+}
+
+// Streaming variant of chat(): requests `stream: true` and parses the
+// OpenAI-compatible SSE format (lines `data: {...}`, terminated by
+// `data: [DONE]`). Calls onDelta(text) for each content fragment as it
+// arrives and resolves with { content } containing the full concatenated
+// text once the stream ends. Used for chat streaming only.
+function postStream(urlString, headers, body, timeoutMs, onDelta) {
+  return new Promise((resolve, reject) => {
+    let url;
+    try {
+      url = new URL(urlString);
+    } catch (err) {
+      reject(new Error(`Invalid provider URL: ${urlString}`));
+      return;
+    }
+
+    const payload = JSON.stringify(body);
+    const transport = url.protocol === "https:" ? https : http;
+    const options = {
+      method: "POST",
+      hostname: url.hostname,
+      port: url.port || (url.protocol === "https:" ? 443 : 80),
+      path: `${url.pathname}${url.search}`,
+      headers: Object.assign({
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(payload)
+      }, headers || {})
+    };
+
+    const startedAt = Date.now();
+    let firstTokenAt = null;
+    let usage = null;
+
+    const req = transport.request(options, (res) => {
+      res.setEncoding("utf8");
+
+      if (res.statusCode < 200 || res.statusCode >= 300) {
+        let errData = "";
+        res.on("data", chunk => { errData += chunk; });
+        res.on("end", () => {
+          reject(new Error(`Provider request failed (${res.statusCode}): ${errData.slice(0, 500)}`));
+        });
+        return;
+      }
+
+      let buf = "";
+      let full = "";
+      res.on("data", (chunk) => {
+        buf += chunk;
+        const lines = buf.split("\n");
+        buf = lines.pop(); // keep the last (possibly partial) line for next time
+
+        lines.forEach((line) => {
+          line = line.trim();
+          if (!line.startsWith("data:")) { return; }
+          const dataStr = line.slice(5).trim();
+          if (!dataStr || dataStr === "[DONE]") { return; }
+
+          let evt;
+          try {
+            evt = JSON.parse(dataStr);
+          } catch (e) {
+            return; // ignore malformed/partial SSE chunk
+          }
+
+          if (evt && evt.usage) { usage = evt.usage; }
+
+          const delta = evt && evt.choices && evt.choices[0] && evt.choices[0].delta
+            ? evt.choices[0].delta.content
+            : "";
+          if (delta) {
+            if (firstTokenAt === null) { firstTokenAt = Date.now(); }
+            full += delta;
+            onDelta(delta);
+          }
+        });
+      });
+      res.on("end", () => {
+        const endedAt = Date.now();
+        resolve({
+          content: full,
+          ttftMs: firstTokenAt !== null ? firstTokenAt - startedAt : null,
+          totalMs: endedAt - startedAt,
+          usage: usage
+        });
+      });
+    });
+
+    req.on("error", reject);
+    req.setTimeout(timeoutMs || 180000, () => {
+      req.destroy(new Error(`Provider request timed out after ${timeoutMs || 180000}ms`));
+    });
+
+    req.write(payload);
+    req.end();
+  });
+}
+
+async function chatStream(settings, messages, onDelta) {
+  const baseUrl = String(settings.baseUrl || "").replace(/\/+$/, "");
+  if (!baseUrl) throw new Error("Provider base URL is required.");
+  if (!settings.model) throw new Error("Model is required.");
+
+  const headers = {};
+  if (settings.apiKey) {
+    headers.Authorization = `Bearer ${settings.apiKey}`;
+  }
+
+  const temperature = settings.temperature !== undefined ? Number(settings.temperature) : 0.2;
+
+  // stream_options.include_usage asks OpenAI-compatible servers to emit a
+  // final SSE chunk carrying token usage (no delta) before [DONE]. Providers
+  // that don't support it just ignore the option; postStream treats a
+  // missing usage field as null either way.
+  const result = await postStream(`${baseUrl}/v1/chat/completions`, headers, {
+    model: settings.model,
+    messages,
+    temperature,
+    stream: true,
+    stream_options: { include_usage: true }
+  }, 180000, onDelta);
+
+  return {
+    content: result.content || "",
+    timing: { ttftMs: result.ttftMs, totalMs: result.totalMs },
+    usage: result.usage
+  };
+}
+
+// ---------------------------------------------------------------------
+// B-cap1: sends a minimal request with a trivial tool definition and a
+// prompt that should trigger a tool call, to detect whether this provider
+// supports OpenAI-style function/tool calling. Returns
+// { supportsTools: boolean }, never throws — a provider that doesn't
+// understand "tools" typically 400s on the request itself, which is a clean
+// "no" for capability purposes (connectivity is checked separately by
+// chat()).
+//
+// CAUTION: a pass here means the provider is ELIGIBLE for agentic
+// tool-calling features, not a reliability guarantee — the agentic path must
+// still handle malformed mid-conversation tool calls by falling back to the
+// envelope for that turn.
+// ---------------------------------------------------------------------
+async function probeTools(settings) {
+  const baseUrl = String(settings.baseUrl || "").replace(/\/+$/, "");
+  if (!baseUrl) throw new Error("Provider base URL is required.");
+  if (!settings.model) throw new Error("Model is required.");
+
+  const headers = {};
+  if (settings.apiKey) {
+    headers.Authorization = `Bearer ${settings.apiKey}`;
+  }
+
+  const tools = [{
+    type: "function",
+    function: {
+      name: "ping",
+      description: "Respond to a connectivity probe. Takes no arguments.",
+      parameters: { type: "object", properties: {}, additionalProperties: false }
+    }
+  }];
+
+  try {
+    const response = await postJson(`${baseUrl}/v1/chat/completions`, headers, {
+      model: settings.model,
+      messages: [
+        { role: "system", content: "You are being tested for tool/function-calling support." },
+        { role: "user", content: "Call the \"ping\" tool now with no arguments. Do not reply with text." }
+      ],
+      tools: tools,
+      tool_choice: "auto",
+      temperature: 0,
+      stream: false
+    }, 30000);
+
+    const message = response && response.choices && response.choices[0] && response.choices[0].message;
+    const toolCalls = message && Array.isArray(message.tool_calls) ? message.tool_calls : [];
+    const wellFormed = toolCalls.length > 0 && toolCalls.every(function (call) {
+      return !!(call && call.function && typeof call.function.name === "string" &&
+        typeof call.function.arguments === "string");
+    });
+
+    return { supportsTools: wellFormed };
+  } catch (err) {
+    return { supportsTools: false, error: err.message };
+  }
+}
+
+module.exports = { chat, chatStream, probeTools, listModels };

package/lib/storage.js

@@ -0,0 +1,265 @@
+const fs = require("fs");
+const path = require("path");
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+function createStorage(userDir) {
+  const baseDir = path.join(userDir, "flowpilot");
+  const chatsDir = path.join(baseDir, "chats");
+  const backupsDir = path.join(baseDir, "backups");
+  const settingsFile = path.join(baseDir, "settings.json");
+  const auditFile = path.join(baseDir, "audit.log");
+
+  // A provider profile. Each has its own model since model names differ
+  // across providers (LocalAI vs cloud).
+  function defaultProvider() {
+    return {
+      id: "default",
+      providerName: "LocalAI",
+      baseUrl: "http://localhost:8080",
+      apiKey: "",
+      model: "",
+      temperature: 0.2
+    };
+  }
+
+  const defaultSettings = {
+    // Multiple provider profiles; one is active at a time. The list shape is
+    // deliberately ready for a future side-by-side compare mode.
+    providers: [defaultProvider()],
+    activeProviderId: "default",
+    maxContextChars: 12000,
+    defaultContextMode: "selected",
+    allowConfigContext: false,
+    logFullContext: false,
+    streamingEnabled: true,
+    // First-run welcome/warning shows until the user saves settings once.
+    firstRunAcknowledged: false,
+    // Context-size advisory thresholds, in estimated tokens (~chars/4).
+    // Advisory only; never blocks sending.
+    contextWarnTokens: 4000,
+    contextHighTokens: 8000,
+    // How many recent chat exchanges (user+assistant pairs) the frontend
+    // includes as history with each request. Older turns are dropped
+    // client-side and the model is told when that happened.
+    historyMaxExchanges: 10,
+    // Lets the user silence the recurring secrets/size reminder bar after
+    // typing an explicit acknowledgement in settings.
+    suppressContextWarnings: false,
+    // User-defined intent buttons: array of { label, text }.
+    customIntents: [],
+    systemPrompt: require("./default-system-prompt")
+  };
+
+  // Older builds persisted a "Phase 1, READ-ONLY mode" system prompt into
+  // settings.json once and never updated it — Object.assign in getSettings
+  // lets that stale persisted copy win forever, even after
+  // default-system-prompt.js is fixed. Detect that stale text (by a phrase
+  // unique to it) and swap in the current default instead. Applied both when
+  // reading settings AND when saving them, so a browser tab that still has
+  // the stale text loaded in the System Prompt textarea can't re-persist it.
+  function fixStaleSystemPrompt(systemPrompt) {
+    if (typeof systemPrompt !== "string" || !systemPrompt.trim()) {
+      return defaultSettings.systemPrompt;
+    }
+    if (systemPrompt.indexOf("operating in READ-ONLY mode") !== -1) {
+      return defaultSettings.systemPrompt;
+    }
+    return systemPrompt;
+  }
+
+  // Migrate an old flat-provider settings object (providerName/baseUrl/etc at
+  // top level) into the new providers-list shape, preserving the user's
+  // configured provider. Idempotent: leaves new-format settings untouched.
+  function migrate(parsed) {
+    if (!parsed || typeof parsed !== "object") { return parsed; }
+    if (Array.isArray(parsed.providers) && parsed.providers.length) {
+      return parsed; // already new format
+    }
+    if (parsed.providerName || parsed.baseUrl || parsed.model) {
+      const migrated = Object.assign({}, parsed);
+      migrated.providers = [{
+        id: "default",
+        providerName: parsed.providerName || "LocalAI",
+        baseUrl: parsed.baseUrl || "http://localhost:8080",
+        apiKey: parsed.apiKey || "",
+        model: parsed.model || "",
+        temperature: parsed.temperature !== undefined ? parsed.temperature : 0.2
+      }];
+      migrated.activeProviderId = "default";
+      // Remove the now-relocated flat fields.
+      delete migrated.providerName;
+      delete migrated.baseUrl;
+      delete migrated.apiKey;
+      delete migrated.model;
+      delete migrated.temperature;
+      return migrated;
+    }
+    return parsed;
+  }
+
+  // Returns the currently active provider profile (or the first, or a default).
+  function getActiveProvider(settings) {
+    const list = Array.isArray(settings.providers) ? settings.providers : [];
+    if (!list.length) { return defaultProvider(); }
+    const found = list.filter(function (p) { return p.id === settings.activeProviderId; })[0];
+    return found || list[0];
+  }
+
+  function init() {
+    ensureDir(baseDir);
+    ensureDir(chatsDir);
+    ensureDir(backupsDir);
+
+    if (!fs.existsSync(settingsFile)) {
+      fs.writeFileSync(settingsFile, JSON.stringify(defaultSettings, null, 2), "utf8");
+    }
+
+    if (!fs.existsSync(auditFile)) {
+      fs.writeFileSync(auditFile, "", "utf8");
+    }
+  }
+
+  function getSettings() {
+    init();
+
+    try {
+      const raw = fs.readFileSync(settingsFile, "utf8");
+      const parsed = migrate(JSON.parse(raw));
+      // Merge top-level app settings with defaults, but take the providers
+      // list verbatim from the file (don't let defaults overwrite it).
+      const merged = Object.assign({}, defaultSettings, parsed);
+      if (Array.isArray(parsed.providers) && parsed.providers.length) {
+        merged.providers = parsed.providers;
+      }
+      merged.systemPrompt = fixStaleSystemPrompt(merged.systemPrompt);
+      return merged;
+    } catch (err) {
+      return Object.assign({}, defaultSettings, {
+        _error: err.message
+      });
+    }
+  }
+
+  function saveSettings(settings) {
+    init();
+
+    let current = {};
+    try {
+      const raw = fs.readFileSync(settingsFile, "utf8");
+      current = migrate(JSON.parse(raw));
+    } catch (err) {
+      current = {};
+    }
+
+    const merged = Object.assign({}, defaultSettings, current, settings || {});
+    merged.systemPrompt = fixStaleSystemPrompt(merged.systemPrompt);
+    delete merged._error;
+    // If the caller sent a providers list, it wins outright (Object.assign
+    // already did this, but be explicit for clarity/safety).
+    if (settings && Array.isArray(settings.providers)) {
+      merged.providers = settings.providers;
+    }
+    // Saving settings is an explicit user action; mark first-run complete so
+    // the welcome/warning stops showing.
+    merged.firstRunAcknowledged = true;
+
+    fs.writeFileSync(settingsFile, JSON.stringify(merged, null, 2), "utf8");
+
+    return merged;
+  }
+
+  function appendAudit(entry) {
+    init();
+
+    const line = JSON.stringify({
+      timestamp: new Date().toISOString(),
+      ...entry
+    });
+
+    fs.appendFileSync(auditFile, line + "\n", "utf8");
+  }
+
+  // Per-conversation transcripts: one JSON Lines file per conversation,
+  // keyed by a frontend-generated conversationId. Callers must pass an
+  // already-validated id (flowpilot.js's sanitizeConversationId) — this is
+  // just file I/O.
+  function transcriptFile(conversationId) {
+    return path.join(chatsDir, `${conversationId}.jsonl`);
+  }
+
+  function appendTranscript(conversationId, entry) {
+    init();
+    fs.appendFileSync(transcriptFile(conversationId), JSON.stringify(entry) + "\n", "utf8");
+  }
+
+  // Removes a conversation's transcript file (e.g. user deletes it from the
+  // conversation list). Best-effort — a missing file is not an error.
+  function deleteTranscript(conversationId) {
+    init();
+    try { fs.unlinkSync(transcriptFile(conversationId)); } catch (err) { /* already gone */ }
+  }
+
+  function readTranscript(conversationId) {
+    init();
+    const file = transcriptFile(conversationId);
+    if (!fs.existsSync(file)) { return []; }
+
+    let raw;
+    try {
+      raw = fs.readFileSync(file, "utf8");
+    } catch (err) {
+      return [];
+    }
+
+    return raw.split("\n").filter(Boolean).map(function (line) {
+      try { return JSON.parse(line); } catch (err) { return null; }
+    }).filter(Boolean);
+  }
+
+  // Used by Recall to search across OTHER conversations' transcripts. Lists
+  // every persisted conversationId (one per chats/*.jsonl file).
+  function listConversationIds() {
+    init();
+    let files;
+    try {
+      files = fs.readdirSync(chatsDir);
+    } catch (err) {
+      return [];
+    }
+    return files
+      .filter(function (f) { return f.endsWith(".jsonl"); })
+      .map(function (f) { return f.slice(0, -6); });
+  }
+
+  // Always the CURRENT contents of lib/default-system-prompt.js — never the
+  // stale copy that may be persisted in settings.json. Lets the Settings UI
+  // offer a "Reset to default" action that picks up prompt updates shipped
+  // in later FlowPilot versions, even though a snapshot was saved once.
+  function getDefaultSystemPrompt() {
+    return defaultSettings.systemPrompt;
+  }
+
+  init();
+
+  return {
+    baseDir,
+    chatsDir,
+    backupsDir,
+    settingsFile,
+    auditFile,
+    getSettings,
+    saveSettings,
+    getActiveProvider,
+    getDefaultSystemPrompt,
+    appendAudit,
+    appendTranscript,
+    readTranscript,
+    deleteTranscript,
+    listConversationIds
+  };
+}
+
+module.exports = createStorage;

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@manny-est/node-red-flowpilot",
+  "version": "0.2.1",
+  "description": "FlowPilot for Node-RED - an AI-powered development assistant sidebar",
+  "main": "flowpilot.js",
+  "keywords": [
+    "node-red",
+    "ai",
+    "localai",
+    "openai",
+    "flowpilot"
+  ],
+  "author": "manny-est",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/manny-est/flowpilot.git"
+  },
+  "homepage": "https://github.com/manny-est/flowpilot#readme",
+  "bugs": {
+    "url": "https://github.com/manny-est/flowpilot/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "node-red": {
+    "version": ">=4.0.0",
+    "nodes": {
+      "flowpilot": "flowpilot.js"
+    },
+    "plugins": {
+      "flowpilot": "flowpilot.html"
+    }
+  },
+  "engines": {
+    "node": ">=16"
+  },
+  "files": [
+    "flowpilot.js",
+    "flowpilot.html",
+    "lib",
+    "icons",
+    "examples",
+    "README.md",
+    "USER-GUIDE.md",
+    "PROJECT-OVERVIEW.md",
+    "LICENSE"
+  ]
+}