@xano/xanoscript-language-server 11.8.4 → 11.9.0

package/server.js

@@ -1,3 +1,7 @@
+import { readdirSync, readFileSync } from "fs";
+import { extname, join } from "path";
+import { setImmediate } from "timers";
+import { fileURLToPath, pathToFileURL } from "url";
 import {
   createConnection,
   ProposedFeatures,
@@ -5,11 +9,30 @@ import {
   TextDocuments,
 } from "vscode-languageserver/node.js";
 import { TextDocument } from "vscode-languageserver-textdocument";
+import { documentCache } from "./cache/documentCache.js";
 import { onCompletion } from "./onCompletion/onCompletion.js";
+import { findDefinition } from "./onDefinition/onDefinition.js";
 import { onDidChangeContent } from "./onDidChangeContent/onDidChangeContent.js";
 import { onHover } from "./onHover/onHover.js";
 import { onSemanticCheck } from "./onSemanticCheck/onSemanticCheck.js";
 import { TOKEN_TYPES } from "./onSemanticCheck/tokens.js";
+import { xanoscriptParser } from "./parser/parser.js";
+import { getSchemeFromContent } from "./utils.js";
+import { workspaceIndex } from "./workspace/workspaceIndex.js";
+
+// Feature toggles for debugging memory issues.
+// Disable one at a time to isolate the culprit.
+const FEATURES = {
+  workspaceScan: true, // initial .xs file scan on startup
+  fileWatcher: true, // onDidChangeWatchedFiles
+  completion: true, // onCompletion (content assist)
+  semanticTokens: true, // semantic highlighting
+  hover: true, // onHover
+  definition: true, // onDefinition (go-to-definition)
+  diagnostics: true, // onDidChangeContent (parse + diagnostics)
+  crossFileValidation: true, // cross-file reference validation
+  variableValidation: true, // variable usage validation
+};
 
 // Create a connection to the VS Code client
 const connection = createConnection(ProposedFeatures.all);
@@ -18,40 +41,205 @@ const connection = createConnection(ProposedFeatures.all);
 export const documents = new TextDocuments(TextDocument);
 
 // Initialize the server
-connection.onInitialize(() => {
+connection.onInitialize((params) => {
   connection.console.log("XanoScript Language Server initialized");
 
+  // Scan workspace for .xs files to populate the index
+  if (FEATURES.workspaceScan) {
+    if (params.workspaceFolders) {
+      for (const folder of params.workspaceFolders) {
+        scanWorkspaceFolder(folder.uri);
+      }
+    } else if (params.rootUri) {
+      scanWorkspaceFolder(params.rootUri);
+    }
+  }
+
   return {
     capabilities: {
       completionProvider: {
-        resolveProvider: false, // We won't implement resolve for now (additional details for completions)
-        triggerCharacters: [".", ":", "$", "|"], // Trigger completion on '.', ':', '$', '|' (e.g., for $variables and filters)
+        resolveProvider: false,
+        triggerCharacters: [".", ":", "$", "|"],
       },
-      textDocumentSync: 1, // Full sync mode: server receives entire document content on change
+      textDocumentSync: 1,
       semanticTokensProvider: {
         documentSelector: [{ language: "xanoscript" }],
         legend: {
           tokenTypes: TOKEN_TYPES,
-          tokenModifiers: [], // Optional, like 'static' or 'deprecated'
+          tokenModifiers: [],
+        },
+        full: true,
+      },
+      hoverProvider: true,
+      definitionProvider: true,
+      workspace: {
+        workspaceFolders: {
+          supported: true,
         },
-        full: true, // Support full document highlighting
       },
-      hoverProvider: true, // Enable hover support
     },
   };
 });
 
-connection.onHover((params) => onHover(params, documents));
-connection.onCompletion((params) => onCompletion(params, documents));
-connection.onRequest("textDocument/semanticTokens/full", (params) =>
-  onSemanticCheck(params, documents, SemanticTokensBuilder),
-);
-documents.onDidChangeContent((params) =>
-  onDidChangeContent(params, connection),
-);
-connection.onDidOpenTextDocument((params) => {
-  console.log("Document opened:", params.textDocument.uri);
-  // Existing handler logic
+/**
+ * Recursively scan a folder for .xs files and add them to the workspace index.
+ * Uses lightweight regex-only indexing (no parsing) and yields to the event loop
+ * in batches to avoid blocking and allow GC to run.
+ * @param {string} folderUri - file:// URI of the folder
+ */
+function scanWorkspaceFolder(folderUri) {
+  try {
+    const folderPath = fileURLToPath(folderUri);
+    const filePaths = collectXsFiles(folderPath);
+    indexFilesInBatches(filePaths);
+  } catch (err) {
+    connection.console.error(`Error scanning workspace: ${err.message}`);
+  }
+}
+
+/**
+ * Collect all .xs file paths recursively (fast, no parsing).
+ */
+function collectXsFiles(dirPath) {
+  const result = [];
+  const stack = [dirPath];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      if (
+        entry.isDirectory() &&
+        !entry.name.startsWith(".") &&
+        entry.name !== "node_modules"
+      ) {
+        stack.push(fullPath);
+      } else if (entry.isFile() && extname(entry.name) === ".xs") {
+        result.push(fullPath);
+      }
+    }
+  }
+  return result;
+}
+
+/** Number of files to index per event-loop tick */
+const SCAN_BATCH_SIZE = 50;
+
+/**
+ * Index files in batches, yielding to the event loop between batches
+ * so GC can run and the server stays responsive.
+ */
+function indexFilesInBatches(filePaths) {
+  let offset = 0;
+  function processBatch() {
+    const end = Math.min(offset + SCAN_BATCH_SIZE, filePaths.length);
+    for (let i = offset; i < end; i++) {
+      try {
+        const content = readFileSync(filePaths[i], "utf-8");
+        const uri = pathToFileURL(filePaths[i]).toString();
+        workspaceIndex.addFile(uri, content);
+      } catch {
+        // Skip unreadable files
+      }
+    }
+    offset = end;
+    if (offset < filePaths.length) {
+      setImmediate(processBatch);
+    }
+  }
+  if (filePaths.length > 0) {
+    processBatch();
+  }
+}
+
+connection.onDidChangeWatchedFiles((params) => {
+  if (!FEATURES.fileWatcher) return;
+  connection.console.log(`[FILEWATCHER] ${params.changes.length} changes`);
+  for (const change of params.changes) {
+    if (!change.uri.endsWith(".xs")) continue;
+    switch (change.type) {
+      case 3: // Deleted
+        workspaceIndex.removeFile(change.uri);
+        break;
+      case 1: // Created
+      case 2: // Changed (from external editor)
+        try {
+          const filePath = fileURLToPath(change.uri);
+          const content = readFileSync(filePath, "utf-8");
+          const scheme = getSchemeFromContent(content);
+          const parser = xanoscriptParser(content, scheme);
+          workspaceIndex.addParsed(change.uri, content, parser.__symbolTable);
+        } catch {
+          // Skip unreadable
+        }
+        break;
+    }
+  }
+});
+
+connection.onHover((params) => {
+  if (!FEATURES.hover) return null;
+  connection.console.log(
+    `[HOVER] ${params.textDocument.uri} pos=${JSON.stringify(params.position)}`,
+  );
+  return onHover(params, documents);
+});
+connection.onCompletion((params) => {
+  if (!FEATURES.completion) return null;
+  connection.console.log(
+    `[COMPLETION] ${params.textDocument.uri} pos=${JSON.stringify(params.position)}`,
+  );
+  return onCompletion(params, documents);
+});
+connection.onDefinition((params) => {
+  if (!FEATURES.definition) return null;
+  connection.console.log(`[DEFINITION] ${params.textDocument.uri}`);
+  const document = documents.get(params.textDocument.uri);
+  if (!document) return null;
+
+  const text = document.getText();
+  const offset = document.offsetAt(params.position);
+  const cached = documentCache.getOrParse(document.uri, document.version, text);
+  const symbolTable = cached.parser?.__symbolTable;
+  const result = findDefinition(text, offset, workspaceIndex, symbolTable);
+
+  if (!result) return null;
+
+  // Same-file variable definition (has offset)
+  if (result.offset != null) {
+    const pos = document.positionAt(result.offset);
+    return {
+      uri: document.uri,
+      range: { start: pos, end: pos },
+    };
+  }
+
+  // Cross-file definition (has uri)
+  return {
+    uri: result.uri,
+    range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } },
+  };
+});
+connection.onRequest("textDocument/semanticTokens/full", (params) => {
+  if (!FEATURES.semanticTokens) return null;
+  connection.console.log(`[SEMANTIC] ${params.textDocument.uri}`);
+  return onSemanticCheck(params, documents, SemanticTokensBuilder);
+});
+documents.onDidChangeContent((params) => {
+  if (!FEATURES.diagnostics) return;
+  connection.console.log(
+    `[DIAGNOSTICS] ${params.document.uri} v${params.document.version}`,
+  );
+  onDidChangeContent(params, connection, FEATURES);
+});
+documents.onDidClose((params) => {
+  const uri = params.document.uri;
+  documentCache.invalidate(uri);
 });
 
 // Bind the document manager to the connection and start listening

package/utils.js

@@ -54,3 +54,35 @@ export function getSchemeFromContent(source) {
   const firstWord = source.match(firstWordRegex)?.[1];
   return schemeByFirstWord[firstWord] || "cfn";
 }
+
+// Reverse map: scheme codes back to user-facing object type keywords
+const typeByScheme = {
+  cfn: "function",
+  api: "query",
+  db: "table",
+  // All other schemes map 1:1 (addon, agent, task, etc.)
+};
+
+// Extends firstWordRegex to also capture the name (quoted or bare identifier)
+const objectInfoRegex = /^(?:\s|\/\/[^\n]*\n)*(\w+)\s+(?:"([^"]+)"|(\w+))/;
+
+/**
+ * Extract the object type and name from XanoScript file content.
+ * Reuses schemeByFirstWord for type detection, extends the regex to capture the name.
+ * @param {string} source - File content
+ * @returns {{ type: string, name: string } | null}
+ */
+export function getObjectInfoFromContent(source) {
+  const match = source.match(objectInfoRegex);
+  if (!match) return null;
+
+  const firstWord = match[1];
+  const scheme = schemeByFirstWord[firstWord];
+  if (!scheme) return null;
+
+  const name = match[2] ?? match[3];
+  if (!name) return null;
+
+  const type = typeByScheme[scheme] ?? scheme;
+  return { type, name };
+}

package/utils.spec.js

@@ -1,6 +1,10 @@
 import { expect } from "chai";
 import { describe, it } from "mocha";
-import { getSchemeFromContent, Iterator } from "./utils.js";
+import {
+  getObjectInfoFromContent,
+  getSchemeFromContent,
+  Iterator,
+} from "./utils.js";
 
 describe("getSchemeFromContent", () => {
   describe("basic keyword detection", () => {
@@ -242,6 +246,94 @@ table products {
   });
 });
 
+describe("getObjectInfoFromContent", () => {
+  it("should extract function with identifier name", () => {
+    const result = getObjectInfoFromContent(
+      "function my_func {\n  stack {\n  }\n}"
+    );
+    expect(result).to.deep.equal({ type: "function", name: "my_func" });
+  });
+
+  it("should extract function with string literal name", () => {
+    const result = getObjectInfoFromContent('function "my func" {\n}');
+    expect(result).to.deep.equal({ type: "function", name: "my func" });
+  });
+
+  it("should extract table", () => {
+    const result = getObjectInfoFromContent(
+      "table users {\n  schema {\n  }\n}"
+    );
+    expect(result).to.deep.equal({ type: "table", name: "users" });
+  });
+
+  it("should extract query with verb", () => {
+    const result = getObjectInfoFromContent('query "/users" GET {\n}');
+    expect(result).to.deep.equal({ type: "query", name: "/users" });
+  });
+
+  it("should extract task", () => {
+    const result = getObjectInfoFromContent('task "daily_cleanup" {\n}');
+    expect(result).to.deep.equal({ type: "task", name: "daily_cleanup" });
+  });
+
+  it("should extract api_group", () => {
+    const result = getObjectInfoFromContent('api_group "users" {\n}');
+    expect(result).to.deep.equal({ type: "api_group", name: "users" });
+  });
+
+  it("should handle leading comments", () => {
+    const result = getObjectInfoFromContent(
+      "// my function\nfunction my_func {\n}"
+    );
+    expect(result).to.deep.equal({ type: "function", name: "my_func" });
+  });
+
+  it("should handle leading whitespace and multiple comments", () => {
+    const result = getObjectInfoFromContent(
+      "\n// comment 1\n// comment 2\ntable users {\n}"
+    );
+    expect(result).to.deep.equal({ type: "table", name: "users" });
+  });
+
+  it("should return null for empty content", () => {
+    expect(getObjectInfoFromContent("")).to.be.null;
+  });
+
+  it("should return null for content with no name token", () => {
+    expect(getObjectInfoFromContent("function")).to.be.null;
+  });
+
+  it("should extract workflow_test", () => {
+    const result = getObjectInfoFromContent(
+      'workflow_test "login flow" {\n}'
+    );
+    expect(result).to.deep.equal({
+      type: "workflow_test",
+      name: "login flow",
+    });
+  });
+
+  it("should extract table_trigger", () => {
+    const result = getObjectInfoFromContent(
+      'table_trigger "on_user_create" {\n}'
+    );
+    expect(result).to.deep.equal({
+      type: "table_trigger",
+      name: "on_user_create",
+    });
+  });
+
+  it("should extract middleware", () => {
+    const result = getObjectInfoFromContent('middleware "auth_check" {\n}');
+    expect(result).to.deep.equal({ type: "middleware", name: "auth_check" });
+  });
+
+  it("should extract addon", () => {
+    const result = getObjectInfoFromContent('addon "my_addon" {\n}');
+    expect(result).to.deep.equal({ type: "addon", name: "my_addon" });
+  });
+});
+
 describe("Iterator", () => {
   describe("basic functionality", () => {
     it("should iterate through items", () => {

package/workspace/crossFileValidator.js

@@ -0,0 +1,166 @@
+/**
+ * Levenshtein distance between two strings.
+ */
+function levenshtein(a, b) {
+  const m = a.length;
+  const n = b.length;
+  const dp = Array.from({ length: m + 1 }, () => new Array(n + 1));
+  for (let i = 0; i <= m; i++) dp[i][0] = i;
+  for (let j = 0; j <= n; j++) dp[0][j] = j;
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      dp[i][j] =
+        a[i - 1] === b[j - 1]
+          ? dp[i - 1][j - 1]
+          : 1 + Math.min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]);
+    }
+  }
+  return dp[m][n];
+}
+
+/**
+ * Find the closest match from candidates using prefix check + edit distance.
+ */
+function findClosestMatch(input, candidates) {
+  const inputLower = input.toLowerCase();
+
+  // Prefix match: if input is a prefix of exactly one candidate, suggest it
+  const prefixMatches = candidates.filter((c) =>
+    c.toLowerCase().startsWith(inputLower)
+  );
+  if (prefixMatches.length === 1) return prefixMatches[0];
+
+  // Levenshtein fallback
+  const maxDistance = Math.max(3, Math.floor(input.length / 2));
+  let best = null;
+  let bestDist = Infinity;
+  for (const candidate of candidates) {
+    const dist = levenshtein(inputLower, candidate.toLowerCase());
+    if (dist < bestDist) {
+      bestDist = dist;
+      best = candidate;
+    }
+  }
+  return bestDist <= maxDistance ? best : null;
+}
+
+/**
+ * Validate cross-file references against the workspace index.
+ * @param {Array<{refType: string, name: string, startOffset: number, endOffset: number}>} references
+ * @param {import('./workspaceIndex.js').WorkspaceIndex} index
+ * @returns {Array<{message: string, startOffset: number, endOffset: number}>} warnings
+ */
+export function crossFileValidate(references, index) {
+  if (!references || references.length === 0) return [];
+
+  const warnings = [];
+  for (const ref of references) {
+    if (!ref.name) {
+      warnings.push({
+        message: `Empty ${ref.refType} reference`,
+        startOffset: ref.startOffset,
+        endOffset: ref.endOffset,
+      });
+      continue;
+    }
+
+    if (!index.has(ref.refType, ref.name)) {
+      let message = `Unknown ${ref.refType} "${ref.name}"`;
+      const allNames = index.getAllNames(ref.refType);
+      const suggestion = findClosestMatch(ref.name, allNames);
+      if (suggestion) {
+        message += `. Did you mean "${suggestion}"?`;
+      }
+      warnings.push({
+        message,
+        startOffset: ref.startOffset,
+        endOffset: ref.endOffset,
+      });
+      continue;
+    }
+
+    // Validate field_name against table columns
+    if (ref.refType === "table" && ref.fieldName) {
+      const entry = index.get(ref.refType, ref.name);
+      const columns = entry?.inputs;
+      if (columns && Object.keys(columns).length > 0) {
+        const columnNames = Object.keys(columns);
+        if (!columnNames.includes(ref.fieldName)) {
+          let message = `Unknown column "${ref.fieldName}" in table "${ref.name}"`;
+          const suggestion = findClosestMatch(ref.fieldName, columnNames);
+          if (suggestion) {
+            message += `. Did you mean "${suggestion}"?`;
+          }
+          warnings.push({
+            message,
+            startOffset: ref.fieldNameStartOffset ?? ref.startOffset,
+            endOffset: ref.fieldNameEndOffset ?? ref.endOffset,
+          });
+        }
+      }
+    }
+
+    // Validate data keys against table columns
+    if (ref.refType === "table" && ref.dataKeys) {
+      const entry = index.get(ref.refType, ref.name);
+      const columns = entry?.inputs;
+      if (columns && Object.keys(columns).length > 0) {
+        const columnNames = Object.keys(columns);
+        for (const dk of ref.dataKeys) {
+          if (!columnNames.includes(dk.name)) {
+            let message = `Unknown column "${dk.name}" in table "${ref.name}"`;
+            const suggestion = findClosestMatch(dk.name, columnNames);
+            if (suggestion) {
+              message += `. Did you mean "${suggestion}"?`;
+            }
+            warnings.push({
+              message,
+              startOffset: dk.startOffset ?? ref.startOffset,
+              endOffset: dk.endOffset ?? ref.endOffset,
+            });
+          }
+        }
+      }
+    }
+
+    // Validate input keys if args are present
+    if (ref.args) {
+      const entry = index.get(ref.refType, ref.name);
+      const declaredInputs = entry?.inputs;
+      if (!declaredInputs || Object.keys(declaredInputs).length === 0) continue;
+
+      const declaredKeys = Object.keys(declaredInputs);
+      for (const [argName, argInfo] of Object.entries(ref.args)) {
+        if (!(argName in declaredInputs)) {
+          let message = `Unknown input "${argName}" for ${ref.refType} "${ref.name}"`;
+          const suggestion = findClosestMatch(argName, declaredKeys);
+          if (suggestion) {
+            message += `. Did you mean "${suggestion}"?`;
+          }
+          warnings.push({
+            message,
+            startOffset: argInfo?.startOffset ?? ref.startOffset,
+            endOffset: argInfo?.endOffset ?? ref.endOffset,
+          });
+        } else if (argInfo?.type) {
+          // Type check: compare literal type against declared type
+          const declared = declaredInputs[argName];
+          const isEnumWithText =
+            declared?.type === "enum" && argInfo.type === "text";
+          if (
+            declared?.type &&
+            declared.type !== argInfo.type &&
+            !isEnumWithText
+          ) {
+            warnings.push({
+              message: `Type mismatch for input "${argName}": expected ${declared.type}, got ${argInfo.type}`,
+              startOffset: argInfo?.startOffset ?? ref.startOffset,
+              endOffset: argInfo?.endOffset ?? ref.endOffset,
+            });
+          }
+        }
+      }
+    }
+  }
+  return warnings;
+}