@xano/xanoscript-language-server 11.8.3 → 11.8.5

package/parser/variableScanner.js

@@ -0,0 +1,64 @@
+import { LongFormVariable,ShortFormVariable } from "../lexer/variables.js";
+import { BLACKLISTED_VARIABLE_NAMES } from "./generic/assignableVariableProperty.js";
+
+const SHORT_FORM = ShortFormVariable.name;
+const LONG_FORM = LongFormVariable.name;
+
+const BUILTIN_NAMES = new Set(BLACKLISTED_VARIABLE_NAMES.map((n) => `$${n}`));
+// Also skip $$ (temporary variable)
+BUILTIN_NAMES.add("$$");
+
+/**
+ * Scan the token stream for variable usages (ShortFormVariable and LongFormVariable tokens).
+ * Excludes tokens at declaration offsets and built-in variable names.
+ * @param {import('chevrotain').IToken[]} tokens - The token stream
+ * @param {Array<{name: string, startOffset: number | null}>} declarations - Known declaration sites
+ * @returns {Array<{name: string, startOffset: number, endOffset: number}>}
+ */
+export function scanVariableUsages(tokens, declarations) {
+  // Build set of declaration offsets to skip
+  const declOffsets = new Set();
+  for (const decl of declarations) {
+    if (decl.startOffset != null) {
+      declOffsets.add(decl.startOffset);
+    }
+  }
+
+  const usages = [];
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    const typeName = token.tokenType.name;
+
+    if (typeName === SHORT_FORM) {
+      // Skip if this is a declaration site
+      if (declOffsets.has(token.startOffset)) continue;
+      // Skip built-in variable names
+      if (BUILTIN_NAMES.has(token.image)) continue;
+
+      usages.push({
+        name: token.image,
+        startOffset: token.startOffset,
+        endOffset: token.endOffset,
+      });
+    } else if (typeName === LONG_FORM) {
+      // $var.foo → resolve to $foo
+      // Next tokens should be Dot + Identifier
+      if (i + 2 < tokens.length) {
+        const identToken = tokens[i + 2];
+        const name = `$${identToken.image}`;
+
+        // Skip if this is a declaration site (check the identifier token)
+        if (declOffsets.has(identToken.startOffset)) continue;
+
+        usages.push({
+          name,
+          startOffset: token.startOffset,
+          endOffset: identToken.endOffset,
+        });
+      }
+    }
+  }
+
+  return usages;
+}

package/parser/variableValidator.js

@@ -0,0 +1,44 @@
+import { scanVariableUsages } from "./variableScanner.js";
+
+/**
+ * Validate variable declarations vs usages.
+ * Returns undefined-variable warnings and unused-variable hints.
+ * @param {Object} symbolTable - The parser's __symbolTable
+ * @param {import('chevrotain').IToken[]} tokens - The token stream from parser.input
+ * @returns {{ warnings: Array<{message: string, startOffset: number, endOffset: number}>, hints: Array<{message: string, startOffset: number, endOffset: number}> }}
+ */
+export function validateVariables(symbolTable, tokens) {
+  const warnings = [];
+  const hints = [];
+
+  const declarations = symbolTable.varDeclarations || [];
+  const declaredNames = new Set(declarations.map((d) => d.name));
+
+  const usages = scanVariableUsages(tokens, declarations);
+  const usedNames = new Set(usages.map((u) => u.name));
+
+  // Undefined: used but not declared
+  for (const usage of usages) {
+    if (!declaredNames.has(usage.name)) {
+      warnings.push({
+        message: `Unknown variable '${usage.name}'`,
+        startOffset: usage.startOffset,
+        endOffset: usage.endOffset,
+      });
+    }
+  }
+
+  // Unused: declared but never used
+  for (const decl of declarations) {
+    if (decl.startOffset == null) continue;
+    if (!usedNames.has(decl.name)) {
+      hints.push({
+        message: `Variable '${decl.name}' is declared but never used`,
+        startOffset: decl.startOffset,
+        endOffset: decl.endOffset,
+      });
+    }
+  }
+
+  return { warnings, hints };
+}

package/parser/variableValidator.spec.js

@@ -0,0 +1,179 @@
+import { expect } from "chai";
+import { readFileSync } from "fs";
+import { before, describe, it } from "mocha";
+import { lexDocument } from "../lexer/lexer.js";
+import { xanoscriptParser } from "./parser.js";
+import { scanVariableUsages } from "./variableScanner.js";
+import { validateVariables } from "./variableValidator.js";
+
+describe("variable tracking - coverage_check.xs", () => {
+  let varDeclarations;
+  let usages;
+  let result;
+  let text;
+
+  before(() => {
+    text = readFileSync(
+      new URL("./tests/variable_test/coverage_check.xs", import.meta.url),
+      "utf8",
+    );
+    const lexResult = lexDocument(text);
+    const parser = xanoscriptParser(text);
+    expect(parser.errors).to.be.empty;
+    varDeclarations = [...parser.__symbolTable.varDeclarations];
+    usages = scanVariableUsages(lexResult.tokens, varDeclarations);
+    result = validateVariables(parser.__symbolTable, lexResult.tokens);
+  });
+
+  function lineOf(offset) {
+    return text.substring(0, offset).split("\n").length;
+  }
+
+  function declLines(name) {
+    return varDeclarations
+      .filter((d) => d.name === name)
+      .map((d) => lineOf(d.startOffset));
+  }
+
+  function usageLines(name) {
+    return usages.filter((u) => u.name === name).map((u) => lineOf(u.startOffset));
+  }
+
+  // --- Declarations: each variable should be declared exactly once ---
+
+  it("$vehicle declared once at line 26 via 'as $vehicle'", () => {
+    expect(declLines("$vehicle")).to.deep.equal([26]);
+  });
+
+  it("$unused_variable declared once at line 35 via 'var'", () => {
+    expect(declLines("$unused_variable")).to.deep.equal([35]);
+  });
+
+  it("$vehicle_response declared once at line 40 via 'var'", () => {
+    expect(declLines("$vehicle_response")).to.deep.equal([40]);
+  });
+
+  it("$policy declared once at line 63 via 'as $policy'", () => {
+    expect(declLines("$policy")).to.deep.equal([63]);
+  });
+
+  it("$policy_response declared once at line 66 via 'var' (NOT again at line 87 via var.update)", () => {
+    expect(declLines("$policy_response")).to.deep.equal([66]);
+  });
+
+  it("$coverages_response declared once at line 70 via 'var'", () => {
+    expect(declLines("$coverages_response")).to.deep.equal([70]);
+  });
+
+  it("$claims_summary declared once at line 74 via 'var' (NOT again at line 188 via var.update)", () => {
+    expect(declLines("$claims_summary")).to.deep.equal([74]);
+  });
+
+  it("$policy_coverages declared once at line 105 via 'as'", () => {
+    expect(declLines("$policy_coverages")).to.deep.equal([105]);
+  });
+
+  it("$pc declared once at line 109 via 'each as $pc'", () => {
+    expect(declLines("$pc")).to.deep.equal([109]);
+  });
+
+  it("$ctype declared once at line 113 via 'as $ctype'", () => {
+    expect(declLines("$ctype")).to.deep.equal([113]);
+  });
+
+  it("$coverage_item declared once at line 116 via 'var'", () => {
+    expect(declLines("$coverage_item")).to.deep.equal([116]);
+  });
+
+  it("$all_claims declared once at line 142 via 'as'", () => {
+    expect(declLines("$all_claims")).to.deep.equal([142]);
+  });
+
+  it("$total_claims_count declared once at line 145 via 'var'", () => {
+    expect(declLines("$total_claims_count")).to.deep.equal([145]);
+  });
+
+  it("$two_years_ago declared once at line 150 via 'var'", () => {
+    expect(declLines("$two_years_ago")).to.deep.equal([150]);
+  });
+
+  it("$recent_claims_list declared once at line 155 via 'var'", () => {
+    expect(declLines("$recent_claims_list")).to.deep.equal([155]);
+  });
+
+  it("$claim declared once at line 160 via 'each as $claim'", () => {
+    expect(declLines("$claim")).to.deep.equal([160]);
+  });
+
+  it("$claim_date_ts declared once at line 162 via 'var'", () => {
+    expect(declLines("$claim_date_ts")).to.deep.equal([162]);
+  });
+
+  it("$recent_claim_item declared once at line 169 via 'var'", () => {
+    expect(declLines("$recent_claim_item")).to.deep.equal([169]);
+  });
+
+  it("$queried_by declared once at line 202 via 'var' (NOT again at line 217 via var.update)", () => {
+    expect(declLines("$queried_by")).to.deep.equal([202]);
+  });
+
+  it("$shop declared once at line 212 via 'as'", () => {
+    expect(declLines("$shop")).to.deep.equal([212]);
+  });
+
+  it("$result declared once at line 230 via 'var' (NOT again at line 242 via var.update)", () => {
+    expect(declLines("$result")).to.deep.equal([230]);
+  });
+
+  // --- Usages: var.update targets count as usages ---
+
+  it("$vehicle used at lines 29, 42-46, 57", () => {
+    expect(usageLines("$vehicle")).to.deep.equal([29, 42, 43, 44, 45, 46, 57]);
+  });
+
+  it("$policy_response used at line 87 (var.update) and 233", () => {
+    expect(usageLines("$policy_response")).to.include(87);
+    expect(usageLines("$policy_response")).to.include(233);
+  });
+
+  it("$claims_summary used at line 188 (var.update) and 235", () => {
+    expect(usageLines("$claims_summary")).to.include(188);
+    expect(usageLines("$claims_summary")).to.include(235);
+  });
+
+  it("$queried_by used at line 217 (var.update), 241, 243", () => {
+    expect(usageLines("$queried_by")).to.include(217);
+    expect(usageLines("$queried_by")).to.include(241);
+    expect(usageLines("$queried_by")).to.include(243);
+  });
+
+  it("$result used at line 242 (var.update), 243, 251", () => {
+    expect(usageLines("$result")).to.include(242);
+    expect(usageLines("$result")).to.include(243);
+    expect(usageLines("$result")).to.include(251);
+  });
+
+  it("$coverages_response used at lines 127 and 234", () => {
+    expect(usageLines("$coverages_response")).to.include(127);
+    expect(usageLines("$coverages_response")).to.include(234);
+  });
+
+  it("$pc used at lines 112, 120-122", () => {
+    expect(usageLines("$pc")).to.deep.equal([112, 120, 121, 122]);
+  });
+
+  it("$claim used at lines 163, 171-175", () => {
+    expect(usageLines("$claim")).to.deep.equal([163, 171, 172, 173, 174, 175]);
+  });
+
+  // --- Validation results ---
+
+  it("should produce zero undefined-variable warnings", () => {
+    expect(result.warnings).to.be.empty;
+  });
+
+  it("should flag only $unused_variable as unused", () => {
+    expect(result.hints).to.have.lengthOf(1);
+    expect(result.hints[0].message).to.include("$unused_variable");
+  });
+});

package/server.js

@@ -1,3 +1,7 @@
+import { readdirSync, readFileSync } from "fs";
+import { extname, join } from "path";
+import { setImmediate } from "timers";
+import { fileURLToPath, pathToFileURL } from "url";
 import {
   createConnection,
   ProposedFeatures,
@@ -5,11 +9,16 @@ import {
   TextDocuments,
 } from "vscode-languageserver/node.js";
 import { TextDocument } from "vscode-languageserver-textdocument";
+import { documentCache } from "./cache/documentCache.js";
 import { onCompletion } from "./onCompletion/onCompletion.js";
+import { findDefinition } from "./onDefinition/onDefinition.js";
 import { onDidChangeContent } from "./onDidChangeContent/onDidChangeContent.js";
 import { onHover } from "./onHover/onHover.js";
 import { onSemanticCheck } from "./onSemanticCheck/onSemanticCheck.js";
 import { TOKEN_TYPES } from "./onSemanticCheck/tokens.js";
+import { xanoscriptParser } from "./parser/parser.js";
+import { getSchemeFromContent } from "./utils.js";
+import { workspaceIndex } from "./workspace/workspaceIndex.js";
 
 // Create a connection to the VS Code client
 const connection = createConnection(ProposedFeatures.all);
@@ -18,40 +27,185 @@ const connection = createConnection(ProposedFeatures.all);
 export const documents = new TextDocuments(TextDocument);
 
 // Initialize the server
-connection.onInitialize(() => {
+connection.onInitialize((params) => {
   connection.console.log("XanoScript Language Server initialized");
 
+  // Scan workspace for .xs files to populate the index
+  if (params.workspaceFolders) {
+    for (const folder of params.workspaceFolders) {
+      scanWorkspaceFolder(folder.uri);
+    }
+  } else if (params.rootUri) {
+    scanWorkspaceFolder(params.rootUri);
+  }
+
   return {
     capabilities: {
       completionProvider: {
-        resolveProvider: false, // We won't implement resolve for now (additional details for completions)
-        triggerCharacters: [".", ":", "$", "|"], // Trigger completion on '.', ':', '$', '|' (e.g., for $variables and filters)
+        resolveProvider: false,
+        triggerCharacters: [".", ":", "$", "|"],
       },
-      textDocumentSync: 1, // Full sync mode: server receives entire document content on change
+      textDocumentSync: 1,
       semanticTokensProvider: {
         documentSelector: [{ language: "xanoscript" }],
         legend: {
           tokenTypes: TOKEN_TYPES,
-          tokenModifiers: [], // Optional, like 'static' or 'deprecated'
+          tokenModifiers: [],
+        },
+        full: true,
+      },
+      hoverProvider: true,
+      definitionProvider: true,
+      workspace: {
+        workspaceFolders: {
+          supported: true,
         },
-        full: true, // Support full document highlighting
       },
-      hoverProvider: true, // Enable hover support
     },
   };
 });
 
+/**
+ * Recursively scan a folder for .xs files and add them to the workspace index.
+ * Uses lightweight regex-only indexing (no parsing) and yields to the event loop
+ * in batches to avoid blocking and allow GC to run.
+ * @param {string} folderUri - file:// URI of the folder
+ */
+function scanWorkspaceFolder(folderUri) {
+  try {
+    const folderPath = fileURLToPath(folderUri);
+    const filePaths = collectXsFiles(folderPath);
+    indexFilesInBatches(filePaths);
+  } catch (err) {
+    connection.console.error(`Error scanning workspace: ${err.message}`);
+  }
+}
+
+/**
+ * Collect all .xs file paths recursively (fast, no parsing).
+ */
+function collectXsFiles(dirPath) {
+  const result = [];
+  const stack = [dirPath];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      if (
+        entry.isDirectory() &&
+        !entry.name.startsWith(".") &&
+        entry.name !== "node_modules"
+      ) {
+        stack.push(fullPath);
+      } else if (entry.isFile() && extname(entry.name) === ".xs") {
+        result.push(fullPath);
+      }
+    }
+  }
+  return result;
+}
+
+/** Number of files to index per event-loop tick */
+const SCAN_BATCH_SIZE = 50;
+
+/**
+ * Index files in batches, yielding to the event loop between batches
+ * so GC can run and the server stays responsive.
+ */
+function indexFilesInBatches(filePaths) {
+  let offset = 0;
+  function processBatch() {
+    const end = Math.min(offset + SCAN_BATCH_SIZE, filePaths.length);
+    for (let i = offset; i < end; i++) {
+      try {
+        const content = readFileSync(filePaths[i], "utf-8");
+        const uri = pathToFileURL(filePaths[i]).toString();
+        workspaceIndex.addFile(uri, content);
+      } catch {
+        // Skip unreadable files
+      }
+    }
+    offset = end;
+    if (offset < filePaths.length) {
+      setImmediate(processBatch);
+    }
+  }
+  if (filePaths.length > 0) {
+    processBatch();
+  }
+}
+
+connection.onDidChangeWatchedFiles((params) => {
+  for (const change of params.changes) {
+    if (!change.uri.endsWith(".xs")) continue;
+    switch (change.type) {
+      case 3: // Deleted
+        workspaceIndex.removeFile(change.uri);
+        break;
+      case 1: // Created
+      case 2: // Changed (from external editor)
+        try {
+          const filePath = fileURLToPath(change.uri);
+          const content = readFileSync(filePath, "utf-8");
+          const scheme = getSchemeFromContent(content);
+          const parser = xanoscriptParser(content, scheme);
+          workspaceIndex.addParsed(change.uri, content, parser.__symbolTable);
+        } catch {
+          // Skip unreadable
+        }
+        break;
+    }
+  }
+});
+
 connection.onHover((params) => onHover(params, documents));
 connection.onCompletion((params) => onCompletion(params, documents));
+connection.onDefinition((params) => {
+  const document = documents.get(params.textDocument.uri);
+  if (!document) return null;
+
+  const text = document.getText();
+  const offset = document.offsetAt(params.position);
+  const cached = documentCache.getOrParse(
+    document.uri,
+    document.version,
+    text,
+  );
+  const symbolTable = cached.parser?.__symbolTable;
+  const result = findDefinition(text, offset, workspaceIndex, symbolTable);
+
+  if (!result) return null;
+
+  // Same-file variable definition (has offset)
+  if (result.offset != null) {
+    const pos = document.positionAt(result.offset);
+    return {
+      uri: document.uri,
+      range: { start: pos, end: pos },
+    };
+  }
+
+  // Cross-file definition (has uri)
+  return {
+    uri: result.uri,
+    range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } },
+  };
+});
 connection.onRequest("textDocument/semanticTokens/full", (params) =>
   onSemanticCheck(params, documents, SemanticTokensBuilder),
 );
 documents.onDidChangeContent((params) =>
   onDidChangeContent(params, connection),
 );
-connection.onDidOpenTextDocument((params) => {
-  console.log("Document opened:", params.textDocument.uri);
-  // Existing handler logic
+documents.onDidClose((params) => {
+  const uri = params.document.uri;
+  documentCache.invalidate(uri);
 });
 
 // Bind the document manager to the connection and start listening

package/utils.js

@@ -54,3 +54,35 @@ export function getSchemeFromContent(source) {
   const firstWord = source.match(firstWordRegex)?.[1];
   return schemeByFirstWord[firstWord] || "cfn";
 }
+
+// Reverse map: scheme codes back to user-facing object type keywords
+const typeByScheme = {
+  cfn: "function",
+  api: "query",
+  db: "table",
+  // All other schemes map 1:1 (addon, agent, task, etc.)
+};
+
+// Extends firstWordRegex to also capture the name (quoted or bare identifier)
+const objectInfoRegex = /^(?:\s|\/\/[^\n]*\n)*(\w+)\s+(?:"([^"]+)"|(\w+))/;
+
+/**
+ * Extract the object type and name from XanoScript file content.
+ * Reuses schemeByFirstWord for type detection, extends the regex to capture the name.
+ * @param {string} source - File content
+ * @returns {{ type: string, name: string } | null}
+ */
+export function getObjectInfoFromContent(source) {
+  const match = source.match(objectInfoRegex);
+  if (!match) return null;
+
+  const firstWord = match[1];
+  const scheme = schemeByFirstWord[firstWord];
+  if (!scheme) return null;
+
+  const name = match[2] ?? match[3];
+  if (!name) return null;
+
+  const type = typeByScheme[scheme] ?? scheme;
+  return { type, name };
+}

package/utils.spec.js

@@ -1,6 +1,10 @@
 import { expect } from "chai";
 import { describe, it } from "mocha";
-import { getSchemeFromContent, Iterator } from "./utils.js";
+import {
+  getObjectInfoFromContent,
+  getSchemeFromContent,
+  Iterator,
+} from "./utils.js";
 
 describe("getSchemeFromContent", () => {
   describe("basic keyword detection", () => {
@@ -242,6 +246,94 @@ table products {
   });
 });
 
+describe("getObjectInfoFromContent", () => {
+  it("should extract function with identifier name", () => {
+    const result = getObjectInfoFromContent(
+      "function my_func {\n  stack {\n  }\n}"
+    );
+    expect(result).to.deep.equal({ type: "function", name: "my_func" });
+  });
+
+  it("should extract function with string literal name", () => {
+    const result = getObjectInfoFromContent('function "my func" {\n}');
+    expect(result).to.deep.equal({ type: "function", name: "my func" });
+  });
+
+  it("should extract table", () => {
+    const result = getObjectInfoFromContent(
+      "table users {\n  schema {\n  }\n}"
+    );
+    expect(result).to.deep.equal({ type: "table", name: "users" });
+  });
+
+  it("should extract query with verb", () => {
+    const result = getObjectInfoFromContent('query "/users" GET {\n}');
+    expect(result).to.deep.equal({ type: "query", name: "/users" });
+  });
+
+  it("should extract task", () => {
+    const result = getObjectInfoFromContent('task "daily_cleanup" {\n}');
+    expect(result).to.deep.equal({ type: "task", name: "daily_cleanup" });
+  });
+
+  it("should extract api_group", () => {
+    const result = getObjectInfoFromContent('api_group "users" {\n}');
+    expect(result).to.deep.equal({ type: "api_group", name: "users" });
+  });
+
+  it("should handle leading comments", () => {
+    const result = getObjectInfoFromContent(
+      "// my function\nfunction my_func {\n}"
+    );
+    expect(result).to.deep.equal({ type: "function", name: "my_func" });
+  });
+
+  it("should handle leading whitespace and multiple comments", () => {
+    const result = getObjectInfoFromContent(
+      "\n// comment 1\n// comment 2\ntable users {\n}"
+    );
+    expect(result).to.deep.equal({ type: "table", name: "users" });
+  });
+
+  it("should return null for empty content", () => {
+    expect(getObjectInfoFromContent("")).to.be.null;
+  });
+
+  it("should return null for content with no name token", () => {
+    expect(getObjectInfoFromContent("function")).to.be.null;
+  });
+
+  it("should extract workflow_test", () => {
+    const result = getObjectInfoFromContent(
+      'workflow_test "login flow" {\n}'
+    );
+    expect(result).to.deep.equal({
+      type: "workflow_test",
+      name: "login flow",
+    });
+  });
+
+  it("should extract table_trigger", () => {
+    const result = getObjectInfoFromContent(
+      'table_trigger "on_user_create" {\n}'
+    );
+    expect(result).to.deep.equal({
+      type: "table_trigger",
+      name: "on_user_create",
+    });
+  });
+
+  it("should extract middleware", () => {
+    const result = getObjectInfoFromContent('middleware "auth_check" {\n}');
+    expect(result).to.deep.equal({ type: "middleware", name: "auth_check" });
+  });
+
+  it("should extract addon", () => {
+    const result = getObjectInfoFromContent('addon "my_addon" {\n}');
+    expect(result).to.deep.equal({ type: "addon", name: "my_addon" });
+  });
+});
+
 describe("Iterator", () => {
   describe("basic functionality", () => {
     it("should iterate through items", () => {