@razdolbai/merls 1.1.1 → 1.2.1

package/.serena/memories/core.md

@@ -1,8 +1,7 @@
-- Completed MVP repo; current durable docs are \README.md\, \PLAN.md\, \AGENTS.md\.
-- Purpose: standalone LSP server for Merlin-style 6502 assembly, primary editor target coc.nvim, stdio transport.
-- Scope invariant: 6502-only. 65816 syntax is out of scope and should be diagnosed as unsupported.
-- Implementation plan and phase breakdown live in \PLAN.md\. All planned tasks are completed.
-- Contributor rules and doc-maintenance requirement live in \AGENTS.md\.
-- For stack and workspace layout, read \mem:tech_stack\.
-- For code/test/doc conventions, read \mem:conventions\.
+- Completed MVP repo; current durable docs are \README.md\, \AGENTS.md\.
+- Purpose: standalone LSP server for Merlin-style 6502 assembly, primary editor target coc.nvim, stdio transport. Published to npm as `@razdolbai/merls`.
+- Scope invariant: 6502-only. 65816 syntax is out of scope and should be diagnosed as unsupported.
+- Contributor rules and doc-maintenance requirement live in \AGENTS.md\.
+- For stack and workspace layout, read \mem:tech_stack\.
+- For code/test/doc conventions, read \mem:conventions\.
 - For done criteria and required verification/docs updates, read \mem:task_completion\.

package/.serena/memories/suggested_commands.md

@@ -1,4 +1,4 @@
-- Windows shell is PowerShell; use PowerShell forms when documenting local commands.
-- Current useful repo commands are read-only/doc-oriented: `Get-ChildItem`, `Get-Content README.md`, `Get-Content PLAN.md`, `Get-Content AGENTS.md`.
-- Planned commands after Phase 1 scaffolding: `npm install`, `npm run build`, `npm test`, `npm run dev` (or equivalent stdio server runner).
-- Do not assume npm scripts exist until `package.json` is added.
+- Windows shell is PowerShell; use PowerShell forms when documenting local commands.
+- Current useful repo commands are read-only/doc-oriented: `Get-ChildItem`, `Get-Content README.md`, `Get-Content AGENTS.md`.
+- Available commands: `npm install`, `npm run build`, `npm test`, `npm run dev` (run the TypeScript compiler in watch mode).
+- A local publishing script is available at `.\publish.ps1`.

package/AGENTS.md

@@ -25,6 +25,7 @@ The current `workspace/symbol` provider is wired through `src/server.ts` and `sr
 The current `textDocument/definition` and `textDocument/references` handlers are wired through `src/server.ts` and `src/lsp/symbol-navigation.ts`.
 The current `textDocument/hover` handler is wired through `src/server.ts` and `src/lsp/hover.ts`.
 The current `textDocument/completion` handler is wired through `src/server.ts` and `src/lsp/completion.ts`.
+The current `textDocument/semanticTokens` handler is wired through `src/server.ts` and `src/lsp/semantic-tokens.ts`.
 The current `textDocument/publishDiagnostics` path is wired through `src/server.ts` and `src/lsp/diagnostics.ts`, with full-document sync on open/change so editor clients receive live parser and resolver diagnostics.
 The current packaged CLI entrypoint lives under `src/cli.ts` and is covered by a compiled stdio launch-contract integration test.
 The coc.nvim smoke test now lives under `test/smoke/` with a headless Vim runner (`run-smoke.ps1`) and a minimal vimrc that isolates the test from the user's real Vim configuration.

package/README.md

@@ -15,25 +15,27 @@ The MVP LSP feature set for Merlin-style 6502 assembly is complete and published
 
 ## Scope
 
-### In scope
-
-- 6502-only Merlin-style assembly
-- parser-based diagnostics for the MVP
-- core LSP features such as:
-  - diagnostics
-  - hover
-  - completion
-  - go to definition
-  - find references
-  - document symbols
-  - workspace symbols
-
-### Out of scope
-
-- 65816 support
-- assembler-backed diagnostics in the first implementation pass
+### In scope
+
+- 6502-only Merlin-style assembly
+- parser-based diagnostics for the MVP
+- core LSP features such as:
+  - diagnostics
+  - hover
+  - completion
+  - go to definition
+  - find references
+  - document symbols
+  - workspace symbols
+  - semantic tokens (syntax highlighting)
+
+### Out of scope
+
+- 65816 support
+- assembler-backed diagnostics in the first implementation pass
 - coc.nvim-specific plugin code for the MVP
 
+
 ## Implementation
 
 - **runtime:** Node.js

package/dist/src/lsp/semantic-tokens.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.semanticTokensLegend = void 0;
+exports.buildSemanticTokens = buildSemanticTokens;
+const vscode_languageserver_1 = require("vscode-languageserver");
+const lexer_1 = require("../asm/lexer");
+const tokenTypesList = [
+    vscode_languageserver_1.SemanticTokenTypes.comment,
+    vscode_languageserver_1.SemanticTokenTypes.function,
+    vscode_languageserver_1.SemanticTokenTypes.macro,
+    vscode_languageserver_1.SemanticTokenTypes.keyword,
+    vscode_languageserver_1.SemanticTokenTypes.string,
+    vscode_languageserver_1.SemanticTokenTypes.number,
+    vscode_languageserver_1.SemanticTokenTypes.operator,
+    vscode_languageserver_1.SemanticTokenTypes.variable
+];
+exports.semanticTokensLegend = {
+    tokenTypes: tokenTypesList,
+    tokenModifiers: []
+};
+const tokenTypeMap = {
+    comment: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.comment),
+    label: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.function),
+    localLabel: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.function),
+    directive: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.macro),
+    mnemonic: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.keyword),
+    string: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.string),
+    numericLiteral: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.number),
+    modifier: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.operator),
+    expressionOperator: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.operator),
+    identifier: tokenTypesList.indexOf(vscode_languageserver_1.SemanticTokenTypes.variable)
+};
+function buildSemanticTokens(source) {
+    const lexedSource = (0, lexer_1.lexSource)(source);
+    const builder = new vscode_languageserver_1.SemanticTokensBuilder();
+    for (const line of lexedSource.lines) {
+        for (const token of line.tokens) {
+            const typeIndex = tokenTypeMap[token.kind];
+            const length = token.end - token.start;
+            builder.push(line.line, token.start, length, typeIndex, 0);
+        }
+    }
+    return builder.build();
+}

package/dist/src/server.js

@@ -9,6 +9,7 @@ const diagnostics_1 = require("./lsp/diagnostics");
 const hover_1 = require("./lsp/hover");
 const symbol_navigation_1 = require("./lsp/symbol-navigation");
 const workspace_symbols_1 = require("./lsp/workspace-symbols");
+const semantic_tokens_1 = require("./lsp/semantic-tokens");
 function createServerConnection(inputStream = process.stdin, outputStream = process.stdout) {
     return (0, node_1.createConnection)(node_1.ProposedFeatures.all, inputStream, outputStream);
 }
@@ -23,6 +24,10 @@ function startServer(inputStream = process.stdin, outputStream = process.stdout)
             hoverProvider: true,
             referencesProvider: true,
             workspaceSymbolProvider: true,
+            semanticTokensProvider: {
+                legend: semantic_tokens_1.semanticTokensLegend,
+                full: true
+            },
             textDocumentSync: {
                 openClose: true,
                 change: node_1.TextDocumentSyncKind.Full
@@ -61,6 +66,13 @@ function startServer(inputStream = process.stdin, outputStream = process.stdout)
     connection.onReferences((params) => (0, symbol_navigation_1.findReferences)(openDocuments, params.textDocument.uri, params.position.line, params.context.includeDeclaration));
     connection.onHover((params) => (0, hover_1.buildHover)(openDocuments, params.textDocument.uri, params.position.line, params.position.character));
     connection.onCompletion((params) => (0, completion_1.buildCompletionItems)(openDocuments, params.textDocument.uri, params.position.line));
+    connection.languages.semanticTokens.on((params) => {
+        const source = openDocuments.get(params.textDocument.uri);
+        if (source === undefined) {
+            return { data: [] };
+        }
+        return (0, semantic_tokens_1.buildSemanticTokens)(source);
+    });
     function publishDiagnostics() {
         for (const [uri, diagnostics] of (0, diagnostics_1.collectDiagnosticsByUri)(openDocuments).entries()) {
             connection.sendDiagnostics({

package/dist/test/coc-config.test.js

@@ -11,11 +11,10 @@ function runCocConfigTest() {
     const settingsPath = node_path_1.default.resolve(process.cwd(), "examples/coc-settings.json");
     const settings = JSON.parse(node_fs_1.default.readFileSync(settingsPath, "utf8"));
     const serverConfig = settings.languageserver?.merls;
-    strict_1.default.equal(serverConfig?.command, "node");
+    strict_1.default.equal(serverConfig?.command, "merls");
     strict_1.default.deepEqual(serverConfig?.args, [
-        "C:/Users/alexe/Projects/merls/dist/src/cli.js",
         "--stdio"
     ]);
     strict_1.default.deepEqual(serverConfig?.filetypes, ["asm"]);
-    strict_1.default.deepEqual(serverConfig?.rootPatterns, [".git", "package.json"]);
+    strict_1.default.deepEqual(serverConfig?.rootPatterns, [".git"]);
 }

package/dist/test/run-tests.js

@@ -22,6 +22,7 @@ const symbols_test_1 = require("./symbols.test");
 const syntax_shape_test_1 = require("./syntax-shape.test");
 const workspace_test_1 = require("./workspace.test");
 const workspace_symbol_test_1 = require("./workspace-symbol.test");
+const semantic_tokens_test_1 = require("./semantic-tokens.test");
 const tests = [
     {
         name: "workspace bootstrap exposes the project name",
@@ -110,6 +111,10 @@ const tests = [
     {
         name: "server publishes and clears diagnostics for open Merlin documents",
         run: publish_diagnostics_test_1.runPublishDiagnosticsTest
+    },
+    {
+        name: "server returns semantic tokens for highlighting",
+        run: semantic_tokens_test_1.runSemanticTokensTest
     }
 ];
 async function main() {

package/dist/test/semantic-tokens.test.js

@@ -0,0 +1,104 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runSemanticTokensTest = runSemanticTokensTest;
+const strict_1 = __importDefault(require("node:assert/strict"));
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_child_process_1 = require("node:child_process");
+function encodeMessage(message) {
+    const body = JSON.stringify(message);
+    return `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n\r\n${body}`;
+}
+function decodeMessages(streamBuffer) {
+    const messages = [];
+    let buffer = streamBuffer;
+    for (;;) {
+        const separator = buffer.indexOf("\r\n\r\n");
+        if (separator === -1) {
+            return { messages, rest: buffer };
+        }
+        const header = buffer.slice(0, separator);
+        const match = /Content-Length: (\d+)/i.exec(header);
+        if (!match) {
+            throw new Error(`Missing Content-Length header: ${header}`);
+        }
+        const length = Number(match[1]);
+        const body = buffer.slice(separator + 4);
+        if (Buffer.byteLength(body, "utf8") < length) {
+            return { messages, rest: buffer };
+        }
+        messages.push(JSON.parse(body.slice(0, length)));
+        buffer = body.slice(length);
+    }
+}
+async function runSemanticTokensTest() {
+    const serverPath = node_path_1.default.resolve(__dirname, "../src/server.js");
+    const mainPath = node_path_1.default.resolve(process.cwd(), "test/fixtures/valid/merlin32-main-6502.asm");
+    const mainUri = `file://${mainPath.replace(/\\/g, "/")}`;
+    const text = node_fs_1.default.readFileSync(mainPath, "utf8");
+    const child = (0, node_child_process_1.spawn)(process.execPath, [serverPath], {
+        stdio: ["pipe", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let nextId = 1;
+    const pending = new Map();
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        stdout += chunk;
+        const decoded = decodeMessages(stdout);
+        stdout = decoded.rest;
+        for (const message of decoded.messages) {
+            if (message.id !== undefined) {
+                pending.get(message.id)?.(message);
+                pending.delete(message.id);
+            }
+        }
+    });
+    function sendRequest(method, params) {
+        const id = nextId++;
+        child.stdin.write(encodeMessage({
+            id,
+            jsonrpc: "2.0",
+            method,
+            params
+        }));
+        return new Promise((resolve) => {
+            pending.set(id, resolve);
+        });
+    }
+    function sendNotification(method, params) {
+        child.stdin.write(encodeMessage({
+            jsonrpc: "2.0",
+            method,
+            params
+        }));
+    }
+    try {
+        await sendRequest("initialize", {
+            capabilities: {},
+            processId: process.pid,
+            rootUri: `file://${node_path_1.default.resolve(process.cwd()).replace(/\\/g, "/")}`
+        });
+        sendNotification("initialized", {});
+        sendNotification("textDocument/didOpen", {
+            textDocument: {
+                uri: mainUri,
+                languageId: "asm",
+                version: 1,
+                text
+            }
+        });
+        const semanticTokensResponse = await sendRequest("textDocument/semanticTokens/full", {
+            textDocument: { uri: mainUri }
+        });
+        const result = semanticTokensResponse.result;
+        strict_1.default.ok(Array.isArray(result.data), "Expected data to be an array");
+        strict_1.default.ok(result.data.length > 0, "Expected non-empty semantic tokens array");
+    }
+    finally {
+        child.kill();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@razdolbai/merls",
-  "version": "1.1.1",
+  "version": "1.2.1",
   "description": "Language server for Merlin-style 6502 assembly.",
   "license": "MIT",
   "type": "commonjs",

package/src/lsp/semantic-tokens.ts

@@ -0,0 +1,52 @@
+import { SemanticTokens, SemanticTokensBuilder, SemanticTokensLegend, SemanticTokenTypes } from "vscode-languageserver";
+import { lexSource, type TokenKind } from "../asm/lexer";
+
+const tokenTypesList = [
+  SemanticTokenTypes.comment,
+  SemanticTokenTypes.function,
+  SemanticTokenTypes.macro,
+  SemanticTokenTypes.keyword,
+  SemanticTokenTypes.string,
+  SemanticTokenTypes.number,
+  SemanticTokenTypes.operator,
+  SemanticTokenTypes.variable
+];
+
+export const semanticTokensLegend: SemanticTokensLegend = {
+  tokenTypes: tokenTypesList,
+  tokenModifiers: []
+};
+
+const tokenTypeMap: Record<TokenKind, number> = {
+  comment: tokenTypesList.indexOf(SemanticTokenTypes.comment),
+  label: tokenTypesList.indexOf(SemanticTokenTypes.function),
+  localLabel: tokenTypesList.indexOf(SemanticTokenTypes.function),
+  directive: tokenTypesList.indexOf(SemanticTokenTypes.macro),
+  mnemonic: tokenTypesList.indexOf(SemanticTokenTypes.keyword),
+  string: tokenTypesList.indexOf(SemanticTokenTypes.string),
+  numericLiteral: tokenTypesList.indexOf(SemanticTokenTypes.number),
+  modifier: tokenTypesList.indexOf(SemanticTokenTypes.operator),
+  expressionOperator: tokenTypesList.indexOf(SemanticTokenTypes.operator),
+  identifier: tokenTypesList.indexOf(SemanticTokenTypes.variable)
+};
+
+export function buildSemanticTokens(source: string): SemanticTokens {
+  const lexedSource = lexSource(source);
+  const builder = new SemanticTokensBuilder();
+
+  for (const line of lexedSource.lines) {
+    for (const token of line.tokens) {
+      const typeIndex = tokenTypeMap[token.kind];
+      const length = token.end - token.start;
+      builder.push(
+        line.line,
+        token.start,
+        length,
+        typeIndex,
+        0
+      );
+    }
+  }
+
+  return builder.build();
+}

package/src/server.ts

@@ -11,6 +11,7 @@ import { collectDiagnosticsByUri } from "./lsp/diagnostics";
 import { buildHover } from "./lsp/hover";
 import { findDefinition, findReferences } from "./lsp/symbol-navigation";
 import { buildWorkspaceSymbols } from "./lsp/workspace-symbols";
+import { buildSemanticTokens, semanticTokensLegend } from "./lsp/semantic-tokens";
 
 export function createServerConnection(
   inputStream: NodeJS.ReadableStream = process.stdin,
@@ -34,6 +35,10 @@ export function startServer(
       hoverProvider: true,
       referencesProvider: true,
       workspaceSymbolProvider: true,
+      semanticTokensProvider: {
+        legend: semanticTokensLegend,
+        full: true
+      },
       textDocumentSync: {
         openClose: true,
         change: TextDocumentSyncKind.Full
@@ -102,6 +107,13 @@ export function startServer(
       params.position.line
     )
   );
+  connection.languages.semanticTokens.on((params) => {
+    const source = openDocuments.get(params.textDocument.uri);
+    if (source === undefined) {
+      return { data: [] };
+    }
+    return buildSemanticTokens(source);
+  });
 
   function publishDiagnostics(): void {
     for (const [uri, diagnostics] of collectDiagnosticsByUri(openDocuments).entries()) {

package/test/coc-config.test.ts

@@ -18,11 +18,10 @@ export function runCocConfigTest(): void {
   const settings = JSON.parse(fs.readFileSync(settingsPath, "utf8")) as CocSettings;
   const serverConfig = settings.languageserver?.merls;
 
-  assert.equal(serverConfig?.command, "node");
+  assert.equal(serverConfig?.command, "merls");
   assert.deepEqual(serverConfig?.args, [
-    "C:/Users/alexe/Projects/merls/dist/src/cli.js",
     "--stdio"
   ]);
   assert.deepEqual(serverConfig?.filetypes, ["asm"]);
-  assert.deepEqual(serverConfig?.rootPatterns, [".git", "package.json"]);
+  assert.deepEqual(serverConfig?.rootPatterns, [".git"]);
 }

package/test/run-tests.ts

@@ -20,6 +20,7 @@ import { runSymbolsTest } from "./symbols.test";
 import { runSyntaxShapeTest } from "./syntax-shape.test";
 import { runWorkspaceGraphTest } from "./workspace.test";
 import { runWorkspaceSymbolTest } from "./workspace-symbol.test";
+import { runSemanticTokensTest } from "./semantic-tokens.test";
 
 type TestCase = {
   name: string;
@@ -114,6 +115,10 @@ const tests: TestCase[] = [
   {
     name: "server publishes and clears diagnostics for open Merlin documents",
     run: runPublishDiagnosticsTest
+  },
+  {
+    name: "server returns semantic tokens for highlighting",
+    run: runSemanticTokensTest
   }
 ];
 

package/test/semantic-tokens.test.ts

@@ -0,0 +1,128 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+
+type JsonRpcMessage = {
+  id?: number;
+  jsonrpc: "2.0";
+  result?: unknown;
+};
+
+function encodeMessage(message: object): string {
+  const body = JSON.stringify(message);
+  return `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n\r\n${body}`;
+}
+
+function decodeMessages(streamBuffer: string): { messages: JsonRpcMessage[]; rest: string } {
+  const messages: JsonRpcMessage[] = [];
+  let buffer = streamBuffer;
+
+  for (;;) {
+    const separator = buffer.indexOf("\r\n\r\n");
+    if (separator === -1) {
+      return { messages, rest: buffer };
+    }
+
+    const header = buffer.slice(0, separator);
+    const match = /Content-Length: (\d+)/i.exec(header);
+    if (!match) {
+      throw new Error(`Missing Content-Length header: ${header}`);
+    }
+
+    const length = Number(match[1]);
+    const body = buffer.slice(separator + 4);
+    if (Buffer.byteLength(body, "utf8") < length) {
+      return { messages, rest: buffer };
+    }
+
+    messages.push(JSON.parse(body.slice(0, length)) as JsonRpcMessage);
+    buffer = body.slice(length);
+  }
+}
+
+export async function runSemanticTokensTest(): Promise<void> {
+  const serverPath = path.resolve(__dirname, "../src/server.js");
+  const mainPath = path.resolve(
+    process.cwd(),
+    "test/fixtures/valid/merlin32-main-6502.asm"
+  );
+  const mainUri = `file://${mainPath.replace(/\\/g, "/")}`;
+  const text = fs.readFileSync(mainPath, "utf8");
+  const child = spawn(process.execPath, [serverPath], {
+    stdio: ["pipe", "pipe", "pipe"]
+  });
+
+  let stdout = "";
+  let nextId = 1;
+  const pending = new Map<number, (message: JsonRpcMessage) => void>();
+
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk: string) => {
+    stdout += chunk;
+    const decoded = decodeMessages(stdout);
+    stdout = decoded.rest;
+
+    for (const message of decoded.messages) {
+      if (message.id !== undefined) {
+        pending.get(message.id)?.(message);
+        pending.delete(message.id);
+      }
+    }
+  });
+
+  function sendRequest(method: string, params: object): Promise<JsonRpcMessage> {
+    const id = nextId++;
+    child.stdin.write(
+      encodeMessage({
+        id,
+        jsonrpc: "2.0",
+        method,
+        params
+      })
+    );
+
+    return new Promise((resolve) => {
+      pending.set(id, resolve);
+    });
+  }
+
+  function sendNotification(method: string, params: object): void {
+    child.stdin.write(
+      encodeMessage({
+        jsonrpc: "2.0",
+        method,
+        params
+      })
+    );
+  }
+
+  try {
+    await sendRequest("initialize", {
+      capabilities: {},
+      processId: process.pid,
+      rootUri: `file://${path.resolve(process.cwd()).replace(/\\/g, "/")}`
+    });
+
+    sendNotification("initialized", {});
+    sendNotification("textDocument/didOpen", {
+      textDocument: {
+        uri: mainUri,
+        languageId: "asm",
+        version: 1,
+        text
+      }
+    });
+
+    const semanticTokensResponse = await sendRequest("textDocument/semanticTokens/full", {
+      textDocument: { uri: mainUri }
+    });
+
+    const result = semanticTokensResponse.result as { data: number[] };
+    assert.ok(Array.isArray(result.data), "Expected data to be an array");
+    assert.ok(result.data.length > 0, "Expected non-empty semantic tokens array");
+
+  } finally {
+    child.kill();
+  }
+}