@razdolbai/merls 0.1.0

package/test/lexer.test.ts

@@ -0,0 +1,87 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+
+import {
+  type LexedLine,
+  type Token,
+  lexSource
+} from "../src/asm/lexer";
+
+function summarizeTokens(tokens: readonly Token[]): readonly (readonly [string, string])[] {
+  return tokens.map((token) => [token.kind, token.lexeme] as const);
+}
+
+function findLine(lines: readonly LexedLine[], source: string): LexedLine {
+  const line = lines.find((candidate) => candidate.text === source);
+  assert.ok(line, `expected to find line: ${source}`);
+  return line;
+}
+
+export function runLexerTest(): void {
+  const mainFixturePath = path.resolve(
+    process.cwd(),
+    "test/fixtures/valid/merlin32-main-6502.asm"
+  );
+  const linkFixturePath = path.resolve(
+    process.cwd(),
+    "test/fixtures/valid/merlin32-linkscript.asm"
+  );
+
+  const mainFixture = fs.readFileSync(mainFixturePath, "utf8");
+  const linkFixture = fs.readFileSync(linkFixturePath, "utf8");
+
+  const mainLines = lexSource(mainFixture).lines;
+  const linkLines = lexSource(linkFixture).lines;
+
+  assert.deepEqual(
+    summarizeTokens(findLine(mainLines, "; Source: apple2accumulator/merlin32").tokens),
+    [["comment", "; Source: apple2accumulator/merlin32"]]
+  );
+
+  assert.deepEqual(
+    summarizeTokens(findLine(mainLines, "TEXT    =   $FB39").tokens),
+    [
+      ["label", "TEXT"],
+      ["expressionOperator", "="],
+      ["numericLiteral", "$FB39"]
+    ]
+  );
+
+  assert.deepEqual(
+    summarizeTokens(findLine(mainLines, "        DUM 0").tokens),
+    [
+      ["directive", "DUM"],
+      ["numericLiteral", "0"]
+    ]
+  );
+
+  assert.deepEqual(
+    summarizeTokens(findLine(mainLines, "        adc ($80,x)").tokens),
+    [
+      ["mnemonic", "adc"],
+      ["expressionOperator", "("],
+      ["numericLiteral", "$80"],
+      ["expressionOperator", ","],
+      ["identifier", "x"],
+      ["expressionOperator", ")"]
+    ]
+  );
+
+  assert.deepEqual(
+    summarizeTokens(findLine(mainLines, "GetKey  ldx $C000").tokens),
+    [
+      ["label", "GetKey"],
+      ["mnemonic", "ldx"],
+      ["numericLiteral", "$C000"]
+    ]
+  );
+
+  assert.deepEqual(
+    summarizeTokens(findLine(linkLines, "    asm \"merlin32-main-6502.asm\"").tokens),
+    [
+      ["directive", "asm"],
+      ["string", "\"merlin32-main-6502.asm\""]
+    ]
+  );
+}

package/test/line-parser.test.ts

@@ -0,0 +1,69 @@
+import assert from "node:assert/strict";
+
+import { parseSourceLines } from "../src/asm/parser";
+
+export function runLineParserTest(): void {
+  const source = [
+    "TEXT    =   $FB39",
+    "        adc (_tmp+dum1+1,x)",
+    "dum0    ds  1",
+    "        hex 2C",
+    "        adc ("
+  ].join("\n");
+
+  const lines = parseSourceLines(source);
+
+  assert.deepEqual(lines[0], {
+    shape: "equate",
+    text: "TEXT    =   $FB39",
+    label: "TEXT",
+    expression: {
+      kind: "numericLiteral",
+      value: "$FB39"
+    }
+  });
+
+  assert.deepEqual(lines[1], {
+    shape: "instruction",
+    text: "        adc (_tmp+dum1+1,x)",
+    label: null,
+    mnemonic: "adc",
+    operand: {
+      immediate: false,
+      indirect: true,
+      indexRegister: "x",
+      expression: {
+        kind: "binary",
+        operator: "+",
+        left: {
+          kind: "binary",
+          operator: "+",
+          left: { kind: "identifier", value: "_tmp" },
+          right: { kind: "identifier", value: "dum1" }
+        },
+        right: { kind: "numericLiteral", value: "1" }
+      }
+    }
+  });
+
+  assert.deepEqual(lines[2], {
+    shape: "directive",
+    text: "dum0    ds  1",
+    label: "dum0",
+    directive: "ds",
+    operand: {
+      kind: "numericLiteral",
+      value: "1"
+    }
+  });
+
+  assert.deepEqual(lines[3], {
+    shape: "data",
+    text: "        hex 2C",
+    label: null,
+    directive: "hex",
+    payload: "2C"
+  });
+
+  assert.equal(lines[4]?.shape, "malformed");
+}

package/test/local-labels.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+
+import { parseDocument } from "../src/asm/document";
+import { resolveLocalLabels } from "../src/asm/local-labels";
+
+export function runLocalLabelScopeTest(): void {
+  const fixturePath = path.resolve(
+    process.cwd(),
+    "test/fixtures/valid/merlin32-main-6502.asm"
+  );
+  const source = fs.readFileSync(fixturePath, "utf8");
+
+  const document = parseDocument(source);
+  const scope = resolveLocalLabels(document);
+
+  assert.deepEqual(scope.definitions.get("]loop@69"), {
+    name: "]loop",
+    line: 71,
+    anchor: "GetKey",
+    qualifiedName: "]loop@69"
+  });
+
+  assert.deepEqual(scope.references.get("]loop@73"), {
+    name: "]loop",
+    line: 73,
+    anchor: "GetKey",
+    qualifiedName: "]loop@69",
+    targetLine: 71
+  });
+
+  assert.deepEqual(scope.definitions.get(":err@69"), {
+    name: ":err",
+    line: 82,
+    anchor: "GetKey",
+    qualifiedName: ":err@69"
+  });
+
+  assert.deepEqual(scope.references.get(":good@81"), {
+    name: ":good",
+    line: 81,
+    anchor: "GetKey",
+    qualifiedName: ":good@69",
+    targetLine: 84
+  });
+}

package/test/metadata.test.ts

@@ -0,0 +1,27 @@
+import assert from "node:assert/strict";
+
+import {
+  directiveTable,
+  opcodeTable
+} from "../src/asm/metadata";
+
+export function runMetadataTableTest(): void {
+  assert.equal(opcodeTable.size, 56);
+  assert.deepEqual(opcodeTable.get("lda")?.modes, [
+    "immediate",
+    "zeroPage",
+    "zeroPageX",
+    "absolute",
+    "absoluteX",
+    "absoluteY",
+    "indexedIndirect",
+    "indirectIndexed"
+  ]);
+  assert.equal(opcodeTable.has("mvn"), false);
+
+  assert.equal(directiveTable.get("org")?.supported, true);
+  assert.equal(directiveTable.get("dum")?.supported, true);
+  assert.equal(directiveTable.get("xc")?.supported, false);
+  assert.equal(directiveTable.get("mx")?.supported, false);
+  assert.equal(directiveTable.get("put")?.kind, "include");
+}

package/test/publish-diagnostics.test.ts

@@ -0,0 +1,206 @@
+import assert from "node:assert/strict";
+import path from "node:path";
+import { spawn } from "node:child_process";
+
+type JsonRpcMessage = {
+  id?: number;
+  jsonrpc: "2.0";
+  method?: string;
+  params?: unknown;
+  result?: unknown;
+};
+
+type PublishedDiagnostic = {
+  message: string;
+  severity?: number;
+  range: {
+    start: {
+      line: number;
+      character: number;
+    };
+    end: {
+      line: number;
+      character: number;
+    };
+  };
+};
+
+type PublishDiagnosticsParams = {
+  uri: string;
+  diagnostics: PublishedDiagnostic[];
+};
+
+function encodeMessage(message: object): string {
+  const body = JSON.stringify(message);
+  return `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n\r\n${body}`;
+}
+
+function decodeMessages(streamBuffer: string): { messages: JsonRpcMessage[]; rest: string } {
+  const messages: JsonRpcMessage[] = [];
+  let buffer = streamBuffer;
+
+  for (;;) {
+    const separator = buffer.indexOf("\r\n\r\n");
+    if (separator === -1) {
+      return { messages, rest: buffer };
+    }
+
+    const header = buffer.slice(0, separator);
+    const match = /Content-Length: (\d+)/i.exec(header);
+    if (!match) {
+      throw new Error(`Missing Content-Length header: ${header}`);
+    }
+
+    const length = Number(match[1]);
+    const body = buffer.slice(separator + 4);
+    if (Buffer.byteLength(body, "utf8") < length) {
+      return { messages, rest: buffer };
+    }
+
+    messages.push(JSON.parse(body.slice(0, length)) as JsonRpcMessage);
+    buffer = body.slice(length);
+  }
+}
+
+export async function runPublishDiagnosticsTest(): Promise<void> {
+  const serverPath = path.resolve(__dirname, "../src/server.js");
+  const documentPath = path.resolve(
+    process.cwd(),
+    "test/fixtures/invalid/publish-diagnostics.asm"
+  );
+  const documentUri = `file://${documentPath.replace(/\\/g, "/")}`;
+  const brokenText = ["dup     equ 1", "        lda missing", "dup     equ 2", "        adc ("].join(
+    "\n"
+  );
+  const fixedText = ["dup     equ 1", "        lda dup", "        adc #1"].join("\n");
+  const child = spawn(process.execPath, [serverPath], {
+    stdio: ["pipe", "pipe", "pipe"]
+  });
+
+  let stdout = "";
+  let nextId = 1;
+  const pending = new Map<number, (message: JsonRpcMessage) => void>();
+  const diagnosticWaiters: Array<(params: PublishDiagnosticsParams) => void> = [];
+
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk: string) => {
+    stdout += chunk;
+    const decoded = decodeMessages(stdout);
+    stdout = decoded.rest;
+
+    for (const message of decoded.messages) {
+      if (message.id !== undefined) {
+        pending.get(message.id)?.(message);
+        pending.delete(message.id);
+        continue;
+      }
+
+      if (message.method === "textDocument/publishDiagnostics") {
+        const params = message.params as PublishDiagnosticsParams;
+        const waiter = diagnosticWaiters.shift();
+        waiter?.(params);
+      }
+    }
+  });
+
+  function sendRequest(method: string, params: object): Promise<JsonRpcMessage> {
+    const id = nextId++;
+    child.stdin.write(
+      encodeMessage({
+        id,
+        jsonrpc: "2.0",
+        method,
+        params
+      })
+    );
+
+    return new Promise((resolve) => {
+      pending.set(id, resolve);
+    });
+  }
+
+  function sendNotification(method: string, params: object): void {
+    child.stdin.write(
+      encodeMessage({
+        jsonrpc: "2.0",
+        method,
+        params
+      })
+    );
+  }
+
+  function waitForDiagnostics(): Promise<PublishDiagnosticsParams> {
+    return new Promise((resolve) => {
+      diagnosticWaiters.push(resolve);
+    });
+  }
+
+  try {
+    await sendRequest("initialize", {
+      capabilities: {},
+      processId: process.pid,
+      rootUri: `file://${path.resolve(process.cwd()).replace(/\\/g, "/")}`
+    });
+
+    sendNotification("initialized", {});
+
+    const openedDiagnostics = waitForDiagnostics();
+    sendNotification("textDocument/didOpen", {
+      textDocument: {
+        uri: documentUri,
+        languageId: "asm",
+        version: 1,
+        text: brokenText
+      }
+    });
+
+    const firstPublish = await openedDiagnostics;
+    assert.equal(firstPublish.uri, documentUri);
+    assert.equal(
+      firstPublish.diagnostics.some(
+        (diagnostic) =>
+          diagnostic.message.includes("Unresolved reference missing") &&
+          diagnostic.severity === 1 &&
+          diagnostic.range.start.line === 1
+      ),
+      true
+    );
+    assert.equal(
+      firstPublish.diagnostics.some(
+        (diagnostic) =>
+          diagnostic.message.includes("Duplicate symbol dup") &&
+          diagnostic.severity === 1 &&
+          diagnostic.range.start.line === 2
+      ),
+      true
+    );
+    assert.equal(
+      firstPublish.diagnostics.some(
+        (diagnostic) =>
+          diagnostic.message.includes("expected expression token") &&
+          diagnostic.severity === 1 &&
+          diagnostic.range.start.line === 3
+      ),
+      true
+    );
+
+    const changedDiagnostics = waitForDiagnostics();
+    sendNotification("textDocument/didChange", {
+      textDocument: {
+        uri: documentUri,
+        version: 2
+      },
+      contentChanges: [
+        {
+          text: fixedText
+        }
+      ]
+    });
+
+    const secondPublish = await changedDiagnostics;
+    assert.equal(secondPublish.uri, documentUri);
+    assert.deepEqual(secondPublish.diagnostics, []);
+  } finally {
+    child.kill();
+  }
+}

package/test/run-tests.ts

@@ -0,0 +1,139 @@
+import { runBootstrapTest } from "./bootstrap.test";
+import { runCliContractTest } from "./cli-contract.test";
+import { runCocConfigTest } from "./coc-config.test";
+import { runCompletionTest } from "./completion.test";
+import { runDefinitionReferencesTest } from "./definition-references.test";
+import { runDiagnosticsTest } from "./diagnostics.test";
+import { runDocumentModelTest } from "./document-model.test";
+import { runDocumentSymbolTest } from "./document-symbol.test";
+import { runExpressionTest } from "./expression.test";
+import { runFixtureCorpusTest } from "./fixture-corpus.test";
+import { runHoverTest } from "./hover.test";
+import { runLexerTest } from "./lexer.test";
+import { runLineParserTest } from "./line-parser.test";
+import { runLocalLabelScopeTest } from "./local-labels.test";
+import { runMetadataTableTest } from "./metadata.test";
+import { runPublishDiagnosticsTest } from "./publish-diagnostics.test";
+import { runInitializeHandshakeTest } from "./server-initialize.test";
+import { runServerEntrypointTest } from "./server-entrypoint.test";
+import { runSymbolsTest } from "./symbols.test";
+import { runSyntaxShapeTest } from "./syntax-shape.test";
+import { runWorkspaceGraphTest } from "./workspace.test";
+import { runWorkspaceSymbolTest } from "./workspace-symbol.test";
+
+type TestCase = {
+  name: string;
+  run: () => void | Promise<void>;
+};
+
+const tests: TestCase[] = [
+  {
+    name: "workspace bootstrap exposes the project name",
+    run: runBootstrapTest
+  },
+  {
+    name: "cli contract exposes a packaged stdio entrypoint",
+    run: runCliContractTest
+  },
+  {
+    name: "coc.nvim example targets the packaged stdio CLI contract",
+    run: runCocConfigTest
+  },
+  {
+    name: "server entrypoint exposes callable startup helpers",
+    run: runServerEntrypointTest
+  },
+  {
+    name: "server process answers initialize",
+    run: runInitializeHandshakeTest
+  },
+  {
+    name: "positive fixture corpus includes upstream Merlin32 samples",
+    run: runFixtureCorpusTest
+  },
+  {
+    name: "shared opcode and directive metadata covers 6502 and Merlin syntax",
+    run: runMetadataTableTest
+  },
+  {
+    name: "token kinds and line shapes cover Merlin syntax categories",
+    run: runSyntaxShapeTest
+  },
+  {
+    name: "lexer tokenizes fixture comments labels mnemonics directives and literals",
+    run: runLexerTest
+  },
+  {
+    name: "expression parser handles numeric forms modifiers arithmetic and indexed operands",
+    run: runExpressionTest
+  },
+  {
+    name: "line parser recognizes equates instructions directives data and malformed lines",
+    run: runLineParserTest
+  },
+  {
+    name: "document model preserves line structure and tolerates malformed lines",
+    run: runDocumentModelTest
+  },
+  {
+    name: "symbol collection indexes labels equates and named data definitions",
+    run: runSymbolsTest
+  },
+  {
+    name: "local label scope resolves Merlin local definitions and references",
+    run: runLocalLabelScopeTest
+  },
+  {
+    name: "workspace indexing follows Merlin include directives and merges symbols",
+    run: runWorkspaceGraphTest
+  },
+  {
+    name: "diagnostics report duplicates unresolved refs malformed lines and unsupported 65816 syntax",
+    run: runDiagnosticsTest
+  },
+  {
+    name: "server returns document symbols for Merlin labels equates and data definitions",
+    run: runDocumentSymbolTest
+  },
+  {
+    name: "server returns workspace symbols across open Merlin documents",
+    run: runWorkspaceSymbolTest
+  },
+  {
+    name: "server resolves definitions and references for Merlin symbols",
+    run: runDefinitionReferencesTest
+  },
+  {
+    name: "server returns hover information for opcodes directives and symbols",
+    run: runHoverTest
+  },
+  {
+    name: "server returns opcode directive and symbol completions",
+    run: runCompletionTest
+  },
+  {
+    name: "server publishes and clears diagnostics for open Merlin documents",
+    run: runPublishDiagnosticsTest
+  }
+];
+
+async function main(): Promise<void> {
+  let failures = 0;
+
+  for (const test of tests) {
+    try {
+      await test.run();
+      console.log(`PASS ${test.name}`);
+    } catch (error) {
+      failures += 1;
+      console.error(`FAIL ${test.name}`);
+      console.error(error);
+    }
+  }
+
+  if (failures > 0) {
+    process.exitCode = 1;
+  }
+}
+
+void main();

package/test/server-entrypoint.test.ts

@@ -0,0 +1,11 @@
+import assert from "node:assert/strict";
+
+import { createServerConnection, startServer } from "../src/server";
+
+export function runServerEntrypointTest(): void {
+  assert.equal(typeof startServer, "function");
+
+  const connection = createServerConnection();
+  assert.equal(typeof connection.listen, "function");
+  connection.dispose();
+}

package/test/server-initialize.test.ts

@@ -0,0 +1,101 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import path from "node:path";
+
+type JsonRpcMessage = {
+  id?: number;
+  jsonrpc: "2.0";
+  method?: string;
+  result?: {
+    capabilities?: {
+      definitionProvider?: boolean;
+      referencesProvider?: boolean;
+      textDocumentSync?: {
+        change?: number;
+        openClose?: boolean;
+      };
+    };
+  };
+};
+
+function encodeMessage(message: object): string {
+  const body = JSON.stringify(message);
+  return `Content-Length: ${Buffer.byteLength(body, "utf8")}\r\n\r\n${body}`;
+}
+
+export async function runInitializeHandshakeTest(): Promise<void> {
+  const serverPath = path.resolve(__dirname, "../src/server.js");
+  const child = spawn(process.execPath, [serverPath], {
+    stdio: ["pipe", "pipe", "pipe"]
+  });
+
+  const response = new Promise<JsonRpcMessage>((resolve, reject) => {
+    let stdout = "";
+    let stderr = "";
+
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+
+    child.stdout.on("data", (chunk: string) => {
+      stdout += chunk;
+
+      const separator = stdout.indexOf("\r\n\r\n");
+      if (separator === -1) {
+        return;
+      }
+
+      const header = stdout.slice(0, separator);
+      const match = /Content-Length: (\d+)/i.exec(header);
+      if (!match) {
+        reject(new Error(`Missing Content-Length header in response: ${stdout}`));
+        return;
+      }
+
+      const length = Number(match[1]);
+      const body = stdout.slice(separator + 4);
+      if (Buffer.byteLength(body, "utf8") < length) {
+        return;
+      }
+
+      resolve(JSON.parse(body.slice(0, length)) as JsonRpcMessage);
+    });
+
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+    });
+
+    child.once("error", reject);
+    child.once("exit", (code) => {
+      reject(new Error(`Server exited before initialize response. code=${code}, stderr=${stderr}`));
+    });
+  });
+
+  child.stdin.write(
+    encodeMessage({
+      id: 1,
+      jsonrpc: "2.0",
+      method: "initialize",
+      params: {
+        capabilities: {},
+        clientInfo: {
+          name: "merls-test"
+        },
+        processId: process.pid,
+        rootUri: null
+      }
+    })
+  );
+
+  try {
+    const message = await response;
+    assert.equal(message.id, 1);
+    assert.equal(message.jsonrpc, "2.0");
+    assert.equal(typeof message.result?.capabilities, "object");
+    assert.equal(message.result?.capabilities?.textDocumentSync?.openClose, true);
+    assert.equal(message.result?.capabilities?.textDocumentSync?.change, 1);
+    assert.equal(message.result?.capabilities?.definitionProvider, true);
+    assert.equal(message.result?.capabilities?.referencesProvider, true);
+  } finally {
+    child.kill();
+  }
+}