@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/sql-types.ts

@@ -133,7 +133,7 @@ function originOf(entity: Declarable, exportName: string, projectDir: string): s
 }
 
 /** The names a relation's `.columns` record holds at run time, in order. */
-const runtimeColumns = (entity: Declarable | undefined): string[] => Object.keys((entity as { columns?: object } | undefined)?.columns ?? {});
+export const runtimeColumns = (entity: Declarable | undefined): string[] => Object.keys((entity as { columns?: object } | undefined)?.columns ?? {});
 
 /** Whether `expr` is the column `column` alone, bare or qualified (`o.user_id`). */
 function isBareColumn(expr: string, quoted: string): boolean {
@@ -148,7 +148,7 @@ interface LineageJson {
 }
 
 /** A view's column that passes one column through: that column's export and name. */
-function passthrough(o: OutputObject, name: string, quote: (c: string) => string): { export: string; column: string } | undefined {
+export function passthrough(o: OutputObject, name: string, quote: (c: string) => string): { export: string; column: string } | undefined {
   const edge = ((o.lineage as LineageJson[] | undefined) ?? []).find((e) => e.output === name);
   const ref = edge?.from.length === 1 ? edge.from[0] : undefined;
   if (!edge || !ref) return undefined;
@@ -177,7 +177,7 @@ function spell(canonical: string, t: ParsedType): string {
 const arrayOf = (ts: string): string => `${/[|&]/.test(ts) && !/^\{.*\}$/s.test(ts) ? `(${ts})` : ts}[]`;
 
 /** A Postgres type's spelling, row type, and whether the type itself forbids NULL (a serial, a NOT NULL domain). */
-function pgType(p: PgProject, text: string | undefined, depth = 0): { sqlType: string; ts: string; notNull: boolean } {
+export function pgType(p: PgProject, text: string | undefined, depth = 0): { sqlType: string; ts: string; notNull: boolean } {
   const t = parseType(text);
   if (!t || depth > 5) return { sqlType: text?.trim() || "unknown", ts: "unknown", notNull: false };
   const r = resolveType(p, t, { serial: true });
@@ -320,7 +320,7 @@ function renderEntry(e: RelationEntry, lines: string[]): void {
 }
 
 /** The dialect a build's entities declare, else the config's, else ClickHouse. */
-function dialectOf(entities: ReadonlyMap<string, Declarable>, config: Record<string, unknown>): SqlDialect {
+export function dialectOf(entities: ReadonlyMap<string, Declarable>, config: Record<string, unknown>): SqlDialect {
   try {
     return filesDialect(entities, config);
   } catch {
@@ -328,6 +328,12 @@ function dialectOf(entities: ReadonlyMap<string, Declarable>, config: Record<str
   }
 }
 
+/** The build's sql output as `chant build` writes it: the objects, and the Postgres major they were built for. */
+export function serializedObjects(entities: ReadonlyMap<string, Declarable>, config: Record<string, unknown>): { objects: OutputObject[]; postgresMajor?: number } {
+  const out = sqlSerializer.serialize(new Map(entities), undefined, { config }) as SerializerResult | string;
+  return JSON.parse(typeof out === "string" ? out : out.primary) as { objects: OutputObject[]; postgresMajor?: number };
+}
+
 /**
  * The `sql-types.d.ts` for a build's entities: one module that augments the
  * dialect's `SqlSchema` with each table, view and dictionary, in key order.
@@ -340,8 +346,7 @@ export function renderSqlTypes(entities: ReadonlyMap<string, Declarable>, opts:
   const sqlEntities = new Map([...entities].filter(([, e]) => isPostgresObject(e) || isClickHouseObject(e)));
   let entries: RelationEntry[] = [];
   if (sqlEntities.size > 0) {
-    const out = sqlSerializer.serialize(new Map(entities), undefined, { config }) as SerializerResult | string;
-    const doc = JSON.parse(typeof out === "string" ? out : out.primary) as { objects: OutputObject[]; postgresMajor?: number };
+    const doc = serializedObjects(entities, config);
     entries = dialect === "postgres" ? postgresEntries(doc.objects, doc.postgresMajor ?? PINNED_MAJOR, entities, projectDir) : clickhouseEntries(doc.objects, entities, projectDir);
   }
   const byKey = new Map<string, RelationEntry>();

package/src/sqlserver/fixture.test.ts

@@ -0,0 +1,186 @@
+/**
+ * The first two Done-when boxes of #4398, on a public sample schema: an
+ * excerpt of Microsoft's WideWorldImporters (MIT, vendored in
+ * `test-fixtures/sqlserver/` with its licence). The schema imports, every
+ * non-mechanical decision is a SARIF diagnostic, and the output builds and
+ * passes the sql lexicon's lint rules and post-synth checks.
+ */
+
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, describe, expect, test } from "vitest";
+import { build } from "@intentius/chant/build";
+import { runLint } from "@intentius/chant/lint/engine";
+import { runPostSynthChecks } from "@intentius/chant/lint/post-synth";
+import { importExitCode, importSarif } from "@intentius/chant/cli/commands/import-report";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { sqlPlugin } from "../plugin";
+import { sqlTemplateGenerator } from "../import-generator";
+import { postSynthChecks } from "../lint/post-synth";
+import { CLICKHOUSE_DDL_FILE, sqlSerializer } from "../serializer";
+import { splitStatements } from "../clickhouse/import/ir";
+import { REVIEW_RULES } from "../clickhouse/review-catalog";
+
+const fixtures = join(import.meta.dirname, "..", "..", "test-fixtures", "sqlserver");
+const source = readFileSync(join(fixtures, "wwi-excerpt.sql"), "utf8");
+const parse = (text: string, options: Record<string, string | boolean> = { from: "sqlserver" }) => sqlPlugin.templateParser!(options).parse(text);
+const ir = parse(source);
+const diagnostics = ir.diagnostics ?? [];
+const ddlOf = (name: string) => (ir.resources.find((r) => (r.properties as { name: string }).name === name)!.properties as { ddl: string }).ddl;
+
+describe("the vendored sample", () => {
+  test("carries its licence, source path and commit", () => {
+    const licence = readFileSync(join(fixtures, "LICENSE-sql-server-samples.txt"), "utf8");
+    expect(licence).toContain("MIT License");
+    expect(licence).toContain("Copyright (c) Microsoft Corporation");
+    expect(source).toContain("Licence: MIT. Copyright (c) Microsoft Corporation");
+    expect(source).toContain("samples/databases/wide-world-importers/wwi-ssdt/wwi-ssdt/");
+    expect(source).toMatch(/Commit:\s+[0-9a-f]{40}/);
+  });
+});
+
+describe("importing it", () => {
+  test("every table, view and schema becomes one object, databases first", () => {
+    expect(ir.resources.map((r) => `${r.type.replace("ClickHouse::", "")} ${(r.properties as { name: string }).name}`)).toEqual([
+      "Database Warehouse",
+      "Database Sales",
+      "Database WebApi",
+      "Table StockItemTransactions",
+      "Table CustomerTransactions",
+      "Table Orders",
+      "Table Colors",
+      "Table ColdRoomTemperatures",
+      "View Colors",
+    ]);
+    expect(ir.warnings ?? []).toEqual([]);
+  });
+
+  test("the columnstore table, the clustered-index table and the primary-key tables get the sort key the catalog picks", () => {
+    expect(ddlOf("StockItemTransactions")).toContain("ORDER BY StockItemTransactionID");
+    expect(ddlOf("CustomerTransactions")).toContain("ORDER BY TransactionDate");
+    expect(ddlOf("Orders")).toContain("ORDER BY OrderID");
+  });
+
+  test("the generated declarations name their dependencies", () => {
+    const [file] = sqlTemplateGenerator.generate(ir);
+    expect(file!.content).toContain("CREATE TABLE ${warehouseDb}.StockItemTransactions");
+    expect(file!.content).toContain("FROM ${warehouseColors}");
+  });
+});
+
+describe("every non-mechanical decision is a diagnostic", () => {
+  const at = (rule: string) => diagnostics.filter((d) => d.ruleId === rule);
+  const columnsOf = (ddl: string) => [...ddl.matchAll(/^ {4}(\w+) (.+?),?$/gm)].map((m) => ({ name: m[1]!, rest: m[2]! }));
+
+  test("each Nullable column in the DDL has a SQLRV003 diagnostic naming the table and column", () => {
+    const wanted: string[] = [];
+    for (const r of ir.resources) {
+      const p = r.properties as { name: string; database?: string; ddl: string };
+      if (r.type !== "ClickHouse::Table") continue;
+      for (const c of columnsOf(p.ddl)) if (c.rest.startsWith("Nullable(")) wanted.push(`${p.database ?? ""}.${p.name}.${c.name}`.replace(/^\./, ""));
+    }
+    expect(wanted.length).toBeGreaterThan(10);
+    expect(at("SQLRV003").map((d) => d.message.split(":")[0])).toEqual(expect.arrayContaining(wanted));
+    expect(at("SQLRV003")).toHaveLength(wanted.length);
+  });
+
+  test("each table has exactly one sort key diagnostic and one engine diagnostic", () => {
+    expect(at("SQLRV001").map((d) => d.entity).sort()).toEqual(["Sales.CustomerTransactions", "Sales.Orders", "Warehouse.ColdRoomTemperatures", "Warehouse.Colors", "Warehouse.StockItemTransactions"]);
+    expect(at("SQLRV002")).toHaveLength(5);
+  });
+
+  test("each foreign key and unique constraint in the source is a comment in the DDL and a SQLRV005 diagnostic", () => {
+    const sourceCount = [...source.matchAll(/FOREIGN KEY|\bUNIQUE NONCLUSTERED/g)].length;
+    const commentCount = ir.resources.reduce((n, r) => n + [...((r.properties as { ddl: string }).ddl.matchAll(/^-- (?:FOREIGN KEY|UNIQUE)/gm))].length, 0);
+    expect(sourceCount).toBeGreaterThan(15);
+    expect(commentCount).toBe(sourceCount);
+    expect(at("SQLRV005")).toHaveLength(sourceCount);
+  });
+
+  test("the identity column, the sequence defaults, the computed column and the period columns are SQLRV006", () => {
+    const messages = at("SQLRV006").map((d) => d.message);
+    expect(messages.filter((m) => m.includes("draws from a sequence"))).toHaveLength([...source.matchAll(/NEXT VALUE FOR/g)].length);
+    expect(messages.some((m) => m.includes("Identity column ColdRoomTemperatureID"))).toBe(true);
+    expect(messages.some((m) => m.includes("Computed column IsFinalized"))).toBe(true);
+    expect(messages.filter((m) => m.includes("filled by the system"))).toHaveLength(4);
+  });
+
+  test("the sysdatetime() defaults, the view and the indexes that are not carried are diagnostics", () => {
+    expect(at("SQLRV008")).toHaveLength([...source.matchAll(/DEFAULT \(sysdatetime\(\)\)/g)].length);
+    expect(at("SQLRV010")).toHaveLength(1);
+    const notCarried = at("SQLRV009").filter((d) => d.message.includes("is not carried") && d.message.includes("index"));
+    expect(notCarried).toHaveLength([...source.matchAll(/CREATE NONCLUSTERED INDEX/g)].length + 1);
+  });
+
+  test("the diagnostics use only rules in the catalog, each with a line in the source", () => {
+    for (const d of diagnostics) {
+      expect(Object.keys(REVIEW_RULES)).toContain(d.ruleId);
+      expect(d.line, d.message).toBeGreaterThan(0);
+    }
+  });
+
+  test("the SARIF report has one result per diagnostic with its rule, level and region", () => {
+    const sarif = JSON.parse(importSarif("wwi-excerpt.sql", diagnostics)) as {
+      version: string;
+      runs: Array<{ results: Array<{ ruleId: string; level: string; message: { text: string }; locations: Array<{ physicalLocation: { artifactLocation: { uri: string }; region: { startLine: number } } }> }> }>;
+    };
+    expect(sarif.version).toBe("2.1.0");
+    const results = sarif.runs[0]!.results;
+    expect(results).toHaveLength(diagnostics.length);
+    expect(new Set(results.map((r) => r.level))).toEqual(new Set(["warning", "note"]));
+    expect(results[0]!.locations[0]!.physicalLocation.artifactLocation.uri).toContain("wwi-excerpt.sql");
+    expect(results.find((r) => r.ruleId === "SQLRV001")!.message.text).toContain("Evidence:");
+    expect(results.every((r) => r.locations[0]!.physicalLocation.region.startLine > 0)).toBe(true);
+  });
+});
+
+describe("--strict", () => {
+  test("exits non-zero while a diagnostic to review is open, and zero without --strict", () => {
+    expect(importExitCode(diagnostics, true)).toBe(1);
+    expect(importExitCode(diagnostics, false)).toBe(0);
+  });
+
+  test("a schema whose only diagnostics are notes passes --strict", () => {
+    const notes = diagnostics.filter((d) => d.severity === "info");
+    expect(notes.length).toBeGreaterThan(0);
+    expect(importExitCode(notes, true)).toBe(0);
+  });
+});
+
+describe("refusing the sample's function", () => {
+  const refused = readFileSync(join(fixtures, "wwi-refused.sql"), "utf8");
+
+  test("the import fails and names the function and its line", () => {
+    expect(() => parse(refused)).toThrow(/function Application\.DetermineCustomerAccess \(line \d+\)/);
+  });
+
+  test("with skipUndeclarable it imports the rest and warns by name", () => {
+    const withTable = `${source}\nGO\n${refused}`;
+    const r = parse(withTable, { from: "sqlserver", skipUndeclarable: true });
+    expect(r.resources).toHaveLength(ir.resources.length);
+    expect(r.diagnostics!.some((d) => d.severity === "warning" && d.message.includes("function Application.DetermineCustomerAccess"))).toBe(true);
+  });
+});
+
+// The output builds and lints.
+const dir = join(import.meta.dirname, "..", "..", `.sqlserver-import-tmp-${process.pid}`);
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+describe("the output builds and lints clean", () => {
+  test("the declarations build to one statement per object and pass the lexicon's lint and checks", async () => {
+    const [file] = sqlTemplateGenerator.generate(ir);
+    mkdirSync(join(dir, "src"), { recursive: true });
+    writeFileSync(join(dir, "chant.config.ts"), 'export default { lexicons: ["sql"] };\n');
+    writeFileSync(join(dir, "src", "schema.ts"), file!.content);
+
+    const result = await build(join(dir, "src"), [sqlSerializer], undefined, { fold: true, intrinsics: sqlPlugin.intrinsics!(), lexicons: ["sql"] });
+    expect(result.errors).toEqual([]);
+    const statements = splitStatements((result.outputs.get("sql") as SerializerResult).files![CLICKHOUSE_DDL_FILE]!);
+    expect(statements).toHaveLength(ir.resources.length);
+    expect(statements.find((s) => s.includes("Warehouse.StockItemTransactions"))).toContain("ENGINE = MergeTree");
+
+    expect(runPostSynthChecks(postSynthChecks, result)).toEqual([]);
+    const lint = await runLint([join(dir, "src", "schema.ts")], sqlPlugin.lintRules!(), undefined, sqlPlugin.intrinsics!());
+    expect(lint.diagnostics.filter((d) => d.severity === "error" || d.severity === "warning")).toEqual([]);
+  });
+});

package/src/sqlserver/import-cli.test.ts

@@ -0,0 +1,61 @@
+/**
+ * `chant import` end to end in-process, for the third Done-when box of #4398:
+ * `--strict` exits non-zero while a diagnostic is open, `--report` writes the
+ * SARIF, and without `--strict` the same import exits 0.
+ */
+
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
+import { parseArgs } from "@intentius/chant/cli/parse-args";
+import { runImport } from "@intentius/chant/cli/handlers/misc";
+
+const dir = mkdtempSync(join(tmpdir(), "sqlserver-import-cli-"));
+const input = join(dir, "schema.sql");
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+beforeAll(() => writeFileSync(input, "CREATE TABLE [dbo].[T] ([Id] INT NOT NULL, [Note] NVARCHAR(40) NULL, CONSTRAINT [PK_T] PRIMARY KEY CLUSTERED ([Id]));\nGO\n"));
+
+async function run(extra: string[], name: string) {
+  const out = join(dir, name);
+  const args = parseArgs(["import", input, "--lexicon", "sql", "--parser-option", "from=sqlserver", "--output", out, ...extra]);
+  const spy = vi.spyOn(console, "error").mockImplementation(() => {});
+  const log = vi.spyOn(console, "log").mockImplementation(() => {});
+  try {
+    const code = await runImport({ args, plugins: [], serializers: [] });
+    return { code, out, stderr: spy.mock.calls.map((c) => String(c[0])).join("\n") };
+  } finally {
+    spy.mockRestore();
+    log.mockRestore();
+  }
+}
+
+describe("chant import --parser-option from=sqlserver", () => {
+  test("exits 0 without --strict, writes the declarations and prints the review summary", async () => {
+    const r = await run([], "plain");
+    expect(r.code).toBe(0);
+    expect(existsSync(join(r.out, "schema.ts"))).toBe(true);
+    expect(r.stderr).toContain("Review: 3 to review");
+    expect(r.stderr).toContain("SQLRV001 x1");
+  });
+
+  test("--strict exits 1 while a diagnostic is open", async () => {
+    expect((await run(["--strict"], "strict")).code).toBe(1);
+  });
+
+  test("--report writes SARIF with the three diagnostics, with or without --strict", async () => {
+    const report = join(dir, "review.sarif");
+    const r = await run(["--report", report], "report");
+    expect(r.code).toBe(0);
+    const sarif = JSON.parse(readFileSync(report, "utf8")) as { runs: Array<{ results: Array<{ ruleId: string; locations: Array<{ physicalLocation: { artifactLocation: { uri: string } } }> }> }> };
+    expect(sarif.runs[0]!.results.map((x) => x.ruleId).sort()).toEqual(["SQLRV001", "SQLRV002", "SQLRV003"]);
+    expect(sarif.runs[0]!.results[0]!.locations[0]!.physicalLocation.artifactLocation.uri).toContain("schema.sql");
+  });
+
+  test("a refused statement fails the import and names it", async () => {
+    writeFileSync(input, "CREATE TABLE T (Id int NOT NULL);\nGO\nCREATE TRIGGER trg_t ON T AFTER INSERT AS SELECT 1\n");
+    const r = await run([], "refused");
+    expect(r.code).toBe(1);
+    expect(r.stderr).toContain("trigger trg_t (line 3)");
+  });
+});

package/src/sqlserver/lexer.ts

@@ -0,0 +1,260 @@
+/**
+ * A T-SQL tokenizer for the statements `chant import` reads from SQL Server.
+ *
+ * T-SQL differs from the lexers the other dialects use in four places: an
+ * identifier can be `[bracketed]` (a `]` inside is written `]]`) or
+ * "double quoted", a string can carry an `N` prefix, a line holding only `GO`
+ * ends a batch (it is a client command, not a statement), and statements need
+ * no terminating `;`. Comments (`--` and nestable `/* *\/`) are dropped; each
+ * token keeps its 1-based line and column in the source.
+ */
+
+export type TokenKind = "word" | "qident" | "string" | "number" | "variable" | "punct" | "go";
+
+export interface Token {
+  kind: TokenKind;
+  /** As written: a `qident` keeps its brackets; a string its quotes. */
+  text: string;
+  /** A `qident`'s name, a string's value, a word's text. */
+  value: string;
+  line: number;
+  column: number;
+  /** Offset of the first character. */
+  start: number;
+  /** Offset after the last. */
+  end: number;
+}
+
+export class TsqlLexError extends Error {
+  constructor(
+    message: string,
+    readonly line: number,
+    readonly column: number,
+  ) {
+    super(`${message} (line ${String(line)}, column ${String(column)})`);
+    this.name = "TsqlLexError";
+  }
+}
+
+const WORD_START = /[A-Za-z_#@\u0080-￿]/;
+const WORD_PART = /[A-Za-z0-9_#$@\u0080-￿]/;
+
+/** Tokenize a file. A `GO` on a line of its own (optionally with a count) becomes one `go` token. */
+export function tokenize(text: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+  let line = 1;
+  let lineStart = 0;
+  const n = text.length;
+  // A byte order mark starts files saved by SSMS and Visual Studio.
+  if (text.charCodeAt(0) === 0xfeff) {
+    i = 1;
+    lineStart = 1;
+  }
+  const push = (kind: TokenKind, start: number, end: number, value: string, startLine: number, startCol: number) =>
+    tokens.push({ kind, text: text.slice(start, end), value, line: startLine, column: startCol, start, end });
+
+  while (i < n) {
+    const c = text[i]!;
+    if (c === "\n") {
+      line++;
+      i++;
+      lineStart = i;
+      continue;
+    }
+    if (/\s/.test(c)) {
+      i++;
+      continue;
+    }
+    const col = i - lineStart + 1;
+    if (c === "-" && text[i + 1] === "-") {
+      while (i < n && text[i] !== "\n") i++;
+      continue;
+    }
+    if (c === "/" && text[i + 1] === "*") {
+      let depth = 1;
+      const startLine = line;
+      const startCol = col;
+      i += 2;
+      while (i < n && depth > 0) {
+        if (text[i] === "/" && text[i + 1] === "*") {
+          depth++;
+          i += 2;
+        } else if (text[i] === "*" && text[i + 1] === "/") {
+          depth--;
+          i += 2;
+        } else {
+          if (text[i] === "\n") {
+            line++;
+            lineStart = i + 1;
+          }
+          i++;
+        }
+      }
+      if (depth > 0) throw new TsqlLexError("unterminated /* comment", startLine, startCol);
+      continue;
+    }
+    if (c === "[") {
+      const start = i;
+      let value = "";
+      i++;
+      for (;;) {
+        if (i >= n) throw new TsqlLexError("unterminated [identifier]", line, col);
+        if (text[i] === "]") {
+          if (text[i + 1] === "]") {
+            value += "]";
+            i += 2;
+            continue;
+          }
+          i++;
+          break;
+        }
+        if (text[i] === "\n") {
+          line++;
+          lineStart = i + 1;
+        }
+        value += text[i];
+        i++;
+      }
+      push("qident", start, i, value, line, col);
+      continue;
+    }
+    if (c === '"') {
+      const start = i;
+      let value = "";
+      i++;
+      for (;;) {
+        if (i >= n) throw new TsqlLexError('unterminated "identifier"', line, col);
+        if (text[i] === '"') {
+          if (text[i + 1] === '"') {
+            value += '"';
+            i += 2;
+            continue;
+          }
+          i++;
+          break;
+        }
+        value += text[i];
+        i++;
+      }
+      push("qident", start, i, value, line, col);
+      continue;
+    }
+    if (c === "'" || ((c === "N" || c === "n") && text[i + 1] === "'")) {
+      const start = i;
+      const startLine = line;
+      if (c !== "'") i++;
+      i++;
+      let value = "";
+      for (;;) {
+        if (i >= n) throw new TsqlLexError("unterminated string", startLine, col);
+        if (text[i] === "'") {
+          if (text[i + 1] === "'") {
+            value += "'";
+            i += 2;
+            continue;
+          }
+          i++;
+          break;
+        }
+        if (text[i] === "\n") {
+          line++;
+          lineStart = i + 1;
+        }
+        value += text[i];
+        i++;
+      }
+      push("string", start, i, value, startLine, col);
+      continue;
+    }
+    if (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(text[i + 1] ?? ""))) {
+      const start = i;
+      if (c === "0" && (text[i + 1] === "x" || text[i + 1] === "X")) {
+        i += 2;
+        while (i < n && /[0-9A-Fa-f]/.test(text[i]!)) i++;
+      } else {
+        while (i < n && /[0-9]/.test(text[i]!)) i++;
+        if (text[i] === "." ) {
+          i++;
+          while (i < n && /[0-9]/.test(text[i]!)) i++;
+        }
+        if ((text[i] === "e" || text[i] === "E") && /[0-9+-]/.test(text[i + 1] ?? "")) {
+          i += 2;
+          while (i < n && /[0-9]/.test(text[i]!)) i++;
+        }
+      }
+      push("number", start, i, text.slice(start, i), line, col);
+      continue;
+    }
+    if (c === "@" && text[i + 1] !== undefined && WORD_PART.test(text[i + 1]!)) {
+      const start = i;
+      i++;
+      while (i < n && WORD_PART.test(text[i]!)) i++;
+      push("variable", start, i, text.slice(start, i), line, col);
+      continue;
+    }
+    if (WORD_START.test(c)) {
+      const start = i;
+      i++;
+      while (i < n && WORD_PART.test(text[i]!)) i++;
+      const word = text.slice(start, i);
+      push("word", start, i, word, line, col);
+      continue;
+    }
+    // Two-character operators are one token so a `<>` is not a `<` and a `>`.
+    const two = text.slice(i, i + 2);
+    if (["<>", ">=", "<=", "!=", "+=", "-=", "*=", "/=", "||", "::"].includes(two)) {
+      push("punct", i, i + 2, two, line, col);
+      i += 2;
+      continue;
+    }
+    push("punct", i, i + 1, c, line, col);
+    i++;
+  }
+  return markBatchSeparators(tokens, text);
+}
+
+/** `GO` alone on its line (or `GO 5`) is a separator, not a word. */
+function markBatchSeparators(tokens: Token[], text: string): Token[] {
+  const out: Token[] = [];
+  for (let k = 0; k < tokens.length; k++) {
+    const t = tokens[k]!;
+    if (t.kind === "word" && t.value.toUpperCase() === "GO") {
+      const prev = tokens[k - 1];
+      const next = tokens[k + 1];
+      const alone = (a: Token | undefined) => a === undefined || a.line !== t.line;
+      const count = next && next.kind === "number" && next.line === t.line ? next : undefined;
+      const after = count ? tokens[k + 2] : next;
+      // The line before `GO` ends the earlier statement, and a `GO` mid-line is a name.
+      if (alone(prev) && alone(after) && !/[A-Za-z0-9_]/.test(text.slice(t.end, t.end + 1))) {
+        out.push({ ...t, kind: "go" });
+        if (count) k++;
+        continue;
+      }
+    }
+    out.push(t);
+  }
+  return out;
+}
+
+/** Tokens split into batches at `go` tokens; empty batches are dropped. */
+export function batches(tokens: readonly Token[]): Token[][] {
+  const out: Token[][] = [];
+  let cur: Token[] = [];
+  for (const t of tokens) {
+    if (t.kind === "go") {
+      if (cur.length > 0) out.push(cur);
+      cur = [];
+    } else cur.push(t);
+  }
+  if (cur.length > 0) out.push(cur);
+  return out;
+}
+
+export const upper = (t: Token | undefined): string => (t && t.kind === "word" ? t.value.toUpperCase() : "");
+
+/** The source text of a run of tokens, from the first token's start to the last token's end. */
+export function sourceOf(text: string, tokens: readonly Token[]): string {
+  if (tokens.length === 0) return "";
+  return text.slice(tokens[0]!.start, tokens[tokens.length - 1]!.end);
+}