@lix-js/sdk 0.5.0-preview.1 → 0.5.1

package/src/engine/preprocessor/types.ts

@@ -0,0 +1,103 @@
+import type { SegmentedStatementNode } from "./sql-parser/nodes.js";
+import type { LixEngine } from "../boot.js";
+import type { CelEnvironment } from "../cel-environment/cel-environment.js";
+import type { VersionInheritanceMap } from "./inheritance/version-inheritance-cache.js";
+
+/**
+ * Controls which subset of preprocessing stages to execute.
+ */
+export type PreprocessMode = "full" | "vtable-select-only" | "none";
+
+/**
+ * Shared trace entry type for documenting per-step behaviour.
+ *
+ * @example
+ * ```ts
+ * const entry: PreprocessorTraceEntry = { step: "rewrite", payload: null };
+ * ```
+ */
+export type PreprocessorTraceEntry = {
+	readonly step: string;
+	readonly payload: any;
+};
+
+/**
+ * Ordered collection of preprocessing trace entries.
+ */
+export type PreprocessorTrace = PreprocessorTraceEntry[];
+
+/**
+ * Shared context passed to each v3 preprocessor stage.
+ *
+ * @example
+ * ```ts
+ * const context: PreprocessorContext = {
+ *   getStoredSchemas: () => new Map(),
+ *   getCacheTables: () => new Map(),
+ *   hasOpenTransaction: () => false,
+ * };
+ * ```
+ */
+export type PreprocessorContext = {
+	readonly getStoredSchemas?: () => Map<string, unknown>;
+	readonly getCacheTables?: () => Map<string, unknown>;
+	readonly getSqlViews?: () => Map<string, string>;
+	readonly hasOpenTransaction?: () => boolean;
+	readonly getCelEnvironment?: () => CelEnvironment;
+	readonly getEngine?: () => LixEngine;
+	readonly getVersionInheritance?: () => VersionInheritanceMap;
+	readonly getActiveVersionId?: () => string | null;
+	readonly trace?: PreprocessorTrace;
+};
+
+/**
+ * Context shape for individual preprocessing steps.
+ */
+export type PreprocessorStepContext = PreprocessorContext & {
+	readonly statements: readonly SegmentedStatementNode[];
+	readonly parameters?: ReadonlyArray<unknown>;
+};
+
+/**
+ * Signature for a single preprocessing stage.
+ *
+ * @example
+ * ```ts
+ * const noop: PreprocessorStep = (context) => context.statements;
+ * ```
+ */
+export type PreprocessorStep = (
+	context: PreprocessorStepContext
+) => readonly SegmentedStatementNode[];
+
+/**
+ * Arguments accepted by the v3 preprocessor entry point.
+ */
+export type PreprocessorArgs = {
+	readonly sql: string;
+	readonly parameters: ReadonlyArray<unknown>;
+	readonly trace?: boolean;
+	readonly mode?: PreprocessMode;
+};
+
+/**
+ * Shape of the function returned by `createPreprocessor`.
+ */
+export type PreprocessorFn = (args: PreprocessorArgs) => PreprocessorResult;
+
+/**
+ * Shared result produced after preprocessing SQL.
+ *
+ * @example
+ * ```ts
+ * const result: PreprocessorResult = {
+ *   sql: "SELECT 1",
+ *   parameters: [],
+ * };
+ * ```
+ */
+export type PreprocessorResult = {
+	readonly sql: string;
+	readonly parameters: ReadonlyArray<unknown>;
+	readonly trace?: PreprocessorTrace;
+};

package/src/engine/sql-parser/tokenizer.test.ts

@@ -0,0 +1,408 @@
+import { expect, test } from "vitest";
+import {
+	tokenize,
+	FROM,
+	WHERE,
+	LIMIT,
+	ORDER,
+	BY,
+	GROUP,
+	HAVING,
+	WINDOW,
+	OFFSET,
+	FETCH,
+	AS,
+	JOIN,
+	Ident,
+	QIdent,
+	Dot,
+	SQStr,
+	Num,
+	QMark,
+	ColonName,
+	DollarName,
+	DollarNumber,
+	AtName,
+	Minus,
+	Equals,
+	WITH,
+	LParen,
+	RParen,
+	type Token,
+} from "./tokenizer.js";
+
+test("tokenizes FROM lix_internal_state_vtable with alias and LIMIT 1", () => {
+	const sql = `SELECT json_extract(s.snapshot_content,'$.value.nano_id') AS nano_id
+                 FROM lix_internal_state_vtable s
+                 WHERE s.schema_key = 'lix_key_value'
+                 AND s.snapshot_content IS NOT NULL
+                 LIMIT 1`;
+
+	const tokens = tokenize(sql);
+	const iFrom = tokens.findIndex((tt) => tt.tokenType === FROM);
+	expect(iFrom).toBeGreaterThanOrEqual(0);
+
+	const tableTok = requireToken(tokens[iFrom + 1], "table ident");
+	const aliasTok = requireToken(tokens[iFrom + 2], "table alias");
+	expect(tableTok.tokenType).toBe(Ident);
+	expect(tableTok.image).toBe("lix_internal_state_vtable");
+	expect(aliasTok.tokenType).toBe(Ident);
+	expect(aliasTok.image).toBe("s");
+
+	const iWhere = tokens.findIndex((tt) => tt.tokenType === WHERE);
+	expect(iWhere).toBeGreaterThanOrEqual(0);
+
+	const whereAlias = requireToken(tokens[iWhere + 1], "where alias");
+	const dotTok = requireToken(tokens[iWhere + 2], "qualified dot");
+	const columnTok = requireToken(tokens[iWhere + 3], "schema_key column");
+	expect(whereAlias.image).toBe("s");
+	expect(dotTok.tokenType).toBe(Dot);
+	expect(columnTok.image.toLowerCase()).toBe("schema_key");
+
+	const lit = requireToken(
+		tokens.find(
+			(tt) => tt.tokenType === SQStr && tt.image === `'lix_key_value'`
+		),
+		"schema key literal"
+	);
+	expect(lit.image).toBe(`'lix_key_value'`);
+
+	const iLimit = tokens.findIndex((tt) => tt.tokenType === LIMIT);
+	expect(iLimit).toBeGreaterThanOrEqual(0);
+	const limitValue = requireToken(tokens[iLimit + 1], "limit value");
+	expect(limitValue.tokenType).toBe(Num);
+	expect(limitValue.image).toBe("1");
+});
+
+test("tokenizes ORDER BY clause without turning ORDER into an identifier", () => {
+	const sql = `SELECT path FROM file ORDER BY path DESC`;
+	const tokens = tokenize(sql);
+
+	const orderIx = tokens.findIndex((t) => t.tokenType === ORDER);
+	const byIx = tokens.findIndex((t) => t.tokenType === BY);
+
+	expect(orderIx).toBeGreaterThanOrEqual(0);
+	expect(byIx).toBe(orderIx + 1);
+
+	const orderColumn = requireToken(tokens[orderIx + 2], "order by column");
+	expect(orderColumn.tokenType).toBe(Ident);
+	expect(orderColumn.image).toBe("path");
+
+	const identHits = tokens.filter((t) => t.tokenType === Ident);
+	expect(identHits.map((t) => t.image.toLowerCase())).not.toContain("order");
+});
+
+test("tokenizes GROUP BY / HAVING with dedicated keyword tokens", () => {
+	const sql = `SELECT key, COUNT(*) FROM stored_schema GROUP BY key HAVING COUNT(*) > 1`;
+	const tokens = tokenize(sql);
+
+	const groupIx = tokens.findIndex((t) => t.tokenType === GROUP);
+	const byIx = tokens.findIndex((t) => t.tokenType === BY);
+	const havingIx = tokens.findIndex((t) => t.tokenType === HAVING);
+
+	expect(groupIx).toBeGreaterThanOrEqual(0);
+	expect(byIx).toBe(groupIx + 1);
+	expect(havingIx).toBeGreaterThan(groupIx);
+
+	const groupedColumn = requireToken(tokens[groupIx + 2], "group by column");
+	expect(groupedColumn.tokenType).toBe(Ident);
+	expect(groupedColumn.image).toBe("key");
+
+	const identImages = tokens
+		.filter((t) => t.tokenType === Ident)
+		.map((t) => t.image.toLowerCase());
+	expect(identImages).not.toContain("group");
+	expect(identImages).not.toContain("having");
+});
+
+test("tokenizes WINDOW clause and OFFSET/FETCH keywords", () => {
+	const sql = `SELECT * FROM file WINDOW w AS (PARTITION BY path ORDER BY updated_at) OFFSET 10 ROWS FETCH NEXT 5 ROWS ONLY`;
+	const tokens = tokenize(sql);
+
+	const windowIx = tokens.findIndex((t) => t.tokenType === WINDOW);
+	const offsetIx = tokens.findIndex((t) => t.tokenType === OFFSET);
+	const fetchIx = tokens.findIndex((t) => t.tokenType === FETCH);
+
+	expect(windowIx).toBeGreaterThanOrEqual(0);
+	expect(offsetIx).toBeGreaterThan(windowIx);
+	expect(fetchIx).toBeGreaterThan(offsetIx);
+
+	const windowAlias = requireToken(tokens[windowIx + 1], "window alias");
+	expect(windowAlias.tokenType).toBe(Ident);
+	expect(windowAlias.image).toBe("w");
+
+	const identImages = tokens
+		.filter((t) => t.tokenType === Ident)
+		.map((t) => t.image.toLowerCase());
+	expect(identImages).not.toContain("window");
+	expect(identImages).not.toContain("offset");
+	expect(identImages).not.toContain("fetch");
+});
+
+test("handles quoted table and alias", () => {
+	const sql = `SELECT * FROM "lix_internal_state_vtable" AS "S"`;
+	const tok = tokenize(sql);
+
+	const fromIx = tok.findIndex((t) => t.tokenType === FROM);
+	expect(fromIx).toBeGreaterThanOrEqual(0);
+
+	const tableTok = requireToken(tok[fromIx + 1], "quoted table");
+	const asTok = requireToken(tok[fromIx + 2], "AS keyword");
+	const aliasTok = requireToken(tok[fromIx + 3], "quoted alias");
+
+	expect(tableTok.tokenType).toBe(QIdent);
+	expect(tableTok.image).toBe(`"lix_internal_state_vtable"`);
+	expect(asTok.tokenType).toBe(AS);
+	expect(aliasTok.tokenType).toBe(QIdent);
+	expect(aliasTok.image).toBe(`"S"`);
+
+	expect(sliceBy(sql, tableTok)).toBe(`"lix_internal_state_vtable"`);
+	expect(sliceBy(sql, aliasTok)).toBe(`"S"`);
+});
+
+test("does not produce identifiers from comments or strings", () => {
+	const sql = `
+      /* FROM lix_internal_state_vtable should be ignored */
+      SELECT 'lix_internal_state_vtable in a string' AS note
+      FROM lix_internal_state_vtable -- trailing comment has lix_internal_state_vtable
+    `;
+	const tok = tokenize(sql);
+
+	const hits = tok.filter(
+		(t) => t.tokenType === Ident && t.image === "lix_internal_state_vtable"
+	);
+	expect(hits.length).toBe(1);
+
+	const str = requireToken(
+		tok.find((t) => t.tokenType === SQStr),
+		"string literal token"
+	);
+	expect(str.image).toBe(`'lix_internal_state_vtable in a string'`);
+});
+
+test('captures contiguous range "lix_internal_state_vtable AS v"', () => {
+	const sql = `SELECT * FROM   lix_internal_state_vtable   AS   v  WHERE 1=1`;
+	const tok = tokenize(sql);
+	const iFrom = tok.findIndex((t) => t.tokenType === FROM);
+	expect(iFrom).toBeGreaterThanOrEqual(0);
+
+	const tableTok = requireToken(tok[iFrom + 1], "table ident");
+	const asTok = requireToken(tok[iFrom + 2], "AS keyword");
+	const aliasTok = requireToken(tok[iFrom + 3], "alias token");
+
+	const start = ensureOffset(tableTok.startOffset, tableTok.image);
+	const end = ensureOffset(aliasTok.endOffset, aliasTok.image);
+	const frag = sql.slice(start, end + 1);
+
+	expect(asTok.image.toUpperCase()).toBe("AS");
+	expect(frag).toContain("lix_internal_state_vtable");
+	expect(frag).toMatch(/lix_internal_state_vtable[\s\S]*AS[\s\S]*v/);
+});
+
+test("tokenizes positional and named placeholders", () => {
+	const sql = `SELECT *
+    FROM lix_internal_state_vtable r
+    WHERE r.schema_key = ?
+      AND r.snapshot_content = :named
+      AND r.created_by = @user
+      AND r.updated_by = $param`;
+	const tok = tokenize(sql);
+
+	const mark = requireToken(
+		tok.find((t) => t.tokenType === QMark),
+		"question placeholder"
+	);
+	const colon = requireToken(
+		tok.find((t) => t.tokenType === ColonName),
+		"colon placeholder"
+	);
+	const at = requireToken(
+		tok.find((t) => t.tokenType === AtName),
+		"at placeholder"
+	);
+	const dollar = requireToken(
+		tok.find((t) => t.tokenType === DollarName),
+		"dollar placeholder"
+	);
+
+	expect(sliceBy(sql, mark)).toBe("?");
+	expect(sliceBy(sql, colon)).toBe(":named");
+	expect(sliceBy(sql, at)).toBe("@user");
+	expect(sliceBy(sql, dollar)).toBe("$param");
+});
+
+test("does not tokenize placeholders inside strings", () => {
+	const sql = `SELECT '$param :x @y ?' AS s FROM lix_internal_state_vtable`;
+	const tok = tokenize(sql);
+
+	const placeholderHits = tok.filter((t) =>
+		[DollarName, ColonName, AtName, QMark].includes(t.tokenType)
+	);
+	expect(placeholderHits.length).toBe(0);
+
+	const str = requireToken(
+		tok.find((t) => t.tokenType === SQStr),
+		"string literal token"
+	);
+	expect(str.image).toBe(`'$param :x @y ?'`);
+});
+
+test("keeps json path string intact", () => {
+	const sql = `SELECT json_extract(iv.snapshot_content, '$.value') FROM lix_internal_state_vtable iv`;
+	const tok = tokenize(sql);
+
+	const str = requireToken(
+		tok.find((t) => t.tokenType === SQStr),
+		"json path literal"
+	);
+	expect(str.image).toBe(`'$.value'`);
+});
+
+test("skips block comments and preserves offsets", () => {
+	const sql = `SELECT /* block comment */ *\n    FROM\n      /* comment with lix_internal_state_vtable */\n      lix_internal_state_vtable r\n    WHERE /* inline */ r.schema_key = 'reader_key'\n  `;
+	const tokens = tokenize(sql);
+	expectMonotonicOffsets(tokens);
+
+	const tableTok = requireToken(
+		tokens.find(
+			(t) => t.tokenType === Ident && t.image === "lix_internal_state_vtable"
+		),
+		"real table identifier"
+	);
+	expect(sliceBy(sql, tableTok)).toBe("lix_internal_state_vtable");
+
+	const literal = requireToken(
+		tokens.find((t) => t.tokenType === SQStr && t.image === `'reader_key'`),
+		"reader key literal"
+	);
+	expect(sliceBy(sql, literal)).toBe(`'reader_key'`);
+});
+
+test("tokenizes numeric literals and signed limits", () => {
+	const sql = `SELECT 42, 3.14, 1.2e3, 4E-2 FROM lix_internal_state_vtable r LIMIT -1`;
+	const tokens = tokenize(sql);
+
+	const numImages = tokens
+		.filter((t) => t.tokenType === Num)
+		.map((t) => t.image);
+	expect(numImages).toContain("42");
+	expect(numImages).toContain("3.14");
+	expect(numImages).toContain("1.2e3");
+	expect(numImages).toContain("4E-2");
+	expect(numImages).toContain("1");
+
+	const minus = requireToken(
+		tokens.find((t) => t.tokenType === Minus),
+		"minus token"
+	);
+	expect(minus.image).toBe("-");
+});
+
+test("distinguishes dollar-number params", () => {
+	const sql = `SELECT * FROM lix_internal_state_vtable r WHERE r.schema_key = $1 AND r.created_by = $param`;
+	const tokens = tokenize(sql);
+
+	const dollarNumber = requireToken(
+		tokens.find((t) => t.tokenType === DollarNumber),
+		"dollar number placeholder"
+	);
+	expect(dollarNumber.image).toBe("$1");
+
+	const dollarNames = tokens
+		.filter((t) => t.tokenType === DollarName)
+		.map((t) => t.image);
+	expect(dollarNames).toContain("$param");
+	expect(dollarNames).not.toContain("$1");
+});
+
+test("tokenizes join with quoted reader and mock table", () => {
+	const sql = `SELECT * FROM "lix_internal_state_vtable" r JOIN mock_other_table m ON m.reader_id = r.reader_id`;
+	const tokens = tokenize(sql);
+
+	const fromIx = tokens.findIndex((t) => t.tokenType === FROM);
+	expect(fromIx).toBeGreaterThanOrEqual(0);
+	const joinIx = tokens.findIndex((t) => t.tokenType === JOIN);
+	expect(joinIx).toBeGreaterThan(fromIx);
+
+	const readerTok = requireToken(tokens[fromIx + 1], "quoted reader");
+	expect(readerTok.tokenType).toBe(QIdent);
+	expect(readerTok.image).toBe(`"lix_internal_state_vtable"`);
+
+	const mockTok = requireToken(tokens[joinIx + 1], "mock table ident");
+	expect(mockTok.tokenType).toBe(Ident);
+	expect(mockTok.image).toBe("mock_other_table");
+
+	const equalsTok = requireToken(
+		tokens.find((t) => t.tokenType === Equals),
+		"equals token in join predicate"
+	);
+	expect(equalsTok.image).toBe("=");
+});
+
+test("tokenizes CTE with nested select", () => {
+	const sql = `WITH reader AS (SELECT * FROM lix_internal_state_vtable) SELECT * FROM reader`;
+	const tokens = tokenize(sql);
+
+	const withIx = tokens.findIndex((t) => t.tokenType === WITH);
+	expect(withIx).toBeGreaterThanOrEqual(0);
+
+	const lParenCount = tokens.filter((t) => t.tokenType === LParen).length;
+	const rParenCount = tokens.filter((t) => t.tokenType === RParen).length;
+	expect(lParenCount).toBe(rParenCount);
+
+	const innerTable = requireToken(
+		tokens.find(
+			(t) => t.tokenType === Ident && t.image === "lix_internal_state_vtable"
+		),
+		"inner table"
+	);
+	expect(innerTable.image).toBe("lix_internal_state_vtable");
+});
+
+test("continues tokenizing after unknown characters", () => {
+	const sql = `SELECT ☺ FROM lix_internal_state_vtable`; // smiley is intentionally unsupported
+	const tokens = tokenize(sql);
+
+	const tableTok = requireToken(
+		tokens.find(
+			(t) => t.tokenType === Ident && t.image === "lix_internal_state_vtable"
+		),
+		"table token after unknown char"
+	);
+	expect(tableTok.image).toBe("lix_internal_state_vtable");
+
+	// We expect at least one token even though the smiley is unrecognized
+	expect(tokens.length).toBeGreaterThan(0);
+});
+
+function sliceBy(source: string, token: Token) {
+	const start = ensureOffset(token.startOffset, token.image);
+	const end = ensureOffset(token.endOffset, token.image);
+	return source.slice(start, end + 1);
+}
+
+function expectMonotonicOffsets(tokens: Token[]) {
+	let lastEnd = -1;
+	for (const token of tokens) {
+		const start = ensureOffset(token.startOffset, token.image);
+		expect(start).toBeGreaterThan(lastEnd);
+		lastEnd = ensureOffset(token.endOffset, token.image);
+	}
+}
+
+function ensureOffset(value: number | undefined, label: string): number {
+	if (value == null) {
+		throw new Error(`Token ${label} missing offset information`);
+	}
+	return value;
+}
+
+function requireToken<T extends Token>(
+	value: T | undefined,
+	message: string
+): T {
+	if (!value) {
+		throw new Error(`Expected token: ${message}`);
+	}
+	return value;
+}

package/src/engine/sql-parser/tokenizer.ts

@@ -0,0 +1,205 @@
+import { createToken, Lexer, type IToken, type TokenType } from "chevrotain";
+
+/**
+ * Resilient Chevrotain-based tokenizer for the SQL fragments the rewriter cares about.
+ *
+ * The lexer skips whitespace and comments and recognizes identifiers (quoted/unquoted),
+ * SQLite-style named parameters, basic punctuation, and the subset of keywords that
+ * drive lix_internal_state_vtable rewriting.
+ *
+ * @example
+ * ```ts
+ * import { tokenize, FROM, Ident } from "./tokenizer";
+ * const sql = "SELECT * FROM lix_internal_state_vtable";
+ * const tokens = tokenize(sql);
+ * const fromIx = tokens.findIndex((t) => t.tokenType === FROM);
+ * console.log(tokens[fromIx + 1].tokenType === Ident); // true
+ * ```
+ */
+
+const WhiteSpace: TokenType = createToken({
+	name: "WhiteSpace",
+	pattern: /\s+/,
+	group: Lexer.SKIPPED,
+	line_breaks: true,
+});
+
+const LineComment: TokenType = createToken({
+	name: "LineComment",
+	pattern: /--[^\n]*/,
+	group: Lexer.SKIPPED,
+	line_breaks: true,
+});
+
+const BlockComment: TokenType = createToken({
+	name: "BlockComment",
+	pattern: /\/\*[\s\S]*?\*\//,
+	group: Lexer.SKIPPED,
+	line_breaks: true,
+});
+
+export const Comma: TokenType = createToken({ name: "Comma", pattern: /,/ });
+export const Dot: TokenType = createToken({ name: "Dot", pattern: /\./ });
+export const LParen: TokenType = createToken({ name: "LParen", pattern: /\(/ });
+export const RParen: TokenType = createToken({ name: "RParen", pattern: /\)/ });
+export const Star: TokenType = createToken({ name: "Star", pattern: /\*/ });
+
+export const SQStr: TokenType = createToken({
+	name: "SQStr",
+	pattern: /'(?:[^']|'')*'/,
+	line_breaks: true,
+});
+
+export const Num: TokenType = createToken({
+	name: "Num",
+	pattern: /\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/,
+});
+
+export const QMarkNumber: TokenType = createToken({
+	name: "QMarkNumber",
+	pattern: /\?\d+/,
+});
+export const QMark: TokenType = createToken({ name: "QMark", pattern: /\?/ });
+export const DollarName: TokenType = createToken({
+	name: "DollarName",
+	pattern: /\$[A-Za-z_][\w$]*/,
+});
+export const DollarNumber: TokenType = createToken({
+	name: "DollarNumber",
+	pattern: /\$\d+/,
+});
+export const AtName: TokenType = createToken({
+	name: "AtName",
+	pattern: /@[A-Za-z_][\w$]*/,
+});
+export const ColonName: TokenType = createToken({
+	name: "ColonName",
+	pattern: /:[A-Za-z_][\w$]*/,
+});
+
+export const Equals: TokenType = createToken({ name: "Equals", pattern: /=/ });
+export const Minus: TokenType = createToken({ name: "Minus", pattern: /-/ });
+export const Plus: TokenType = createToken({ name: "Plus", pattern: /\+/ });
+export const Pipe: TokenType = createToken({ name: "Pipe", pattern: /\|\|/ });
+export const Semicolon: TokenType = createToken({
+	name: "Semicolon",
+	pattern: /;/,
+});
+
+export const QIdent: TokenType = createToken({
+	name: "QIdent",
+	pattern: /"(?:[^"]|"")*"/,
+	line_breaks: true,
+});
+
+export const Ident: TokenType = createToken({
+	name: "Ident",
+	pattern: /[A-Za-z_][\w$]*/,
+});
+
+function keyword(name: string): TokenType {
+	return createToken({
+		name,
+		pattern: new RegExp(name, "i"),
+		longer_alt: Ident,
+	});
+}
+
+export const SELECT: TokenType = keyword("SELECT");
+export const FROM: TokenType = keyword("FROM");
+export const JOIN: TokenType = keyword("JOIN");
+export const AS: TokenType = keyword("AS");
+export const WHERE: TokenType = keyword("WHERE");
+export const LIMIT: TokenType = keyword("LIMIT");
+export const ORDER: TokenType = keyword("ORDER");
+export const BY: TokenType = keyword("BY");
+export const GROUP: TokenType = keyword("GROUP");
+export const HAVING: TokenType = keyword("HAVING");
+export const WINDOW: TokenType = keyword("WINDOW");
+export const OFFSET: TokenType = keyword("OFFSET");
+export const FETCH: TokenType = keyword("FETCH");
+export const ON: TokenType = keyword("ON");
+export const AND: TokenType = keyword("AND");
+export const OR: TokenType = keyword("OR");
+export const IS: TokenType = keyword("IS");
+export const NOT: TokenType = keyword("NOT");
+export const NULL: TokenType = keyword("NULL");
+export const CAST: TokenType = keyword("CAST");
+export const WITH: TokenType = keyword("WITH");
+export const INSERT: TokenType = keyword("INSERT");
+export const UPDATE: TokenType = keyword("UPDATE");
+export const DELETE: TokenType = keyword("DELETE");
+
+const allTokens: TokenType[] = [
+	WhiteSpace,
+	LineComment,
+	BlockComment,
+	Comma,
+	Dot,
+	LParen,
+	RParen,
+	Star,
+	Equals,
+	Minus,
+	Plus,
+	Pipe,
+	Semicolon,
+	SQStr,
+	Num,
+	QMarkNumber,
+	QMark,
+	DollarName,
+	DollarNumber,
+	AtName,
+	ColonName,
+	QIdent,
+	SELECT,
+	FROM,
+	JOIN,
+	AS,
+	WHERE,
+	LIMIT,
+	ORDER,
+	BY,
+	GROUP,
+	HAVING,
+	WINDOW,
+	OFFSET,
+	FETCH,
+	ON,
+	AND,
+	OR,
+	IS,
+	NOT,
+	NULL,
+	CAST,
+	WITH,
+	INSERT,
+	UPDATE,
+	DELETE,
+	Ident,
+];
+
+const lexer = new Lexer(allTokens, { ensureOptimizations: true });
+
+export type Token = IToken;
+
+/**
+ * Tokenize an input SQL string into a flat sequence of Chevrotain tokens.
+ *
+ * Errors are tolerated; if the lexer encounters unrecognized characters the caller
+ * can decide how to handle the partial output.
+ *
+ * @example
+ * ```ts
+ * const [first] = tokenize("SELECT 1");
+ * console.log(first.image); // SELECT
+ * ```
+ */
+export function tokenize(input: string): Token[] {
+	const { tokens, errors } = lexer.tokenize(input);
+	if (errors.length > 0) {
+		// Preserve partial tokenization; downstream code can fall back gracefully.
+	}
+	return tokens;
+}