@texuli/blocks 0.0.0-stage → 0.1.0-next.0

package/dist/code/grammars.js

@@ -0,0 +1,83 @@
+/** Languages with a grammar. Anything else renders as plain text. */
+export const CODE_GRAMMAR_LANGUAGES = [
+    'ts',
+    'js',
+    'json',
+    'bash',
+    'python',
+    'html',
+    'css',
+    'sql',
+    'yaml',
+];
+const ALIASES = {
+    typescript: 'ts',
+    tsx: 'ts',
+    mts: 'ts',
+    cts: 'ts',
+    javascript: 'js',
+    jsx: 'js',
+    mjs: 'js',
+    cjs: 'js',
+    jsonc: 'json',
+    json5: 'json',
+    sh: 'bash',
+    shell: 'bash',
+    zsh: 'bash',
+    console: 'bash',
+    py: 'python',
+    python3: 'python',
+    htm: 'html',
+    xml: 'html',
+    svg: 'html',
+    svelte: 'html',
+    vue: 'html',
+    postgres: 'sql',
+    postgresql: 'sql',
+    sqlite: 'sql',
+    mysql: 'sql',
+    yml: 'yaml',
+};
+/** A language id or alias (`typescript`, `sh`, `yml`…) → a grammar language, or `null`. */
+export function resolveCodeLanguage(lang) {
+    const key = lang.trim().toLowerCase();
+    if (CODE_GRAMMAR_LANGUAGES.includes(key)) {
+        return key;
+    }
+    return Object.hasOwn(ALIASES, key) ? ALIASES[key] : null;
+}
+/*
+ * One lazy chunk per grammar file: a code block downloads only the grammar of
+ * its own language, while the tokenizer core stays in the code family chunk.
+ */
+const LOADERS = {
+    ts: () => import("./languages/javascript.js").then((m) => m.typescriptGrammar),
+    js: () => import("./languages/javascript.js").then((m) => m.javascriptGrammar),
+    json: () => import("./languages/json.js").then((m) => m.jsonGrammar),
+    bash: () => import("./languages/bash.js").then((m) => m.bashGrammar),
+    python: () => import("./languages/python.js").then((m) => m.pythonGrammar),
+    html: () => import("./languages/html.js").then((m) => m.htmlGrammar),
+    css: () => import("./languages/css.js").then((m) => m.cssGrammar),
+    sql: () => import("./languages/sql.js").then((m) => m.sqlGrammar),
+    yaml: () => import("./languages/yaml.js").then((m) => m.yamlGrammar),
+};
+const cache = new Map();
+/**
+ * The grammar for a language id or alias, loaded on demand and cached; `null`
+ * for `text` and unknown languages. A failed load is not cached, so a later
+ * render can retry.
+ */
+export function loadCodeGrammar(lang) {
+    const resolved = lang === undefined ? null : resolveCodeLanguage(lang);
+    if (resolved === null)
+        return Promise.resolve(null);
+    const cached = cache.get(resolved);
+    if (cached)
+        return cached;
+    const pending = LOADERS[resolved]().then((grammar) => grammar, () => {
+        cache.delete(resolved);
+        return null;
+    });
+    cache.set(resolved, pending);
+    return pending;
+}

package/dist/code/highlight.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Stepped line highlights for code blocks. A step is either a line spec
+ * string (`"1-2,4"`, the code block schema's form) or a list of line numbers
+ * and `[start, end]` ranges (`[[1, 2], 4]`). Lines are 1-based.
+ */
+import { parseLineSpec } from '../core/schema/line-spec.js';
+export { parseLineSpec };
+export type HighlightStep = string | readonly (number | readonly [number, number])[];
+/**
+ * The highlighted lines of every step: sorted, unique, clamped to
+ * `lineCount`. A malformed step highlights nothing rather than everything.
+ */
+export declare function resolveHighlightSteps(steps: readonly HighlightStep[], lineCount: number): number[][];
+/** Lines in a piece of code, counting a trailing empty line like editors do. */
+export declare function countCodeLines(code: string): number;

package/dist/code/highlight.js

@@ -0,0 +1,53 @@
+/**
+ * Stepped line highlights for code blocks. A step is either a line spec
+ * string (`"1-2,4"`, the code block schema's form) or a list of line numbers
+ * and `[start, end]` ranges (`[[1, 2], 4]`). Lines are 1-based.
+ */
+import { parseLineSpec } from "../core/schema/line-spec.js";
+export { parseLineSpec };
+function rangesOf(step) {
+    if (typeof step === 'string')
+        return parseLineSpec(step);
+    if (!Array.isArray(step))
+        return null;
+    const ranges = [];
+    for (const item of step) {
+        if (typeof item === 'number') {
+            if (!Number.isInteger(item) || item < 1)
+                return null;
+            ranges.push([item, item]);
+        }
+        else if (Array.isArray(item) && item.length === 2) {
+            const [a, b] = item;
+            if (!Number.isInteger(a) || !Number.isInteger(b) || a < 1 || b < a)
+                return null;
+            ranges.push([a, b]);
+        }
+        else {
+            return null;
+        }
+    }
+    return ranges;
+}
+/**
+ * The highlighted lines of every step: sorted, unique, clamped to
+ * `lineCount`. A malformed step highlights nothing rather than everything.
+ */
+export function resolveHighlightSteps(steps, lineCount) {
+    const max = Math.max(0, Math.floor(lineCount));
+    return steps.map((step) => {
+        const ranges = rangesOf(step);
+        if (!ranges)
+            return [];
+        const lines = new Set();
+        for (const [start, end] of ranges) {
+            for (let line = start; line <= Math.min(end, max); line += 1)
+                lines.add(line);
+        }
+        return [...lines].sort((a, b) => a - b);
+    });
+}
+/** Lines in a piece of code, counting a trailing empty line like editors do. */
+export function countCodeLines(code) {
+    return code.split('\n').length;
+}

package/dist/code/index.d.ts

@@ -0,0 +1,13 @@
+/// <reference path="../css-modules.d.ts" />
+/**
+ * `@texuli/blocks/code` (T-blocks-code): the code family's light entry, with
+ * the guard, the lazy renderer entry, labels, language resolution and token
+ * styles. It loads no grammar and no Svelte. The tokenizer, every grammar and
+ * the highlight helpers live in `@texuli/blocks/code/tokenizer`; the
+ * renderer in `@texuli/blocks/code/renderers` or through `loadCodeFamily()`.
+ */
+export { type CodeBlockCheck, checkCodeBlock } from './check.js';
+export { CODE_BLOCK_TYPES, type CodeBlockViewProps, type CodeFamilyBlock, type CodeFamilyBlockType, isCodeFamilyBlock, loadCodeFamily, } from './family.js';
+export { CODE_GRAMMAR_LANGUAGES, type CodeGrammarLanguage, loadCodeGrammar, resolveCodeLanguage, } from './grammars.js';
+export { type CodeBlockLabelOverrides, type CodeBlockLabels, DEFAULT_CODE_BLOCK_LABELS, fillCodeLabel, resolveCodeBlockLabels, } from './labels.js';
+export { activeHighlightIndex, CODE_COLOR_ROLES, CODE_TOKEN_STYLES, type CodeColorRole, type CodeTokenStyle, codeTokenClass, } from './token-styles.js';

package/dist/code/index.js

@@ -0,0 +1,12 @@
+/**
+ * `@texuli/blocks/code` (T-blocks-code): the code family's light entry, with
+ * the guard, the lazy renderer entry, labels, language resolution and token
+ * styles. It loads no grammar and no Svelte. The tokenizer, every grammar and
+ * the highlight helpers live in `@texuli/blocks/code/tokenizer`; the
+ * renderer in `@texuli/blocks/code/renderers` or through `loadCodeFamily()`.
+ */
+export { checkCodeBlock } from "./check.js";
+export { CODE_BLOCK_TYPES, isCodeFamilyBlock, loadCodeFamily, } from "./family.js";
+export { CODE_GRAMMAR_LANGUAGES, loadCodeGrammar, resolveCodeLanguage, } from "./grammars.js";
+export { DEFAULT_CODE_BLOCK_LABELS, fillCodeLabel, resolveCodeBlockLabels, } from "./labels.js";
+export { activeHighlightIndex, CODE_COLOR_ROLES, CODE_TOKEN_STYLES, codeTokenClass, } from "./token-styles.js";

package/dist/code/labels.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Every visible or announced string the code family renders that does not
+ * come from the block itself. Hosts pass translations; English is the
+ * default. `{name}` in a template is replaced by the block's name (its
+ * filename, or its language in capitals).
+ */
+export interface CodeBlockLabels {
+    /** Shown in place of a block that failed validation. */
+    invalid: string;
+    /** Name of a code block with neither a filename nor a language. */
+    code: string;
+    /** Visible copy button text, and its state texts. */
+    copy: string;
+    copied: string;
+    copyFailed: string;
+    /** Accessible name of the copy button (template with `{name}`). */
+    copyName: string;
+    /** Announced after a copy succeeds or fails. */
+    copiedAnnouncement: string;
+    copyFailedAnnouncement: string;
+}
+export declare const DEFAULT_CODE_BLOCK_LABELS: Readonly<CodeBlockLabels>;
+export type CodeBlockLabelOverrides = Partial<CodeBlockLabels>;
+/** Merges host overrides over the English defaults; non-string overrides are ignored. */
+export declare function resolveCodeBlockLabels(overrides?: CodeBlockLabelOverrides): CodeBlockLabels;
+/** Replaces every `{name}` in a label template (plain text, never markup). */
+export declare function fillCodeLabel(template: string, name: string): string;

package/dist/code/labels.js

@@ -0,0 +1,24 @@
+export const DEFAULT_CODE_BLOCK_LABELS = /* @__PURE__ */ Object.freeze({
+    invalid: 'This block could not be shown.',
+    code: 'Code',
+    copy: 'Copy',
+    copied: 'Copied',
+    copyFailed: 'Copy failed',
+    copyName: 'Copy {name} code',
+    copiedAnnouncement: 'Code copied to the clipboard.',
+    copyFailedAnnouncement: 'The code could not be copied.',
+});
+/** Merges host overrides over the English defaults; non-string overrides are ignored. */
+export function resolveCodeBlockLabels(overrides = {}) {
+    const labels = { ...DEFAULT_CODE_BLOCK_LABELS };
+    for (const key of Object.keys(labels)) {
+        const value = overrides[key];
+        if (typeof value === 'string')
+            labels[key] = value;
+    }
+    return labels;
+}
+/** Replaces every `{name}` in a label template (plain text, never markup). */
+export function fillCodeLabel(template, name) {
+    return template.split('{name}').join(name);
+}

package/dist/code/languages/bash.d.ts

@@ -0,0 +1,3 @@
+import { type Grammar } from '../tokenizer.js';
+/** Shell scripts: commands, flags, variables, quoting and comments. */
+export declare const bashGrammar: Grammar;

package/dist/code/languages/bash.js

@@ -0,0 +1,48 @@
+import { sticky } from "../tokenizer.js";
+import { HASH_COMMENT, WHITESPACE, words } from "./shared.js";
+const KEYWORDS = words('if', 'then', 'else', 'elif', 'fi', 'for', 'while', 'until', 'do', 'done', 'case', 'esac', 'in', 'function', 'select', 'return', 'break', 'continue', 'export', 'local', 'readonly', 'declare', 'unset', 'shift', 'source', 'exit');
+/** Tokens after which the next word is a command name. */
+const COMMAND_STARTERS = words('|', '||', '&&', ';', '(', '{', '`', '$(', 'then', 'do', 'else', 'elif', '!');
+/** True when only spaces or tabs separate `start` from a line start (bounded). */
+function atLineStart(input, start) {
+    for (let i = start - 1, n = 0; i >= 0 && n < 64; i -= 1, n += 1) {
+        const char = input[i];
+        if (char === '\n')
+            return true;
+        if (char !== ' ' && char !== '\t')
+            return false;
+    }
+    return start <= 64 && input.slice(0, start).trim() === '';
+}
+function word(text, ctx) {
+    if (KEYWORDS.has(text))
+        return 'keyword';
+    const start = ctx.end - text.length;
+    if (ctx.prev === null || COMMAND_STARTERS.has(ctx.prev.text) || atLineStart(ctx.input, start)) {
+        return 'function';
+    }
+    return 'plain';
+}
+/** Shell scripts: commands, flags, variables, quoting and comments. */
+export const bashGrammar = {
+    name: 'bash',
+    start: 'code',
+    states: {
+        code: [
+            WHITESPACE,
+            HASH_COMMENT,
+            // Single quotes have no escapes; both quotes may span lines.
+            { re: sticky(String.raw `'[^']*(?:'|(?![\s\S]))`), kind: 'string' },
+            // `\\[\s\S]?` lets a lone trailing backslash match too: an unterminated
+            // string always runs to the end instead of failing and being rescanned.
+            { re: sticky(String.raw `"(?:[^"\\]|\\[\s\S]?)*(?:"|(?![\s\S]))`), kind: 'string' },
+            { re: sticky(String.raw `\$\{[^}\n]*\}?|\$(?:[A-Za-z_]\w*|[0-9@#?$!*-])`), kind: 'variable' },
+            { re: sticky(String.raw `\$\(`), kind: 'punctuation' },
+            { re: sticky(String.raw `(?<=^|[ \t])--?[A-Za-z][\w-]*`), kind: 'attr' },
+            { re: sticky(String.raw `[A-Za-z_./~][\w./~:@%+-]*`), kind: word },
+            { re: sticky(String.raw `\d+`), kind: 'number' },
+            { re: sticky(String.raw `\|\||&&|[|&;<>!=]+`), kind: 'operator' },
+            { re: sticky(String.raw `[(){}[\]\x60\\]`), kind: 'punctuation' },
+        ],
+    },
+};

package/dist/code/languages/css.d.ts

@@ -0,0 +1,10 @@
+import { type Grammar } from '../tokenizer.js';
+/**
+ * CSS in four states: selectors (outside a block), declarations (inside a
+ * block), the colon after a property, and values. Inside a block, a word
+ * followed by `:` is a property unless a `{` follows within 200 characters
+ * before any `;` or `}` (then it is a nested selector such as `a:hover {`);
+ * the bound keeps scanning linear. After a nested block's `}` the grammar
+ * returns to selectors, which still reads the next selector correctly.
+ */
+export declare const cssGrammar: Grammar;

package/dist/code/languages/css.js

@@ -0,0 +1,67 @@
+import { sticky } from "../tokenizer.js";
+import { BLOCK_COMMENT, DOUBLE_QUOTED, SINGLE_QUOTED, WHITESPACE } from "./shared.js";
+const COMMON = [
+    WHITESPACE,
+    BLOCK_COMMENT,
+    DOUBLE_QUOTED,
+    SINGLE_QUOTED,
+    { re: sticky(String.raw `@[\w-]+`), kind: 'keyword' },
+];
+const NUMBER = {
+    re: sticky(String.raw `-?(?:\d+\.?\d*|\.\d+)(?:%|[A-Za-z]+)?`),
+    kind: 'number',
+};
+/** Selector pieces, shared by the top level and rules nested in a block. */
+const SELECTOR = [
+    { re: sticky(String.raw `[.#][\w-]+`), kind: 'attr' },
+    { re: sticky(String.raw `::?[A-Za-z][\w-]*`), kind: 'keyword' },
+    { re: sticky(String.raw `\{`), kind: 'punctuation', next: 'declaration' },
+];
+/**
+ * CSS in four states: selectors (outside a block), declarations (inside a
+ * block), the colon after a property, and values. Inside a block, a word
+ * followed by `:` is a property unless a `{` follows within 200 characters
+ * before any `;` or `}` (then it is a nested selector such as `a:hover {`);
+ * the bound keeps scanning linear. After a nested block's `}` the grammar
+ * returns to selectors, which still reads the next selector correctly.
+ */
+export const cssGrammar = {
+    name: 'css',
+    start: 'selector',
+    states: {
+        selector: [
+            ...COMMON,
+            ...SELECTOR,
+            // Media features such as `(max-width: 600px)`.
+            { re: sticky(String.raw `[A-Za-z][\w-]*(?=[ \t]*:[ \t])`), kind: 'property' },
+            { re: sticky(String.raw `[A-Za-z][\w-]*|\*`), kind: 'tag' },
+            NUMBER,
+            { re: sticky(String.raw `[>+~,()[\]=:]|\}`), kind: 'punctuation' },
+        ],
+        declaration: [
+            ...COMMON,
+            {
+                re: sticky(String.raw `(?:--[\w-]+|-?[A-Za-z][\w-]*)(?=[ \t]*:)(?![^;{}]{0,200}\{)`),
+                kind: 'property',
+                next: 'colon',
+            },
+            { re: sticky(String.raw `\}`), kind: 'punctuation', next: 'selector' },
+            ...SELECTOR,
+            { re: sticky(String.raw `[A-Za-z][\w-]*|\*`), kind: 'tag' },
+            { re: sticky(String.raw `[;,>+~()[\]=]`), kind: 'punctuation' },
+        ],
+        colon: [WHITESPACE, { re: sticky(':'), kind: 'punctuation', next: 'value' }],
+        value: [
+            ...COMMON,
+            { re: sticky(String.raw `#[\da-fA-F]{3,8}(?![\w-])`), kind: 'number' },
+            NUMBER,
+            { re: sticky('!important'), kind: 'keyword' },
+            { re: sticky(String.raw `--[\w-]+`), kind: 'variable' },
+            { re: sticky(String.raw `[A-Za-z_-][\w-]*(?=\()`), kind: 'function' },
+            { re: sticky(String.raw `[A-Za-z_-][\w-]*`), kind: 'constant' },
+            { re: sticky(';'), kind: 'punctuation', next: 'declaration' },
+            { re: sticky(String.raw `\}`), kind: 'punctuation', next: 'selector' },
+            { re: sticky('[(),/*+:]'), kind: 'punctuation' },
+        ],
+    },
+};

package/dist/code/languages/html.d.ts

@@ -0,0 +1,6 @@
+import { type Grammar } from '../tokenizer.js';
+/**
+ * HTML markup: comments, doctype, tags, attributes and entities. Embedded
+ * `<script>` and `<style>` bodies are not re-tokenized as JS or CSS.
+ */
+export declare const htmlGrammar: Grammar;

package/dist/code/languages/html.js

@@ -0,0 +1,35 @@
+import { sticky } from "../tokenizer.js";
+import { WHITESPACE } from "./shared.js";
+/**
+ * HTML markup: comments, doctype, tags, attributes and entities. Embedded
+ * `<script>` and `<style>` bodies are not re-tokenized as JS or CSS.
+ */
+export const htmlGrammar = {
+    name: 'html',
+    start: 'text',
+    states: {
+        text: [
+            { re: sticky(String.raw `<!--[\s\S]*?(?:-->|(?![\s\S]))`), kind: 'comment' },
+            { re: sticky('<![^>]*>?'), kind: 'keyword' },
+            { re: sticky(String.raw `<\/?(?=[A-Za-z])`), kind: 'punctuation', next: 'tagName' },
+            {
+                re: sticky(String.raw `&(?:#\d{1,7}|#[xX][\da-fA-F]{1,6}|[A-Za-z][A-Za-z\d]{0,31});`),
+                kind: 'constant',
+            },
+            { re: sticky('[^<&]+'), kind: 'plain' },
+        ],
+        tagName: [{ re: sticky(String.raw `[A-Za-z][\w:-]*`), kind: 'tag', next: 'tag' }],
+        tag: [
+            WHITESPACE,
+            { re: sticky(String.raw `\/?>`), kind: 'punctuation', next: 'text' },
+            { re: sticky(String.raw `"[^"]*(?:"|(?![\s\S]))`), kind: 'string' },
+            { re: sticky(String.raw `'[^']*(?:'|(?![\s\S]))`), kind: 'string' },
+            { re: sticky('='), kind: 'operator' },
+            {
+                re: sticky(String.raw `[^\s"'>/=]+`),
+                kind: (_text, ctx) => (ctx.prev?.text === '=' ? 'string' : 'attr'),
+            },
+            { re: sticky('/'), kind: 'punctuation' },
+        ],
+    },
+};

package/dist/code/languages/javascript.d.ts

@@ -0,0 +1,3 @@
+import { type Grammar } from '../tokenizer.js';
+export declare const javascriptGrammar: Grammar;
+export declare const typescriptGrammar: Grammar;

package/dist/code/languages/javascript.js

@@ -0,0 +1,113 @@
+import { classifyWord, sticky } from "../tokenizer.js";
+import { BLOCK_COMMENT, C_NUMBER, DOUBLE_QUOTED, SINGLE_QUOTED, SLASH_COMMENT, WHITESPACE, words, } from "./shared.js";
+const JS_KEYWORDS = [
+    'as',
+    'async',
+    'await',
+    'break',
+    'case',
+    'catch',
+    'class',
+    'const',
+    'continue',
+    'debugger',
+    'default',
+    'delete',
+    'do',
+    'else',
+    'export',
+    'extends',
+    'finally',
+    'for',
+    'from',
+    'function',
+    'if',
+    'import',
+    'in',
+    'instanceof',
+    'let',
+    'new',
+    'of',
+    'return',
+    'static',
+    'super',
+    'switch',
+    'this',
+    'throw',
+    'try',
+    'typeof',
+    'var',
+    'void',
+    'while',
+    'with',
+    'yield',
+];
+const TS_KEYWORDS = [
+    'abstract',
+    'asserts',
+    'declare',
+    'enum',
+    'implements',
+    'infer',
+    'interface',
+    'is',
+    'keyof',
+    'module',
+    'namespace',
+    'override',
+    'private',
+    'protected',
+    'public',
+    'readonly',
+    'satisfies',
+    'type',
+    'unique',
+];
+const CONSTANTS = words('true', 'false', 'null', 'undefined', 'NaN', 'Infinity');
+const TYPE_INTRODUCERS = words('class', 'interface', 'type', 'enum', 'extends', 'implements', 'new');
+const FUNCTION_INTRODUCERS = words('function');
+/** `` `…` ``, possibly across lines; `${…}` stays part of the string. */
+const TEMPLATE = {
+    re: sticky(String.raw `\x60(?:[^\x60\\]|\\[\s\S]?)*(?:\x60|(?![\s\S]))`),
+    kind: 'string',
+};
+function grammar(name, keywords, types) {
+    const word = classifyWord({
+        keywords,
+        constants: CONSTANTS,
+        types,
+        typeIntroducers: TYPE_INTRODUCERS,
+        functionIntroducers: FUNCTION_INTRODUCERS,
+    });
+    const capitalizedTypes = name === 'ts';
+    return {
+        name,
+        start: 'code',
+        states: {
+            code: [
+                WHITESPACE,
+                SLASH_COMMENT,
+                BLOCK_COMMENT,
+                DOUBLE_QUOTED,
+                SINGLE_QUOTED,
+                TEMPLATE,
+                {
+                    re: sticky(String.raw `[A-Za-z_$][\w$]*`),
+                    kind: (text, ctx) => {
+                        const kind = word(text, ctx);
+                        // In TypeScript a capitalized name that is not called is usually a type.
+                        if (kind === 'plain' && capitalizedTypes && /^[A-Z]/.test(text))
+                            return 'type';
+                        return kind;
+                    },
+                },
+                C_NUMBER,
+                { re: sticky(String.raw `@[A-Za-z_$][\w$]*`), kind: 'attr' },
+                { re: sticky(String.raw `\?\.|[+\-*/%=!<>&|^~?:]+`), kind: 'operator' },
+                { re: sticky(String.raw `[{}()[\];,.]`), kind: 'punctuation' },
+            ],
+        },
+    };
+}
+export const javascriptGrammar = grammar('js', words(...JS_KEYWORDS), words());
+export const typescriptGrammar = grammar('ts', words(...JS_KEYWORDS, ...TS_KEYWORDS), words('string', 'number', 'boolean', 'any', 'unknown', 'never', 'object', 'symbol', 'bigint'));

package/dist/code/languages/json.d.ts

@@ -0,0 +1,3 @@
+import { type Grammar } from '../tokenizer.js';
+/** JSON, plus the comments JSONC allows. A string followed by `:` is a property. */
+export declare const jsonGrammar: Grammar;

package/dist/code/languages/json.js

@@ -0,0 +1,24 @@
+import { nextNonBlank, sticky } from "../tokenizer.js";
+import { BLOCK_COMMENT, SLASH_COMMENT, WHITESPACE } from "./shared.js";
+/** JSON, plus the comments JSONC allows. A string followed by `:` is a property. */
+export const jsonGrammar = {
+    name: 'json',
+    start: 'value',
+    states: {
+        value: [
+            WHITESPACE,
+            SLASH_COMMENT,
+            BLOCK_COMMENT,
+            {
+                re: sticky(String.raw `"(?:[^"\\\n]|\\.?)*(?:"|$)`),
+                kind: (_text, ctx) => (nextNonBlank(ctx.input, ctx.end) === ':' ? 'property' : 'string'),
+            },
+            { re: sticky(String.raw `-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?`), kind: 'number' },
+            {
+                re: sticky(String.raw `[A-Za-z_]\w*`),
+                kind: (text) => text === 'true' || text === 'false' || text === 'null' ? 'constant' : 'plain',
+            },
+            { re: sticky(String.raw `[{}[\],:]`), kind: 'punctuation' },
+        ],
+    },
+};

package/dist/code/languages/python.d.ts

@@ -0,0 +1,3 @@
+import { type Grammar } from '../tokenizer.js';
+/** Python 3: triple-quoted and prefixed strings, decorators, keywords and builtins. */
+export declare const pythonGrammar: Grammar;

package/dist/code/languages/python.js

@@ -0,0 +1,35 @@
+import { classifyWord, sticky } from "../tokenizer.js";
+import { C_NUMBER, HASH_COMMENT, WHITESPACE, words } from "./shared.js";
+const word = classifyWord({
+    keywords: words('and', 'as', 'assert', 'async', 'await', 'break', 'case', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'match', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'),
+    constants: words('True', 'False', 'None', 'Ellipsis', 'NotImplemented'),
+    builtins: words('abs', 'all', 'any', 'bool', 'dict', 'enumerate', 'filter', 'float', 'int', 'isinstance', 'len', 'list', 'map', 'max', 'min', 'open', 'print', 'range', 'round', 'set', 'sorted', 'str', 'sum', 'super', 'tuple', 'type', 'zip'),
+    typeIntroducers: words('class'),
+    functionIntroducers: words('def'),
+});
+const PREFIX = '(?:[rRbBuUfF]{1,2})?';
+/** Python 3: triple-quoted and prefixed strings, decorators, keywords and builtins. */
+export const pythonGrammar = {
+    name: 'python',
+    start: 'code',
+    states: {
+        code: [
+            WHITESPACE,
+            HASH_COMMENT,
+            // Triple quotes may span lines; unterminated ones run to the end.
+            {
+                re: sticky(`${PREFIX}${String.raw `(?:"""[\s\S]*?(?:"""|(?![\s\S]))|'''[\s\S]*?(?:'''|(?![\s\S])))`}`),
+                kind: 'string',
+            },
+            {
+                re: sticky(`${PREFIX}${String.raw `(?:"(?:[^"\\\n]|\\.?)*(?:"|$)|'(?:[^'\\\n]|\\.?)*(?:'|$))`}`),
+                kind: 'string',
+            },
+            { re: sticky(String.raw `(?<=^[ \t]*)@[A-Za-z_][\w.]*`), kind: 'attr' },
+            { re: sticky(String.raw `[A-Za-z_]\w*`), kind: word },
+            C_NUMBER,
+            { re: sticky(String.raw `[+\-*/%=!<>&|^~:@]+`), kind: 'operator' },
+            { re: sticky(String.raw `[()[\]{},.;]`), kind: 'punctuation' },
+        ],
+    },
+};

package/dist/code/languages/shared.d.ts

@@ -0,0 +1,22 @@
+import { type GrammarRule } from '../tokenizer.js';
+/**
+ * Rules several grammars share. Each opening delimiter always matches: an
+ * unterminated single-line string ends at its line break (`$` with the `m`
+ * flag), an unterminated block comment or multi-line string at the end of
+ * the input (`$` without it). See tokenizer.ts for why that keeps scanning
+ * linear.
+ */
+export declare const WHITESPACE: GrammarRule;
+/** `"…"` on one line, with backslash escapes. */
+export declare const DOUBLE_QUOTED: GrammarRule;
+/** `'…'` on one line, with backslash escapes. */
+export declare const SINGLE_QUOTED: GrammarRule;
+/** `/* … *\/`, possibly across lines. */
+export declare const BLOCK_COMMENT: GrammarRule;
+/** `// …` to the end of the line. */
+export declare const SLASH_COMMENT: GrammarRule;
+/** `# …` at a line start or after whitespace (not `$#` or `a#b`). */
+export declare const HASH_COMMENT: GrammarRule;
+/** Decimal, hex, octal and binary numbers with `_` separators and exponents. */
+export declare const C_NUMBER: GrammarRule;
+export declare const words: (...list: string[]) => ReadonlySet<string>;