@zuilib/text-editor 0.7.2 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog — @zuilib/text-editor
 
+## 0.8.0
+
+- **Syntax highlighting** for fenced code blocks, driven by the package's
+  own grammars instead of Prism: JavaScript/TypeScript, Kotlin, Java, C/C++,
+  Python, Go, Rust, JSON, YAML, shell, SQL, CSS, HTML/XML, diff and
+  markdown. Colors are CSS custom properties (`--zui-code-<type>` on
+  `.zui-code`) with light and dark defaults; override them to retheme.
+  `registerCodeLanguage(grammar)` adds a language from a data-only grammar;
+  `tokenizeCode` and `codeTokenizer` are exported for reuse
+- Untagged code blocks are highlighted as `plain` and export as a bare
+  ``` fence (previously they came back as ```javascript). `CODE_BLOCK` is
+  the exported transformer that does this
+- Unknown languages get the same line structure as known ones, so cursor
+  movement and indentation behave identically
+
 ## 0.7.2
 
 - Canvas properties are one **color** setting (outline plus its matching

package/dist/index.d.ts

@@ -3,6 +3,7 @@ import { ReactNode, ReactElement } from 'react';
 import { LexicalEditor, TextFormatType, ElementNode, NodeKey, EditorConfig, SerializedElementNode, LexicalNode, DecoratorNode, Spread, SerializedLexicalNode, DOMExportOutput, DOMConversionMap } from 'lexical';
 import { TableNode } from '@lexical/table';
 import { MultilineElementTransformer, ElementTransformer } from '@lexical/markdown';
+import { registerCodeHighlighting } from '@lexical/code';
 
 /**
  * Horizontal sizing of a block that would otherwise span the editor:
@@ -975,4 +976,280 @@ type MermaidOptions = Readonly<{
  */
 declare function drawingToMermaid(data: DrawingData, options?: MermaidOptions): string;
 
-export { $createDrawingNode, $createFrontmatterNode, $getSelectedTable, $getTableDensity, $getTableSettings, $getTableWidth, $isDrawingNode, $isFrontmatterNode, $isTableWidthExplicit, $setTableDensity, $setTableSettings, $setTableWidth, BLOCK_WIDTHS, BOX_DEFINITIONS, BOX_TYPES, type Binding, type BindingSide, type BlockWidth, type BlockWidthDefaults, type BoxDefinition, type BoxType, COLOR_PRESETS, CONNECTOR_TYPES, type ColorName, type ConnectorType, DEFAULT_TABLE_SETTINGS, DIAGRAM, DRAWING, DRAWING_DATA_JSON_SCHEMA, DRAWING_SKELETON_JSON_SCHEMA, type DrawingData, DrawingNode, type DrawingShape, type DrawingShapeType, type DrawingSkeleton, EMPTY_DRAWING, type EditorContentProps, type EditorMode, type EditorRootProps, FILL_COLORS, FRONTMATTER, FormatButtons, FrontmatterNode, HistoryButtons, InsertButtons, MarkdownEditor, type MarkdownEditorApi, type Props as MarkdownEditorProps, type MermaidDirection, type MermaidOptions, type Point, SHAPE_TYPES, SIDE_FIXED_POINTS, STROKE_COLORS, type SerializedDrawingNode, type SerializedFrontmatterNode, type SkeletonBox, type SkeletonConnector, type SkeletonEnd, type SkeletonText, TABLE, TABLE_WIDTH_MARKER, type TableDensity, type TableSettings, type TableSettingsOptions, type TextField, Toolbar, ToolbarButton, ToolbarDivider, type ToolbarItems, anchorPoint, bindEndpoints, boxOutline, connectorPoints, drawingToMermaid, expandSkeleton, findBoxAt, fixedPointFor, formatTableSettingsMarker, isBlockWidth, isDrawingSkeleton, makeBinding, normalizeDrawingData, parseDrawingData, parseDrawingSkeleton, parseTableSettingsMarker, resolveBindings, routeElbow, serializeDrawingData, useMarkdownEditor };
+/**
+ * `@lexical/markdown`'s CODE transformer, except that a block whose language
+ * is the highlighter's default (`plain`, assigned to every untagged block)
+ * exports as a bare ``` fence — markdown in, markdown out.
+ */
+declare const CODE_BLOCK: MultilineElementTransformer;
+
+/** The built-in grammars. Importing this module registers them. */
+declare const BUILTIN_CODE_LANGUAGES: readonly [Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>, Readonly<{
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    identifier?: string;
+    caseInsensitive?: boolean;
+    callIsFunction?: boolean;
+    capitalizedIsType?: boolean;
+}>];
+
+/**
+ * Semantic token vocabulary for code highlighting. Closed on purpose: every
+ * grammar maps its syntax onto these names, and the theme maps each name onto
+ * one class (`zui-token-<type>`) and one color custom property
+ * (`--zui-code-<type>`). Adding a language never adds a color.
+ */
+declare const CODE_TOKEN_TYPES: readonly ["comment", "string", "number", "keyword", "builtin", "type", "function", "property", "variable", "constant", "operator", "punctuation", "tag", "attr", "regex", "meta", "inserted", "deleted"];
+type CodeTokenType = (typeof CODE_TOKEN_TYPES)[number];
+/** A run of source text. `text` is uncolored (identifiers, whitespace, …). */
+type CodeToken = Readonly<{
+    type: CodeTokenType | 'text';
+    text: string;
+}>;
+/**
+ * One lexer rule: a regular-expression *source* (no flags, no capturing
+ * groups — use `(?:…)`, no backreferences) tried at the current position.
+ * Rules are tried in order; the first one that matches wins.
+ */
+type CodeRule = Readonly<{
+    type: CodeTokenType;
+    match: string;
+}>;
+/**
+ * A language definition. Pure data: the lexer compiles it once (cached per
+ * object identity) and walks the source with it.
+ *
+ * Identifiers not consumed by a rule are classified by the word lists, then
+ * by the two heuristics, and otherwise left as plain text.
+ */
+type CodeGrammar = Readonly<{
+    /** Canonical name, as written after ``` in markdown */
+    name: string;
+    aliases?: readonly string[];
+    rules: readonly CodeRule[];
+    keywords?: readonly string[];
+    builtins?: readonly string[];
+    types?: readonly string[];
+    constants?: readonly string[];
+    /** Regex source for identifiers (default: `[A-Za-z_$][\w$]*`) */
+    identifier?: string;
+    /** Word lists and rules match case-insensitively (SQL) */
+    caseInsensitive?: boolean;
+    /** An identifier directly followed by `(` is a `function` */
+    callIsFunction?: boolean;
+    /**
+     * A Capitalized identifier is a `type`; a SCREAMING_CASE one is a
+     * `constant`
+     */
+    capitalizedIsType?: boolean;
+}>;
+
+/**
+ * Splits `code` into a flat list of tokens covering the whole input, in
+ * order; adjacent tokens of the same type are merged. Never throws on user
+ * input and always makes progress (an empty match counts as no match).
+ */
+declare function tokenizeCode(code: string, grammar: CodeGrammar): CodeToken[];
+
+/**
+ * Language registry: canonical names and aliases, lower-cased, mapped to a
+ * grammar. Unknown names resolve to `plain` (line structure, no colors) so
+ * every fenced block gets the same editing behavior.
+ */
+declare const PLAIN_LANGUAGE = "plain";
+/** Adds (or replaces) a language. Call before the editor mounts. */
+declare function registerCodeLanguage(grammar: CodeGrammar): void;
+declare function hasCodeLanguage(name: string): boolean;
+/** Resolves a fence name (canonical, alias, or `diff-<lang>`) to a grammar. */
+declare function resolveCodeLanguage(name: string | null | undefined): CodeGrammar;
+/** Canonical names of every registered language, sorted */
+declare function getCodeLanguages(): string[];
+
+type LexicalTokenizer = NonNullable<Parameters<typeof registerCodeHighlighting>[1]>;
+/**
+ * The tokenizer handed to `@lexical/code`. It never touches Prism: the
+ * registry's grammars produce the tokens, and `$tokenize` turns them into the
+ * flat CodeHighlightNode / LineBreakNode / TabNode structure CodeNode expects.
+ */
+declare const codeTokenizer: LexicalTokenizer;
+/**
+ * Registers syntax highlighting for every CodeNode in the editor, driven by
+ * the language registry. Unknown languages render as plain text with the
+ * same line structure, so editing behaves identically.
+ */
+declare function registerCodeBlockHighlighting(editor: LexicalEditor): () => void;
+
+export { $createDrawingNode, $createFrontmatterNode, $getSelectedTable, $getTableDensity, $getTableSettings, $getTableWidth, $isDrawingNode, $isFrontmatterNode, $isTableWidthExplicit, $setTableDensity, $setTableSettings, $setTableWidth, BLOCK_WIDTHS, BOX_DEFINITIONS, BOX_TYPES, BUILTIN_CODE_LANGUAGES, type Binding, type BindingSide, type BlockWidth, type BlockWidthDefaults, type BoxDefinition, type BoxType, CODE_BLOCK, CODE_TOKEN_TYPES, COLOR_PRESETS, CONNECTOR_TYPES, type CodeGrammar, type CodeRule, type CodeToken, type CodeTokenType, type ColorName, type ConnectorType, DEFAULT_TABLE_SETTINGS, DIAGRAM, DRAWING, DRAWING_DATA_JSON_SCHEMA, DRAWING_SKELETON_JSON_SCHEMA, type DrawingData, DrawingNode, type DrawingShape, type DrawingShapeType, type DrawingSkeleton, EMPTY_DRAWING, type EditorContentProps, type EditorMode, type EditorRootProps, FILL_COLORS, FRONTMATTER, FormatButtons, FrontmatterNode, HistoryButtons, InsertButtons, MarkdownEditor, type MarkdownEditorApi, type Props as MarkdownEditorProps, type MermaidDirection, type MermaidOptions, PLAIN_LANGUAGE, type Point, SHAPE_TYPES, SIDE_FIXED_POINTS, STROKE_COLORS, type SerializedDrawingNode, type SerializedFrontmatterNode, type SkeletonBox, type SkeletonConnector, type SkeletonEnd, type SkeletonText, TABLE, TABLE_WIDTH_MARKER, type TableDensity, type TableSettings, type TableSettingsOptions, type TextField, Toolbar, ToolbarButton, ToolbarDivider, type ToolbarItems, anchorPoint, bindEndpoints, boxOutline, codeTokenizer, connectorPoints, drawingToMermaid, expandSkeleton, findBoxAt, fixedPointFor, formatTableSettingsMarker, getCodeLanguages, hasCodeLanguage, isBlockWidth, isDrawingSkeleton, makeBinding, normalizeDrawingData, parseDrawingData, parseDrawingSkeleton, parseTableSettingsMarker, registerCodeBlockHighlighting, registerCodeLanguage, resolveBindings, resolveCodeLanguage, routeElbow, serializeDrawingData, tokenizeCode, useMarkdownEditor };