@synclineapi/editor 4.0.3 → 4.0.4

package/README.md

@@ -22,6 +22,7 @@ Ships as both an **ES module** and **UMD bundle**, runs entirely inside a **Shad
   - [Editing](#editing)
   - [Features](#features-config)
   - [Syntax & Autocomplete](#syntax--autocomplete-customisation)
+  - [Custom Syntax Highlighter — provideTokens](#custom-syntax-highlighter--providetokens)
   - [Token Colors](#token-colors-config)
   - [Theme](#theme-config)
   - [Callbacks](#callbacks)
@@ -61,6 +62,7 @@ Ships as both an **ES module** and **UMD bundle**, runs entirely inside a **Shad
 - [Dynamic Completion Provider](#dynamic-completion-provider)
   - [provideCompletions](#providecompletions)
   - [CompletionContext](#completioncontext)
+- [Custom Syntax Highlighter — provideTokens](#custom-syntax-highlighter--providetokens)
 - [Events & Callbacks](#events--callbacks)
 - [Advanced Features](#advanced-features)
   - [Multi-cursor](#multi-cursor)
@@ -89,6 +91,7 @@ Ships as both an **ES module** and **UMD bundle**, runs entirely inside a **Shad
 - [Keyboard Shortcuts](#keyboard-shortcuts)
 - [Recipes](#recipes)
   - [Monaco-style Editor Embed](#monaco-style-editor-embed)
+  - [Markdown Editor](#markdown-editor)
   - [DSL / SQL Editor](#dsl--sql-editor)
   - [Read-Only Code Viewer](#read-only-code-viewer)
   - [Custom Theme from Scratch](#custom-theme-from-scratch-recipe)
@@ -109,6 +112,7 @@ Ships as both an **ES module** and **UMD bundle**, runs entirely inside a **Shad
 | **Shadow DOM** | Fully encapsulated styles — no CSS leakage in either direction |
 | **Dual build** | ES module + UMD bundle, full TypeScript declarations |
 | **Syntax highlighting** | TypeScript, JavaScript, CSS, JSON, Markdown — nine token classes |
+| **`provideTokens` callback** | Fully override the built-in tokeniser for any language — pass `(line, language) => TokenSegment[]` |
 | **Token color overrides** | Override individual token colours on top of any theme without replacing it |
 | **6 built-in themes** | VR Dark, VS Code Dark+, Monokai, Dracula, GitHub Light, Solarized Light |
 | **Custom themes** | Full `ThemeDefinition` API — every CSS variable exposed |
@@ -274,6 +278,7 @@ Default `autoClosePairs`:
 | `replaceBuiltins` | `boolean` | `false` | When `true`, `completions` replaces the built-in language keywords/types entirely. |
 | `provideCompletions` | `(ctx: CompletionContext) => CompletionItem[] \| null` | `undefined` | Dynamic callback — called on every popup open; return `null` to fall through to defaults. |
 | `provideHover` | `(ctx: HoverContext) => HoverDoc \| null` | `undefined` | Dynamic hover callback — return a `HoverDoc` to show a tooltip for any word not covered by built-in docs or the `completions` array. |
+| `provideTokens` | `(line: string, language: string) => TokenSegment[]` | `undefined` | Fully override syntax tokenisation. Called for every visible line on every render. Return an array of `TokenSegment` objects. When set, the built-in tokeniser is **never called** — the callback owns all highlighting for every language. See [Custom Syntax Highlighter](#custom-syntax-highlighter--providetokens). |
 | `maxCompletions` | `number` | `14` | Maximum items shown in the autocomplete popup. |
 
 ### Token Colors Config
@@ -1039,6 +1044,120 @@ createEditor(container, { emmet: false }); // disable
 
 ---
 
+## Custom Syntax Highlighter — `provideTokens`
+
+When the built-in tokeniser doesn't cover your language, supply a `provideTokens` callback and own syntax highlighting entirely:
+
+```ts
+provideTokens?: (line: string, language: string) => TokenSegment[]
+```
+
+- Called for **every visible line** on every render pass.
+- When set, the built-in tokeniser is **never called** — your callback is the sole source of token segments.
+- Any line not fully covered by your segments renders as plain text for those characters.
+- Runs synchronously on the render thread — keep it fast (no I/O, no heavy regex backtracking).
+- The `language` argument is the active `EditorConfig.language` string — use it when you register one callback for multiple languages.
+
+### `TokenSegment` shape
+
+```ts
+interface TokenSegment {
+  cls:   TokenClass;  // which colour to apply — see table below
+  start: number;      // start character index in the line (inclusive)
+  end:   number;      // end character index in the line (exclusive)
+}
+```
+
+Segments may overlap; later segments in the array take priority. Gaps between segments render as plain text (`--tok-op` colour).
+
+### `TokenClass` values
+
+| `cls` | CSS variable | Typical use |
+|---|---|---|
+| `'kw'` | `--tok-kw` | Keywords, control flow |
+| `'str'` | `--tok-str` | String and template literals |
+| `'cmt'` | `--tok-cmt` | Comments |
+| `'fn'` | `--tok-fn` | Function names, calls |
+| `'num'` | `--tok-num` | Numeric literals |
+| `'cls'` | `--tok-cls` | Class / type names |
+| `'op'` | `--tok-op` | Operators, punctuation, plain text |
+| `'typ'` | `--tok-typ` | Type annotations |
+| `'dec'` | `--tok-dec` | Decorators / annotations |
+
+### Markdown example
+
+The built-in `'markdown'` language produces no highlighting — it runs the generic path. Use `provideTokens` to add full Markdown highlighting:
+
+```ts
+import { createEditor } from 'syncline-editor';
+import type { TokenSegment } from 'syncline-editor';
+
+createEditor(container, {
+  language: 'markdown',
+  provideTokens: (line): TokenSegment[] => {
+    const segs: TokenSegment[] = [];
+
+    // ATX headings  # / ## / … → keyword colour
+    if (/^#{1,6}(\s|$)/.test(line)) {
+      segs.push({ cls: 'kw', start: 0, end: line.length });
+      return segs;
+    }
+
+    // Fenced code block delimiter  ``` or ~~~
+    if (/^(`{3,}|~{3,})/.test(line)) {
+      segs.push({ cls: 'cmt', start: 0, end: line.length });
+      return segs;
+    }
+
+    // Blockquote  > …
+    if (/^>/.test(line)) {
+      segs.push({ cls: 'cmt', start: 0, end: line.length });
+      return segs;
+    }
+
+    // Inline patterns — scan with regex
+    const inlineRules: [RegExp, TokenSegment['cls']][] = [
+      [/`[^`]+`/g,           'str'],   // inline code
+      [/\*\*[^*]+\*\*/g,     'kw'],    // bold
+      [/\*[^*]+\*/g,         'typ'],   // italic
+      [/~~[^~]+~~/g,         'cmt'],   // strikethrough
+      [/!?\[[^\]]*\]\([^)]*\)/g, 'fn'], // links / images
+      [/^([-*+]|\d+\.)\s/,  'dec'],   // list marker
+    ];
+
+    for (const [re, cls] of inlineRules) {
+      let m: RegExpExecArray | null;
+      const g = new RegExp(re.source, 'g');
+      while ((m = g.exec(line)) !== null) {
+        segs.push({ cls, start: m.index, end: m.index + m[0].length });
+      }
+    }
+
+    return segs;
+  },
+});
+```
+
+### Switching the callback at runtime
+
+`updateConfig` clears the token cache before applying the new callback, so the next render is always fresh:
+
+```ts
+// Switch from Markdown highlighting to a plain-text view
+editor.updateConfig({ provideTokens: undefined });
+
+// Swap to a different highlighter entirely
+editor.updateConfig({ provideTokens: myNewHighlighter });
+```
+
+### Performance tips
+
+- Avoid constructing new `RegExp` objects inside the callback on every call — define them outside and reuse.
+- Return an **empty array** `[]` for lines you intentionally want as plain text rather than pushing an `op` segment for the whole line.
+- For large files, keep per-line work to O(line-length) — the callback is invoked once per visible row per frame.
+
+---
+
 ## Dynamic Completion Provider
 
 Use `provideCompletions` for fully context-aware completions that change based on the cursor position, current prefix, or document content. When this callback returns a non-null array, it **completely overrides** all other completion sources.
@@ -1407,15 +1526,17 @@ createEditor(container, {
 | `Ctrl / Cmd + Z` | Undo | `undoBatchMs`, `maxUndoHistory` |
 | `Ctrl / Cmd + Y` or `Ctrl + Shift + Z` | Redo | — |
 | `Ctrl / Cmd + A` | Select all | — |
-| `Ctrl / Cmd + C` | Copy | — |
-| `Ctrl / Cmd + X` | Cut | `readOnly` |
-| `Ctrl / Cmd + V` | Paste | `readOnly` |
+| `Ctrl / Cmd + C` | Copy selection; **with no selection copies the entire current line** (pasting inserts it above the cursor line) | — |
+| `Ctrl / Cmd + X` | Cut selection; **with no selection cuts the entire current line** | `readOnly` |
+| `Ctrl / Cmd + V` | Paste; if clipboard holds a whole-line copy, inserts the line above the cursor line and shifts cursor down | `readOnly` |
 | `Ctrl / Cmd + F` | Open find bar | `find` |
 | `Ctrl / Cmd + H` | Open find + replace | `findReplace` |
 | `Ctrl / Cmd + G` | Open Go to Line prompt | `goToLine` |
-| `Ctrl / Cmd + D` | Select next occurrence | `multiCursor` |
+| `Ctrl / Cmd + D` | Select next occurrence (first press selects current word; each subsequent press adds next occurrence) | `multiCursor` |
 | `Ctrl / Cmd + Shift + D` | Duplicate line | `readOnly` |
 | `Ctrl / Cmd + K` | Delete line | `readOnly` |
+| `Cmd + Enter` | Insert new line below cursor (without moving to end of line) | `readOnly` |
+| `Cmd + Shift + Enter` | Insert new line above cursor | `readOnly` |
 | `Ctrl / Cmd + /` | Toggle line comment | `lineCommentToken`, `readOnly` |
 | `Alt / Option + Z` | Toggle word wrap | — |
 | `Alt / Option + ↑` | Move current line (or selected block) up | `readOnly` |
@@ -1446,12 +1567,84 @@ createEditor(container, {
 
 All mutating shortcuts are silently blocked when `readOnly: true`. Navigation, selection, and `Ctrl+C` always work.
 
+> **Linewise copy/cut/paste** (when there is no active selection) follows VS Code exactly: `Ctrl/Cmd+C` copies the whole line including its newline; `Ctrl/Cmd+X` cuts it; `Ctrl/Cmd+V` inserts the line *above* the current line and moves the cursor to the original line — so the document is shifted down by one but the cursor text is unchanged.
+
 ---
 
 ## Recipes
 
 Real-world patterns you can copy and adapt.
 
+### Markdown Editor
+
+Full-featured Markdown editor with custom syntax highlighting via `provideTokens`, appropriate language settings, and bracket pairs disabled (Markdown doesn't use them):
+
+```ts
+import { createEditor } from 'syncline-editor';
+import type { TokenSegment } from 'syncline-editor';
+
+function markdownTokens(line: string): TokenSegment[] {
+  const segs: TokenSegment[] = [];
+
+  if (/^#{1,6}(\s|$)/.test(line))
+    return [{ cls: 'kw', start: 0, end: line.length }];
+  if (/^(`{3,}|~{3,})/.test(line))
+    return [{ cls: 'cmt', start: 0, end: line.length }];
+  if (/^>/.test(line))
+    return [{ cls: 'cmt', start: 0, end: line.length }];
+
+  const rules: [RegExp, TokenSegment['cls']][] = [
+    [/`[^`]+`/g,               'str'],
+    [/\*\*[^*]+\*\*/g,         'kw' ],
+    [/\*[^*]+\*/g,             'typ'],
+    [/~~[^~]+~~/g,             'cmt'],
+    [/!?\[[^\]]*\]\([^)]*\)/g, 'fn' ],
+    [/^([-*+]|\d+\.)\s/,      'dec'],
+  ];
+
+  for (const [re, cls] of rules) {
+    let m: RegExpExecArray | null;
+    const g = new RegExp(re.source, 'g');
+    while ((m = g.exec(line)) !== null)
+      segs.push({ cls, start: m.index, end: m.index + m[0].length });
+  }
+  return segs;
+}
+
+const editor = createEditor(document.getElementById('md-editor')!, {
+  language:         'markdown',
+  theme:            'github-light',
+  provideTokens:    markdownTokens,
+  lineCommentToken: '',          // no line-comment toggle
+  autoClosePairs:   { '(': ')', '[': ']', '`': '`', '*': '*', '_': '_' },
+  bracketMatching:  false,       // not useful in prose
+  emmet:            false,
+  wordWrap:         true,
+  wrapColumn:       80,
+  showMinimap:      false,
+  tabSize:          2,
+  onChange: (value) => preview.setContent(value),
+});
+```
+
+```css
+#md-editor { width: 100%; height: 100vh; }
+```
+
+**Updating the highlighter without recreating the editor:**
+
+```ts
+// Extend the highlighter at runtime — e.g. add frontmatter support
+editor.updateConfig({
+  provideTokens: (line) => {
+    if (/^---$/.test(line)) return [{ cls: 'dec', start: 0, end: 3 }];
+    return markdownTokens(line);
+  },
+});
+```
+
+---
+
 ### Monaco-style Editor Embed
 
 A feature-complete editor inside a fixed-height container, closely resembling a VS Code embed:
@@ -1676,6 +1869,10 @@ import type {
   // Token colours
   TokenColors,
 
+  // Custom tokeniser
+  TokenSegment,
+  TokenClass,
+
   // Themes
   ThemeDefinition,
   ThemeTokens,
@@ -1747,6 +1944,22 @@ interface CompletionContext {
 }
 ```
 
+### `TokenSegment`
+
+```ts
+interface TokenSegment {
+  cls:   TokenClass;  // which token colour class to apply
+  start: number;      // start character index in the line (inclusive)
+  end:   number;      // end character index in the line (exclusive)
+}
+```
+
+### `TokenClass`
+
+```ts
+type TokenClass = 'kw' | 'str' | 'cmt' | 'fn' | 'num' | 'cls' | 'op' | 'typ' | 'dec';
+```
+
 ### `TokenColors`
 
 ```ts