@lexical/code-shiki 0.45.0 → 0.45.1-dev.0

package/dist/LexicalCodeShiki.dev.js

@@ -153,14 +153,12 @@ function mapTokensToLexicalStructure(tokens, diff) {
           text = text.slice(1);
         }
       }
-      const parts = text.split('\t');
-      parts.forEach((part, pidx) => {
-        if (pidx) {
-          nodes.push(lexical.$createTabNode());
-        }
-        if (part !== '') {
+      const style = core.stringifyTokenStyle(token.htmlStyle || core.getTokenStyleObject(token));
+      lexical.tokenizeRawText(text, {
+        linebreak: () => nodes.push(lexical.$createLineBreakNode()),
+        tab: () => nodes.push(lexical.$createTabNode()),
+        text: part => {
           const node = codeCore.$createCodeHighlightNode(part);
-          const style = core.stringifyTokenStyle(token.htmlStyle || core.getTokenStyleObject(token));
           node.setStyle(style);
           nodes.push(node);
         }

package/dist/LexicalCodeShiki.dev.mjs

@@ -8,7 +8,7 @@
 
 import { $isCodeNode, $createCodeHighlightNode, CodeExtension, CodeIndentExtension, DEFAULT_CODE_LANGUAGE, CodeNode, CodeHighlightNode, $plainifyCodeContent, $isCodeHighlightNode, registerCodeIndentation } from '@lexical/code-core';
 import { effect, namedSignals } from '@lexical/extension';
-import { $getNodeByKey, $createLineBreakNode, $createTabNode, defineExtension, safeCast, TextNode, mergeRegister, $isLineBreakNode, $onUpdate, $createTextNode, $getSelection, $isRangeSelection, $isTextNode, $isTabNode } from 'lexical';
+import { $getNodeByKey, $createLineBreakNode, tokenizeRawText, $createTabNode, defineExtension, safeCast, TextNode, mergeRegister, $isLineBreakNode, $onUpdate, $createTextNode, $getSelection, $isRangeSelection, $isTextNode, $isTabNode } from 'lexical';
 import { createHighlighterCoreSync, isSpecialTheme, isSpecialLang, stringifyTokenStyle, getTokenStyleObject } from '@shikijs/core';
 import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript';
 import { bundledLanguagesInfo } from 'shiki/langs';
@@ -151,14 +151,12 @@ function mapTokensToLexicalStructure(tokens, diff) {
           text = text.slice(1);
         }
       }
-      const parts = text.split('\t');
-      parts.forEach((part, pidx) => {
-        if (pidx) {
-          nodes.push($createTabNode());
-        }
-        if (part !== '') {
+      const style = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token));
+      tokenizeRawText(text, {
+        linebreak: () => nodes.push($createLineBreakNode()),
+        tab: () => nodes.push($createTabNode()),
+        text: part => {
           const node = $createCodeHighlightNode(part);
-          const style = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token));
           node.setStyle(style);
           nodes.push(node);
         }

package/dist/LexicalCodeShiki.prod.js

@@ -6,4 +6,4 @@
  *
  */
 
-"use strict";var e=require("@lexical/code-core"),t=require("@lexical/extension"),n=require("lexical"),i=require("@shikijs/core"),o=require("@shikijs/engine-javascript"),r=require("shiki/langs"),d=require("shiki/themes");const s=i.createHighlighterCoreSync({engine:o.createJavaScriptRegexEngine(),langs:[],themes:[]});function g(e){const t=/^diff-([\w-]+)/i.exec(e);return t?t[1]:null}function a(e){const t=g(e)||e;return!!i.isSpecialLang(t)||s.getLoadedLanguages().includes(t)}function l(t,i,o){const d=g(t),l=d||t;if(!a(l)){const d=r.bundledLanguagesInfo.find(e=>e.id===l||e.aliases&&e.aliases.includes(l));if(d)return s.loadLanguage(d.import()).then(()=>{i&&o&&i.update(()=>{const i=n.$getNodeByKey(o);e.$isCodeNode(i)&&i.getLanguage()===t&&!i.getIsSyntaxHighlightSupported()&&i.setIsSyntaxHighlightSupported(!0)})})}}function u(e){const t=e;return!!i.isSpecialTheme(t)||s.getLoadedThemes().includes(t)}function c(t,i,o){if(!u(t)){const r=d.bundledThemesInfo.find(e=>e.id===t);if(r)return s.loadTheme(r.import()).then(()=>{i&&o&&i.update(()=>{const t=n.$getNodeByKey(o);e.$isCodeNode(t)&&t.markDirty()})})}}function h(t,o){const r=/^diff-([\w-]+)/i.exec(o),d=t.getTextContent(),g=s.codeToTokens(d,{lang:r?r[1]:o,theme:t.getTheme()||"poimandres"}),{tokens:a,bg:l,fg:u}=g;let c="";return l&&(c+=`background-color: ${l};`),u&&(c+=`color: ${u};`),t.getStyle()!==c&&t.setStyle(c),function(t,o){const r=[];return t.forEach((t,d)=>{d&&r.push(n.$createLineBreakNode()),t.forEach((t,d)=>{let s=t.content;if(o&&0===d&&s.length>0){const t=["+","-",">","<"," "],n=["inserted","deleted","inserted","deleted","unchanged"],i=t.indexOf(s[0]);-1!==i&&(r.push(e.$createCodeHighlightNode(t[i],n[i])),s=s.slice(1))}s.split("\t").forEach((o,d)=>{if(d&&r.push(n.$createTabNode()),""!==o){const n=e.$createCodeHighlightNode(o),d=i.stringifyTokenStyle(t.htmlStyle||i.getTokenStyleObject(t));n.setStyle(d),r.push(n)}})})}),r}(a,!!r)}const f={$tokenize(t,n){const i=n||this.defaultLanguage;return null===i?e.$plainifyCodeContent(t.getTextContent()):h(t,i)},defaultLanguage:e.DEFAULT_CODE_LANGUAGE,defaultTheme:"one-light"};function p(t,i,o,r){const d=r.getParent();e.$isCodeNode(d)?m(t,i,o,d):e.$isCodeHighlightNode(r)&&r.replace(n.$createTextNode(r.__text))}function C(e,t){const i=t.getElementByKey(e.getKey());if(null===i)return;const o=e.getChildren(),r=o.length;if(r===i.__cachedChildrenLength)return;i.__cachedChildrenLength=r;let d="1",s=1;for(let e=0;e<r;e++)n.$isLineBreakNode(o[e])&&(d+="\n"+ ++s);i.setAttribute("data-gutter",d)}function m(t,i,o,r){const d=r.getKey(),{nodesCurrentlyHighlighting:s}=o;let g=r.getLanguage();g||null===i.defaultLanguage||(g=i.defaultLanguage,r.setLanguage(g));let h=r.getTheme();h||(h=i.defaultTheme,r.setTheme(h));let f=!1;u(h)||(c(h,t,d),g&&(f=!0)),g?a(g)?r.getIsSyntaxHighlightSupported()||r.setIsSyntaxHighlightSupported(!0):(r.getIsSyntaxHighlightSupported()&&r.setIsSyntaxHighlightSupported(!1),l(g,t,d),f=!0):r.getIsSyntaxHighlightSupported()&&r.setIsSyntaxHighlightSupported(!1),f||s.has(d)||(s.add(d),o.didTransform||(o.didTransform=!0,n.$onUpdate(()=>{o.didTransform=!1,s.clear()})),function(t,i){const o=n.$getNodeByKey(t);if(!e.$isCodeNode(o)||!o.isAttached())return;const r=n.$getSelection();if(!n.$isRangeSelection(r))return void i();const d=r.anchor,s=d.offset,g="element"===d.type&&n.$isLineBreakNode(o.getChildAtIndex(d.offset-1));let a=0;if(!g){const e=d.getNode();a=s+e.getPreviousSiblings().reduce((e,t)=>e+t.getTextContentSize(),0)}if(!i())return;if(g)return void d.getNode().select(s,s);o.getChildren().some(e=>{const t=n.$isTextNode(e);if(t||n.$isLineBreakNode(e)){const n=e.getTextContentSize();if(t&&n>=a)return e.select(a,a),!0;a-=n}return!1})}(d,()=>{const t=n.$getNodeByKey(d);if(!e.$isCodeNode(t)||!t.isAttached())return!1;const o=t.getLanguage()||i.defaultLanguage,s=i.$tokenize(t,o??void 0),g=function(e,t){let n=0;for(;n<e.length&&x(e[n],t[n]);)n++;const i=e.length,o=t.length,r=Math.min(i,o)-n;let d=0;for(;d<r;)if(d++,!x(e[i-d],t[o-d])){d--;break}const s=n,g=i-d,a=t.slice(n,o-d);return{from:s,nodesForReplacement:a,to:g}}(t.getChildren(),s),{from:a,to:l,nodesForReplacement:u}=g;return!(a===l&&!u.length)&&(r.splice(a,l-a,u),!0)}))}function x(t,i){return e.$isCodeHighlightNode(t)&&e.$isCodeHighlightNode(i)&&t.__text===i.__text&&t.__highlightType===i.__highlightType&&t.__style===i.__style||n.$isTabNode(t)&&n.$isTabNode(i)||n.$isLineBreakNode(t)&&n.$isLineBreakNode(i)}function y(t,i){const o=[];!0!==t._headless&&o.push(t.registerMutationListener(e.CodeNode,e=>{t.getEditorState().read(()=>{for(const[i,o]of e)if("destroyed"!==o){const e=n.$getNodeByKey(i);null!==e&&C(e,t)}})},{skipInitialization:!1}));const r={didTransform:!1,nodesCurrentlyHighlighting:new Set};return o.push(t.registerNodeTransform(e.CodeNode,m.bind(null,t,i,r)),t.registerNodeTransform(n.TextNode,p.bind(null,t,i,r)),t.registerNodeTransform(e.CodeHighlightNode,p.bind(null,t,i,r))),n.mergeRegister(...o)}const N=n.defineExtension({build:(e,n)=>t.namedSignals(n),config:n.safeCast({disabled:!1,tokenizer:f}),dependencies:[e.CodeExtension,e.CodeIndentExtension],name:"@lexical/code-shiki",register:(e,n,i)=>{const o=i.getOutput();return t.effect(()=>{if(!o.disabled.value)return y(e,o.tokenizer.value)})}}),S=n.defineExtension({config:n.safeCast(f),dependencies:[N],init:(e,t,n)=>{n.getDependency(N).config.tokenizer=t},name:"@lexical/code-shiki/legacy"});exports.CodeHighlighterShikiExtension=S,exports.CodeShikiExtension=N,exports.ShikiTokenizer=f,exports.getCodeLanguageOptions=function(){return r.bundledLanguagesInfo.map(e=>[e.id,e.name])},exports.getCodeThemeOptions=function(){return d.bundledThemesInfo.map(e=>[e.id,e.displayName])},exports.isCodeLanguageLoaded=a,exports.loadCodeLanguage=l,exports.loadCodeTheme=c,exports.normalizeCodeLanguage=function(e){const t=e,n=r.bundledLanguagesInfo.find(e=>e.id===t||e.aliases&&e.aliases.includes(t));return n?n.id:e},exports.registerCodeHighlighting=function(t,i=f){if(!t.hasNodes([e.CodeNode,e.CodeHighlightNode]))throw new Error("CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor");return n.mergeRegister(y(t,i),e.registerCodeIndentation(t))};
+"use strict";var e=require("@lexical/code-core"),t=require("@lexical/extension"),n=require("lexical"),i=require("@shikijs/core"),o=require("@shikijs/engine-javascript"),r=require("shiki/langs"),d=require("shiki/themes");const s=i.createHighlighterCoreSync({engine:o.createJavaScriptRegexEngine(),langs:[],themes:[]});function g(e){const t=/^diff-([\w-]+)/i.exec(e);return t?t[1]:null}function a(e){const t=g(e)||e;return!!i.isSpecialLang(t)||s.getLoadedLanguages().includes(t)}function l(t,i,o){const d=g(t),l=d||t;if(!a(l)){const d=r.bundledLanguagesInfo.find(e=>e.id===l||e.aliases&&e.aliases.includes(l));if(d)return s.loadLanguage(d.import()).then(()=>{i&&o&&i.update(()=>{const i=n.$getNodeByKey(o);e.$isCodeNode(i)&&i.getLanguage()===t&&!i.getIsSyntaxHighlightSupported()&&i.setIsSyntaxHighlightSupported(!0)})})}}function u(e){const t=e;return!!i.isSpecialTheme(t)||s.getLoadedThemes().includes(t)}function c(t,i,o){if(!u(t)){const r=d.bundledThemesInfo.find(e=>e.id===t);if(r)return s.loadTheme(r.import()).then(()=>{i&&o&&i.update(()=>{const t=n.$getNodeByKey(o);e.$isCodeNode(t)&&t.markDirty()})})}}function h(t,o){const r=/^diff-([\w-]+)/i.exec(o),d=t.getTextContent(),g=s.codeToTokens(d,{lang:r?r[1]:o,theme:t.getTheme()||"poimandres"}),{tokens:a,bg:l,fg:u}=g;let c="";return l&&(c+=`background-color: ${l};`),u&&(c+=`color: ${u};`),t.getStyle()!==c&&t.setStyle(c),function(t,o){const r=[];return t.forEach((t,d)=>{d&&r.push(n.$createLineBreakNode()),t.forEach((t,d)=>{let s=t.content;if(o&&0===d&&s.length>0){const t=["+","-",">","<"," "],n=["inserted","deleted","inserted","deleted","unchanged"],i=t.indexOf(s[0]);-1!==i&&(r.push(e.$createCodeHighlightNode(t[i],n[i])),s=s.slice(1))}const g=i.stringifyTokenStyle(t.htmlStyle||i.getTokenStyleObject(t));n.tokenizeRawText(s,{linebreak:()=>r.push(n.$createLineBreakNode()),tab:()=>r.push(n.$createTabNode()),text:t=>{const n=e.$createCodeHighlightNode(t);n.setStyle(g),r.push(n)}})})}),r}(a,!!r)}const f={$tokenize(t,n){const i=n||this.defaultLanguage;return null===i?e.$plainifyCodeContent(t.getTextContent()):h(t,i)},defaultLanguage:e.DEFAULT_CODE_LANGUAGE,defaultTheme:"one-light"};function p(t,i,o,r){const d=r.getParent();e.$isCodeNode(d)?x(t,i,o,d):e.$isCodeHighlightNode(r)&&r.replace(n.$createTextNode(r.__text))}function C(e,t){const i=t.getElementByKey(e.getKey());if(null===i)return;const o=e.getChildren(),r=o.length;if(r===i.__cachedChildrenLength)return;i.__cachedChildrenLength=r;let d="1",s=1;for(let e=0;e<r;e++)n.$isLineBreakNode(o[e])&&(d+="\n"+ ++s);i.setAttribute("data-gutter",d)}function x(t,i,o,r){const d=r.getKey(),{nodesCurrentlyHighlighting:s}=o;let g=r.getLanguage();g||null===i.defaultLanguage||(g=i.defaultLanguage,r.setLanguage(g));let h=r.getTheme();h||(h=i.defaultTheme,r.setTheme(h));let f=!1;u(h)||(c(h,t,d),g&&(f=!0)),g?a(g)?r.getIsSyntaxHighlightSupported()||r.setIsSyntaxHighlightSupported(!0):(r.getIsSyntaxHighlightSupported()&&r.setIsSyntaxHighlightSupported(!1),l(g,t,d),f=!0):r.getIsSyntaxHighlightSupported()&&r.setIsSyntaxHighlightSupported(!1),f||s.has(d)||(s.add(d),o.didTransform||(o.didTransform=!0,n.$onUpdate(()=>{o.didTransform=!1,s.clear()})),function(t,i){const o=n.$getNodeByKey(t);if(!e.$isCodeNode(o)||!o.isAttached())return;const r=n.$getSelection();if(!n.$isRangeSelection(r))return void i();const d=r.anchor,s=d.offset,g="element"===d.type&&n.$isLineBreakNode(o.getChildAtIndex(d.offset-1));let a=0;if(!g){const e=d.getNode();a=s+e.getPreviousSiblings().reduce((e,t)=>e+t.getTextContentSize(),0)}if(!i())return;if(g)return void d.getNode().select(s,s);o.getChildren().some(e=>{const t=n.$isTextNode(e);if(t||n.$isLineBreakNode(e)){const n=e.getTextContentSize();if(t&&n>=a)return e.select(a,a),!0;a-=n}return!1})}(d,()=>{const t=n.$getNodeByKey(d);if(!e.$isCodeNode(t)||!t.isAttached())return!1;const o=t.getLanguage()||i.defaultLanguage,s=i.$tokenize(t,o??void 0),g=function(e,t){let n=0;for(;n<e.length&&m(e[n],t[n]);)n++;const i=e.length,o=t.length,r=Math.min(i,o)-n;let d=0;for(;d<r;)if(d++,!m(e[i-d],t[o-d])){d--;break}const s=n,g=i-d,a=t.slice(n,o-d);return{from:s,nodesForReplacement:a,to:g}}(t.getChildren(),s),{from:a,to:l,nodesForReplacement:u}=g;return!(a===l&&!u.length)&&(r.splice(a,l-a,u),!0)}))}function m(t,i){return e.$isCodeHighlightNode(t)&&e.$isCodeHighlightNode(i)&&t.__text===i.__text&&t.__highlightType===i.__highlightType&&t.__style===i.__style||n.$isTabNode(t)&&n.$isTabNode(i)||n.$isLineBreakNode(t)&&n.$isLineBreakNode(i)}function y(t,i){const o=[];!0!==t._headless&&o.push(t.registerMutationListener(e.CodeNode,e=>{t.getEditorState().read(()=>{for(const[i,o]of e)if("destroyed"!==o){const e=n.$getNodeByKey(i);null!==e&&C(e,t)}})},{skipInitialization:!1}));const r={didTransform:!1,nodesCurrentlyHighlighting:new Set};return o.push(t.registerNodeTransform(e.CodeNode,x.bind(null,t,i,r)),t.registerNodeTransform(n.TextNode,p.bind(null,t,i,r)),t.registerNodeTransform(e.CodeHighlightNode,p.bind(null,t,i,r))),n.mergeRegister(...o)}const N=n.defineExtension({build:(e,n)=>t.namedSignals(n),config:n.safeCast({disabled:!1,tokenizer:f}),dependencies:[e.CodeExtension,e.CodeIndentExtension],name:"@lexical/code-shiki",register:(e,n,i)=>{const o=i.getOutput();return t.effect(()=>{if(!o.disabled.value)return y(e,o.tokenizer.value)})}}),T=n.defineExtension({config:n.safeCast(f),dependencies:[N],init:(e,t,n)=>{n.getDependency(N).config.tokenizer=t},name:"@lexical/code-shiki/legacy"});exports.CodeHighlighterShikiExtension=T,exports.CodeShikiExtension=N,exports.ShikiTokenizer=f,exports.getCodeLanguageOptions=function(){return r.bundledLanguagesInfo.map(e=>[e.id,e.name])},exports.getCodeThemeOptions=function(){return d.bundledThemesInfo.map(e=>[e.id,e.displayName])},exports.isCodeLanguageLoaded=a,exports.loadCodeLanguage=l,exports.loadCodeTheme=c,exports.normalizeCodeLanguage=function(e){const t=e,n=r.bundledLanguagesInfo.find(e=>e.id===t||e.aliases&&e.aliases.includes(t));return n?n.id:e},exports.registerCodeHighlighting=function(t,i=f){if(!t.hasNodes([e.CodeNode,e.CodeHighlightNode]))throw new Error("CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor");return n.mergeRegister(y(t,i),e.registerCodeIndentation(t))};

package/dist/LexicalCodeShiki.prod.mjs

@@ -6,4 +6,4 @@
  *
  */
 
-import{$isCodeNode as e,$createCodeHighlightNode as t,CodeExtension as n,CodeIndentExtension as i,DEFAULT_CODE_LANGUAGE as o,CodeNode as r,CodeHighlightNode as s,$plainifyCodeContent as l,$isCodeHighlightNode as d,registerCodeIndentation as g}from"@lexical/code-core";import{effect as a,namedSignals as u}from"@lexical/extension";import{$getNodeByKey as c,$createLineBreakNode as f,$createTabNode as h,defineExtension as p,safeCast as m,TextNode as y,mergeRegister as x,$isLineBreakNode as S,$onUpdate as T,$createTextNode as _,$getSelection as k,$isRangeSelection as L,$isTextNode as C,$isTabNode as H}from"lexical";import{createHighlighterCoreSync as b,isSpecialTheme as I,isSpecialLang as N,stringifyTokenStyle as z,getTokenStyleObject as v}from"@shikijs/core";import{createJavaScriptRegexEngine as E}from"@shikijs/engine-javascript";import{bundledLanguagesInfo as w}from"shiki/langs";import{bundledThemesInfo as A}from"shiki/themes";const $=b({engine:E(),langs:[],themes:[]});function j(e){const t=/^diff-([\w-]+)/i.exec(e);return t?t[1]:null}function K(e){const t=j(e)||e;return!!N(t)||$.getLoadedLanguages().includes(t)}function P(t,n,i){const o=j(t),r=o||t;if(!K(r)){const o=w.find(e=>e.id===r||e.aliases&&e.aliases.includes(r));if(o)return $.loadLanguage(o.import()).then(()=>{n&&i&&n.update(()=>{const n=c(i);e(n)&&n.getLanguage()===t&&!n.getIsSyntaxHighlightSupported()&&n.setIsSyntaxHighlightSupported(!0)})})}}function D(e){const t=e;return!!I(t)||$.getLoadedThemes().includes(t)}function F(t,n,i){if(!D(t)){const o=A.find(e=>e.id===t);if(o)return $.loadTheme(o.import()).then(()=>{n&&i&&n.update(()=>{const t=c(i);e(t)&&t.markDirty()})})}}function M(){return w.map(e=>[e.id,e.name])}function O(){return A.map(e=>[e.id,e.displayName])}function R(e){const t=e,n=w.find(e=>e.id===t||e.aliases&&e.aliases.includes(t));return n?n.id:e}function B(e,n){const i=/^diff-([\w-]+)/i.exec(n),o=e.getTextContent(),r=$.codeToTokens(o,{lang:i?i[1]:n,theme:e.getTheme()||"poimandres"}),{tokens:s,bg:l,fg:d}=r;let g="";return l&&(g+=`background-color: ${l};`),d&&(g+=`color: ${d};`),e.getStyle()!==g&&e.setStyle(g),function(e,n){const i=[];return e.forEach((e,o)=>{o&&i.push(f()),e.forEach((e,o)=>{let r=e.content;if(n&&0===o&&r.length>0){const e=["+","-",">","<"," "],n=["inserted","deleted","inserted","deleted","unchanged"],o=e.indexOf(r[0]);-1!==o&&(i.push(t(e[o],n[o])),r=r.slice(1))}r.split("\t").forEach((n,o)=>{if(o&&i.push(h()),""!==n){const o=t(n),r=z(e.htmlStyle||v(e));o.setStyle(r),i.push(o)}})})}),i}(s,!!i)}const q={$tokenize(e,t){const n=t||this.defaultLanguage;return null===n?l(e.getTextContent()):B(e,n)},defaultLanguage:o,defaultTheme:"one-light"};function G(t,n,i,o){const r=o.getParent();e(r)?Q(t,n,i,r):d(o)&&o.replace(_(o.__text))}function J(e,t){const n=t.getElementByKey(e.getKey());if(null===n)return;const i=e.getChildren(),o=i.length;if(o===n.__cachedChildrenLength)return;n.__cachedChildrenLength=o;let r="1",s=1;for(let e=0;e<o;e++)S(i[e])&&(r+="\n"+ ++s);n.setAttribute("data-gutter",r)}function Q(t,n,i,o){const r=o.getKey(),{nodesCurrentlyHighlighting:s}=i;let l=o.getLanguage();l||null===n.defaultLanguage||(l=n.defaultLanguage,o.setLanguage(l));let d=o.getTheme();d||(d=n.defaultTheme,o.setTheme(d));let g=!1;D(d)||(F(d,t,r),l&&(g=!0)),l?K(l)?o.getIsSyntaxHighlightSupported()||o.setIsSyntaxHighlightSupported(!0):(o.getIsSyntaxHighlightSupported()&&o.setIsSyntaxHighlightSupported(!1),P(l,t,r),g=!0):o.getIsSyntaxHighlightSupported()&&o.setIsSyntaxHighlightSupported(!1),g||s.has(r)||(s.add(r),i.didTransform||(i.didTransform=!0,T(()=>{i.didTransform=!1,s.clear()})),function(t,n){const i=c(t);if(!e(i)||!i.isAttached())return;const o=k();if(!L(o))return void n();const r=o.anchor,s=r.offset,l="element"===r.type&&S(i.getChildAtIndex(r.offset-1));let d=0;if(!l){const e=r.getNode();d=s+e.getPreviousSiblings().reduce((e,t)=>e+t.getTextContentSize(),0)}if(!n())return;if(l)return void r.getNode().select(s,s);i.getChildren().some(e=>{const t=C(e);if(t||S(e)){const n=e.getTextContentSize();if(t&&n>=d)return e.select(d,d),!0;d-=n}return!1})}(r,()=>{const t=c(r);if(!e(t)||!t.isAttached())return!1;const i=t.getLanguage()||n.defaultLanguage,s=n.$tokenize(t,i??void 0),l=function(e,t){let n=0;for(;n<e.length&&U(e[n],t[n]);)n++;const i=e.length,o=t.length,r=Math.min(i,o)-n;let s=0;for(;s<r;)if(s++,!U(e[i-s],t[o-s])){s--;break}const l=n,d=i-s,g=t.slice(n,o-s);return{from:l,nodesForReplacement:g,to:d}}(t.getChildren(),s),{from:d,to:g,nodesForReplacement:a}=l;return!(d===g&&!a.length)&&(o.splice(d,g-d,a),!0)}))}function U(e,t){return d(e)&&d(t)&&e.__text===t.__text&&e.__highlightType===t.__highlightType&&e.__style===t.__style||H(e)&&H(t)||S(e)&&S(t)}function V(e,t){const n=[];!0!==e._headless&&n.push(e.registerMutationListener(r,t=>{e.getEditorState().read(()=>{for(const[n,i]of t)if("destroyed"!==i){const t=c(n);null!==t&&J(t,e)}})},{skipInitialization:!1}));const i={didTransform:!1,nodesCurrentlyHighlighting:new Set};return n.push(e.registerNodeTransform(r,Q.bind(null,e,t,i)),e.registerNodeTransform(y,G.bind(null,e,t,i)),e.registerNodeTransform(s,G.bind(null,e,t,i))),x(...n)}function W(e,t=q){if(!e.hasNodes([r,s]))throw new Error("CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor");return x(V(e,t),g(e))}const X=p({build:(e,t)=>u(t),config:m({disabled:!1,tokenizer:q}),dependencies:[n,i],name:"@lexical/code-shiki",register:(e,t,n)=>{const i=n.getOutput();return a(()=>{if(!i.disabled.value)return V(e,i.tokenizer.value)})}}),Y=p({config:m(q),dependencies:[X],init:(e,t,n)=>{n.getDependency(X).config.tokenizer=t},name:"@lexical/code-shiki/legacy"});export{Y as CodeHighlighterShikiExtension,X as CodeShikiExtension,q as ShikiTokenizer,M as getCodeLanguageOptions,O as getCodeThemeOptions,K as isCodeLanguageLoaded,P as loadCodeLanguage,F as loadCodeTheme,R as normalizeCodeLanguage,W as registerCodeHighlighting};
+import{$isCodeNode as e,$createCodeHighlightNode as t,CodeExtension as n,CodeIndentExtension as i,DEFAULT_CODE_LANGUAGE as o,CodeNode as r,CodeHighlightNode as s,$plainifyCodeContent as l,$isCodeHighlightNode as d,registerCodeIndentation as g}from"@lexical/code-core";import{effect as a,namedSignals as u}from"@lexical/extension";import{$getNodeByKey as c,$createLineBreakNode as h,tokenizeRawText as f,$createTabNode as p,defineExtension as m,safeCast as y,TextNode as x,mergeRegister as S,$isLineBreakNode as T,$onUpdate as _,$createTextNode as k,$getSelection as L,$isRangeSelection as C,$isTextNode as b,$isTabNode as H}from"lexical";import{createHighlighterCoreSync as I,isSpecialTheme as N,isSpecialLang as z,stringifyTokenStyle as v,getTokenStyleObject as w}from"@shikijs/core";import{createJavaScriptRegexEngine as E}from"@shikijs/engine-javascript";import{bundledLanguagesInfo as A}from"shiki/langs";import{bundledThemesInfo as $}from"shiki/themes";const j=I({engine:E(),langs:[],themes:[]});function K(e){const t=/^diff-([\w-]+)/i.exec(e);return t?t[1]:null}function P(e){const t=K(e)||e;return!!z(t)||j.getLoadedLanguages().includes(t)}function D(t,n,i){const o=K(t),r=o||t;if(!P(r)){const o=A.find(e=>e.id===r||e.aliases&&e.aliases.includes(r));if(o)return j.loadLanguage(o.import()).then(()=>{n&&i&&n.update(()=>{const n=c(i);e(n)&&n.getLanguage()===t&&!n.getIsSyntaxHighlightSupported()&&n.setIsSyntaxHighlightSupported(!0)})})}}function F(e){const t=e;return!!N(t)||j.getLoadedThemes().includes(t)}function M(t,n,i){if(!F(t)){const o=$.find(e=>e.id===t);if(o)return j.loadTheme(o.import()).then(()=>{n&&i&&n.update(()=>{const t=c(i);e(t)&&t.markDirty()})})}}function O(){return A.map(e=>[e.id,e.name])}function R(){return $.map(e=>[e.id,e.displayName])}function B(e){const t=e,n=A.find(e=>e.id===t||e.aliases&&e.aliases.includes(t));return n?n.id:e}function q(e,n){const i=/^diff-([\w-]+)/i.exec(n),o=e.getTextContent(),r=j.codeToTokens(o,{lang:i?i[1]:n,theme:e.getTheme()||"poimandres"}),{tokens:s,bg:l,fg:d}=r;let g="";return l&&(g+=`background-color: ${l};`),d&&(g+=`color: ${d};`),e.getStyle()!==g&&e.setStyle(g),function(e,n){const i=[];return e.forEach((e,o)=>{o&&i.push(h()),e.forEach((e,o)=>{let r=e.content;if(n&&0===o&&r.length>0){const e=["+","-",">","<"," "],n=["inserted","deleted","inserted","deleted","unchanged"],o=e.indexOf(r[0]);-1!==o&&(i.push(t(e[o],n[o])),r=r.slice(1))}const s=v(e.htmlStyle||w(e));f(r,{linebreak:()=>i.push(h()),tab:()=>i.push(p()),text:e=>{const n=t(e);n.setStyle(s),i.push(n)}})})}),i}(s,!!i)}const G={$tokenize(e,t){const n=t||this.defaultLanguage;return null===n?l(e.getTextContent()):q(e,n)},defaultLanguage:o,defaultTheme:"one-light"};function J(t,n,i,o){const r=o.getParent();e(r)?U(t,n,i,r):d(o)&&o.replace(k(o.__text))}function Q(e,t){const n=t.getElementByKey(e.getKey());if(null===n)return;const i=e.getChildren(),o=i.length;if(o===n.__cachedChildrenLength)return;n.__cachedChildrenLength=o;let r="1",s=1;for(let e=0;e<o;e++)T(i[e])&&(r+="\n"+ ++s);n.setAttribute("data-gutter",r)}function U(t,n,i,o){const r=o.getKey(),{nodesCurrentlyHighlighting:s}=i;let l=o.getLanguage();l||null===n.defaultLanguage||(l=n.defaultLanguage,o.setLanguage(l));let d=o.getTheme();d||(d=n.defaultTheme,o.setTheme(d));let g=!1;F(d)||(M(d,t,r),l&&(g=!0)),l?P(l)?o.getIsSyntaxHighlightSupported()||o.setIsSyntaxHighlightSupported(!0):(o.getIsSyntaxHighlightSupported()&&o.setIsSyntaxHighlightSupported(!1),D(l,t,r),g=!0):o.getIsSyntaxHighlightSupported()&&o.setIsSyntaxHighlightSupported(!1),g||s.has(r)||(s.add(r),i.didTransform||(i.didTransform=!0,_(()=>{i.didTransform=!1,s.clear()})),function(t,n){const i=c(t);if(!e(i)||!i.isAttached())return;const o=L();if(!C(o))return void n();const r=o.anchor,s=r.offset,l="element"===r.type&&T(i.getChildAtIndex(r.offset-1));let d=0;if(!l){const e=r.getNode();d=s+e.getPreviousSiblings().reduce((e,t)=>e+t.getTextContentSize(),0)}if(!n())return;if(l)return void r.getNode().select(s,s);i.getChildren().some(e=>{const t=b(e);if(t||T(e)){const n=e.getTextContentSize();if(t&&n>=d)return e.select(d,d),!0;d-=n}return!1})}(r,()=>{const t=c(r);if(!e(t)||!t.isAttached())return!1;const i=t.getLanguage()||n.defaultLanguage,s=n.$tokenize(t,i??void 0),l=function(e,t){let n=0;for(;n<e.length&&V(e[n],t[n]);)n++;const i=e.length,o=t.length,r=Math.min(i,o)-n;let s=0;for(;s<r;)if(s++,!V(e[i-s],t[o-s])){s--;break}const l=n,d=i-s,g=t.slice(n,o-s);return{from:l,nodesForReplacement:g,to:d}}(t.getChildren(),s),{from:d,to:g,nodesForReplacement:a}=l;return!(d===g&&!a.length)&&(o.splice(d,g-d,a),!0)}))}function V(e,t){return d(e)&&d(t)&&e.__text===t.__text&&e.__highlightType===t.__highlightType&&e.__style===t.__style||H(e)&&H(t)||T(e)&&T(t)}function W(e,t){const n=[];!0!==e._headless&&n.push(e.registerMutationListener(r,t=>{e.getEditorState().read(()=>{for(const[n,i]of t)if("destroyed"!==i){const t=c(n);null!==t&&Q(t,e)}})},{skipInitialization:!1}));const i={didTransform:!1,nodesCurrentlyHighlighting:new Set};return n.push(e.registerNodeTransform(r,U.bind(null,e,t,i)),e.registerNodeTransform(x,J.bind(null,e,t,i)),e.registerNodeTransform(s,J.bind(null,e,t,i))),S(...n)}function X(e,t=G){if(!e.hasNodes([r,s]))throw new Error("CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor");return S(W(e,t),g(e))}const Y=m({build:(e,t)=>u(t),config:y({disabled:!1,tokenizer:G}),dependencies:[n,i],name:"@lexical/code-shiki",register:(e,t,n)=>{const i=n.getOutput();return a(()=>{if(!i.disabled.value)return W(e,i.tokenizer.value)})}}),Z=m({config:y(G),dependencies:[Y],init:(e,t,n)=>{n.getDependency(Y).config.tokenizer=t},name:"@lexical/code-shiki/legacy"});export{Z as CodeHighlighterShikiExtension,Y as CodeShikiExtension,G as ShikiTokenizer,O as getCodeLanguageOptions,R as getCodeThemeOptions,P as isCodeLanguageLoaded,D as loadCodeLanguage,M as loadCodeTheme,B as normalizeCodeLanguage,X as registerCodeHighlighting};

package/package.json

@@ -8,7 +8,7 @@
     "code"
   ],
   "license": "MIT",
-  "version": "0.45.0",
+  "version": "0.45.1-dev.0",
   "main": "./dist/LexicalCodeShiki.js",
   "types": "./dist/index.d.ts",
   "dependencies": {
@@ -17,9 +17,9 @@
     "@shikijs/langs": "^4.0.2",
     "@shikijs/themes": "^4.0.2",
     "shiki": "^4.0.2",
-    "@lexical/code-core": "0.45.0",
-    "lexical": "0.45.0",
-    "@lexical/extension": "0.45.0"
+    "@lexical/extension": "0.45.1-dev.0",
+    "@lexical/code-core": "0.45.1-dev.0",
+    "lexical": "0.45.1-dev.0"
   },
   "repository": {
     "type": "git",

package/src/FacadeShiki.ts

@@ -19,7 +19,12 @@ import {
   stringifyTokenStyle,
 } from '@shikijs/core';
 import {createJavaScriptRegexEngine} from '@shikijs/engine-javascript';
-import {$createLineBreakNode, $createTabNode, $getNodeByKey} from 'lexical';
+import {
+  $createLineBreakNode,
+  $createTabNode,
+  $getNodeByKey,
+  tokenizeRawText,
+} from 'lexical';
 import {bundledLanguagesInfo} from 'shiki/langs';
 import {bundledThemesInfo} from 'shiki/themes';
 
@@ -195,19 +200,17 @@ function mapTokensToLexicalStructure(
         }
       }
 
-      const parts = text.split('\t');
-      parts.forEach((part: string, pidx: number) => {
-        if (pidx) {
-          nodes.push($createTabNode());
-        }
-        if (part !== '') {
+      const style = stringifyTokenStyle(
+        token.htmlStyle || getTokenStyleObject(token),
+      );
+      tokenizeRawText(text, {
+        linebreak: () => nodes.push($createLineBreakNode()),
+        tab: () => nodes.push($createTabNode()),
+        text: (part: string) => {
           const node = $createCodeHighlightNode(part);
-          const style = stringifyTokenStyle(
-            token.htmlStyle || getTokenStyleObject(token),
-          );
           node.setStyle(style);
           nodes.push(node);
-        }
+        },
       });
     });
   });