@coze-editor/lezer-parser-jinja2 0.1.0-alpha.09ffeb

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 coze-dev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@coze-editor/lezer-parser-jinja2",
+  "version": "0.1.0-alpha.09ffeb",
+  "description": "lezer-parser-jinja2",
+  "license": "MIT",
+  "author": "fengzilong",
+  "maintainers": [],
+  "sideEffects": [
+    "**/*.css",
+    "**/*.less",
+    "**/*.sass",
+    "**/*.scss"
+  ],
+  "main": "./src/index.js",
+  "module": "./src/index.js",
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "build": "lezer-generator src/jinja2.grammar -o src/index.js",
+    "jinja:test": "npm run build & npx tsx src/test.ts",
+    "lint": "eslint"
+  },
+  "dependencies": {
+    "@lezer/highlight": "~1.2.0",
+    "@lezer/lr": "^1.4.0"
+  },
+  "devDependencies": {
+    "@coze-arch/ts-config": "workspace:*",
+    "@coze-editor/eslint-config": "workspace:*",
+    "@lezer-unofficial/printer": "^1.0.1",
+    "@lezer/generator": "^1.7.0",
+    "@types/node": "^22",
+    "eslint": "9.14.0",
+    "typescript": "^5.8.2"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  }
+}

package/src/highlight.js

@@ -0,0 +1,9 @@
+//  Copyright (c) 2025 coze-dev
+//  SPDX-License-Identifier: MIT
+
+import { styleTags, tags as t } from '@lezer/highlight'
+
+export const jinjaHighlighting = styleTags({
+  'JinjaExpressionStart JinjaExpressionEnd JinjaStatementStart JinjaStatementEnd': t.angleBracket,
+  'JinjaCommentStart JinjaCommentContent JinjaCommentEnd': t.blockComment,
+})

package/src/index.d.ts

@@ -0,0 +1,10 @@
+//  Copyright (c) 2025 coze-dev
+//  SPDX-License-Identifier: MIT
+
+import { LRParser } from '@lezer/lr'
+
+declare const parser: LRParser
+
+export {
+  parser,
+}

package/src/index.js

@@ -0,0 +1,21 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+import {LRParser} from "@lezer/lr"
+import {tokenizeSingleQuoteString, tokenizeDoubleQuoteString, tokenizeExpression, tokenizeStatement, tokenizeComment, tokenizeRawText, tokenizeText} from "./tokens"
+import {jinjaHighlighting} from "./highlight"
+const spec_JinjaIdentifier = {__proto__:null,if:76, elif:78, else:80, endif:82, is:84, not:86, set:88, for:90, in:92, endfor:94, extends:96, macro:98, endmacro:100, block:102, endblock:104, call:106, endcall:108, include:110, from:112, import:114, as:116, raw:42, endraw:46}
+export const parser = LRParser.deserialize({
+  version: 14,
+  states: "&`QYO%WOOOhO&zO'#CcO#jQ#xO'#CfO#qO$fO'#CnO%pQ,^O'#CtOOO!b'#DR'#DROOO!b'#Cu'#CuQYO%WOOOOO`'#Cv'#CvO%zO&zO,58}OOO!b,58},58}O&SOPO'#CjO&[OQO'#CjOOQS'#Cw'#CwO&dQ#xO,59QOOO!b,59Q,59QOOQ['#Ch'#ChO&kQ7[O'#CwOOO!b,59Y,59YO#tO#tO,59YO&pQ#tO'#CrOOQW'#Cx'#CxO&uQ,^O,59`OOO!b,59`,59`O&|Q#|O,59ZOOO!b-E6s-E6sOOO`-E6t-E6tOOO!b1G.i1G.iOOQ[,59U,59UO'ROPO,59UO'WOQO,59UOOQS-E6u-E6uOOO!b1G.l1G.lOOQS,59c,59cOOO!b1G.t1G.tO']Q#|O,59^OOQW-E6v-E6vOOO!b1G.z1G.zOOOp1G.u1G.uOOQ[1G.p1G.pOOO!b1G.x1G.x",
+  stateData: "'m~OsOStOS~OTTOWPOZQOdSO~ORYOXWO~O]]O_]Oa]Ov`Ow`Ox`Oy`Oz`O{`O|`O}`O!O`O!P`O!Q`O!R`O!S`O!T`O!U`O!V`O!W`O!X`O!Y`O!Z`O![`O!]ZO!^[O!_aO!`]O~OP_O~PpOScOddO~O]eO_eOv`Ow`Ox`Oy`Oz`O{`O|`O}`O!O`O!P`O!Q`O!R`O!S`O!T`O!U`O!V`O!W`O!X`O!Y`O!Z`O![`O!]ZO!^[O!`eO!aeO~OQgOehO~P#yORkOXWO~OnmOolO~OpnOqlO~OPpO~PpO`qO~OgsO~OQuO~P#yOQvO~OowO~OqwO~OQxO~Os`dt]_!`!aa]~",
+  goto: "!{vPPPPPPPwPPwP{P{PPPw!TPP!XPw!_!e!k!qPPPPPPPP!wTTOVS]Q^TeSfTROVQbRRrcQVORiVQXPRjXQ^QRo^QfSRtfTUOV",
+  nodeNames: "⚠ JinjaExpressionEnd JinjaStatementEnd JinjaCommentEnd RawText JinjaText JinjaProgram JinjaComment JinjaCommentStart JinjaCommentContent JinjaExpression JinjaExpressionStart JinjaKeyword JinjaIdentifier JinjaStringLiteral JinjaNumberLiteral JinjaFilterName JinjaExpressionUnknownContent JinjaRawBlock JinjaRawOpenStatement JinjaStatementStart JinjaKeyword JinjaRawCloseStatement JinjaKeyword JinjaStatement",
+  maxTerm: 63,
+  propSources: [jinjaHighlighting],
+  skippedNodes: [0],
+  repeatNodeCount: 4,
+  tokenData: "!)f_R!^OX$}XY;_YZ>QZ[;_[]$}]^?e^p$}pq;_qrAOrsBcst'|tuCPuvEUvw$}wxFZxyAOyzAOz{AO{|Fw|}AO}!OFw!O!PGv!P!QAO!Q![KO![!]AO!]!^$}!^!_AO!_!`AO!`!aAO!a!bAO!b!c$}!c!}CP!}#OAO#O#P$}#P#QAO#Q#RAO#R#SCP#S#T$}#T#oCP#o#pLr#p#q!$}#q#r8f#r#sAO#s$f$}$f$g;_$g#BYCP#BY#BZ!&Y#BZ$ISCP$IS$I_!&Y$I_$JTCP$JT$JU!&Y$JU$KVCP$KV$KW!&Y$KW&FUCP&FU&FV!&Y&FV;'SCP;'S;=`EO<%l?HTCP?HT?HU!&Y?HUOCPV%WbXQ!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}Q&eVXQOs&`st&zt#o&`#o#p'g#p;'S&`;'S;=`'a<%lO&`Q&}UO#q&`#r;'S&`;'S;=`'a<%l~&`~O&`~~&`Q'dP;=`<%l&`Q'jUOs&`t;'S&`;'S;=`'a<%l~&`~O&`~~&`V(Tb!aSaPOr$}rs&`su$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#q*`#q#r5W#r;'S$};'S;=`5Q<%l~$}~O$}~~&`V)d^XQaPOr$}rs*`st'|tw$}wx*`x#o$}#o#p-k#p#q*`#q#r0V#r;'S$};'S;=`5Q<%l~$}~O$}~~.rU*g`XQ!aSOr*`rs&`st+itu*`uv,svw*`wx&`x{*`{|&`|}*`}!O&`!O#o*`#o#p-k#p;'S*`;'S;=`0P<%lO*`U+na!aSOr*`rs&`su*`uv,svw*`wx&`x{*`{|&`|}*`}!O&`!O#q*`#q#r.r#r;'S*`;'S;=`0P<%l~*`~O*`~~&`U,xZXQOs*`st+it#o*`#o#p-k#p#q*`#q#r&`#r;'S*`;'S;=`0P<%l~*`~O*`~~.rU-p`!aSOr*`rs&`st.rtu*`uv,svw*`wx&`x{*`{|&`|}*`}!O&`!O;'S*`;'S;=`0P<%l~*`~O*`~~&`S.wX!aSOr.rsu.ruv/dvw.rx{.r|}.r!O;'S.r;'S;=`/y<%lO.rS/gUO#q.r#r;'S.r;'S;=`/y<%l~.r~O.r~~.rS/|P;=`<%l.rU0SP;=`<%l*`R0[ZXQOs0}st1zt#o0}#o#p4S#p#q0}#q#r&`#r;'S0};'S;=`3|<%l~0}~O0}~~3[R1U[XQaPOr0}rs&`st1ztw0}wx&`x#o0}#o#p'g#p#q&`#q#r0V#r;'S0};'S;=`3|<%lO0}R2P[aPOr0}rs&`sw0}wx&`x#o0}#o#q&`#q#r2u#r;'S0};'S;=`3|<%l~0}~O0}~~&`P2xUO#q3[#r;'S3[;'S;=`3v<%l~3[~O3[~~3[P3aVaPOr3[sw3[x#o3[#q#r2u#r;'S3[;'S;=`3v<%lO3[P3yP;=`<%l3[R4PP;=`<%l0}R4X]aPOr0}rs&`st3[tw0}wx&`x#o0}#o#q&`#q#r0V#r;'S0};'S;=`3|<%l~0}~O0}~~&`V5TP;=`<%l$}T5]a!aSOr6brs3[su6buv7evw6bwx3[x{6b{|3[|}6b}!O3[!O#q6b#q#r.r#r;'S6b;'S;=`8`<%l~6b~O6b~~3[T6i^!aSaPOr6bsu6buv7evw6bx{6b{|3[|}6b}!O3[!O#o6b#o#q.r#q#r5W#r;'S6b;'S;=`8`<%lO6bT7j[aPOr6brs.rsw6bwx.rx#o6b#o#q.r#q#r2u#r;'S6b;'S;=`8`<%l~6b~O6b~~.rT8cP;=`<%l6bV8mdXQ!aSOr$}rs0}st'|tu$}uv)]vw$}wx0}x{$}{|0}|}$}}!O0}!O#o$}#o#p9{#p#q$}#q#r*`#r;'S$};'S;=`5Q<%l~$}~O$}~~3[V:Sc!aSaPOr$}rs&`st6btu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#q*`#q#r8f#r;'S$};'S;=`5Q<%l~$}~O$}~~&`_;jvXQs]!aSaPOX$}XY;_YZ$}Z[;_[p$}pq;_qr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#p-k#p#q*`#q#r8f#r$f$}$f$g;_$g#BY$}#BY#BZ;_#BZ$IS$}$IS$I_;_$I_$JT$}$JT$JU;_$JU$KV$}$KV$KW;_$KW&FU$}&FU&FV;_&FV;'S$};'S;=`5Q<%l?HT$}?HT?HU;_?HUO$}_>]bXQt]!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}_?pdXQt]!aSaPOY$}YZ>QZr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}VAZbXQ!`T!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}VBjV!^TXQOs&`st&zt#o&`#o#p'g#p;'S&`;'S;=`'a<%lO&`_C^jXQ`W]T!aSaPOr$}rs&`st'|tuCPuv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O!Q$}!Q![CP![!c$}!c!}CP!}#R$}#R#SCP#S#T$}#T#oCP#o#p-k#p#q*`#q#r8f#r$g$}$g;'SCP;'S;=`EO<%lOCP_ERP;=`<%lCPVE_^XQ!`TaPOr$}rs*`st'|tw$}wx*`x#o$}#o#p-k#p#q*`#q#r0V#r;'S$};'S;=`5Q<%l~$}~O$}~~.rVFbV!]TXQOs&`st&zt#o&`#o#p'g#p;'S&`;'S;=`'a<%lO&`VGQ[XQ!`TaPOr0}rs&`st1ztw0}wx&`x#o0}#o#p'g#p#q&`#q#r0V#r;'S0};'S;=`3|<%lO0}VHPdXQ!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O!Q$}!Q![I_![#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}VIjfXQ_T!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O!Q$}!Q![I_![#R$}#R#SI_#S#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}VKZgXQ_T!aSaPOr$}rs&`st'|tu$}uv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O!PI_!P!Q$}!Q![KO![#R$}#R#SKO#S#o$}#o#p-k#p#q*`#q#r8f#r;'S$};'S;=`5Q<%lO$}VLwb!aSOr*`rs&`stNPtu*`uvNsvw*`wx&`x{*`{|&`|}*`}!O&`!O#o*`#o#p!#U#p;'S*`;'S;=`0P<%l~*`~O*`~~&`TNWXWP!aSOr.rsu.ruv/dvw.rx{.r|}.r!O;'S.r;'S;=`/y<%lO.rVNz_XQdPOs*`st+it{*`{|! y|}*`}!O! y!O#o*`#o#p-k#p#q*`#q#r&`#r;'S*`;'S;=`0P<%l~*`~O*`~~.rV!!S`XQdP!aSOr*`rs&`st+itu*`uv,svw*`wx&`x{*`{|&`|}*`}!O&`!O#o*`#o#p-k#p;'S*`;'S;=`0P<%lO*`V!#_`XQZP!aSOr*`rs&`st+itu*`uv,svw*`wx&`x{*`{|!$a|}*`}!O!$a!O#o*`#o#p-k#p;'S*`;'S;=`0P<%lO*`R!$hVXQZPOs&`st&zt#o&`#o#p'g#p;'S&`;'S;=`'a<%lO&`V!%W`!_PXQ!aSOr*`rs&`st+itu*`uv,svw*`wx&`x{*`{|&`|}*`}!O&`!O#o*`#o#p-k#p;'S*`;'S;=`0P<%lO*`_!&i}XQs]`W]T!aSaPOX$}XY;_YZ$}Z[;_[p$}pq;_qr$}rs&`st'|tuCPuv)]vw$}wx&`x{$}{|0}|}$}}!O0}!O!Q$}!Q![CP![!c$}!c!}CP!}#R$}#R#SCP#S#T$}#T#oCP#o#p-k#p#q*`#q#r8f#r$f$}$f$g;_$g#BYCP#BY#BZ!&Y#BZ$ISCP$IS$I_!&Y$I_$JTCP$JT$JU!&Y$JU$KVCP$KV$KW!&Y$KW&FUCP&FU&FV!&Y&FV;'SCP;'S;=`EO<%l?HTCP?HT?HU!&Y?HUOCP",
+  tokenizers: [tokenizeSingleQuoteString, tokenizeDoubleQuoteString, tokenizeExpression, tokenizeStatement, tokenizeComment, tokenizeRawText, tokenizeText, 0, 1, 2, 3],
+  topRules: {"JinjaProgram":[0,6]},
+  specialized: [{term: 13, get: (value) => spec_JinjaIdentifier[value] || -1}],
+  tokenPrec: 294
+})

package/src/index.terms.js

@@ -0,0 +1,28 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const
+  stringContentSingleQuote = 30,
+  stringEndSingleQuote = 31,
+  stringContentDoubleQuote = 32,
+  stringEndDoubleQuote = 33,
+  JinjaExpressionEnd = 1,
+  JinjaStatementEnd = 2,
+  JinjaCommentEnd = 3,
+  RawText = 4,
+  JinjaText = 5,
+  JinjaProgram = 6,
+  JinjaComment = 7,
+  JinjaCommentStart = 8,
+  JinjaCommentContent = 9,
+  JinjaExpression = 10,
+  JinjaExpressionStart = 11,
+  JinjaKeyword = 23,
+  JinjaIdentifier = 13,
+  JinjaStringLiteral = 14,
+  JinjaNumberLiteral = 15,
+  JinjaFilterName = 16,
+  JinjaExpressionUnknownContent = 17,
+  JinjaRawBlock = 18,
+  JinjaRawOpenStatement = 19,
+  JinjaStatementStart = 20,
+  JinjaRawCloseStatement = 22,
+  JinjaStatement = 24

package/src/jinja2.grammar

@@ -0,0 +1,175 @@
+@top JinjaProgram { entity* }
+
+entity {
+  JinjaText |
+  JinjaComment |
+  JinjaExpression |
+  JinjaRawBlock |
+  JinjaStatement
+}
+
+JinjaComment {
+  JinjaCommentStart JinjaCommentContent* JinjaCommentEnd
+}
+
+@skip { spaces | newline } {
+  JinjaExpression {
+    JinjaExpressionStart (JinjaKeyword | JinjaIdentifier | JinjaStringLiteral | JinjaNumberLiteral | jinjaFilter | jinjaSign | JinjaExpressionUnknownContent)* JinjaExpressionEnd
+  }
+
+  jinjaFilter {
+    "|"
+    JinjaFilterName
+  }
+
+  JinjaStatement {
+    JinjaStatementStart (JinjaKeyword | JinjaIdentifier | JinjaStringLiteral | JinjaNumberLiteral | jinjaSign | jinjaStatementUnknownContent)* JinjaStatementEnd
+  }
+
+  JinjaRawOpenStatement {
+    JinjaStatementStart namedKw<"raw", JinjaKeyword> JinjaStatementEnd
+  }
+
+  JinjaRawCloseStatement {
+    JinjaStatementStart namedKw<"endraw", JinjaKeyword> JinjaStatementEnd
+  }
+}
+
+JinjaKeyword {
+  kw<"if"> | kw<"elif"> | kw<"else"> | kw<"endif"> |
+  kw<"is"> | kw<"not"> |
+  kw<"set"> |
+  kw<"for"> | kw<"in"> | kw<"endfor"> |
+  kw<"extends"> |
+  kw<"macro"> | kw<"endmacro"> |
+  kw<"block"> | kw<"endblock"> |
+  kw<"call"> | kw<"endcall"> |
+  kw<"include"> |
+  kw<"from"> | kw<"import"> | kw<"as">
+}
+
+JinjaStringLiteral {
+  "'" stringContentSingleQuote? stringEndSingleQuote |
+  '"' stringContentDoubleQuote? stringEndDoubleQuote
+}
+
+@external tokens tokenizeSingleQuoteString from "./tokens" {
+  stringContentSingleQuote,
+  stringEndSingleQuote
+}
+
+@external tokens tokenizeDoubleQuoteString from "./tokens" {
+  stringContentDoubleQuote,
+  stringEndDoubleQuote
+}
+
+@external tokens tokenizeExpression from "./tokens" {
+  JinjaExpressionEnd
+}
+
+@external tokens tokenizeStatement from "./tokens" {
+  JinjaStatementEnd
+}
+
+@external tokens tokenizeComment from "./tokens" {
+  JinjaCommentEnd
+}
+
+JinjaRawBlock {
+  JinjaRawOpenStatement RawText? JinjaRawCloseStatement
+}
+
+@external tokens tokenizeRawText from "./tokens" {
+  RawText
+}
+
+@external tokens tokenizeText from "./tokens" {
+  JinjaText
+}
+
+@tokens {
+  newline {
+    "\r" | "\n" | "\r\n"
+  }
+
+  JinjaStatementStart { "{%" $[-+]? }
+
+  jinjaStatementUnknownContent {
+    ![-+'"%] jinjaStatementUnknownContent? |
+    "%" (@eof | ![}]) jinjaStatementUnknownContent?
+  }
+
+  JinjaExpressionStart { "{{" $[-+]? }
+  JinjaExpressionUnknownContent {
+    ![{'"}|] JinjaExpressionUnknownContent? |
+    "}" (@eof | ![}]) JinjaExpressionUnknownContent?
+  }
+
+  JinjaCommentStart { "{#" }
+  JinjaCommentContent {
+    ![#{] JinjaCommentContent? |
+    "#" (@eof | ![}]) JinjaCommentContent? |
+    "{" (@eof | ![#]) JinjaCommentContent?
+  }
+
+  identifierChar { @asciiLetter | $[_$\u{a1}-\u{10ffff}] }
+  word { identifierChar (identifierChar | @digit)* }
+  JinjaIdentifier { word }
+
+  spaces { $[\u0009 \u000b\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000\ufeff]+ }
+
+  jinjaSign {
+    "-" |
+    "+" |
+    "*" |
+    "/" |
+    "=" |
+    "%" |
+    ">" |
+    "<" |
+    "(" |
+    ")" |
+    "[" |
+    "]" |
+    "," |
+    "!" |
+    "~" |
+    "?" |
+    ":" |
+    "," |
+    "^"
+  }
+
+  JinjaNumberLiteral {
+    @digit ("_" | @digit)* ("." ("_" | @digit)*)? | "." @digit ("_" | @digit)*
+  }
+
+  JinjaFilterName { word }
+
+  @precedence {
+    spaces,
+    JinjaFilterName
+  }
+
+  @precedence {
+    JinjaText,
+    JinjaStatementStart
+  }
+
+  @precedence {
+    spaces,
+    newline,
+    JinjaIdentifier,
+    JinjaStringLiteral,
+    JinjaNumberLiteral,
+    jinjaSign,
+    jinjaStatementUnknownContent,
+    JinjaExpressionUnknownContent,
+    expressionUnknownContent
+  }
+}
+
+kw<term> { @specialize<JinjaIdentifier, term> }
+namedKw<term, name> { @specialize[@name={name}]<JinjaIdentifier, term> }
+
+@external propSource jinjaHighlighting from "./highlight"

package/src/test.ts

@@ -0,0 +1,51 @@
+//  Copyright (c) 2025 coze-dev
+//  SPDX-License-Identifier: MIT
+
+/* eslint-disable */
+import { printTree } from "@lezer-unofficial/printer"
+import { parser } from './index'
+
+// const source = `a {# 注释 #} a {%- for  xxx  -%}hello{{- foo + 1 +}}world{%+ endfor %}  dasdas bb`
+// const source = `{{ '}}' }} {{ '{{' }} {{ "}}" }}`
+// const source = `{#
+// multiple line
+// multiple line
+// multiple line
+// #} haha {# single line #}
+// `
+// const source = `a {% for item in items %} {% if a is defined %}`
+// const source = `a {% if a is * defined(a + 1, "hello") 'world' if a is %}`
+// const source = `
+// a {% hello world %} {% if a is * defined(a + 1, "hello") 'world' if a is %} b
+// {% raw %}
+// {% if a is * defined(a + 1, "hello") 'world' if a is %}
+// {% endraw %} b
+// `.trim()
+// const source = `{% if "hello\\"" x%} foo
+// Empty slot: {#slot id="foo" placeholder="请输入"#}{#/slot#}
+// `.trim()
+// const source = `{% if "a\\"" is true %}`
+// const source = `{{a + 1} ???`
+// const source = `{% hello + 123 + ??? world {#comment#}aaa{#hello#}`
+// 换行处理
+// const source = `slot{% if a is none
+// {#slot id="foo" placeholder="请输入"#}prefilled value{#/slot#}
+// foo
+// `
+// 多余 {
+// const source = `{{#slot id="foo" placeholder="请输入"#}{#/slot#}`
+// const source = `{{}{#slot id="foo" placeholder="请输入"#}{#/slot#}`
+const source = `{{in}}`
+// const source = `{%in%}`
+
+// const source = `{{a + 1 ??? aaa {#comment#}aaa{#hello#}`
+// const source = `{{hello|default('hello')}}`
+// const source = `foo{#hello {# slot #}`
+// const source = `foo{#hell{o#}`
+
+// const source = `{% set designer_type ="`
+
+console.log(
+  'tree',
+  printTree(parser.parse(source), source)
+)

package/src/tokens.js

@@ -0,0 +1,325 @@
+//  Copyright (c) 2025 coze-dev
+//  SPDX-License-Identifier: MIT
+
+import { ExternalTokenizer } from '@lezer/lr'
+import {
+  RawText,
+  stringContentSingleQuote,
+  stringEndSingleQuote,
+  stringContentDoubleQuote,
+  stringEndDoubleQuote,
+  JinjaExpressionEnd,
+  JinjaStatementEnd,
+  JinjaCommentEnd,
+  JinjaText,
+} from './index.terms'
+
+// scan to {% endraw %}
+const sequence = [ '{%', 'endraw', '%}' ]
+const tokenizeRawText = new ExternalTokenizer((input) => {
+  const first = sequence[0].charCodeAt(0)
+  let len = 0
+  let foundSequence = false
+
+  while (true) {
+    if (input.next < 0) {
+      break
+    }
+
+    if (input.next === first) {
+      let n = 0
+      let index = 0
+      for (; index < sequence.length; index++) {
+        const segment = sequence[index]
+        if (match(input, n, segment)) {
+          n = n + segment.length
+          if (index < sequence.length - 1) {
+            const spaceLen = skipSpaces(input, n)
+            n = n + spaceLen
+          }
+        } else {
+          break
+        }
+      }
+
+      // all match
+      if (index === sequence.length) {
+        foundSequence = true
+        break
+      } else {
+        input.advance()
+        len++
+      }
+    } else {
+      input.advance()
+      len++
+    }
+  }
+
+  // not found at end
+  if (!foundSequence && input.next < 0) {
+    return
+  }
+
+  if (len > 0) {
+    input.acceptToken(RawText)
+  }
+})
+
+function match(input, from, text) {
+  let i = 0
+  let len = 0
+
+  while (true) {
+    if (input.peek(from + i) === text.charCodeAt(i) && len < text.length) {
+      i++
+      len++
+    } else {
+      break
+    }
+  }
+
+  return len === text.length
+}
+
+const spaceReg = /^\s+$/
+function skipSpaces(input, from) {
+  let i = from
+  let len = 0
+
+  while (true) {
+    const char = String.fromCharCode(input.peek(i))
+    if (spaceReg.test(char)) {
+      i++
+      len++
+    } else {
+      break
+    }
+  }
+
+  return len
+}
+
+const singleQuote = 39
+const doubleQuote = 34
+const tokenizeSingleQuoteString = scanString(singleQuote)
+const tokenizeDoubleQuoteString = scanString(doubleQuote)
+
+function scanString(quote) {
+  const endToken = quote === singleQuote ?
+    stringEndSingleQuote :
+    stringEndDoubleQuote
+  const contentToken = quote === singleQuote ?
+    stringContentSingleQuote :
+    stringContentDoubleQuote
+
+  return new ExternalTokenizer((input) => {
+    let len = 0
+    while (true) {
+      if (input.next < 0) {
+        break;
+      }
+    
+      if (
+        // \\
+        (input.next === 92 && input.peek(1) === 92) ||
+        // \"
+        (input.next === 92 && input.peek(1) === quote)
+      ) {
+        input.advance(2)
+        len += 2
+        continue
+      }
+
+      if (
+        (input.next === quote) ||
+        // % }
+        (input.next === 37 && input.peek(1) === 125) ||
+        // \n | \r
+        (input.next === 10 || input.next === 13)
+      ) {
+        break
+      }
+
+      len++
+      input.advance()
+    }
+
+    if (len > 0) {
+      input.acceptToken(contentToken)
+    }
+
+    if (input.next === quote) {
+      input.advance()
+    }
+
+    input.acceptToken(endToken)
+  })
+}
+
+// {
+const leftBracket = 123
+// }
+const rightBracket = 125
+// #
+const hash = 35
+// %
+const percent = 37
+const minus = 45
+const plus = 43
+
+const tokenizeExpression = new ExternalTokenizer((input) => {
+  if (input.next === minus || input.next === plus) {
+    // skip spaces between -+ and }}
+    const n = skipSpaces(input, 1)
+
+    if (
+      input.peek(n + 1) === rightBracket &&
+      input.peek(n + 2) === rightBracket
+    ) {
+      input.advance(n + 3)
+      input.acceptToken(JinjaExpressionEnd)
+      return
+    }
+  }
+
+  if (
+    input.next === rightBracket && input.peek(1) === rightBracket
+  ) {
+    input.advance(2)
+    input.acceptToken(JinjaExpressionEnd)
+    return
+  }
+
+  if (input.next === leftBracket && input.peek(1) === hash) {
+    input.acceptToken(JinjaExpressionEnd)
+    return
+  }
+
+  if (input.next < 0) {
+    input.acceptToken(JinjaExpressionEnd)
+    return
+  }
+})
+
+// function debug(input) {
+//   console.log(
+//     'next',
+//     [
+//       String.fromCharCode(input.peek(0)),
+//       String.fromCharCode(input.peek(1)),
+//       String.fromCharCode(input.peek(2)),
+//       String.fromCharCode(input.peek(3)),
+//       String.fromCharCode(input.peek(4)),
+//       String.fromCharCode(input.peek(5)),
+//     ]
+//   )
+// }
+
+const tokenizeStatement = new ExternalTokenizer(input => {
+  if (input.next === minus || input.next === plus) {
+    // skip spaces between -+ and }}
+    const n = skipSpaces(input, 1)
+
+    if (
+      input.peek(n + 1) === percent &&
+      input.peek(n + 2) === rightBracket
+    ) {
+      input.advance(n + 3)
+      input.acceptToken(JinjaStatementEnd)
+      return
+    }
+  }
+
+  if (
+    input.next === percent &&
+    input.peek(1) === rightBracket
+  ) {
+    input.advance(2)
+    input.acceptToken(JinjaStatementEnd)
+    return
+  }
+
+  if (input.next === leftBracket && input.peek(1) === hash) {
+    input.acceptToken(JinjaStatementEnd)
+    return
+  }
+
+  if (input.next < 0) {
+    input.acceptToken(JinjaStatementEnd)
+    return
+  }
+})
+
+const tokenizeComment = new ExternalTokenizer(input => {
+  if (
+    input.next === hash &&
+    input.peek(1) === rightBracket
+  ) {
+    input.advance(2)
+    input.acceptToken(JinjaCommentEnd)
+  } else if (
+    input.next === leftBracket &&
+    input.peek(1) === hash
+  ) {
+    input.acceptToken(JinjaCommentEnd)
+  }
+})
+
+const tokenizeText = new ExternalTokenizer(input => {
+  let offset = 0
+  let len = 0
+
+  while (true) {
+    const next = input.peek(offset)
+
+    // eof
+    if (next < 0) {
+      break
+    }
+
+    // ![{]
+    if (next !== leftBracket) {
+      offset++
+      len++
+    } else if (
+      // { ![{#%]
+      next === leftBracket &&
+      input.peek(offset + 1) !== leftBracket &&
+      input.peek(offset + 1) !== hash &&
+      input.peek(offset + 1) !== percent
+    ) {
+      offset++
+      len++
+    } else if (
+      // {{#
+      next === leftBracket &&
+      input.peek(offset + 1) === leftBracket &&
+      (
+        input.peek(offset + 2) === leftBracket ||
+        input.peek(offset + 2) === hash ||
+        input.peek(offset + 2) === percent
+      )
+    ) {
+      offset++
+      len++
+    } else {
+      break
+    }
+  }
+
+  if (len > 0) {
+    input.advance(len)
+    input.acceptToken(JinjaText)
+  }
+})
+
+export {
+  tokenizeRawText,
+  tokenizeSingleQuoteString,
+  tokenizeDoubleQuoteString,
+  tokenizeExpression,
+  tokenizeStatement,
+  tokenizeComment,
+  tokenizeText,
+}