@fuzdev/fuz_code 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/highlight_manager.d.ts +9 -1
- package/dist/highlight_manager.d.ts.map +1 -1
- package/dist/highlight_manager.js +9 -1
- package/dist/highlight_priorities.gen.d.ts +1 -1
- package/dist/highlight_priorities.gen.d.ts.map +1 -1
- package/dist/svelte_preprocess_fuz_code.d.ts +20 -0
- package/dist/svelte_preprocess_fuz_code.d.ts.map +1 -1
- package/dist/svelte_preprocess_fuz_code.js +49 -42
- package/dist/syntax_styler.d.ts +6 -0
- package/dist/syntax_styler.d.ts.map +1 -1
- package/dist/syntax_styler.js +6 -0
- package/dist/syntax_styler_global.d.ts +10 -0
- package/dist/syntax_styler_global.d.ts.map +1 -1
- package/dist/syntax_styler_global.js +10 -0
- package/dist/tokenize_syntax.d.ts +2 -0
- package/dist/tokenize_syntax.d.ts.map +1 -1
- package/dist/tokenize_syntax.js +2 -0
- package/package.json +10 -5
- package/src/lib/highlight_manager.ts +9 -1
- package/src/lib/highlight_priorities.gen.ts +1 -1
- package/src/lib/svelte_preprocess_fuz_code.ts +53 -55
- package/src/lib/syntax_styler.ts +6 -0
- package/src/lib/syntax_styler_global.ts +10 -0
- package/src/lib/tokenize_syntax.ts +2 -0
|
@@ -5,8 +5,16 @@ export type HighlightMode = 'auto' | 'ranges' | 'html';
|
|
|
5
5
|
*/
|
|
6
6
|
export declare const supports_css_highlight_api: () => boolean;
|
|
7
7
|
/**
|
|
8
|
-
* Manages
|
|
8
|
+
* Manages CSS Custom Highlight API ranges for a single element.
|
|
9
9
|
* Tracks ranges per element and only removes its own ranges when clearing.
|
|
10
|
+
*
|
|
11
|
+
* **Experimental** — limited browser support. Use `Code` for production.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const manager = new HighlightManager();
|
|
16
|
+
* manager.highlight_from_syntax_tokens(element, tokens);
|
|
17
|
+
* ```
|
|
10
18
|
*/
|
|
11
19
|
export declare class HighlightManager {
|
|
12
20
|
#private;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"highlight_manager.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/highlight_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD;;GAEG;AACH,eAAO,MAAM,0BAA0B,QAAO,OACS,CAAC;AAExD
|
|
1
|
+
{"version":3,"file":"highlight_manager.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/highlight_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGzD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD;;GAEG;AACH,eAAO,MAAM,0BAA0B,QAAO,OACS,CAAC;AAExD;;;;;;;;;;;GAWG;AACH,qBAAa,gBAAgB;;IAC5B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;;IAS1C;;OAEG;IACH,4BAA4B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,GAAG,IAAI;IAkD/E;;OAEG;IACH,oBAAoB,IAAI,IAAI;IAmB5B,OAAO,IAAI,IAAI;CAiFf"}
|
|
@@ -4,8 +4,16 @@ import { highlight_priorities } from './highlight_priorities.js';
|
|
|
4
4
|
*/
|
|
5
5
|
export const supports_css_highlight_api = () => !!(globalThis.CSS?.highlights && globalThis.Highlight); // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
|
6
6
|
/**
|
|
7
|
-
* Manages
|
|
7
|
+
* Manages CSS Custom Highlight API ranges for a single element.
|
|
8
8
|
* Tracks ranges per element and only removes its own ranges when clearing.
|
|
9
|
+
*
|
|
10
|
+
* **Experimental** — limited browser support. Use `Code` for production.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* const manager = new HighlightManager();
|
|
15
|
+
* manager.highlight_from_syntax_tokens(element, tokens);
|
|
16
|
+
* ```
|
|
9
17
|
*/
|
|
10
18
|
export class HighlightManager {
|
|
11
19
|
element_ranges;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"highlight_priorities.gen.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/highlight_priorities.gen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"highlight_priorities.gen.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/highlight_priorities.gen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,aAAa,CAAC;AAKrC,cAAc;AACd,eAAO,MAAM,GAAG,EAAE,GAgEjB,CAAC"}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type PreprocessorGroup } from 'svelte/compiler';
|
|
2
2
|
import type { SyntaxStyler } from './syntax_styler.js';
|
|
3
|
+
/**
|
|
4
|
+
* Options for `svelte_preprocess_fuz_code`.
|
|
5
|
+
*/
|
|
3
6
|
export interface PreprocessFuzCodeOptions {
|
|
4
7
|
/** File patterns to exclude. */
|
|
5
8
|
exclude?: Array<string | RegExp>;
|
|
@@ -20,5 +23,22 @@ export interface PreprocessFuzCodeOptions {
|
|
|
20
23
|
*/
|
|
21
24
|
on_error?: 'log' | 'throw';
|
|
22
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Svelte preprocessor that compiles static `Code` component content at build time,
|
|
28
|
+
* replacing runtime syntax highlighting with pre-rendered HTML.
|
|
29
|
+
*
|
|
30
|
+
* @param options preprocessor configuration
|
|
31
|
+
* @returns a Svelte preprocessor group
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* ```ts
|
|
35
|
+
* // svelte.config.js
|
|
36
|
+
* import {svelte_preprocess_fuz_code} from './svelte_preprocess_fuz_code.js';
|
|
37
|
+
*
|
|
38
|
+
* export default {
|
|
39
|
+
* preprocess: [svelte_preprocess_fuz_code(), vitePreprocess()],
|
|
40
|
+
* };
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
23
43
|
export declare const svelte_preprocess_fuz_code: (options?: PreprocessFuzCodeOptions) => PreprocessorGroup;
|
|
24
44
|
//# sourceMappingURL=svelte_preprocess_fuz_code.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"svelte_preprocess_fuz_code.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/svelte_preprocess_fuz_code.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,iBAAiB,EAAW,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"svelte_preprocess_fuz_code.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/svelte_preprocess_fuz_code.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,iBAAiB,EAAW,MAAM,iBAAiB,CAAC;AAgBxE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAErD;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC,gCAAgC;IAChC,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAEjC,0DAA0D;IAC1D,aAAa,CAAC,EAAE,YAAY,CAAC;IAE7B,8CAA8C;IAC9C,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,0BAA0B,GACtC,UAAS,wBAA6B,KACpC,iBA6DF,CAAC"}
|
|
@@ -3,8 +3,25 @@ import MagicString from 'magic-string';
|
|
|
3
3
|
import { walk } from 'zimmerframe';
|
|
4
4
|
import { should_exclude_path } from '@fuzdev/fuz_util/path.js';
|
|
5
5
|
import { escape_js_string } from '@fuzdev/fuz_util/string.js';
|
|
6
|
-
import { find_attribute,
|
|
6
|
+
import { find_attribute, extract_static_string, try_extract_conditional_chain, build_static_bindings, resolve_component_names, handle_preprocess_error, } from '@fuzdev/fuz_util/svelte_preprocess_helpers.js';
|
|
7
7
|
import { syntax_styler_global } from './syntax_styler_global.js';
|
|
8
|
+
/**
|
|
9
|
+
* Svelte preprocessor that compiles static `Code` component content at build time,
|
|
10
|
+
* replacing runtime syntax highlighting with pre-rendered HTML.
|
|
11
|
+
*
|
|
12
|
+
* @param options preprocessor configuration
|
|
13
|
+
* @returns a Svelte preprocessor group
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```ts
|
|
17
|
+
* // svelte.config.js
|
|
18
|
+
* import {svelte_preprocess_fuz_code} from './svelte_preprocess_fuz_code.js';
|
|
19
|
+
*
|
|
20
|
+
* export default {
|
|
21
|
+
* preprocess: [svelte_preprocess_fuz_code(), vitePreprocess()],
|
|
22
|
+
* };
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
8
25
|
export const svelte_preprocess_fuz_code = (options = {}) => {
|
|
9
26
|
const { exclude = [], syntax_styler = syntax_styler_global, cache = true, component_imports = ['@fuzdev/fuz_code/Code.svelte'], on_error = process.env.CI === 'true' ? 'throw' : 'log', } = options;
|
|
10
27
|
// In-memory cache: content+lang hash → highlighted HTML
|
|
@@ -63,7 +80,7 @@ const try_highlight = (text, lang, syntax_styler, options) => {
|
|
|
63
80
|
options.cache?.set(cache_key, html);
|
|
64
81
|
}
|
|
65
82
|
catch (error) {
|
|
66
|
-
|
|
83
|
+
handle_preprocess_error(error, '[fuz-code]', options.filename, options.on_error);
|
|
67
84
|
return null;
|
|
68
85
|
}
|
|
69
86
|
}
|
|
@@ -117,53 +134,43 @@ const find_code_usages = (ast, syntax_styler, code_names, options) => {
|
|
|
117
134
|
});
|
|
118
135
|
return;
|
|
119
136
|
}
|
|
120
|
-
// Try conditional
|
|
121
|
-
const
|
|
122
|
-
if (
|
|
123
|
-
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
137
|
+
// Try conditional chain (handles both simple and nested ternaries)
|
|
138
|
+
const chain = try_extract_conditional_chain(content_attr.value, options.source, options.bindings);
|
|
139
|
+
if (chain) {
|
|
140
|
+
// Highlight all branches
|
|
141
|
+
const highlighted = [];
|
|
142
|
+
let any_changed = false;
|
|
143
|
+
for (const branch of chain) {
|
|
144
|
+
const html = try_highlight(branch.value, lang_value, syntax_styler, options);
|
|
145
|
+
if (html === null)
|
|
146
|
+
return;
|
|
147
|
+
if (html !== branch.value)
|
|
148
|
+
any_changed = true;
|
|
149
|
+
highlighted.push({ html, original: branch.value });
|
|
150
|
+
}
|
|
151
|
+
if (!any_changed)
|
|
128
152
|
return;
|
|
153
|
+
// Build nested ternary expression for dangerous_raw_html
|
|
154
|
+
// chain: [{test_source: 'a', value: ...}, {test_source: 'b', value: ...}, {test_source: null, value: ...}]
|
|
155
|
+
// → a ? 'html_a' : b ? 'html_b' : 'html_c'
|
|
156
|
+
let expr = '';
|
|
157
|
+
for (let i = 0; i < chain.length; i++) {
|
|
158
|
+
const branch = chain[i];
|
|
159
|
+
const html = highlighted[i].html;
|
|
160
|
+
if (branch.test_source !== null) {
|
|
161
|
+
expr += `${branch.test_source} ? '${escape_js_string(html)}' : `;
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
expr += `'${escape_js_string(html)}'`;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
129
167
|
transformations.push({
|
|
130
168
|
start: content_attr.start,
|
|
131
169
|
end: content_attr.end,
|
|
132
|
-
replacement: `dangerous_raw_html={${
|
|
170
|
+
replacement: `dangerous_raw_html={${expr}}`,
|
|
133
171
|
});
|
|
134
172
|
}
|
|
135
173
|
},
|
|
136
174
|
});
|
|
137
175
|
return transformations;
|
|
138
176
|
};
|
|
139
|
-
/**
|
|
140
|
-
* Try to extract a conditional expression where both branches are static strings.
|
|
141
|
-
* Returns the condition source text and both branch values, or `null` if not applicable.
|
|
142
|
-
*/
|
|
143
|
-
const try_extract_conditional = (value, source, bindings) => {
|
|
144
|
-
if (value === true || Array.isArray(value))
|
|
145
|
-
return null;
|
|
146
|
-
const expr = value.expression;
|
|
147
|
-
if (expr.type !== 'ConditionalExpression')
|
|
148
|
-
return null;
|
|
149
|
-
const consequent = evaluate_static_expr(expr.consequent, bindings);
|
|
150
|
-
if (consequent === null)
|
|
151
|
-
return null;
|
|
152
|
-
const alternate = evaluate_static_expr(expr.alternate, bindings);
|
|
153
|
-
if (alternate === null)
|
|
154
|
-
return null;
|
|
155
|
-
const test = expr.test;
|
|
156
|
-
const test_source = source.slice(test.start, test.end);
|
|
157
|
-
return { test_source, consequent, alternate };
|
|
158
|
-
};
|
|
159
|
-
/**
|
|
160
|
-
* Handle errors during highlighting.
|
|
161
|
-
*/
|
|
162
|
-
const handle_error = (error, options) => {
|
|
163
|
-
const message = `[fuz-code] Highlighting failed${options.filename ? ` in ${options.filename}` : ''}: ${error instanceof Error ? error.message : String(error)}`;
|
|
164
|
-
if (options.on_error === 'throw') {
|
|
165
|
-
throw new Error(message);
|
|
166
|
-
}
|
|
167
|
-
// eslint-disable-next-line no-console
|
|
168
|
-
console.error(message);
|
|
169
|
-
};
|
package/dist/syntax_styler.d.ts
CHANGED
|
@@ -46,18 +46,24 @@ export declare class SyntaxStyler {
|
|
|
46
46
|
* @returns HTML string with syntax highlighting using CSS classes (`.token_*`)
|
|
47
47
|
*
|
|
48
48
|
* @example
|
|
49
|
+
* ```ts
|
|
49
50
|
* // Standard usage - uses registered grammar
|
|
50
51
|
* stylize('var foo = true;', 'ts');
|
|
52
|
+
* ```
|
|
51
53
|
*
|
|
52
54
|
* @example
|
|
55
|
+
* ```ts
|
|
53
56
|
* // Custom grammar - overrides registered grammar
|
|
54
57
|
* const customGrammar = { keyword: [...], string: [...] };
|
|
55
58
|
* stylize('var foo = false;', 'ts', customGrammar);
|
|
59
|
+
* ```
|
|
56
60
|
*
|
|
57
61
|
* @example
|
|
62
|
+
* ```ts
|
|
58
63
|
* // Extended grammar - builds on existing grammar
|
|
59
64
|
* const extended = this.extend_grammar('ts', { customToken: [...] });
|
|
60
65
|
* stylize('var foo = 42;', 'ts-extended', extended);
|
|
66
|
+
* ```
|
|
61
67
|
*/
|
|
62
68
|
stylize(text: string, lang: string, grammar?: SyntaxGrammar | undefined): string;
|
|
63
69
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syntax_styler.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGtE,MAAM,MAAM,gBAAgB,GAAG,CAAC,aAAa,EAAE,YAAY,KAAK,IAAI,CAAC;AAErE;;;;;;;GAOG;AACH,qBAAa,YAAY;;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAAC,CAE9C;IAkBF,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI;IAc9E,iBAAiB,CAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,gBAAgB,EAC3B,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GACrB,aAAa;IAahB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa;IAQnC
|
|
1
|
+
{"version":3,"file":"syntax_styler.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGtE,MAAM,MAAM,gBAAgB,GAAG,CAAC,aAAa,EAAE,YAAY,KAAK,IAAI,CAAC;AAErE;;;;;;;GAOG;AACH,qBAAa,YAAY;;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAAC,CAE9C;IAkBF,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI;IAc9E,iBAAiB,CAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,gBAAgB,EAC3B,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GACrB,aAAa;IAahB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa;IAQnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,OAAO,CACN,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,aAAa,GAAG,SAA+B,GACtD,MAAM;IAcT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0EG;IACH,qBAAqB,CACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,gBAAgB,EACxB,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAc,GACpC,aAAa;IAmChB;;;;;;;;OAQG;IACH,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,iBAAiB,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAoDlF;;;;;;;;;;;;;;;;;;OAkBG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,GAAG,aAAa;IAiG3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IAGlC,qBAAqB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAM;IAC9D,oBAAoB,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAM;IAC5D,UAAU,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAM;IAEzC,wBAAwB,CAAC,EAAE,EAAE,0BAA0B,GAAG,IAAI;IAG9D,uBAAuB,CAAC,EAAE,EAAE,yBAAyB,GAAG,IAAI;IAG5D,aAAa,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIzC,wBAAwB,CAAC,GAAG,EAAE,iCAAiC,GAAG,IAAI;IAKtE,uBAAuB,CAAC,GAAG,EAAE,gCAAgC,GAAG,IAAI;IAKpE,aAAa,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;CAKjD;AAED,MAAM,MAAM,qBAAqB,GAC9B,MAAM,GACN,qBAAqB,GACrB,KAAK,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC;AAEzC,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAAC,GAAG;IAClF,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAwBtE,MAAM,MAAM,0BAA0B,GAAG,CAAC,GAAG,EAAE,iCAAiC,KAAK,IAAI,CAAC;AAC1F,MAAM,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,gCAAgC,KAAK,IAAI,CAAC;AACxF,MAAM,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,uBAAuB,KAAK,IAAI,CAAC;AAEtE,MAAM,WAAW,iCAAiC;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,SAAS,CAAC;CAClB;AACD,MAAM,WAAW,gCAAgC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,iBAAiB,CAAC;CAC1B;AACD,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,IAAI,EAAE,MAAM,CAAC;CACb"}
|
package/dist/syntax_styler.js
CHANGED
|
@@ -91,18 +91,24 @@ export class SyntaxStyler {
|
|
|
91
91
|
* @returns HTML string with syntax highlighting using CSS classes (`.token_*`)
|
|
92
92
|
*
|
|
93
93
|
* @example
|
|
94
|
+
* ```ts
|
|
94
95
|
* // Standard usage - uses registered grammar
|
|
95
96
|
* stylize('var foo = true;', 'ts');
|
|
97
|
+
* ```
|
|
96
98
|
*
|
|
97
99
|
* @example
|
|
100
|
+
* ```ts
|
|
98
101
|
* // Custom grammar - overrides registered grammar
|
|
99
102
|
* const customGrammar = { keyword: [...], string: [...] };
|
|
100
103
|
* stylize('var foo = false;', 'ts', customGrammar);
|
|
104
|
+
* ```
|
|
101
105
|
*
|
|
102
106
|
* @example
|
|
107
|
+
* ```ts
|
|
103
108
|
* // Extended grammar - builds on existing grammar
|
|
104
109
|
* const extended = this.extend_grammar('ts', { customToken: [...] });
|
|
105
110
|
* stylize('var foo = 42;', 'ts-extended', extended);
|
|
111
|
+
* ```
|
|
106
112
|
*/
|
|
107
113
|
stylize(text, lang, grammar = this.get_lang(lang)) {
|
|
108
114
|
var ctx = {
|
|
@@ -1,3 +1,13 @@
|
|
|
1
1
|
import { SyntaxStyler } from './syntax_styler.js';
|
|
2
|
+
/**
|
|
3
|
+
* Pre-configured `SyntaxStyler` instance with all built-in grammars registered.
|
|
4
|
+
*
|
|
5
|
+
* @example
|
|
6
|
+
* ```ts
|
|
7
|
+
* import {syntax_styler_global} from './syntax_styler_global.js';
|
|
8
|
+
*
|
|
9
|
+
* const html = syntax_styler_global.stylize('const x = 1;', 'ts');
|
|
10
|
+
* ```
|
|
11
|
+
*/
|
|
2
12
|
export declare const syntax_styler_global: SyntaxStyler;
|
|
3
13
|
//# sourceMappingURL=syntax_styler_global.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syntax_styler_global.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler_global.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAUhD,eAAO,MAAM,oBAAoB,cAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"syntax_styler_global.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler_global.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAUhD;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,cAAqB,CAAC"}
|
|
@@ -7,6 +7,16 @@ import { add_grammar_ts } from './grammar_ts.js';
|
|
|
7
7
|
import { add_grammar_svelte } from './grammar_svelte.js';
|
|
8
8
|
import { add_grammar_json } from './grammar_json.js';
|
|
9
9
|
import { add_grammar_markdown } from './grammar_markdown.js';
|
|
10
|
+
/**
|
|
11
|
+
* Pre-configured `SyntaxStyler` instance with all built-in grammars registered.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* import {syntax_styler_global} from './syntax_styler_global.js';
|
|
16
|
+
*
|
|
17
|
+
* const html = syntax_styler_global.stylize('const x = 1;', 'ts');
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
10
20
|
export const syntax_styler_global = new SyntaxStyler();
|
|
11
21
|
add_grammar_markup(syntax_styler_global);
|
|
12
22
|
add_grammar_css(syntax_styler_global);
|
|
@@ -16,6 +16,7 @@ import { type SyntaxTokenStream } from './syntax_token.js';
|
|
|
16
16
|
* @returns an array of strings and tokens, a token stream
|
|
17
17
|
*
|
|
18
18
|
* @example
|
|
19
|
+
* ```ts
|
|
19
20
|
* var code = `var foo = 0;`;
|
|
20
21
|
* var tokens = tokenize_syntax(code, SyntaxStyler.langs.js);
|
|
21
22
|
* for (var token of tokens) {
|
|
@@ -23,6 +24,7 @@ import { type SyntaxTokenStream } from './syntax_token.js';
|
|
|
23
24
|
* console.log(`Found numeric literal: ${token.content}`);
|
|
24
25
|
* }
|
|
25
26
|
* }
|
|
27
|
+
* ```
|
|
26
28
|
*/
|
|
27
29
|
export declare const tokenize_syntax: (text: string, grammar: SyntaxGrammar) => SyntaxTokenStream;
|
|
28
30
|
//# sourceMappingURL=tokenize_syntax.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize_syntax.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/tokenize_syntax.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAc,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEtE
|
|
1
|
+
{"version":3,"file":"tokenize_syntax.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/tokenize_syntax.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAc,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEtE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,aAAa,KAAG,iBAQtE,CAAC"}
|
package/dist/tokenize_syntax.js
CHANGED
|
@@ -15,6 +15,7 @@ import { SyntaxToken } from './syntax_token.js';
|
|
|
15
15
|
* @returns an array of strings and tokens, a token stream
|
|
16
16
|
*
|
|
17
17
|
* @example
|
|
18
|
+
* ```ts
|
|
18
19
|
* var code = `var foo = 0;`;
|
|
19
20
|
* var tokens = tokenize_syntax(code, SyntaxStyler.langs.js);
|
|
20
21
|
* for (var token of tokens) {
|
|
@@ -22,6 +23,7 @@ import { SyntaxToken } from './syntax_token.js';
|
|
|
22
23
|
* console.log(`Found numeric literal: ${token.content}`);
|
|
23
24
|
* }
|
|
24
25
|
* }
|
|
26
|
+
* ```
|
|
25
27
|
*/
|
|
26
28
|
export const tokenize_syntax = (text, grammar) => {
|
|
27
29
|
// Grammar is already normalized (rest merged, patterns in arrays, etc.)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fuzdev/fuz_code",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.45.0",
|
|
4
4
|
"description": "syntax styling utilities and components for TypeScript, Svelte, and Markdown",
|
|
5
5
|
"glyph": "🎨",
|
|
6
6
|
"logo": "logo.svg",
|
|
@@ -58,15 +58,18 @@
|
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"@changesets/changelog-git": "^0.2.1",
|
|
61
|
-
"@fuzdev/fuz_css": "^0.
|
|
62
|
-
"@fuzdev/fuz_ui": "^0.
|
|
63
|
-
"@fuzdev/fuz_util": "^0.
|
|
61
|
+
"@fuzdev/fuz_css": "^0.48.0",
|
|
62
|
+
"@fuzdev/fuz_ui": "^0.182.1",
|
|
63
|
+
"@fuzdev/fuz_util": "^0.50.1",
|
|
64
|
+
"@fuzdev/gro": "^0.192.1",
|
|
65
|
+
"@jridgewell/trace-mapping": "^0.3.31",
|
|
64
66
|
"@ryanatkn/eslint-config": "^0.9.0",
|
|
65
67
|
"@ryanatkn/gro": "^0.191.0",
|
|
66
68
|
"@sveltejs/adapter-static": "^3.0.10",
|
|
67
69
|
"@sveltejs/kit": "^2.50.1",
|
|
68
70
|
"@sveltejs/package": "^2.5.7",
|
|
69
71
|
"@sveltejs/vite-plugin-svelte": "^6.2.4",
|
|
72
|
+
"@types/estree": "^1.0.8",
|
|
70
73
|
"@types/node": "^24.10.1",
|
|
71
74
|
"@webref/css": "^8.2.0",
|
|
72
75
|
"eslint": "^9.39.1",
|
|
@@ -77,11 +80,13 @@
|
|
|
77
80
|
"prettier-plugin-svelte": "^3.4.1",
|
|
78
81
|
"svelte": "^5.49.1",
|
|
79
82
|
"svelte-check": "^4.3.6",
|
|
83
|
+
"svelte2tsx": "^0.7.47",
|
|
80
84
|
"tslib": "^2.8.1",
|
|
81
85
|
"typescript": "^5.9.3",
|
|
82
86
|
"typescript-eslint": "^8.48.1",
|
|
83
87
|
"vitest": "^4.0.15",
|
|
84
|
-
"zimmerframe": "^1.1.4"
|
|
88
|
+
"zimmerframe": "^1.1.4",
|
|
89
|
+
"zod": "^4.3.6"
|
|
85
90
|
},
|
|
86
91
|
"prettier": {
|
|
87
92
|
"plugins": [
|
|
@@ -10,8 +10,16 @@ export const supports_css_highlight_api = (): boolean =>
|
|
|
10
10
|
!!(globalThis.CSS?.highlights && globalThis.Highlight); // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
|
-
* Manages
|
|
13
|
+
* Manages CSS Custom Highlight API ranges for a single element.
|
|
14
14
|
* Tracks ranges per element and only removes its own ranges when clearing.
|
|
15
|
+
*
|
|
16
|
+
* **Experimental** — limited browser support. Use `Code` for production.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const manager = new HighlightManager();
|
|
21
|
+
* manager.highlight_from_syntax_tokens(element, tokens);
|
|
22
|
+
* ```
|
|
15
23
|
*/
|
|
16
24
|
export class HighlightManager {
|
|
17
25
|
element_ranges: Map<string, Array<Range>>;
|
|
@@ -5,16 +5,20 @@ import {should_exclude_path} from '@fuzdev/fuz_util/path.js';
|
|
|
5
5
|
import {escape_js_string} from '@fuzdev/fuz_util/string.js';
|
|
6
6
|
import {
|
|
7
7
|
find_attribute,
|
|
8
|
-
evaluate_static_expr,
|
|
9
8
|
extract_static_string,
|
|
9
|
+
try_extract_conditional_chain,
|
|
10
10
|
build_static_bindings,
|
|
11
11
|
resolve_component_names,
|
|
12
|
+
handle_preprocess_error,
|
|
12
13
|
type ResolvedComponentImport,
|
|
13
14
|
} from '@fuzdev/fuz_util/svelte_preprocess_helpers.js';
|
|
14
15
|
|
|
15
16
|
import {syntax_styler_global} from './syntax_styler_global.js';
|
|
16
17
|
import type {SyntaxStyler} from './syntax_styler.js';
|
|
17
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Options for `svelte_preprocess_fuz_code`.
|
|
21
|
+
*/
|
|
18
22
|
export interface PreprocessFuzCodeOptions {
|
|
19
23
|
/** File patterns to exclude. */
|
|
20
24
|
exclude?: Array<string | RegExp>;
|
|
@@ -40,6 +44,23 @@ export interface PreprocessFuzCodeOptions {
|
|
|
40
44
|
on_error?: 'log' | 'throw';
|
|
41
45
|
}
|
|
42
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Svelte preprocessor that compiles static `Code` component content at build time,
|
|
49
|
+
* replacing runtime syntax highlighting with pre-rendered HTML.
|
|
50
|
+
*
|
|
51
|
+
* @param options preprocessor configuration
|
|
52
|
+
* @returns a Svelte preprocessor group
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```ts
|
|
56
|
+
* // svelte.config.js
|
|
57
|
+
* import {svelte_preprocess_fuz_code} from '@fuzdev/fuz_code/svelte_preprocess_fuz_code.js';
|
|
58
|
+
*
|
|
59
|
+
* export default {
|
|
60
|
+
* preprocess: [svelte_preprocess_fuz_code(), vitePreprocess()],
|
|
61
|
+
* };
|
|
62
|
+
* ```
|
|
63
|
+
*/
|
|
43
64
|
export const svelte_preprocess_fuz_code = (
|
|
44
65
|
options: PreprocessFuzCodeOptions = {},
|
|
45
66
|
): PreprocessorGroup => {
|
|
@@ -136,7 +157,7 @@ const try_highlight = (
|
|
|
136
157
|
html = syntax_styler.stylize(text, lang);
|
|
137
158
|
options.cache?.set(cache_key, html);
|
|
138
159
|
} catch (error) {
|
|
139
|
-
|
|
160
|
+
handle_preprocess_error(error, '[fuz-code]', options.filename, options.on_error);
|
|
140
161
|
return null;
|
|
141
162
|
}
|
|
142
163
|
}
|
|
@@ -165,7 +186,7 @@ const find_code_usages = (
|
|
|
165
186
|
if (!code_names.has(node.name)) return;
|
|
166
187
|
|
|
167
188
|
// Skip if spread attributes present — can't determine content statically
|
|
168
|
-
if (node.attributes.some((attr
|
|
189
|
+
if (node.attributes.some((attr) => attr.type === 'SpreadAttribute')) return;
|
|
169
190
|
|
|
170
191
|
const content_attr = find_attribute(node, 'content');
|
|
171
192
|
if (!content_attr) return;
|
|
@@ -200,21 +221,42 @@ const find_code_usages = (
|
|
|
200
221
|
return;
|
|
201
222
|
}
|
|
202
223
|
|
|
203
|
-
// Try conditional
|
|
204
|
-
const
|
|
224
|
+
// Try conditional chain (handles both simple and nested ternaries)
|
|
225
|
+
const chain = try_extract_conditional_chain(
|
|
205
226
|
content_attr.value,
|
|
206
227
|
options.source,
|
|
207
228
|
options.bindings,
|
|
208
229
|
);
|
|
209
|
-
if (
|
|
210
|
-
|
|
211
|
-
const
|
|
212
|
-
|
|
213
|
-
|
|
230
|
+
if (chain) {
|
|
231
|
+
// Highlight all branches
|
|
232
|
+
const highlighted: Array<{html: string; original: string}> = [];
|
|
233
|
+
let any_changed = false;
|
|
234
|
+
for (const branch of chain) {
|
|
235
|
+
const html = try_highlight(branch.value, lang_value, syntax_styler, options);
|
|
236
|
+
if (html === null) return;
|
|
237
|
+
if (html !== branch.value) any_changed = true;
|
|
238
|
+
highlighted.push({html, original: branch.value});
|
|
239
|
+
}
|
|
240
|
+
if (!any_changed) return;
|
|
241
|
+
|
|
242
|
+
// Build nested ternary expression for dangerous_raw_html
|
|
243
|
+
// chain: [{test_source: 'a', value: ...}, {test_source: 'b', value: ...}, {test_source: null, value: ...}]
|
|
244
|
+
// → a ? 'html_a' : b ? 'html_b' : 'html_c'
|
|
245
|
+
let expr = '';
|
|
246
|
+
for (let i = 0; i < chain.length; i++) {
|
|
247
|
+
const branch = chain[i]!;
|
|
248
|
+
const html = highlighted[i]!.html;
|
|
249
|
+
if (branch.test_source !== null) {
|
|
250
|
+
expr += `${branch.test_source} ? '${escape_js_string(html)}' : `;
|
|
251
|
+
} else {
|
|
252
|
+
expr += `'${escape_js_string(html)}'`;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
214
256
|
transformations.push({
|
|
215
257
|
start: content_attr.start,
|
|
216
258
|
end: content_attr.end,
|
|
217
|
-
replacement: `dangerous_raw_html={${
|
|
259
|
+
replacement: `dangerous_raw_html={${expr}}`,
|
|
218
260
|
});
|
|
219
261
|
}
|
|
220
262
|
},
|
|
@@ -222,47 +264,3 @@ const find_code_usages = (
|
|
|
222
264
|
|
|
223
265
|
return transformations;
|
|
224
266
|
};
|
|
225
|
-
|
|
226
|
-
type AttributeValue = AST.Attribute['value'];
|
|
227
|
-
|
|
228
|
-
interface ConditionalStaticStrings {
|
|
229
|
-
test_source: string;
|
|
230
|
-
consequent: string;
|
|
231
|
-
alternate: string;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* Try to extract a conditional expression where both branches are static strings.
|
|
236
|
-
* Returns the condition source text and both branch values, or `null` if not applicable.
|
|
237
|
-
*/
|
|
238
|
-
const try_extract_conditional = (
|
|
239
|
-
value: AttributeValue,
|
|
240
|
-
source: string,
|
|
241
|
-
bindings: ReadonlyMap<string, string>,
|
|
242
|
-
): ConditionalStaticStrings | null => {
|
|
243
|
-
if (value === true || Array.isArray(value)) return null;
|
|
244
|
-
const expr = value.expression;
|
|
245
|
-
if (expr.type !== 'ConditionalExpression') return null;
|
|
246
|
-
|
|
247
|
-
const consequent = evaluate_static_expr(expr.consequent, bindings);
|
|
248
|
-
if (consequent === null) return null;
|
|
249
|
-
const alternate = evaluate_static_expr(expr.alternate, bindings);
|
|
250
|
-
if (alternate === null) return null;
|
|
251
|
-
|
|
252
|
-
const test = expr.test as any;
|
|
253
|
-
const test_source = source.slice(test.start, test.end);
|
|
254
|
-
return {test_source, consequent, alternate};
|
|
255
|
-
};
|
|
256
|
-
|
|
257
|
-
/**
|
|
258
|
-
* Handle errors during highlighting.
|
|
259
|
-
*/
|
|
260
|
-
const handle_error = (error: unknown, options: FindCodeUsagesOptions): void => {
|
|
261
|
-
const message = `[fuz-code] Highlighting failed${options.filename ? ` in ${options.filename}` : ''}: ${error instanceof Error ? error.message : String(error)}`;
|
|
262
|
-
|
|
263
|
-
if (options.on_error === 'throw') {
|
|
264
|
-
throw new Error(message);
|
|
265
|
-
}
|
|
266
|
-
// eslint-disable-next-line no-console
|
|
267
|
-
console.error(message);
|
|
268
|
-
};
|
package/src/lib/syntax_styler.ts
CHANGED
|
@@ -104,18 +104,24 @@ export class SyntaxStyler {
|
|
|
104
104
|
* @returns HTML string with syntax highlighting using CSS classes (`.token_*`)
|
|
105
105
|
*
|
|
106
106
|
* @example
|
|
107
|
+
* ```ts
|
|
107
108
|
* // Standard usage - uses registered grammar
|
|
108
109
|
* stylize('var foo = true;', 'ts');
|
|
110
|
+
* ```
|
|
109
111
|
*
|
|
110
112
|
* @example
|
|
113
|
+
* ```ts
|
|
111
114
|
* // Custom grammar - overrides registered grammar
|
|
112
115
|
* const customGrammar = { keyword: [...], string: [...] };
|
|
113
116
|
* stylize('var foo = false;', 'ts', customGrammar);
|
|
117
|
+
* ```
|
|
114
118
|
*
|
|
115
119
|
* @example
|
|
120
|
+
* ```ts
|
|
116
121
|
* // Extended grammar - builds on existing grammar
|
|
117
122
|
* const extended = this.extend_grammar('ts', { customToken: [...] });
|
|
118
123
|
* stylize('var foo = 42;', 'ts-extended', extended);
|
|
124
|
+
* ```
|
|
119
125
|
*/
|
|
120
126
|
stylize(
|
|
121
127
|
text: string,
|
|
@@ -8,6 +8,16 @@ import {add_grammar_svelte} from './grammar_svelte.js';
|
|
|
8
8
|
import {add_grammar_json} from './grammar_json.js';
|
|
9
9
|
import {add_grammar_markdown} from './grammar_markdown.js';
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Pre-configured `SyntaxStyler` instance with all built-in grammars registered.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* import {syntax_styler_global} from '@fuzdev/fuz_code/syntax_styler_global.js';
|
|
17
|
+
*
|
|
18
|
+
* const html = syntax_styler_global.stylize('const x = 1;', 'ts');
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
11
21
|
export const syntax_styler_global = new SyntaxStyler();
|
|
12
22
|
|
|
13
23
|
add_grammar_markup(syntax_styler_global);
|
|
@@ -17,6 +17,7 @@ import {SyntaxToken, type SyntaxTokenStream} from './syntax_token.js';
|
|
|
17
17
|
* @returns an array of strings and tokens, a token stream
|
|
18
18
|
*
|
|
19
19
|
* @example
|
|
20
|
+
* ```ts
|
|
20
21
|
* var code = `var foo = 0;`;
|
|
21
22
|
* var tokens = tokenize_syntax(code, SyntaxStyler.langs.js);
|
|
22
23
|
* for (var token of tokens) {
|
|
@@ -24,6 +25,7 @@ import {SyntaxToken, type SyntaxTokenStream} from './syntax_token.js';
|
|
|
24
25
|
* console.log(`Found numeric literal: ${token.content}`);
|
|
25
26
|
* }
|
|
26
27
|
* }
|
|
28
|
+
* ```
|
|
27
29
|
*/
|
|
28
30
|
export const tokenize_syntax = (text: string, grammar: SyntaxGrammar): SyntaxTokenStream => {
|
|
29
31
|
// Grammar is already normalized (rest merged, patterns in arrays, etc.)
|