@altopelago/aeon-lexer 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -15,10 +15,34 @@ import { tokenize } from '@altopelago/aeon-lexer';
|
|
|
15
15
|
|
|
16
16
|
const result = tokenize('answer = 42');
|
|
17
17
|
|
|
18
|
+
if (result.errors.length > 0) {
|
|
19
|
+
console.error(result.errors);
|
|
20
|
+
}
|
|
21
|
+
|
|
18
22
|
for (const token of result.tokens) {
|
|
19
23
|
console.log(token.kind, token.raw);
|
|
20
24
|
}
|
|
21
25
|
```
|
|
22
26
|
|
|
27
|
+
## What This Package Does
|
|
28
|
+
|
|
29
|
+
- converts AEON source text into lexer tokens
|
|
30
|
+
- preserves raw token text for diagnostics and source-aware tooling
|
|
31
|
+
- reports lexical errors without executing or interpreting document meaning
|
|
32
|
+
|
|
33
|
+
## API
|
|
34
|
+
|
|
35
|
+
- `tokenize(input, options?)`
|
|
36
|
+
- token kinds and token metadata types used by parser and editor tooling
|
|
37
|
+
|
|
38
|
+
## When To Use It
|
|
39
|
+
|
|
23
40
|
Use this package when you need direct token access for tooling, analysis, or editor features.
|
|
24
41
|
If you want the stable application-facing entry point, prefer `@altopelago/aeon-core`.
|
|
42
|
+
|
|
43
|
+
## Notes
|
|
44
|
+
|
|
45
|
+
- The lexer does not build AST nodes.
|
|
46
|
+
- The lexer does not validate AEON schemas or materialize JSON.
|
|
47
|
+
- Downstream packages such as `@altopelago/aeon-parser` and
|
|
48
|
+
`@altopelago/aeon-core` consume the token stream.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@altopelago/aeon-lexer",
|
|
3
|
-
"version": "0.9.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.9.3",
|
|
4
|
+
"description": "Tokenize AEON source text for parsers, editors, and tooling.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./dist/index.js",
|
|
@@ -35,8 +35,10 @@
|
|
|
35
35
|
"keywords": [
|
|
36
36
|
"aeon",
|
|
37
37
|
"aeonite",
|
|
38
|
-
"
|
|
39
|
-
"
|
|
38
|
+
"lexer",
|
|
39
|
+
"tokenizer",
|
|
40
|
+
"tokens",
|
|
41
|
+
"syntax",
|
|
40
42
|
"typescript"
|
|
41
43
|
],
|
|
42
44
|
"publishConfig": {
|