@jotx-labs/core 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -0
- package/dist/__tests__/events.test.d.ts +2 -0
- package/dist/__tests__/events.test.d.ts.map +1 -0
- package/dist/__tests__/events.test.js +63 -0
- package/dist/__tests__/events.test.js.map +1 -0
- package/dist/__tests__/idGenerator.test.d.ts +5 -0
- package/dist/__tests__/idGenerator.test.d.ts.map +1 -0
- package/dist/__tests__/idGenerator.test.js +98 -0
- package/dist/__tests__/idGenerator.test.js.map +1 -0
- package/dist/__tests__/platform.test.d.ts +2 -0
- package/dist/__tests__/platform.test.d.ts.map +1 -0
- package/dist/__tests__/platform.test.js +37 -0
- package/dist/__tests__/platform.test.js.map +1 -0
- package/dist/__tests__/plugin.test.d.ts +2 -0
- package/dist/__tests__/plugin.test.d.ts.map +1 -0
- package/dist/__tests__/plugin.test.js +72 -0
- package/dist/__tests__/plugin.test.js.map +1 -0
- package/dist/converters/MarkdownConverter.d.ts +15 -0
- package/dist/converters/MarkdownConverter.d.ts.map +1 -0
- package/dist/converters/MarkdownConverter.js +519 -0
- package/dist/converters/MarkdownConverter.js.map +1 -0
- package/dist/document/index.d.ts +4 -0
- package/dist/document/index.d.ts.map +1 -0
- package/dist/document/index.js +8 -0
- package/dist/document/index.js.map +1 -0
- package/dist/events/EventEmitter.d.ts +20 -0
- package/dist/events/EventEmitter.d.ts.map +1 -0
- package/dist/events/EventEmitter.js +51 -0
- package/dist/events/EventEmitter.js.map +1 -0
- package/dist/events/index.d.ts +2 -0
- package/dist/events/index.d.ts.map +1 -0
- package/dist/events/index.js +18 -0
- package/dist/events/index.js.map +1 -0
- package/dist/index.d.ts +47 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +118 -0
- package/dist/index.js.map +1 -0
- package/dist/parser/__tests__/parser.test.d.ts +2 -0
- package/dist/parser/__tests__/parser.test.d.ts.map +1 -0
- package/dist/parser/__tests__/parser.test.js +72 -0
- package/dist/parser/__tests__/parser.test.js.map +1 -0
- package/dist/parser/__tests__/serializer.test.d.ts +2 -0
- package/dist/parser/__tests__/serializer.test.d.ts.map +1 -0
- package/dist/parser/__tests__/serializer.test.js +60 -0
- package/dist/parser/__tests__/serializer.test.js.map +1 -0
- package/dist/parser/__tests__/tokenizer.test.d.ts +2 -0
- package/dist/parser/__tests__/tokenizer.test.d.ts.map +1 -0
- package/dist/parser/__tests__/tokenizer.test.js +76 -0
- package/dist/parser/__tests__/tokenizer.test.js.map +1 -0
- package/dist/parser/__tests__/validator.test.d.ts +2 -0
- package/dist/parser/__tests__/validator.test.d.ts.map +1 -0
- package/dist/parser/__tests__/validator.test.js +281 -0
- package/dist/parser/__tests__/validator.test.js.map +1 -0
- package/dist/parser/serializer.d.ts +28 -0
- package/dist/parser/serializer.d.ts.map +1 -0
- package/dist/parser/serializer.js +157 -0
- package/dist/parser/serializer.js.map +1 -0
- package/dist/parser/tokenizer.new.d.ts +31 -0
- package/dist/parser/tokenizer.new.d.ts.map +1 -0
- package/dist/parser/tokenizer.new.js +181 -0
- package/dist/parser/tokenizer.new.js.map +1 -0
- package/dist/parser/v3/BlockParser.d.ts +11 -0
- package/dist/parser/v3/BlockParser.d.ts.map +1 -0
- package/dist/parser/v3/BlockParser.js +93 -0
- package/dist/parser/v3/BlockParser.js.map +1 -0
- package/dist/parser/v3/DocumentParser.d.ts +8 -0
- package/dist/parser/v3/DocumentParser.d.ts.map +1 -0
- package/dist/parser/v3/DocumentParser.js +109 -0
- package/dist/parser/v3/DocumentParser.js.map +1 -0
- package/dist/parser/v3/Parser.d.ts +11 -0
- package/dist/parser/v3/Parser.d.ts.map +1 -0
- package/dist/parser/v3/Parser.js +25 -0
- package/dist/parser/v3/Parser.js.map +1 -0
- package/dist/parser/validator.d.ts +24 -0
- package/dist/parser/validator.d.ts.map +1 -0
- package/dist/parser/validator.js +177 -0
- package/dist/parser/validator.js.map +1 -0
- package/dist/platform/NullPlatformAdapter.d.ts +18 -0
- package/dist/platform/NullPlatformAdapter.d.ts.map +1 -0
- package/dist/platform/NullPlatformAdapter.js +41 -0
- package/dist/platform/NullPlatformAdapter.js.map +1 -0
- package/dist/platform/PlatformAdapter.d.ts +22 -0
- package/dist/platform/PlatformAdapter.d.ts.map +1 -0
- package/dist/platform/PlatformAdapter.js +7 -0
- package/dist/platform/PlatformAdapter.js.map +1 -0
- package/dist/platform/index.d.ts +3 -0
- package/dist/platform/index.d.ts.map +1 -0
- package/dist/platform/index.js +19 -0
- package/dist/platform/index.js.map +1 -0
- package/dist/plugin/PluginSystem.d.ts +46 -0
- package/dist/plugin/PluginSystem.d.ts.map +1 -0
- package/dist/plugin/PluginSystem.js +58 -0
- package/dist/plugin/PluginSystem.js.map +1 -0
- package/dist/plugin/index.d.ts +2 -0
- package/dist/plugin/index.d.ts.map +1 -0
- package/dist/plugin/index.js +18 -0
- package/dist/plugin/index.js.map +1 -0
- package/dist/types/ast.d.ts +118 -0
- package/dist/types/ast.d.ts.map +1 -0
- package/dist/types/ast.js +331 -0
- package/dist/types/ast.js.map +1 -0
- package/dist/types/enums.d.ts +67 -0
- package/dist/types/enums.d.ts.map +1 -0
- package/dist/types/enums.js +278 -0
- package/dist/types/enums.js.map +1 -0
- package/dist/utils/idGenerator.d.ts +34 -0
- package/dist/utils/idGenerator.d.ts.map +1 -0
- package/dist/utils/idGenerator.js +105 -0
- package/dist/utils/idGenerator.js.map +1 -0
- package/dist/version.d.ts +7 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +10 -0
- package/dist/version.js.map +1 -0
- package/package.json +41 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Tokenizer (Lexer) for jotx 3.0
|
|
4
|
+
* Converts raw string into strict tokens for the parser
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.Tokenizer = exports.TokenType = void 0;
|
|
8
|
+
var TokenType;
|
|
9
|
+
(function (TokenType) {
|
|
10
|
+
TokenType["HEADER_DEF"] = "HEADER_DEF";
|
|
11
|
+
TokenType["BLOCK_START"] = "BLOCK_START";
|
|
12
|
+
TokenType["INDENT"] = "INDENT";
|
|
13
|
+
TokenType["DEDENT"] = "DEDENT";
|
|
14
|
+
TokenType["PROPERTY"] = "PROPERTY";
|
|
15
|
+
TokenType["TEXT_START"] = "TEXT_START";
|
|
16
|
+
TokenType["TEXT_CONTENT"] = "TEXT_CONTENT";
|
|
17
|
+
TokenType["TEXT_END"] = "TEXT_END";
|
|
18
|
+
TokenType["COMMENT"] = "COMMENT";
|
|
19
|
+
TokenType["NEWLINE"] = "NEWLINE"; // Empty line
|
|
20
|
+
})(TokenType || (exports.TokenType = TokenType = {}));
|
|
21
|
+
class Tokenizer {
|
|
22
|
+
constructor() {
|
|
23
|
+
this.tokens = [];
|
|
24
|
+
this.indentStack = [0]; // Tack indentation levels (spaces)
|
|
25
|
+
this.currentLine = 0;
|
|
26
|
+
// State for multiline text
|
|
27
|
+
this.inMultilineText = false;
|
|
28
|
+
this.currentMultilineDelimiter = '"""'; // Default, but will be overwritten
|
|
29
|
+
}
|
|
30
|
+
tokenize(input) {
|
|
31
|
+
this.tokens = [];
|
|
32
|
+
this.indentStack = [0];
|
|
33
|
+
this.currentLine = 0;
|
|
34
|
+
this.inMultilineText = false;
|
|
35
|
+
const lines = input.split('\n');
|
|
36
|
+
for (let i = 0; i < lines.length; i++) {
|
|
37
|
+
this.currentLine = i + 1;
|
|
38
|
+
const rawLine = lines[i];
|
|
39
|
+
// 1. Handle Multiline Text Mode
|
|
40
|
+
if (this.inMultilineText) {
|
|
41
|
+
const trimmedRight = rawLine.trimEnd();
|
|
42
|
+
if (rawLine.trim() === this.currentMultilineDelimiter) {
|
|
43
|
+
// Standard closing on own line
|
|
44
|
+
this.tokens.push({ type: TokenType.TEXT_END, line: this.currentLine });
|
|
45
|
+
this.inMultilineText = false;
|
|
46
|
+
}
|
|
47
|
+
else if (trimmedRight.endsWith(this.currentMultilineDelimiter)) {
|
|
48
|
+
// Content + delimiter on same line
|
|
49
|
+
// Preserve indentation/whitespace on left, strip delimiter from right
|
|
50
|
+
const content = trimmedRight.slice(0, -this.currentMultilineDelimiter.length);
|
|
51
|
+
this.tokens.push({ type: TokenType.TEXT_CONTENT, value: content, line: this.currentLine });
|
|
52
|
+
this.tokens.push({ type: TokenType.TEXT_END, line: this.currentLine });
|
|
53
|
+
this.inMultilineText = false;
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
// Just content
|
|
57
|
+
this.tokens.push({ type: TokenType.TEXT_CONTENT, value: rawLine, line: this.currentLine });
|
|
58
|
+
}
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
// 2. Skip empty lines (but maybe meaningful for block separation?)
|
|
62
|
+
// For now, ignore purely empty lines
|
|
63
|
+
if (!rawLine.trim()) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
// 3. Handle Indentation
|
|
67
|
+
const indent = rawLine.search(/\S/);
|
|
68
|
+
if (indent === -1)
|
|
69
|
+
continue; // Should be handled by empty check, but safety
|
|
70
|
+
const currentIndent = this.indentStack[this.indentStack.length - 1];
|
|
71
|
+
if (indent > currentIndent) {
|
|
72
|
+
this.indentStack.push(indent);
|
|
73
|
+
this.tokens.push({ type: TokenType.INDENT, line: this.currentLine });
|
|
74
|
+
}
|
|
75
|
+
else if (indent < currentIndent) {
|
|
76
|
+
while (this.indentStack.length > 1 && this.indentStack[this.indentStack.length - 1] > indent) {
|
|
77
|
+
this.indentStack.pop();
|
|
78
|
+
this.tokens.push({ type: TokenType.DEDENT, line: this.currentLine });
|
|
79
|
+
}
|
|
80
|
+
if (this.indentStack[this.indentStack.length - 1] !== indent) {
|
|
81
|
+
throw new Error(`Line ${this.currentLine}: Invalid indentation level`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const trimmed = rawLine.trim();
|
|
85
|
+
// 4. Tokenize Line Content
|
|
86
|
+
// Header Definition: hdef <type> <id>
|
|
87
|
+
if (trimmed.startsWith('hdef ')) {
|
|
88
|
+
const parts = trimmed.substring(5).split(/\s+/); // Remove "hdef " and split
|
|
89
|
+
this.tokens.push({
|
|
90
|
+
type: TokenType.HEADER_DEF,
|
|
91
|
+
params: parts, // [docType, docId]
|
|
92
|
+
line: this.currentLine
|
|
93
|
+
});
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
// Block Start: def <type> <id>
|
|
97
|
+
if (trimmed.startsWith('def ')) {
|
|
98
|
+
const parts = trimmed.substring(4).split(/\s+/); // Remove "def " and split
|
|
99
|
+
if (parts.length < 2) {
|
|
100
|
+
// Allow optional ID? For parsing simplicity let's require it for now,
|
|
101
|
+
// or auto-gen it in parser if missing.
|
|
102
|
+
// Pattern: def video my_video_1
|
|
103
|
+
}
|
|
104
|
+
this.tokens.push({
|
|
105
|
+
type: TokenType.BLOCK_START,
|
|
106
|
+
params: parts,
|
|
107
|
+
line: this.currentLine
|
|
108
|
+
});
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
// Multiline Text Start: text """ or text """"
|
|
112
|
+
// Regex to capture key and the opening delimiter (3 or more quotes), allowing optional content after
|
|
113
|
+
const multiLineMatch = trimmed.match(/^([a-zA-Z_]+)\s+("{3,})(.*)$/);
|
|
114
|
+
if (multiLineMatch) {
|
|
115
|
+
const key = multiLineMatch[1];
|
|
116
|
+
const delimiter = multiLineMatch[2];
|
|
117
|
+
const initialContent = multiLineMatch[3];
|
|
118
|
+
this.tokens.push({ type: TokenType.TEXT_START, value: key, line: this.currentLine });
|
|
119
|
+
// Store the specific delimiter
|
|
120
|
+
this.currentMultilineDelimiter = delimiter;
|
|
121
|
+
// Check if it's a one-liner: key """value"""
|
|
122
|
+
if (initialContent && initialContent.trimEnd().endsWith(delimiter)) {
|
|
123
|
+
const content = initialContent.trimEnd().slice(0, -delimiter.length);
|
|
124
|
+
if (content) {
|
|
125
|
+
this.tokens.push({ type: TokenType.TEXT_CONTENT, value: content, line: this.currentLine });
|
|
126
|
+
}
|
|
127
|
+
this.tokens.push({ type: TokenType.TEXT_END, line: this.currentLine });
|
|
128
|
+
this.inMultilineText = false;
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
// Multiline mode
|
|
132
|
+
this.inMultilineText = true;
|
|
133
|
+
// If there's content immediately after the delimiter, add it
|
|
134
|
+
if (initialContent) {
|
|
135
|
+
this.tokens.push({ type: TokenType.TEXT_CONTENT, value: initialContent, line: this.currentLine });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
// Property: key "value" or key 123 or key true
|
|
141
|
+
// Regex: ^key "value"$ or ^key 123$
|
|
142
|
+
// Simple property parser
|
|
143
|
+
// Property: key "value" or key 123
|
|
144
|
+
const propMatch = trimmed.match(/^([a-zA-Z_]+)\s+(.+)$/);
|
|
145
|
+
if (propMatch) {
|
|
146
|
+
const key = propMatch[1];
|
|
147
|
+
let value = propMatch[2];
|
|
148
|
+
// Strip inline comments if present
|
|
149
|
+
const commentIdx = value.indexOf('#');
|
|
150
|
+
if (commentIdx !== -1) {
|
|
151
|
+
value = value.substring(0, commentIdx).trim();
|
|
152
|
+
}
|
|
153
|
+
// Strip quotes if string
|
|
154
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
155
|
+
value = value.substring(1, value.length - 1);
|
|
156
|
+
}
|
|
157
|
+
this.tokens.push({
|
|
158
|
+
type: TokenType.PROPERTY,
|
|
159
|
+
value: key,
|
|
160
|
+
params: [value],
|
|
161
|
+
line: this.currentLine
|
|
162
|
+
});
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
// Fallback/Comment
|
|
166
|
+
if (trimmed.startsWith('#')) {
|
|
167
|
+
this.tokens.push({ type: TokenType.COMMENT, value: trimmed, line: this.currentLine });
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
throw new Error(`Line ${this.currentLine}: Syntax error: ${trimmed}`);
|
|
171
|
+
}
|
|
172
|
+
// Close remaining indentation
|
|
173
|
+
while (this.indentStack.length > 1) {
|
|
174
|
+
this.indentStack.pop();
|
|
175
|
+
this.tokens.push({ type: TokenType.DEDENT, line: this.currentLine });
|
|
176
|
+
}
|
|
177
|
+
return this.tokens;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
exports.Tokenizer = Tokenizer;
|
|
181
|
+
//# sourceMappingURL=tokenizer.new.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.new.js","sourceRoot":"","sources":["../../src/parser/tokenizer.new.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,IAAY,SAWX;AAXD,WAAY,SAAS;IACnB,sCAAyB,CAAA;IACzB,wCAA2B,CAAA;IAC3B,8BAAiB,CAAA;IACjB,8BAAiB,CAAA;IACjB,kCAAqB,CAAA;IACrB,sCAAyB,CAAA;IACzB,0CAA6B,CAAA;IAC7B,kCAAqB,CAAA;IACrB,gCAAmB,CAAA;IACnB,gCAAmB,CAAA,CAAU,aAAa;AAC5C,CAAC,EAXW,SAAS,yBAAT,SAAS,QAWpB;AASD,MAAa,SAAS;IAAtB;QACU,WAAM,GAAY,EAAE,CAAC;QACrB,gBAAW,GAAa,CAAC,CAAC,CAAC,CAAC,CAAC,mCAAmC;QAChE,gBAAW,GAAW,CAAC,CAAC;QAEhC,2BAA2B;QACnB,oBAAe,GAAY,KAAK,CAAC;QACjC,8BAAyB,GAAW,KAAK,CAAC,CAAC,mCAAmC;IA0KxF,CAAC;IAxKC,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAE7B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEzB,gCAAgC;YAChC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBAEvC,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,yBAAyB,EAAE,CAAC;oBACtD,+BAA+B;oBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBACvE,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;gBAC/B,CAAC;qBAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAAE,CAAC;oBACjE,mCAAmC;oBACnC,sEAAsE;oBACtE,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC;oBAC9E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBAC3F,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBACvE,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;gBAC/B,CAAC;qBAAM,CAAC;oBACN,eAAe;oBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC7F,CAAC;gBACD,SAAS;YACX,CAAC;YAED,mEAAmE;YACnE,qCAAqC;YACrC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,wBAAwB;YACxB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,KAAK,CAAC,CAAC;gBAAE,SAAS,CAAC,+CAA+C;YAE5E,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEpE,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;gBAC3B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACvE,CAAC;iBAAM,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;gBAClC,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;oBAC7F,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;oBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBACvE,CAAC;gBACD,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;oBAC7D,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,WAAW,6BAA6B,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAE/B,2BAA2B;YAE3B,sCAAsC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,2BAA2B;gBAC5E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,SAAS,CAAC,UAAU;oBAC1B,MAAM,EAAE,KAAK,EAAE,mBAAmB;oBAClC,IAAI,EAAE,IAAI,CAAC,WAAW;iBACvB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,+BAA+B;YAC/B,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,0BAA0B;gBAC3E,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,uEAAuE;oBACvE,wCAAwC;oBACxC,gCAAgC;gBAClC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,SAAS,CAAC,WAAW;oBAC3B,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,IAAI,CAAC,WAAW;iBACvB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,8CAA8C;YAC9C,qGAAqG;YACrG,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YACrE,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBAErF,+BAA+B;gBAC/B,IAAI,CAAC,yBAAyB,GAAG,SAAS,CAAC;gBAE3C,6CAA6C;gBAC7C,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnE,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBACrE,IAAI,OAAO,EAAE,CAAC;wBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBAC7F,CAAC;oBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBACvE,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;gBAC/B,CAAC;qBAAM,CAAC;oBACN,iBAAiB;oBACjB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;oBAC5B,6DAA6D;oBAC7D,IAAI,cAAc,EAAE,CAAC;wBACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBACpG,CAAC;gBACH,CAAC;gBACD,SAAS;YACX,CAAC;YAED,+CAA+C;YAC/C,oCAAoC;YACpC,yBAAyB;YACzB,mCAAmC;YACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACzD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBACzB,IAAI,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAEzB,mCAAmC;gBACnC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;oBACtB,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;gBAChD,CAAC;gBAED,yBAAyB;gBACzB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjD,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC/C,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,SAAS,CAAC,QAAQ;oBACxB,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,CAAC,KAAK,CAAC;oBACf,IAAI,EAAE,IAAI,CAAC,WAAW;iBACvB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,mBAAmB;YACnB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtF,SAAS;YACX,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,WAAW,mBAAmB,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,8BAA8B;QAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF;AAjLD,8BAiLC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Token } from '../tokenizer.new';
|
|
2
|
+
import { BlockNode } from '../../types/ast';
|
|
3
|
+
export declare class BlockParser {
|
|
4
|
+
parseBlock(tokens: Token[], startIndex: number, parentStack: BlockNode[]): {
|
|
5
|
+
node: BlockNode;
|
|
6
|
+
nextIndex: number;
|
|
7
|
+
} | null;
|
|
8
|
+
resolveBlockType(rawType: string, stack: BlockNode[]): string;
|
|
9
|
+
parseProperty(block: BlockNode, key: string, value: any): void;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=BlockParser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BlockParser.d.ts","sourceRoot":"","sources":["../../../src/parser/v3/BlockParser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAa,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,qBAAa,WAAW;IAGpB,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG;QAAE,IAAI,EAAE,SAAS,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAwCxH,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM;IA6B7D,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;CAyB1D"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BlockParser = void 0;
|
|
4
|
+
const tokenizer_new_1 = require("../tokenizer.new");
|
|
5
|
+
const registry_1 = require("@jotx/registry");
|
|
6
|
+
class BlockParser {
|
|
7
|
+
// Returns the parsed block and the next token index
|
|
8
|
+
parseBlock(tokens, startIndex, parentStack) {
|
|
9
|
+
const token = tokens[startIndex];
|
|
10
|
+
if (token.type !== tokenizer_new_1.TokenType.BLOCK_START) {
|
|
11
|
+
return null;
|
|
12
|
+
}
|
|
13
|
+
// 1. Identify Type & ID
|
|
14
|
+
let type = token.params[0];
|
|
15
|
+
const id = token.params[1] || `auto_${Date.now()}_${startIndex}`;
|
|
16
|
+
// 2. Resolve Registry Definition
|
|
17
|
+
const resolvedType = this.resolveBlockType(type, parentStack);
|
|
18
|
+
const block = {
|
|
19
|
+
id,
|
|
20
|
+
type: resolvedType,
|
|
21
|
+
properties: {},
|
|
22
|
+
children: [],
|
|
23
|
+
line: token.line
|
|
24
|
+
};
|
|
25
|
+
// 3. Parse Body (Properties & Children)
|
|
26
|
+
let i = startIndex + 1;
|
|
27
|
+
// We need to track our own scope.
|
|
28
|
+
// In the indent-based tokenizer, children are marked by INDENT/DEDENT
|
|
29
|
+
// But the linear token stream flattens this.
|
|
30
|
+
// Actually, the RegistryParser approach loop was:
|
|
31
|
+
// Iterate all tokens, maintain a `stack` of open blocks.
|
|
32
|
+
// That approach is O(N) single pass and easier than recursive recursive calls
|
|
33
|
+
// because the tokenizer outputs linear tokens including INDENT/DEDENTS.
|
|
34
|
+
// So BlockParser might just be a helper for "resolving type" and "parsing single prop",
|
|
35
|
+
// while DocumentParser (or a main Parser loop) manages the stack.
|
|
36
|
+
// Let's refactor DocumentParser to control the loop, and move the specific logic here.
|
|
37
|
+
return null; // Logic moved to main loop for streaming efficiency
|
|
38
|
+
}
|
|
39
|
+
resolveBlockType(rawType, stack) {
|
|
40
|
+
let def = registry_1.registry.get(rawType) || registry_1.registry.getByKeyword(rawType);
|
|
41
|
+
// Context-aware overriding
|
|
42
|
+
if (rawType === 'item' && stack.length > 0) {
|
|
43
|
+
const parent = stack[stack.length - 1];
|
|
44
|
+
if (parent.type === 'checklist')
|
|
45
|
+
return 'taskItem';
|
|
46
|
+
if (parent.type === 'list')
|
|
47
|
+
return 'listItem';
|
|
48
|
+
}
|
|
49
|
+
let finalType = def ? def.type : rawType;
|
|
50
|
+
// V3 STRICT: Simplified Mappings (CASE-SENSITIVE)
|
|
51
|
+
const SIMPLIFIED_MAP = {
|
|
52
|
+
// No mappings - strict flatcase AST
|
|
53
|
+
};
|
|
54
|
+
if (SIMPLIFIED_MAP[finalType]) {
|
|
55
|
+
finalType = SIMPLIFIED_MAP[finalType];
|
|
56
|
+
}
|
|
57
|
+
// V3 STRICT: Error if not registered (no fuzzy fallback)
|
|
58
|
+
if (!registry_1.registry.get(finalType)) {
|
|
59
|
+
console.error(`Unknown block type '${rawType}'. Block types must be registered in registry and use exact flatcase names.`);
|
|
60
|
+
finalType = 'custom';
|
|
61
|
+
}
|
|
62
|
+
return finalType;
|
|
63
|
+
}
|
|
64
|
+
parseProperty(block, key, value) {
|
|
65
|
+
// V3 STRICT: Property names must be exact flatcase (no fuzzy matching)
|
|
66
|
+
const blockDef = registry_1.registry.get(block.type);
|
|
67
|
+
let finalKey = key;
|
|
68
|
+
if (blockDef && blockDef.schema) {
|
|
69
|
+
// Type Coercion
|
|
70
|
+
const propDef = blockDef.schema.properties[finalKey];
|
|
71
|
+
if (propDef) {
|
|
72
|
+
if (propDef.type === 'boolean') {
|
|
73
|
+
if (value === 'true')
|
|
74
|
+
value = true;
|
|
75
|
+
if (value === 'false')
|
|
76
|
+
value = false;
|
|
77
|
+
}
|
|
78
|
+
if (propDef.type === 'number') {
|
|
79
|
+
const num = parseFloat(value);
|
|
80
|
+
if (!isNaN(num))
|
|
81
|
+
value = num;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
// V3 STRICT: Property not in schema - warn but allow (could be custom)
|
|
86
|
+
console.warn(`Property '${key}' not defined in schema for block type '${block.type}'. Ensure property name is exact flatcase.`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
block.properties[finalKey] = value;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
exports.BlockParser = BlockParser;
|
|
93
|
+
//# sourceMappingURL=BlockParser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BlockParser.js","sourceRoot":"","sources":["../../../src/parser/v3/BlockParser.ts"],"names":[],"mappings":";;;AAAA,oDAAoD;AAEpD,6CAA0C;AAE1C,MAAa,WAAW;IAEpB,oDAAoD;IACpD,UAAU,CAAC,MAAe,EAAE,UAAkB,EAAE,WAAwB;QACpE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAS,CAAC,WAAW,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,wBAAwB;QACxB,IAAI,IAAI,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,EAAE,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC;QAElE,iCAAiC;QACjC,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAc;YACrB,EAAE;YACF,IAAI,EAAE,YAAmB;YACzB,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,EAAE;YACZ,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC;QAEF,wCAAwC;QACxC,IAAI,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC;QAEvB,mCAAmC;QACnC,sEAAsE;QACtE,6CAA6C;QAE7C,kDAAkD;QAClD,yDAAyD;QACzD,+EAA+E;QAC/E,wEAAwE;QAExE,wFAAwF;QACxF,kEAAkE;QAClE,uFAAuF;QAEvF,OAAO,IAAI,CAAC,CAAC,oDAAoD;IACrE,CAAC;IAED,gBAAgB,CAAC,OAAe,EAAE,KAAkB;QAChD,IAAI,GAAG,GAAG,mBAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,mBAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAElE,2BAA2B;QAC3B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW;gBAAE,OAAO,UAAU,CAAC;YACnD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,UAAU,CAAC;QAClD,CAAC;QAED,IAAI,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;QAEzC,kDAAkD;QAClD,MAAM,cAAc,GAA2B;QAC3C,oCAAoC;SACvC,CAAC;QACF,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5B,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,mBAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,uBAAuB,OAAO,6EAA6E,CAAC,CAAC;YAC3H,SAAS,GAAG,QAAQ,CAAC;QACzB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,aAAa,CAAC,KAAgB,EAAE,GAAW,EAAE,KAAU;QACnD,uEAAuE;QACvE,MAAM,QAAQ,GAAG,mBAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,QAAQ,GAAG,GAAG,CAAC;QAEnB,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,gBAAgB;YAChB,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrD,IAAI,OAAO,EAAE,CAAC;gBACV,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC7B,IAAI,KAAK,KAAK,MAAM;wBAAE,KAAK,GAAG,IAAI,CAAC;oBACnC,IAAI,KAAK,KAAK,OAAO;wBAAE,KAAK,GAAG,KAAK,CAAC;gBACzC,CAAC;gBACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;oBAC9B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;wBAAE,KAAK,GAAG,GAAG,CAAC;gBACjC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,uEAAuE;gBACvE,OAAO,CAAC,IAAI,CAAC,aAAa,GAAG,2CAA2C,KAAK,CAAC,IAAI,4CAA4C,CAAC,CAAC;YACpI,CAAC;QACL,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;IACvC,CAAC;CACJ;AAjGD,kCAiGC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DocumentParser.d.ts","sourceRoot":"","sources":["../../../src/parser/v3/DocumentParser.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAG1D,qBAAa,cAAc;IACvB,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,SAAS,CAAY;;IAO7B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY;CA+GrC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DocumentParser = void 0;
|
|
4
|
+
const tokenizer_new_1 = require("../tokenizer.new");
|
|
5
|
+
const BlockParser_1 = require("./BlockParser");
|
|
6
|
+
class DocumentParser {
|
|
7
|
+
constructor() {
|
|
8
|
+
this.blockParser = new BlockParser_1.BlockParser();
|
|
9
|
+
this.tokenizer = new tokenizer_new_1.Tokenizer();
|
|
10
|
+
}
|
|
11
|
+
parse(input) {
|
|
12
|
+
const tokens = this.tokenizer.tokenize(input);
|
|
13
|
+
const rootBlocks = [];
|
|
14
|
+
const stack = [];
|
|
15
|
+
let currentBlock = null;
|
|
16
|
+
let currentPropKey = null;
|
|
17
|
+
// Document metadata from hdef header
|
|
18
|
+
let docType = 'jotx';
|
|
19
|
+
let docId = 'doc';
|
|
20
|
+
let docMetadata = {};
|
|
21
|
+
let parsingDocMetadata = false;
|
|
22
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
23
|
+
const token = tokens[i];
|
|
24
|
+
switch (token.type) {
|
|
25
|
+
case tokenizer_new_1.TokenType.HEADER_DEF: {
|
|
26
|
+
// hdef <type> <id>
|
|
27
|
+
const headerType = token.params[0];
|
|
28
|
+
const headerId = token.params[1] || 'doc';
|
|
29
|
+
docType = headerType;
|
|
30
|
+
docId = headerId;
|
|
31
|
+
parsingDocMetadata = true; // Next properties belong to document metadata
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
case tokenizer_new_1.TokenType.BLOCK_START: {
|
|
35
|
+
// First block encountered - stop parsing doc metadata
|
|
36
|
+
parsingDocMetadata = false;
|
|
37
|
+
const type = token.params[0];
|
|
38
|
+
const id = token.params[1] || `auto_${Date.now()}_${i}`;
|
|
39
|
+
// Delegate Type Resolution
|
|
40
|
+
const finalType = this.blockParser.resolveBlockType(type, stack);
|
|
41
|
+
const newBlock = {
|
|
42
|
+
id,
|
|
43
|
+
type: finalType,
|
|
44
|
+
properties: {},
|
|
45
|
+
children: [],
|
|
46
|
+
line: token.line
|
|
47
|
+
};
|
|
48
|
+
if (stack.length > 0) {
|
|
49
|
+
const parent = stack[stack.length - 1];
|
|
50
|
+
if (!parent.children)
|
|
51
|
+
parent.children = [];
|
|
52
|
+
parent.children.push(newBlock);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
rootBlocks.push(newBlock);
|
|
56
|
+
}
|
|
57
|
+
currentBlock = newBlock;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
case tokenizer_new_1.TokenType.INDENT:
|
|
61
|
+
if (currentBlock) {
|
|
62
|
+
stack.push(currentBlock);
|
|
63
|
+
}
|
|
64
|
+
break;
|
|
65
|
+
case tokenizer_new_1.TokenType.DEDENT:
|
|
66
|
+
stack.pop();
|
|
67
|
+
currentBlock = stack.length > 0 ? stack[stack.length - 1] : null;
|
|
68
|
+
break;
|
|
69
|
+
case tokenizer_new_1.TokenType.PROPERTY:
|
|
70
|
+
// If we're still parsing doc metadata (before first block), add to docMetadata
|
|
71
|
+
if (parsingDocMetadata && token.value) {
|
|
72
|
+
let val = token.params[0];
|
|
73
|
+
// Strip quotes if string
|
|
74
|
+
if (typeof val === 'string' && val.startsWith('"') && val.endsWith('"')) {
|
|
75
|
+
val = val.substring(1, val.length - 1);
|
|
76
|
+
}
|
|
77
|
+
docMetadata[token.value] = val;
|
|
78
|
+
}
|
|
79
|
+
else if (currentBlock && token.value) {
|
|
80
|
+
this.blockParser.parseProperty(currentBlock, token.value, token.params[0]);
|
|
81
|
+
}
|
|
82
|
+
break;
|
|
83
|
+
case tokenizer_new_1.TokenType.TEXT_START:
|
|
84
|
+
if (currentBlock && token.value) {
|
|
85
|
+
currentPropKey = token.value;
|
|
86
|
+
currentBlock.properties[currentPropKey] = "";
|
|
87
|
+
}
|
|
88
|
+
break;
|
|
89
|
+
case tokenizer_new_1.TokenType.TEXT_CONTENT:
|
|
90
|
+
if (currentBlock && currentPropKey) {
|
|
91
|
+
const currentVal = currentBlock.properties[currentPropKey];
|
|
92
|
+
currentBlock.properties[currentPropKey] = currentVal ? currentVal + '\n' + token.value : token.value;
|
|
93
|
+
}
|
|
94
|
+
break;
|
|
95
|
+
case tokenizer_new_1.TokenType.TEXT_END:
|
|
96
|
+
currentPropKey = null;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
id: docId,
|
|
102
|
+
type: docType,
|
|
103
|
+
metadata: docMetadata,
|
|
104
|
+
blocks: rootBlocks
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
exports.DocumentParser = DocumentParser;
|
|
109
|
+
//# sourceMappingURL=DocumentParser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DocumentParser.js","sourceRoot":"","sources":["../../../src/parser/v3/DocumentParser.ts"],"names":[],"mappings":";;;AAAA,oDAA+D;AAE/D,+CAA4C;AAE5C,MAAa,cAAc;IAIvB;QACI,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAW,EAAE,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAS,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,KAAa;QACf,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAgB,EAAE,CAAC;QACnC,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAqB,IAAI,CAAC;QAC1C,IAAI,cAAc,GAAkB,IAAI,CAAC;QAEzC,qCAAqC;QACrC,IAAI,OAAO,GAAW,MAAM,CAAC;QAC7B,IAAI,KAAK,GAAW,KAAK,CAAC;QAC1B,IAAI,WAAW,GAAwB,EAAE,CAAC;QAC1C,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAExB,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACjB,KAAK,yBAAS,CAAC,UAAU,CAAC,CAAC,CAAC;oBACxB,mBAAmB;oBACnB,MAAM,UAAU,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC;oBACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;oBAE3C,OAAO,GAAG,UAAU,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC;oBACjB,kBAAkB,GAAG,IAAI,CAAC,CAAC,8CAA8C;oBACzE,MAAM;gBACV,CAAC;gBAED,KAAK,yBAAS,CAAC,WAAW,CAAC,CAAC,CAAC;oBACzB,sDAAsD;oBACtD,kBAAkB,GAAG,KAAK,CAAC;oBAE3B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC;oBAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;oBAEzD,2BAA2B;oBAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;oBAEjE,MAAM,QAAQ,GAAc;wBACxB,EAAE;wBACF,IAAI,EAAE,SAAgB;wBACtB,UAAU,EAAE,EAAE;wBACd,QAAQ,EAAE,EAAE;wBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;qBACnB,CAAC;oBAEF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACnB,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,QAAQ;4BAAE,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;wBAC3C,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACnC,CAAC;yBAAM,CAAC;wBACJ,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC9B,CAAC;oBACD,YAAY,GAAG,QAAQ,CAAC;oBACxB,MAAM;gBACV,CAAC;gBAED,KAAK,yBAAS,CAAC,MAAM;oBACjB,IAAI,YAAY,EAAE,CAAC;wBACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAC7B,CAAC;oBACD,MAAM;gBAEV,KAAK,yBAAS,CAAC,MAAM;oBACjB,KAAK,CAAC,GAAG,EAAE,CAAC;oBACZ,YAAY,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACjE,MAAM;gBAEV,KAAK,yBAAS,CAAC,QAAQ;oBACnB,+EAA+E;oBAC/E,IAAI,kBAAkB,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;wBACpC,IAAI,GAAG,GAAQ,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC;wBAEhC,yBAAyB;wBACzB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;4BACtE,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC3C,CAAC;wBAED,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;oBACnC,CAAC;yBAAM,IAAI,YAAY,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;wBACrC,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;oBACD,MAAM;gBAEV,KAAK,yBAAS,CAAC,UAAU;oBACrB,IAAI,YAAY,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;wBAC9B,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC;wBAC7B,YAAY,CAAC,UAAU,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;oBACjD,CAAC;oBACD,MAAM;gBAEV,KAAK,yBAAS,CAAC,YAAY;oBACvB,IAAI,YAAY,IAAI,cAAc,EAAE,CAAC;wBACjC,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,cAAc,CAAW,CAAC;wBACrE,YAAY,CAAC,UAAU,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;oBACzG,CAAC;oBACD,MAAM;gBAEV,KAAK,yBAAS,CAAC,QAAQ;oBACnB,cAAc,GAAG,IAAI,CAAC;oBACtB,MAAM;YACd,CAAC;QACL,CAAC;QAED,OAAO;YACH,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,OAAc;YACpB,QAAQ,EAAE,WAAW;YACrB,MAAM,EAAE,UAAU;SACrB,CAAC;IACN,CAAC;CACJ;AAxHD,wCAwHC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { AST } from '../../types/ast';
|
|
2
|
+
/**
|
|
3
|
+
* V3 Parser Entry Point
|
|
4
|
+
* Wraps the modular DocumentParser to match the legacy API.
|
|
5
|
+
*/
|
|
6
|
+
export declare class Parser {
|
|
7
|
+
private docParser;
|
|
8
|
+
constructor();
|
|
9
|
+
parse(input: string): AST;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=Parser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Parser.d.ts","sourceRoot":"","sources":["../../../src/parser/v3/Parser.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAEtC;;;GAGG;AACH,qBAAa,MAAM;IACf,OAAO,CAAC,SAAS,CAAiB;;IAMlC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG;CAY5B"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Parser = void 0;
|
|
4
|
+
const DocumentParser_1 = require("./DocumentParser");
|
|
5
|
+
/**
|
|
6
|
+
* V3 Parser Entry Point
|
|
7
|
+
* Wraps the modular DocumentParser to match the legacy API.
|
|
8
|
+
*/
|
|
9
|
+
class Parser {
|
|
10
|
+
constructor() {
|
|
11
|
+
this.docParser = new DocumentParser_1.DocumentParser();
|
|
12
|
+
}
|
|
13
|
+
parse(input) {
|
|
14
|
+
const document = this.docParser.parse(input);
|
|
15
|
+
// V3 Parser delegates validation to a separate step (see Validator class)
|
|
16
|
+
// so we return empty errors here. The consumer should call validate(ast) if they want errors.
|
|
17
|
+
// This is a cleaner 'separation of concerns'.
|
|
18
|
+
return {
|
|
19
|
+
document,
|
|
20
|
+
errors: []
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.Parser = Parser;
|
|
25
|
+
//# sourceMappingURL=Parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Parser.js","sourceRoot":"","sources":["../../../src/parser/v3/Parser.ts"],"names":[],"mappings":";;;AAAA,qDAAkD;AAGlD;;;GAGG;AACH,MAAa,MAAM;IAGf;QACI,IAAI,CAAC,SAAS,GAAG,IAAI,+BAAc,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,KAAa;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE7C,0EAA0E;QAC1E,8FAA8F;QAC9F,8CAA8C;QAE9C,OAAO;YACH,QAAQ;YACR,MAAM,EAAE,EAAE;SACb,CAAC;IACN,CAAC;CACJ;AAnBD,wBAmBC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { DocumentNode, ValidationError } from '../types/ast';
|
|
2
|
+
/**
|
|
3
|
+
* Jotx Validator
|
|
4
|
+
* Validates AST against Registry Schema
|
|
5
|
+
*/
|
|
6
|
+
export declare class Validator {
|
|
7
|
+
validate(doc: DocumentNode): ValidationError[];
|
|
8
|
+
/**
|
|
9
|
+
* Validate that all blocks in the document are allowed for this document type
|
|
10
|
+
*/
|
|
11
|
+
private validateDocumentSchema;
|
|
12
|
+
/**
|
|
13
|
+
* Validate document metadata (e.g. type enum)
|
|
14
|
+
*/
|
|
15
|
+
private validateDocumentMetadata;
|
|
16
|
+
/**
|
|
17
|
+
* Recursively check if blocks are allowed in this document type
|
|
18
|
+
*/
|
|
19
|
+
private checkBlocksAgainstSchema;
|
|
20
|
+
private validateBlocks;
|
|
21
|
+
private validateBlock;
|
|
22
|
+
}
|
|
23
|
+
export declare function createValidator(): Validator;
|
|
24
|
+
//# sourceMappingURL=validator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validator.d.ts","sourceRoot":"","sources":["../../src/parser/validator.ts"],"names":[],"mappings":"AACA,OAAO,EAAO,YAAY,EAAa,eAAe,EAAuB,MAAM,cAAc,CAAC;AAGlG;;;GAGG;AACH,qBAAa,SAAS;IAEpB,QAAQ,CAAC,GAAG,EAAE,YAAY,GAAG,eAAe,EAAE;IAe9C;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAchC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,aAAa;CA2FtB;AAED,wBAAgB,eAAe,cAE9B"}
|