@lokascript/framework 4.7.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -1
- package/dist/api/create-dsl.d.ts +14 -12
- package/dist/api/create-dsl.d.ts.map +1 -1
- package/dist/api/index.js +888 -1683
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.d.ts +0 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +560 -1133
- package/dist/core/index.js.map +1 -1
- package/dist/core/matching/index.d.ts +2 -0
- package/dist/core/matching/index.d.ts.map +1 -1
- package/dist/core/matching/possessive-keywords.d.ts.map +1 -0
- package/dist/core/matching/stock-confidence.d.ts +3 -4
- package/dist/core/matching/stock-confidence.d.ts.map +1 -1
- package/dist/core/matching/stock-reader.d.ts +8 -1
- package/dist/core/matching/stock-reader.d.ts.map +1 -1
- package/dist/core/matching/stock-reading.d.ts +4 -4
- package/dist/core/{pattern-matching/utils → matching}/type-validation.d.ts +1 -1
- package/dist/core/matching/type-validation.d.ts.map +1 -0
- package/dist/core/types.d.ts +5 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/generation/index.d.ts +1 -1
- package/dist/generation/index.d.ts.map +1 -1
- package/dist/generation/index.js +32 -171
- package/dist/generation/index.js.map +1 -1
- package/dist/generation/profile.d.ts +30 -0
- package/dist/generation/profile.d.ts.map +1 -0
- package/dist/generation/renderer.d.ts +1 -1
- package/dist/generation/renderer.d.ts.map +1 -1
- package/dist/generation/schema-patterns.d.ts +3 -3
- package/dist/index.cjs +969 -1674
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +969 -1668
- package/dist/index.js.map +1 -1
- package/dist/ir/index.js.map +1 -1
- package/dist/multilingual/builders.d.ts +3 -3
- package/dist/multilingual/builders.d.ts.map +1 -1
- package/dist/multilingual/index.js.map +1 -1
- package/dist/parsing/index.d.ts +0 -1
- package/dist/parsing/index.d.ts.map +1 -1
- package/dist/parsing/index.js +0 -1145
- package/dist/parsing/index.js.map +1 -1
- package/dist/testing/gate/index.js +44 -12
- package/dist/testing/gate/index.js.map +1 -1
- package/dist/testing/gate/lint/types.d.ts +1 -1
- package/dist/testing/gate/lint/types.d.ts.map +1 -1
- package/dist/testing/gate/round-trip.d.ts +24 -3
- package/dist/testing/gate/round-trip.d.ts.map +1 -1
- package/package.json +2 -7
- package/src/__test__/test-utils.ts +1 -1
- package/src/api/create-dsl-engine.test.ts +123 -34
- package/src/api/create-dsl.test.ts +7 -0
- package/src/api/create-dsl.ts +56 -39
- package/src/core/index.ts +0 -1
- package/src/core/matching/index.ts +2 -0
- package/src/core/matching/stock-confidence.ts +3 -4
- package/src/core/matching/stock-reader.test.ts +87 -3
- package/src/core/matching/stock-reader.ts +20 -1
- package/src/core/matching/stock-reading.ts +6 -6
- package/src/core/{pattern-matching/utils → matching}/type-validation.test.ts +2 -2
- package/src/core/{pattern-matching/utils → matching}/type-validation.ts +1 -1
- package/src/core/types.ts +5 -0
- package/src/generation/domain-renderer.test.ts +1 -1
- package/src/generation/index.ts +1 -1
- package/src/generation/profile.ts +41 -0
- package/src/generation/renderer.test.ts +1 -1
- package/src/generation/renderer.ts +1 -1
- package/src/generation/schema-patterns.test.ts +41 -1
- package/src/generation/schema-patterns.ts +15 -3
- package/src/multilingual/bridge.test.ts +31 -23
- package/src/multilingual/builders.ts +3 -3
- package/src/parsing/index.ts +0 -1
- package/src/testing/gate/lint/types.ts +1 -1
- package/src/testing/gate/round-trip.test.ts +99 -9
- package/src/testing/gate/round-trip.ts +76 -11
- package/dist/core/pattern-matching/index.d.ts +0 -6
- package/dist/core/pattern-matching/index.d.ts.map +0 -1
- package/dist/core/pattern-matching/index.js +0 -1211
- package/dist/core/pattern-matching/index.js.map +0 -1
- package/dist/core/pattern-matching/pattern-matcher.d.ts +0 -122
- package/dist/core/pattern-matching/pattern-matcher.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/index.d.ts +0 -6
- package/dist/core/pattern-matching/utils/index.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/possessive-keywords.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/type-validation.d.ts.map +0 -1
- package/dist/generation/pattern-generator.d.ts +0 -55
- package/dist/generation/pattern-generator.d.ts.map +0 -1
- package/src/core/pattern-matching/index.ts +0 -6
- package/src/core/pattern-matching/pattern-matcher.test.ts +0 -1031
- package/src/core/pattern-matching/pattern-matcher.ts +0 -765
- package/src/core/pattern-matching/utils/index.ts +0 -6
- package/src/generation/pattern-generator.test.ts +0 -532
- package/src/generation/pattern-generator.ts +0 -327
- /package/dist/core/{pattern-matching/utils → matching}/possessive-keywords.d.ts +0 -0
- /package/src/core/{pattern-matching/utils → matching}/possessive-keywords.ts +0 -0
package/dist/index.js
CHANGED
|
@@ -19,256 +19,670 @@ import {
|
|
|
19
19
|
getRoleValue
|
|
20
20
|
} from "@lokascript/intent";
|
|
21
21
|
|
|
22
|
-
// src/core/
|
|
23
|
-
var
|
|
24
|
-
constructor(
|
|
25
|
-
this.
|
|
26
|
-
this.
|
|
22
|
+
// src/core/tokenization/token-utils.ts
|
|
23
|
+
var TokenStreamImpl = class {
|
|
24
|
+
constructor(tokens, language) {
|
|
25
|
+
this.pos = 0;
|
|
26
|
+
this.tokens = tokens;
|
|
27
|
+
this.language = language;
|
|
27
28
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return
|
|
29
|
+
peek(offset = 0) {
|
|
30
|
+
const index = this.pos + offset;
|
|
31
|
+
if (index < 0 || index >= this.tokens.length) {
|
|
32
|
+
return null;
|
|
32
33
|
}
|
|
33
|
-
return
|
|
34
|
+
return this.tokens[index];
|
|
34
35
|
}
|
|
35
|
-
|
|
36
|
-
if (
|
|
37
|
-
|
|
38
|
-
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
39
|
-
switch (level) {
|
|
40
|
-
case "debug":
|
|
41
|
-
console.debug(`${timestamp} ${prefix} DEBUG:`, message, ...args);
|
|
42
|
-
break;
|
|
43
|
-
case "info":
|
|
44
|
-
console.info(`${timestamp} ${prefix} INFO:`, message, ...args);
|
|
45
|
-
break;
|
|
46
|
-
case "warn":
|
|
47
|
-
console.warn(`${timestamp} ${prefix} WARN:`, message, ...args);
|
|
48
|
-
break;
|
|
49
|
-
case "error":
|
|
50
|
-
console.error(`${timestamp} ${prefix} ERROR:`, message, ...args);
|
|
51
|
-
break;
|
|
36
|
+
advance() {
|
|
37
|
+
if (this.isAtEnd()) {
|
|
38
|
+
throw new Error("Unexpected end of token stream");
|
|
52
39
|
}
|
|
40
|
+
return this.tokens[this.pos++];
|
|
53
41
|
}
|
|
54
|
-
|
|
55
|
-
this.
|
|
42
|
+
isAtEnd() {
|
|
43
|
+
return this.pos >= this.tokens.length;
|
|
56
44
|
}
|
|
57
|
-
|
|
58
|
-
this.
|
|
45
|
+
mark() {
|
|
46
|
+
return { position: this.pos };
|
|
59
47
|
}
|
|
60
|
-
|
|
61
|
-
this.
|
|
48
|
+
reset(mark) {
|
|
49
|
+
this.pos = mark.position;
|
|
62
50
|
}
|
|
63
|
-
|
|
64
|
-
this.
|
|
51
|
+
position() {
|
|
52
|
+
return this.pos;
|
|
65
53
|
}
|
|
66
54
|
/**
|
|
67
|
-
*
|
|
55
|
+
* Get remaining tokens as an array.
|
|
68
56
|
*/
|
|
69
|
-
|
|
70
|
-
return this.
|
|
57
|
+
remaining() {
|
|
58
|
+
return this.tokens.slice(this.pos);
|
|
71
59
|
}
|
|
72
60
|
/**
|
|
73
|
-
*
|
|
61
|
+
* Consume tokens while predicate is true.
|
|
74
62
|
*/
|
|
75
|
-
|
|
76
|
-
|
|
63
|
+
takeWhile(predicate) {
|
|
64
|
+
const result = [];
|
|
65
|
+
while (!this.isAtEnd() && predicate(this.peek())) {
|
|
66
|
+
result.push(this.advance());
|
|
67
|
+
}
|
|
68
|
+
return result;
|
|
77
69
|
}
|
|
78
70
|
/**
|
|
79
|
-
*
|
|
71
|
+
* Skip tokens while predicate is true.
|
|
80
72
|
*/
|
|
81
|
-
|
|
82
|
-
this.
|
|
73
|
+
skipWhile(predicate) {
|
|
74
|
+
while (!this.isAtEnd() && predicate(this.peek())) {
|
|
75
|
+
this.advance();
|
|
76
|
+
}
|
|
83
77
|
}
|
|
84
78
|
};
|
|
85
|
-
function
|
|
86
|
-
return
|
|
79
|
+
function createPosition(start, end) {
|
|
80
|
+
return { start, end };
|
|
87
81
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
82
|
+
function createToken(valueOrParams, kind, position, normalizedOrOptions) {
|
|
83
|
+
if (typeof valueOrParams === "object") {
|
|
84
|
+
const { value: value2, kind: kind2, position: position2, normalized: normalized2, stem, stemConfidence, metadata } = valueOrParams;
|
|
85
|
+
return {
|
|
86
|
+
value: value2,
|
|
87
|
+
kind: kind2,
|
|
88
|
+
position: position2,
|
|
89
|
+
...normalized2 !== void 0 && { normalized: normalized2 },
|
|
90
|
+
...stem !== void 0 && { stem },
|
|
91
|
+
...stemConfidence !== void 0 && { stemConfidence },
|
|
92
|
+
...metadata !== void 0 && { metadata }
|
|
93
|
+
};
|
|
96
94
|
}
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
const value = valueOrParams;
|
|
96
|
+
if (!kind || !position) {
|
|
97
|
+
throw new Error("createToken requires kind and position parameters");
|
|
99
98
|
}
|
|
100
|
-
if (
|
|
101
|
-
return
|
|
99
|
+
if (typeof normalizedOrOptions === "string") {
|
|
100
|
+
return { value, kind, position, normalized: normalizedOrOptions };
|
|
102
101
|
}
|
|
103
|
-
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
102
|
+
if (normalizedOrOptions) {
|
|
103
|
+
const { normalized: normalized2, stem, stemConfidence, metadata } = normalizedOrOptions;
|
|
104
|
+
return {
|
|
105
|
+
value,
|
|
106
|
+
kind,
|
|
107
|
+
position,
|
|
108
|
+
...normalized2 !== void 0 && { normalized: normalized2 },
|
|
109
|
+
...stem !== void 0 && { stem },
|
|
110
|
+
...stemConfidence !== void 0 && { stemConfidence },
|
|
111
|
+
...metadata !== void 0 && { metadata }
|
|
112
|
+
};
|
|
108
113
|
}
|
|
109
|
-
return
|
|
110
|
-
}
|
|
111
|
-
function isCSSSelector(value) {
|
|
112
|
-
return value.startsWith(".") || value.startsWith("#") || value.startsWith("<");
|
|
113
|
-
}
|
|
114
|
-
function isClassName(value) {
|
|
115
|
-
return value.startsWith(".");
|
|
116
|
-
}
|
|
117
|
-
function isIdSelector(value) {
|
|
118
|
-
return value.startsWith("#");
|
|
119
|
-
}
|
|
120
|
-
function isCSSPropertyRef(value) {
|
|
121
|
-
return value.startsWith("*");
|
|
114
|
+
return { value, kind, position };
|
|
122
115
|
}
|
|
123
|
-
function
|
|
124
|
-
|
|
125
|
-
if (durationMatch) {
|
|
126
|
-
return true;
|
|
127
|
-
}
|
|
128
|
-
const num = parseFloat(value);
|
|
129
|
-
return !isNaN(num) && isFinite(num);
|
|
116
|
+
function isWhitespace(char) {
|
|
117
|
+
return /\s/.test(char);
|
|
130
118
|
}
|
|
131
|
-
function
|
|
132
|
-
return
|
|
119
|
+
function isSelectorStart(char) {
|
|
120
|
+
return char === "#" || char === "." || char === "[" || char === "@" || char === "*" || char === "<";
|
|
133
121
|
}
|
|
134
|
-
function
|
|
135
|
-
return
|
|
122
|
+
function isQuote(char) {
|
|
123
|
+
return char === '"' || char === "'" || char === "`" || char === "\u300C" || char === "\u300D";
|
|
136
124
|
}
|
|
137
|
-
function
|
|
138
|
-
|
|
139
|
-
return builtIns.has(value.toLowerCase());
|
|
125
|
+
function isDigit(char) {
|
|
126
|
+
return /\d/.test(char);
|
|
140
127
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
function getPossessiveReference(profile, keyword) {
|
|
144
|
-
return profile?.possessive?.keywords?.[keyword];
|
|
128
|
+
function stripOptionalDiacritics(word) {
|
|
129
|
+
return word.replace(/[\u064b-\u0652\u0670]/g, "");
|
|
145
130
|
}
|
|
146
|
-
function
|
|
147
|
-
return
|
|
131
|
+
function isAsciiLetter(char) {
|
|
132
|
+
return /[a-zA-Z]/.test(char);
|
|
148
133
|
}
|
|
149
|
-
function
|
|
150
|
-
return
|
|
134
|
+
function isAsciiIdentifierChar(char) {
|
|
135
|
+
return /[a-zA-Z0-9_-]/.test(char);
|
|
151
136
|
}
|
|
152
137
|
|
|
153
|
-
// src/core/matching/
|
|
154
|
-
function
|
|
155
|
-
return
|
|
138
|
+
// src/core/matching/match-skeleton.ts
|
|
139
|
+
function groupHasRole(group) {
|
|
140
|
+
return group.tokens.some((t) => t.type === "role" || t.type === "group" && groupHasRole(t));
|
|
156
141
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
142
|
+
var MatchSkeleton = class {
|
|
143
|
+
/**
|
|
144
|
+
* @param makeReader - builds the grammar's reader, handed this matcher's view
|
|
145
|
+
* of the match in progress (one reader per matcher).
|
|
146
|
+
*/
|
|
147
|
+
constructor(makeReader, confidenceModel) {
|
|
148
|
+
/**
|
|
149
|
+
* The pattern tokens still to match after the one being matched, through
|
|
150
|
+
* every enclosing group: a value stops at any marker the pattern still owes.
|
|
151
|
+
*/
|
|
152
|
+
this.patternRest = [];
|
|
153
|
+
/**
|
|
154
|
+
* Whether the role slot being matched directly follows a literal the pattern
|
|
155
|
+
* matched: its marker (`[por {quantity}]`), or the command's verb.
|
|
156
|
+
*/
|
|
157
|
+
this.slotAfterLiteral = false;
|
|
158
|
+
/**
|
|
159
|
+
* Selective memoization cache (Phase 6.1).
|
|
160
|
+
* Caches pattern match results keyed by `tokenPosition:patternId`.
|
|
161
|
+
* Cleared before each top-level matchBest() call.
|
|
162
|
+
*/
|
|
163
|
+
this.matchCache = /* @__PURE__ */ new Map();
|
|
164
|
+
/**
|
|
165
|
+
* Stream index where each role's capture began, for the current
|
|
166
|
+
* `matchPattern` call. Rebuilt per call, written on the same success path as
|
|
167
|
+
* `captured` — see `PatternMatchResult.roleStarts`.
|
|
168
|
+
*/
|
|
169
|
+
this.roleStarts = /* @__PURE__ */ new Map();
|
|
170
|
+
/**
|
|
171
|
+
* Track stem matches for confidence calculation.
|
|
172
|
+
* This is set during matching and read during confidence calculation.
|
|
173
|
+
*/
|
|
174
|
+
this.stemMatchCount = 0;
|
|
175
|
+
this.totalKeywordMatches = 0;
|
|
176
|
+
this.confidenceModel = confidenceModel;
|
|
177
|
+
this.reader = makeReader({
|
|
178
|
+
patternRest: () => this.patternRest,
|
|
179
|
+
slotAfterLiteral: () => this.slotAfterLiteral,
|
|
180
|
+
wouldMatch: (pt, token) => this.patternTokenWouldMatch(pt, token)
|
|
181
|
+
});
|
|
163
182
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
183
|
+
/**
|
|
184
|
+
* Try to match a single pattern against the token stream.
|
|
185
|
+
* Returns the match result or null if no match.
|
|
186
|
+
*/
|
|
187
|
+
matchPattern(tokens, pattern) {
|
|
188
|
+
const mark = tokens.mark();
|
|
189
|
+
const captured = /* @__PURE__ */ new Map();
|
|
190
|
+
this.reader.beginPattern(pattern);
|
|
191
|
+
this.stemMatchCount = 0;
|
|
192
|
+
this.totalKeywordMatches = 0;
|
|
193
|
+
this.roleStarts = /* @__PURE__ */ new Map();
|
|
194
|
+
this.patternRest = [];
|
|
195
|
+
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
196
|
+
if (!success) {
|
|
197
|
+
tokens.reset(mark);
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
const confidence = this.confidenceModel.calculate({
|
|
201
|
+
pattern,
|
|
202
|
+
captured,
|
|
203
|
+
stemMatchCount: this.stemMatchCount,
|
|
204
|
+
totalKeywordMatches: this.totalKeywordMatches
|
|
205
|
+
});
|
|
206
|
+
this.applyExtractionRules(pattern, captured);
|
|
207
|
+
return {
|
|
208
|
+
pattern,
|
|
209
|
+
captured,
|
|
210
|
+
consumedTokens: tokens.position() - mark.position,
|
|
211
|
+
confidence,
|
|
212
|
+
roleStarts: this.roleStarts,
|
|
213
|
+
...this.reader.resultFields?.()
|
|
214
|
+
};
|
|
171
215
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
216
|
+
/**
|
|
217
|
+
* Try to match multiple patterns, return the best match.
|
|
218
|
+
*/
|
|
219
|
+
matchBest(tokens, patterns) {
|
|
220
|
+
this.matchCache.clear();
|
|
221
|
+
const matches = [];
|
|
222
|
+
const startPos = tokens.position();
|
|
223
|
+
for (const pattern of patterns) {
|
|
224
|
+
const mark = tokens.mark();
|
|
225
|
+
const cacheKey = `${startPos}:${pattern.id}`;
|
|
226
|
+
let result;
|
|
227
|
+
if (this.matchCache.has(cacheKey)) {
|
|
228
|
+
result = this.matchCache.get(cacheKey);
|
|
229
|
+
} else {
|
|
230
|
+
result = this.matchPattern(tokens, pattern);
|
|
231
|
+
this.matchCache.set(cacheKey, result);
|
|
232
|
+
}
|
|
233
|
+
if (result) {
|
|
234
|
+
matches.push(result);
|
|
177
235
|
}
|
|
236
|
+
tokens.reset(mark);
|
|
178
237
|
}
|
|
179
|
-
|
|
180
|
-
|
|
238
|
+
if (matches.length === 0) {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
matches.sort((a, b) => {
|
|
242
|
+
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
243
|
+
if (priorityDiff !== 0) return priorityDiff;
|
|
244
|
+
return b.confidence - a.confidence;
|
|
245
|
+
});
|
|
246
|
+
const best = matches[0];
|
|
247
|
+
this.matchPattern(tokens, best.pattern);
|
|
248
|
+
return best;
|
|
181
249
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
250
|
+
/**
|
|
251
|
+
* Match a sequence of pattern tokens against the token stream.
|
|
252
|
+
*/
|
|
253
|
+
matchTokenSequence(tokens, patternTokens, captured, nextAfterSequence) {
|
|
254
|
+
this.reader.skipLeading?.(tokens, patternTokens);
|
|
255
|
+
for (let i = 0; i < patternTokens.length; i++) {
|
|
256
|
+
const patternToken = patternTokens[i];
|
|
257
|
+
this.reader.beforePatternToken?.(tokens, patternTokens, i, captured);
|
|
258
|
+
if (this.reader.maySkipSlot(
|
|
259
|
+
patternToken,
|
|
260
|
+
tokens.peek(),
|
|
261
|
+
patternTokens[i + 1] ?? nextAfterSequence
|
|
262
|
+
)) {
|
|
263
|
+
const skipMark = tokens.mark();
|
|
264
|
+
const speculative = new Map(captured);
|
|
265
|
+
if (this.matchTokenSequence(
|
|
266
|
+
tokens,
|
|
267
|
+
patternTokens.slice(i + 1),
|
|
268
|
+
speculative,
|
|
269
|
+
nextAfterSequence
|
|
270
|
+
) && this.reader.atClauseEnd(tokens)) {
|
|
271
|
+
captured.clear();
|
|
272
|
+
for (const [role, value] of speculative) captured.set(role, value);
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
275
|
+
tokens.reset(skipMark);
|
|
276
|
+
}
|
|
277
|
+
const outerRest = this.patternRest;
|
|
278
|
+
this.patternRest = [...patternTokens.slice(i + 1), ...outerRest];
|
|
279
|
+
this.slotAfterLiteral = patternToken.type === "role" && patternTokens[i - 1]?.type === "literal";
|
|
280
|
+
const matched = this.matchPatternToken(
|
|
281
|
+
tokens,
|
|
282
|
+
patternToken,
|
|
283
|
+
captured,
|
|
284
|
+
patternTokens[i + 1] ?? nextAfterSequence
|
|
285
|
+
);
|
|
286
|
+
this.patternRest = outerRest;
|
|
287
|
+
if (!matched) {
|
|
288
|
+
if (this.isOptional(patternToken)) {
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
return false;
|
|
187
292
|
}
|
|
188
293
|
}
|
|
189
|
-
captured.set(patternToken.role, methodCallValue);
|
|
190
294
|
return true;
|
|
191
295
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
296
|
+
/**
|
|
297
|
+
* Match a single pattern token against the current position in the stream.
|
|
298
|
+
*/
|
|
299
|
+
matchPatternToken(tokens, patternToken, captured, nextPatternToken) {
|
|
300
|
+
switch (patternToken.type) {
|
|
301
|
+
case "literal":
|
|
302
|
+
return this.matchLiteralToken(tokens, patternToken);
|
|
303
|
+
case "role": {
|
|
304
|
+
const roleStart = tokens.position();
|
|
305
|
+
const matchedRole = this.reader.readRole(tokens, patternToken, captured, nextPatternToken);
|
|
306
|
+
if (matchedRole && captured.has(patternToken.role)) {
|
|
307
|
+
this.roleStarts.set(patternToken.role, roleStart);
|
|
308
|
+
}
|
|
309
|
+
return matchedRole;
|
|
197
310
|
}
|
|
311
|
+
case "group":
|
|
312
|
+
return this.matchGroupToken(tokens, patternToken, captured, nextPatternToken);
|
|
313
|
+
default:
|
|
314
|
+
return false;
|
|
198
315
|
}
|
|
199
|
-
captured.set(patternToken.role, possessiveSelectorValue);
|
|
200
|
-
return true;
|
|
201
316
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
317
|
+
/**
|
|
318
|
+
* Match a literal pattern token (keyword or particle).
|
|
319
|
+
*/
|
|
320
|
+
matchLiteralToken(tokens, patternToken) {
|
|
321
|
+
const token = tokens.peek();
|
|
322
|
+
if (!token) return false;
|
|
323
|
+
const matchType = this.getMatchType(token, patternToken.value);
|
|
324
|
+
if (matchType !== "none") {
|
|
325
|
+
this.totalKeywordMatches++;
|
|
326
|
+
if (matchType === "stem") {
|
|
327
|
+
this.stemMatchCount++;
|
|
207
328
|
}
|
|
329
|
+
tokens.advance();
|
|
330
|
+
return true;
|
|
208
331
|
}
|
|
209
|
-
|
|
210
|
-
|
|
332
|
+
if (patternToken.alternatives) {
|
|
333
|
+
for (const alt of patternToken.alternatives) {
|
|
334
|
+
const altMatchType = this.getMatchType(token, alt);
|
|
335
|
+
if (altMatchType !== "none") {
|
|
336
|
+
this.totalKeywordMatches++;
|
|
337
|
+
if (altMatchType === "stem") {
|
|
338
|
+
this.stemMatchCount++;
|
|
339
|
+
}
|
|
340
|
+
tokens.advance();
|
|
341
|
+
return true;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
for (const value of [patternToken.value, ...patternToken.alternatives ?? []]) {
|
|
346
|
+
for (const sep of ["-", "_"]) {
|
|
347
|
+
if (value.includes(sep) && this.tryMatchShatteredCompound(tokens, value, sep)) {
|
|
348
|
+
this.totalKeywordMatches++;
|
|
349
|
+
return true;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return false;
|
|
211
354
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
355
|
+
/**
|
|
356
|
+
* Match a compound keyword (`titik-henti`, `по_умолчанию`) against a
|
|
357
|
+
* shattered token run (`titik` `-` `henti` / `по` `_` `умолчанию`): word
|
|
358
|
+
* segments must be identifier/keyword/particle tokens equal to the expected
|
|
359
|
+
* segments (case-insensitive; particle because prepositions like ru `по` /
|
|
360
|
+
* uk `за` lead some underscore compounds), separators lone `sep` tokens.
|
|
361
|
+
* Consumes the run on success, resets the stream on failure.
|
|
362
|
+
*/
|
|
363
|
+
tryMatchShatteredCompound(tokens, expected, sep) {
|
|
364
|
+
const segments = expected.split(sep);
|
|
365
|
+
if (segments.length < 2 || segments.some((s) => s.length === 0)) return false;
|
|
366
|
+
const mark = tokens.mark();
|
|
367
|
+
for (let i = 0; i < segments.length; i++) {
|
|
368
|
+
if (i > 0) {
|
|
369
|
+
const sepToken = tokens.peek();
|
|
370
|
+
if (!sepToken || sepToken.value !== sep) {
|
|
371
|
+
tokens.reset(mark);
|
|
372
|
+
return false;
|
|
373
|
+
}
|
|
374
|
+
tokens.advance();
|
|
375
|
+
}
|
|
376
|
+
const word = tokens.peek();
|
|
377
|
+
if (!word || word.kind !== "identifier" && word.kind !== "keyword" && word.kind !== "particle" || word.value.toLowerCase() !== segments[i].toLowerCase()) {
|
|
378
|
+
tokens.reset(mark);
|
|
379
|
+
return false;
|
|
217
380
|
}
|
|
381
|
+
tokens.advance();
|
|
218
382
|
}
|
|
219
|
-
captured.set(patternToken.role, selectorPropertyValue);
|
|
220
383
|
return true;
|
|
221
384
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
385
|
+
/**
|
|
386
|
+
* Match a group pattern token (optional sequence).
|
|
387
|
+
*/
|
|
388
|
+
matchGroupToken(tokens, patternToken, captured, nextPatternToken) {
|
|
389
|
+
const mark = tokens.mark();
|
|
390
|
+
const capturedBefore = new Map(captured);
|
|
391
|
+
let success = this.matchTokenSequence(tokens, patternToken.tokens, captured, nextPatternToken);
|
|
392
|
+
if (success && tokens.position() > mark.position && groupHasRole(patternToken) && [...captured].every(([role, value]) => capturedBefore.get(role) === value) && captured.size === capturedBefore.size) {
|
|
393
|
+
success = false;
|
|
394
|
+
}
|
|
395
|
+
if (success) {
|
|
396
|
+
const markerLiterals = [];
|
|
397
|
+
for (const inner of patternToken.tokens) {
|
|
398
|
+
if (inner.type === "group" && inner.renderRequired) {
|
|
399
|
+
for (const t of inner.tokens) {
|
|
400
|
+
if (t.type === "literal") markerLiterals.push(t.value, ...t.alternatives ?? []);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (markerLiterals.length > 0) {
|
|
405
|
+
const addedRole = [...captured.keys()].some((role) => !capturedBefore.has(role));
|
|
406
|
+
if (addedRole) {
|
|
407
|
+
const last = tokens.tokens[tokens.position() - 1];
|
|
408
|
+
const markerSeen = last !== void 0 && tokens.position() > mark.position && markerLiterals.some((lit) => this.getMatchType(last, lit) !== "none");
|
|
409
|
+
if (!markerSeen) success = false;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
if (!success) {
|
|
414
|
+
tokens.reset(mark);
|
|
415
|
+
for (const role of captured.keys()) {
|
|
416
|
+
if (!capturedBefore.has(role)) {
|
|
417
|
+
captured.delete(role);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
for (const [role, value] of capturedBefore) {
|
|
421
|
+
if (captured.get(role) !== value) {
|
|
422
|
+
captured.set(role, value);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
228
425
|
return patternToken.optional || false;
|
|
229
426
|
}
|
|
427
|
+
return true;
|
|
230
428
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
const baseRef = getPossessiveReference(profile, tokenLower);
|
|
242
|
-
if (!baseRef) return null;
|
|
243
|
-
const mark = tokens.mark();
|
|
244
|
-
tokens.advance();
|
|
245
|
-
const propertyToken = tokens.peek();
|
|
246
|
-
if (!propertyToken) {
|
|
247
|
-
tokens.reset(mark);
|
|
248
|
-
return null;
|
|
249
|
-
}
|
|
250
|
-
if (propertyToken.kind === "identifier" || propertyToken.kind === "keyword" && !isStructuralKeyword(propertyToken.value) || propertyToken.kind === "selector" && propertyToken.value.startsWith("*") || propertyToken.kind === "selector" && propertyToken.value.startsWith("@") || propertyToken.kind === "selector" && propertyToken.value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyToken.value)) {
|
|
251
|
-
tokens.advance();
|
|
252
|
-
let propertyName = propertyToken.value;
|
|
253
|
-
if (propertyToken.kind === "selector" && propertyName.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyName)) {
|
|
254
|
-
propertyName = propertyName.substring(1);
|
|
255
|
-
}
|
|
256
|
-
let chainedProps = propertyName;
|
|
257
|
-
while (tokens.peek()?.kind === "selector" && tokens.peek().value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(tokens.peek().value)) {
|
|
258
|
-
chainedProps += tokens.peek().value;
|
|
259
|
-
tokens.advance();
|
|
429
|
+
/**
|
|
430
|
+
* Whether a pattern token (literal, or a group starting with a literal) would
|
|
431
|
+
* match the given stream token. Used to keep the event-head source-clause
|
|
432
|
+
* consumption from stealing a marker the pattern explicitly expects.
|
|
433
|
+
*/
|
|
434
|
+
patternTokenWouldMatch(pt, token) {
|
|
435
|
+
if (!pt) return false;
|
|
436
|
+
if (pt.type === "literal") {
|
|
437
|
+
if (this.getMatchType(token, pt.value) !== "none") return true;
|
|
438
|
+
return !!pt.alternatives?.some((a) => this.getMatchType(token, a) !== "none");
|
|
260
439
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
chainedProps += nextPeek.value;
|
|
264
|
-
tokens.advance();
|
|
440
|
+
if (pt.type === "group") {
|
|
441
|
+
return this.patternTokenWouldMatch(pt.tokens?.[0], token);
|
|
265
442
|
}
|
|
266
|
-
return
|
|
443
|
+
return false;
|
|
267
444
|
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
445
|
+
/**
|
|
446
|
+
* Get the type of match for a token against a value.
|
|
447
|
+
* Used for confidence calculation.
|
|
448
|
+
*/
|
|
449
|
+
getMatchType(token, value) {
|
|
450
|
+
if (token.value === value) return "exact";
|
|
451
|
+
if (token.normalized === value) return "normalized";
|
|
452
|
+
if (token.stem === value && token.stemConfidence !== void 0 && token.stemConfidence >= 0.7) {
|
|
453
|
+
return "stem";
|
|
454
|
+
}
|
|
455
|
+
if (token.kind === "keyword" && token.value.toLowerCase() === value.toLowerCase()) {
|
|
456
|
+
return "case-insensitive";
|
|
457
|
+
}
|
|
458
|
+
if (token.kind === "keyword") {
|
|
459
|
+
const strippedToken = stripOptionalDiacritics(token.value).toLowerCase();
|
|
460
|
+
if (strippedToken && strippedToken === stripOptionalDiacritics(value).toLowerCase()) {
|
|
461
|
+
return "case-insensitive";
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return "none";
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* Apply extraction rules to fill in static values and defaults for missing
|
|
468
|
+
* roles, and re-shape captured roles that declare a `transform` (the
|
|
469
|
+
* ExtractionRule field was documented but unwired — go-qu-url-dest is the
|
|
470
|
+
* first consumer: it re-types the marker-bound capture to the expression
|
|
471
|
+
* shape the en reference produces).
|
|
472
|
+
*/
|
|
473
|
+
applyExtractionRules(pattern, captured) {
|
|
474
|
+
for (const [role, rule] of Object.entries(pattern.extraction)) {
|
|
475
|
+
const existing = captured.get(role);
|
|
476
|
+
if (existing) {
|
|
477
|
+
if (rule.transform) {
|
|
478
|
+
const raw = existing.type === "literal" ? String(existing.value) : String(existing.raw ?? "");
|
|
479
|
+
captured.set(role, rule.transform(raw));
|
|
480
|
+
}
|
|
481
|
+
} else if (rule.value !== void 0) {
|
|
482
|
+
captured.set(
|
|
483
|
+
role,
|
|
484
|
+
rule.valueIsExpression ? { type: "expression", raw: rule.value } : { type: "literal", value: rule.value }
|
|
485
|
+
);
|
|
486
|
+
} else if (rule.default) {
|
|
487
|
+
const implicitDefault = {
|
|
488
|
+
...rule.default,
|
|
489
|
+
implicit: true
|
|
490
|
+
};
|
|
491
|
+
captured.set(role, implicitDefault);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Check if a pattern token is optional.
|
|
497
|
+
*/
|
|
498
|
+
isOptional(patternToken) {
|
|
499
|
+
return patternToken.type !== "literal" && patternToken.optional === true;
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
// src/core/matching/type-validation.ts
|
|
504
|
+
function isTypeCompatible(actualType, expectedTypes) {
|
|
505
|
+
if (!expectedTypes || expectedTypes.length === 0) {
|
|
506
|
+
return true;
|
|
507
|
+
}
|
|
508
|
+
if (expectedTypes.includes(actualType)) {
|
|
509
|
+
return true;
|
|
510
|
+
}
|
|
511
|
+
if (expectedTypes.includes("expression")) {
|
|
512
|
+
return true;
|
|
513
|
+
}
|
|
514
|
+
if (actualType === "property-path") {
|
|
515
|
+
return expectedTypes.some((t) => ["selector", "reference", "expression"].includes(t));
|
|
516
|
+
}
|
|
517
|
+
return false;
|
|
518
|
+
}
|
|
519
|
+
function validateValueType(value, expectedTypes) {
|
|
520
|
+
if (!expectedTypes || expectedTypes.length === 0) {
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
return isTypeCompatible(value.type, expectedTypes);
|
|
524
|
+
}
|
|
525
|
+
function isCSSSelector(value) {
|
|
526
|
+
return value.startsWith(".") || value.startsWith("#") || value.startsWith("<");
|
|
527
|
+
}
|
|
528
|
+
function isClassName(value) {
|
|
529
|
+
return value.startsWith(".");
|
|
530
|
+
}
|
|
531
|
+
function isIdSelector(value) {
|
|
532
|
+
return value.startsWith("#");
|
|
533
|
+
}
|
|
534
|
+
function isCSSPropertyRef(value) {
|
|
535
|
+
return value.startsWith("*");
|
|
536
|
+
}
|
|
537
|
+
function isNumericValue(value) {
|
|
538
|
+
const durationMatch = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
|
|
539
|
+
if (durationMatch) {
|
|
540
|
+
return true;
|
|
541
|
+
}
|
|
542
|
+
const num = parseFloat(value);
|
|
543
|
+
return !isNaN(num) && isFinite(num);
|
|
544
|
+
}
|
|
545
|
+
function isPropertyName(value) {
|
|
546
|
+
return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(value);
|
|
547
|
+
}
|
|
548
|
+
function isVariableRef(value) {
|
|
549
|
+
return value.startsWith(":");
|
|
550
|
+
}
|
|
551
|
+
function isBuiltInReference(value) {
|
|
552
|
+
const builtIns = /* @__PURE__ */ new Set(["me", "you", "it", "result", "event", "target", "body"]);
|
|
553
|
+
return builtIns.has(value.toLowerCase());
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// src/core/matching/possessive-keywords.ts
|
|
557
|
+
function getPossessiveReference(profile, keyword) {
|
|
558
|
+
return profile?.possessive?.keywords?.[keyword];
|
|
559
|
+
}
|
|
560
|
+
function isPossessiveKeyword(profile, keyword) {
|
|
561
|
+
return profile?.possessive?.keywords?.[keyword] !== void 0;
|
|
562
|
+
}
|
|
563
|
+
function getAllPossessiveKeywords(profile) {
|
|
564
|
+
return profile?.possessive?.keywords ?? {};
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// src/core/matching/stock-reading.ts
|
|
568
|
+
function isValidReference(_value) {
|
|
569
|
+
return false;
|
|
570
|
+
}
|
|
571
|
+
function safeToLowerCase(value) {
|
|
572
|
+
if (typeof value === "string") {
|
|
573
|
+
return value.toLowerCase();
|
|
574
|
+
}
|
|
575
|
+
if (value === null || value === void 0) {
|
|
576
|
+
return "";
|
|
577
|
+
}
|
|
578
|
+
return String(value).toLowerCase();
|
|
579
|
+
}
|
|
580
|
+
function readRoleValue(tokens, patternToken, captured, profile) {
|
|
581
|
+
skipNoiseWords(tokens);
|
|
582
|
+
const token = tokens.peek();
|
|
583
|
+
if (!token) {
|
|
584
|
+
return patternToken.optional || false;
|
|
585
|
+
}
|
|
586
|
+
const possessiveValue = tryMatchPossessiveExpression(tokens, profile);
|
|
587
|
+
if (possessiveValue) {
|
|
588
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
589
|
+
if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
|
|
590
|
+
return patternToken.optional || false;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
captured.set(patternToken.role, possessiveValue);
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
const methodCallValue = tryMatchMethodCallExpression(tokens);
|
|
597
|
+
if (methodCallValue) {
|
|
598
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
599
|
+
if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
|
|
600
|
+
return patternToken.optional || false;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
captured.set(patternToken.role, methodCallValue);
|
|
604
|
+
return true;
|
|
605
|
+
}
|
|
606
|
+
const possessiveSelectorValue = tryMatchPossessiveSelectorExpression(tokens);
|
|
607
|
+
if (possessiveSelectorValue) {
|
|
608
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
609
|
+
if (!isTypeCompatible(possessiveSelectorValue.type, patternToken.expectedTypes)) {
|
|
610
|
+
return patternToken.optional || false;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
captured.set(patternToken.role, possessiveSelectorValue);
|
|
614
|
+
return true;
|
|
615
|
+
}
|
|
616
|
+
const propertyAccessValue = tryMatchPropertyAccessExpression(tokens);
|
|
617
|
+
if (propertyAccessValue) {
|
|
618
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
619
|
+
if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
|
|
620
|
+
return patternToken.optional || false;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
captured.set(patternToken.role, propertyAccessValue);
|
|
624
|
+
return true;
|
|
625
|
+
}
|
|
626
|
+
const selectorPropertyValue = tryMatchSelectorPropertyExpression(tokens);
|
|
627
|
+
if (selectorPropertyValue) {
|
|
628
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
629
|
+
if (!isTypeCompatible(selectorPropertyValue.type, patternToken.expectedTypes)) {
|
|
630
|
+
return patternToken.optional || false;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
captured.set(patternToken.role, selectorPropertyValue);
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
const value = tokenToSemanticValue(token);
|
|
637
|
+
if (!value) {
|
|
638
|
+
return patternToken.optional || false;
|
|
639
|
+
}
|
|
640
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
641
|
+
if (!isTypeCompatible(value.type, patternToken.expectedTypes)) {
|
|
642
|
+
return patternToken.optional || false;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
captured.set(patternToken.role, value);
|
|
646
|
+
tokens.advance();
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
649
|
+
function tryMatchPossessiveExpression(tokens, profile) {
|
|
650
|
+
const token = tokens.peek();
|
|
651
|
+
if (!token) return null;
|
|
652
|
+
if (!profile) return null;
|
|
653
|
+
const tokenValue = token.normalized || token.value;
|
|
654
|
+
const tokenLower = safeToLowerCase(tokenValue);
|
|
655
|
+
const baseRef = getPossessiveReference(profile, tokenLower);
|
|
656
|
+
if (!baseRef) return null;
|
|
657
|
+
const mark = tokens.mark();
|
|
658
|
+
tokens.advance();
|
|
659
|
+
const propertyToken = tokens.peek();
|
|
660
|
+
if (!propertyToken) {
|
|
661
|
+
tokens.reset(mark);
|
|
662
|
+
return null;
|
|
663
|
+
}
|
|
664
|
+
if (propertyToken.kind === "identifier" || propertyToken.kind === "keyword" && !isStructuralKeyword(propertyToken.value) || propertyToken.kind === "selector" && propertyToken.value.startsWith("*") || propertyToken.kind === "selector" && propertyToken.value.startsWith("@") || propertyToken.kind === "selector" && propertyToken.value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyToken.value)) {
|
|
665
|
+
tokens.advance();
|
|
666
|
+
let propertyName = propertyToken.value;
|
|
667
|
+
if (propertyToken.kind === "selector" && propertyName.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyName)) {
|
|
668
|
+
propertyName = propertyName.substring(1);
|
|
669
|
+
}
|
|
670
|
+
let chainedProps = propertyName;
|
|
671
|
+
while (tokens.peek()?.kind === "selector" && tokens.peek().value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(tokens.peek().value)) {
|
|
672
|
+
chainedProps += tokens.peek().value;
|
|
673
|
+
tokens.advance();
|
|
674
|
+
}
|
|
675
|
+
const nextPeek = tokens.peek();
|
|
676
|
+
if (nextPeek?.kind === "literal" && nextPeek.value.startsWith("(")) {
|
|
677
|
+
chainedProps += nextPeek.value;
|
|
678
|
+
tokens.advance();
|
|
679
|
+
}
|
|
680
|
+
return createPropertyPath(createReference(baseRef), chainedProps);
|
|
681
|
+
}
|
|
682
|
+
tokens.reset(mark);
|
|
683
|
+
return null;
|
|
684
|
+
}
|
|
685
|
+
function isStructuralKeyword(value) {
|
|
272
686
|
const structural = /* @__PURE__ */ new Set([
|
|
273
687
|
// Prepositions
|
|
274
688
|
"into",
|
|
@@ -481,1195 +895,64 @@ function tokenToSemanticValue(token) {
|
|
|
481
895
|
if (isValidReference(lower)) {
|
|
482
896
|
return createReference(lower);
|
|
483
897
|
}
|
|
484
|
-
return createLiteral(token.normalized || token.value);
|
|
485
|
-
}
|
|
486
|
-
case "identifier": {
|
|
487
|
-
if (typeof token.value === "string" && token.value.startsWith(":")) {
|
|
488
|
-
return createReference(token.value);
|
|
489
|
-
}
|
|
490
|
-
const identLower = safeToLowerCase(token.value);
|
|
491
|
-
if (isValidReference(identLower)) {
|
|
492
|
-
return createReference(identLower);
|
|
493
|
-
}
|
|
494
|
-
return { type: "expression", raw: token.value };
|
|
495
|
-
}
|
|
496
|
-
case "url":
|
|
497
|
-
return createLiteral(token.value, "string");
|
|
498
|
-
default:
|
|
499
|
-
return null;
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
function parseLiteralValue(value) {
|
|
503
|
-
if (value.startsWith('"') || value.startsWith("'") || value.startsWith("`") || value.startsWith("\u300C")) {
|
|
504
|
-
const inner = value.slice(1, -1);
|
|
505
|
-
return createLiteral(inner, "string");
|
|
506
|
-
}
|
|
507
|
-
if (value === "true") return createLiteral(true, "boolean");
|
|
508
|
-
if (value === "false") return createLiteral(false, "boolean");
|
|
509
|
-
const durationMatch = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
|
|
510
|
-
if (durationMatch) {
|
|
511
|
-
const num2 = parseFloat(durationMatch[1]);
|
|
512
|
-
const unit = durationMatch[2];
|
|
513
|
-
if (unit) {
|
|
514
|
-
return createLiteral(value, "duration");
|
|
515
|
-
}
|
|
516
|
-
return createLiteral(num2, "number");
|
|
517
|
-
}
|
|
518
|
-
const num = parseFloat(value);
|
|
519
|
-
if (!isNaN(num)) {
|
|
520
|
-
return createLiteral(num, "number");
|
|
521
|
-
}
|
|
522
|
-
return createLiteral(value, "string");
|
|
523
|
-
}
|
|
524
|
-
var ENGLISH_NOISE_WORDS = /* @__PURE__ */ new Set(["the", "a", "an"]);
|
|
525
|
-
function skipNoiseWords(tokens) {
|
|
526
|
-
const token = tokens.peek();
|
|
527
|
-
if (!token) return;
|
|
528
|
-
const tokenLower = safeToLowerCase(token.value);
|
|
529
|
-
if (ENGLISH_NOISE_WORDS.has(tokenLower)) {
|
|
530
|
-
const mark = tokens.mark();
|
|
531
|
-
tokens.advance();
|
|
532
|
-
const nextToken = tokens.peek();
|
|
533
|
-
if (nextToken && nextToken.kind === "selector") {
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
tokens.reset(mark);
|
|
537
|
-
}
|
|
538
|
-
if (tokenLower === "class") {
|
|
539
|
-
tokens.advance();
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// src/core/matching/stock-confidence.ts
|
|
544
|
-
function scoreParts(ctx) {
|
|
545
|
-
const { pattern, captured } = ctx;
|
|
546
|
-
let score = 0;
|
|
547
|
-
let maxScore = 0;
|
|
548
|
-
const hasDefault = (role) => {
|
|
549
|
-
return pattern.extraction?.[role]?.default !== void 0;
|
|
550
|
-
};
|
|
551
|
-
for (const token of pattern.template.tokens) {
|
|
552
|
-
if (token.type === "role") {
|
|
553
|
-
maxScore += 1;
|
|
554
|
-
if (captured.has(token.role)) {
|
|
555
|
-
score += 1;
|
|
556
|
-
}
|
|
557
|
-
} else if (token.type === "group") {
|
|
558
|
-
for (const subToken of token.tokens) {
|
|
559
|
-
if (subToken.type === "role") {
|
|
560
|
-
const roleHasDefault = hasDefault(subToken.role);
|
|
561
|
-
const weight = 0.8;
|
|
562
|
-
maxScore += weight;
|
|
563
|
-
if (captured.has(subToken.role)) {
|
|
564
|
-
score += weight;
|
|
565
|
-
} else if (roleHasDefault) {
|
|
566
|
-
score += weight * 0.6;
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
const roleCoverage = maxScore > 0 ? score / maxScore : 1;
|
|
573
|
-
let baseConfidence = roleCoverage;
|
|
574
|
-
let stemPenalty = 0;
|
|
575
|
-
if (ctx.stemMatchCount > 0 && ctx.totalKeywordMatches > 0) {
|
|
576
|
-
stemPenalty = ctx.stemMatchCount / ctx.totalKeywordMatches * 0.15;
|
|
577
|
-
baseConfidence = Math.max(0.5, baseConfidence - stemPenalty);
|
|
578
|
-
}
|
|
579
|
-
const vsoBoost = calculateVSOConfidenceBoost(pattern);
|
|
580
|
-
baseConfidence = Math.min(1, baseConfidence + vsoBoost);
|
|
581
|
-
const prepositionAdjustment = arabicPrepositionDisambiguation(pattern, captured);
|
|
582
|
-
baseConfidence = Math.max(0, Math.min(1, baseConfidence + prepositionAdjustment));
|
|
583
|
-
return {
|
|
584
|
-
roleCoverage,
|
|
585
|
-
stemPenalty,
|
|
586
|
-
languageBoost: vsoBoost,
|
|
587
|
-
languageAdjustment: prepositionAdjustment,
|
|
588
|
-
final: baseConfidence
|
|
589
|
-
};
|
|
590
|
-
}
|
|
591
|
-
function calculateVSOConfidenceBoost(pattern) {
|
|
592
|
-
if (pattern.language !== "ar") {
|
|
593
|
-
return 0;
|
|
594
|
-
}
|
|
595
|
-
const firstToken = pattern.template.tokens[0];
|
|
596
|
-
if (!firstToken || firstToken.type !== "literal") {
|
|
597
|
-
return 0;
|
|
598
|
-
}
|
|
599
|
-
const ARABIC_VERBS = /* @__PURE__ */ new Set([
|
|
600
|
-
"\u0628\u062F\u0644",
|
|
601
|
-
"\u063A\u064A\u0631",
|
|
602
|
-
"\u0623\u0636\u0641",
|
|
603
|
-
"\u0623\u0632\u0644",
|
|
604
|
-
"\u0636\u0639",
|
|
605
|
-
"\u0627\u062C\u0639\u0644",
|
|
606
|
-
"\u0639\u064A\u0646",
|
|
607
|
-
"\u0632\u062F",
|
|
608
|
-
"\u0627\u0646\u0642\u0635",
|
|
609
|
-
"\u0633\u062C\u0644",
|
|
610
|
-
"\u0623\u0638\u0647\u0631",
|
|
611
|
-
"\u0623\u062E\u0641",
|
|
612
|
-
"\u0634\u063A\u0644",
|
|
613
|
-
"\u0623\u0631\u0633\u0644",
|
|
614
|
-
"\u0631\u0643\u0632",
|
|
615
|
-
"\u0634\u0648\u0634",
|
|
616
|
-
"\u062A\u0648\u0642\u0641",
|
|
617
|
-
"\u0627\u0646\u0633\u062E",
|
|
618
|
-
"\u0627\u062D\u0630\u0641",
|
|
619
|
-
"\u0627\u0635\u0646\u0639",
|
|
620
|
-
"\u0627\u0646\u062A\u0638\u0631",
|
|
621
|
-
"\u0627\u0646\u062A\u0642\u0627\u0644",
|
|
622
|
-
"\u0623\u0648"
|
|
623
|
-
]);
|
|
624
|
-
if (ARABIC_VERBS.has(firstToken.value)) {
|
|
625
|
-
return 0.15;
|
|
626
|
-
}
|
|
627
|
-
if (firstToken.alternatives) {
|
|
628
|
-
for (const alt of firstToken.alternatives) {
|
|
629
|
-
if (ARABIC_VERBS.has(alt)) {
|
|
630
|
-
return 0.15;
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
}
|
|
634
|
-
return 0;
|
|
635
|
-
}
|
|
636
|
-
function arabicPrepositionDisambiguation(pattern, captured) {
|
|
637
|
-
if (pattern.language !== "ar") {
|
|
638
|
-
return 0;
|
|
639
|
-
}
|
|
640
|
-
let adjustment = 0;
|
|
641
|
-
const PREFERRED_PREPOSITIONS = {
|
|
642
|
-
patient: ["\u0639\u0644\u0649"],
|
|
643
|
-
// element selectors prefer على (on/upon)
|
|
644
|
-
destination: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
645
|
-
// destination prefers إلى (to)
|
|
646
|
-
source: ["\u0645\u0646"],
|
|
647
|
-
// source prefers من (from)
|
|
648
|
-
agent: ["\u0645\u0646"],
|
|
649
|
-
// agent/by prefers من (from/by)
|
|
650
|
-
manner: ["\u0628"],
|
|
651
|
-
// manner prefers ب (with/by)
|
|
652
|
-
style: ["\u0628"],
|
|
653
|
-
// style prefers ب (with)
|
|
654
|
-
goal: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
655
|
-
// target state prefers إلى (to)
|
|
656
|
-
method: ["\u0628"]
|
|
657
|
-
// method prefers ب (with/by)
|
|
658
|
-
};
|
|
659
|
-
for (const [role, value] of captured.entries()) {
|
|
660
|
-
const preferred = PREFERRED_PREPOSITIONS[role];
|
|
661
|
-
if (!preferred || preferred.length === 0) {
|
|
662
|
-
continue;
|
|
663
|
-
}
|
|
664
|
-
const metadata = "metadata" in value ? value.metadata : void 0;
|
|
665
|
-
if (metadata && typeof metadata.prepositionValue === "string") {
|
|
666
|
-
const usedPreposition = metadata.prepositionValue;
|
|
667
|
-
if (preferred.includes(usedPreposition)) {
|
|
668
|
-
adjustment += 0.1;
|
|
669
|
-
} else {
|
|
670
|
-
adjustment -= 0.1;
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
return Math.max(-0.1, Math.min(0.1, adjustment));
|
|
675
|
-
}
|
|
676
|
-
var stockConfidenceModel = {
|
|
677
|
-
calculate: (ctx) => scoreParts(ctx).final,
|
|
678
|
-
calculateWithBreakdown: (ctx) => scoreParts(ctx)
|
|
679
|
-
};
|
|
680
|
-
|
|
681
|
-
// src/core/pattern-matching/pattern-matcher.ts
|
|
682
|
-
var _PatternMatcher = class _PatternMatcher {
|
|
683
|
-
constructor() {
|
|
684
|
-
/** Debug logger */
|
|
685
|
-
this.logger = createLogger("pattern-matcher");
|
|
686
|
-
/**
|
|
687
|
-
* Track stem matches for confidence calculation.
|
|
688
|
-
* This is set during matching and read during confidence calculation.
|
|
689
|
-
*/
|
|
690
|
-
this.stemMatchCount = 0;
|
|
691
|
-
this.totalKeywordMatches = 0;
|
|
692
|
-
}
|
|
693
|
-
/**
|
|
694
|
-
* Try to match a single pattern against the token stream.
|
|
695
|
-
* Returns the match result or null if no match.
|
|
696
|
-
*
|
|
697
|
-
* @param tokens - Token stream to match against
|
|
698
|
-
* @param pattern - Pattern to match
|
|
699
|
-
* @param profile - Optional language profile for possessive handling
|
|
700
|
-
*/
|
|
701
|
-
matchPattern(tokens, pattern, profile) {
|
|
702
|
-
const mark = tokens.mark();
|
|
703
|
-
const captured = /* @__PURE__ */ new Map();
|
|
704
|
-
this.logger.debug("========================================");
|
|
705
|
-
this.logger.debug("matchPattern ENTRY");
|
|
706
|
-
this.logger.debug("Pattern ID:", pattern.id);
|
|
707
|
-
this.logger.debug("Pattern command:", pattern.command);
|
|
708
|
-
this.logger.debug("Pattern language:", pattern.language);
|
|
709
|
-
this.logger.debug("Pattern template:", JSON.stringify(pattern.template, null, 2));
|
|
710
|
-
if (this.logger.isEnabled()) {
|
|
711
|
-
const firstTokens = [];
|
|
712
|
-
for (let i = 0; i < 10; i++) {
|
|
713
|
-
const t = tokens.peek(i);
|
|
714
|
-
if (t)
|
|
715
|
-
firstTokens.push({
|
|
716
|
-
type: t.type,
|
|
717
|
-
value: t.value,
|
|
718
|
-
kind: t.kind
|
|
719
|
-
});
|
|
720
|
-
else break;
|
|
721
|
-
}
|
|
722
|
-
this.logger.debug("Input tokens (first 10):", firstTokens);
|
|
723
|
-
this.logger.debug("Profile code:", profile?.code);
|
|
724
|
-
}
|
|
725
|
-
this.currentProfile = profile;
|
|
726
|
-
this.stemMatchCount = 0;
|
|
727
|
-
this.totalKeywordMatches = 0;
|
|
728
|
-
this.logger.debug("--- Calling matchTokenSequence ---");
|
|
729
|
-
this.logger.debug("Pattern tokens to match:", JSON.stringify(pattern.template.tokens, null, 2));
|
|
730
|
-
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
731
|
-
this.logger.debug("matchTokenSequence returned:", success);
|
|
732
|
-
this.logger.debug(
|
|
733
|
-
"Captured roles:",
|
|
734
|
-
Array.from(captured.entries()).map(([k, v]) => [k, JSON.stringify(v)])
|
|
735
|
-
);
|
|
736
|
-
if (!success) {
|
|
737
|
-
this.logger.debug(">>> MATCH FAILED - resetting token position");
|
|
738
|
-
tokens.reset(mark);
|
|
739
|
-
return null;
|
|
740
|
-
}
|
|
741
|
-
const confidence = stockConfidenceModel.calculate({
|
|
742
|
-
pattern,
|
|
743
|
-
captured,
|
|
744
|
-
stemMatchCount: this.stemMatchCount,
|
|
745
|
-
totalKeywordMatches: this.totalKeywordMatches
|
|
746
|
-
});
|
|
747
|
-
this.applyExtractionRules(pattern, captured);
|
|
748
|
-
return {
|
|
749
|
-
pattern,
|
|
750
|
-
captured,
|
|
751
|
-
consumedTokens: tokens.position() - mark.position,
|
|
752
|
-
confidence
|
|
753
|
-
};
|
|
754
|
-
}
|
|
755
|
-
/**
|
|
756
|
-
* Try to match multiple patterns, return the best match.
|
|
757
|
-
*
|
|
758
|
-
* @param tokens - Token stream to match against
|
|
759
|
-
* @param patterns - Candidate patterns to try
|
|
760
|
-
* @param profile - Optional language profile for possessive handling
|
|
761
|
-
*/
|
|
762
|
-
matchBest(tokens, patterns, profile) {
|
|
763
|
-
const matches = [];
|
|
764
|
-
for (const pattern of patterns) {
|
|
765
|
-
const mark = tokens.mark();
|
|
766
|
-
const result = this.matchPattern(tokens, pattern, profile);
|
|
767
|
-
if (result) {
|
|
768
|
-
matches.push(result);
|
|
769
|
-
}
|
|
770
|
-
tokens.reset(mark);
|
|
771
|
-
}
|
|
772
|
-
if (matches.length === 0) {
|
|
773
|
-
return null;
|
|
774
|
-
}
|
|
775
|
-
matches.sort((a, b) => {
|
|
776
|
-
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
777
|
-
if (priorityDiff !== 0) return priorityDiff;
|
|
778
|
-
const confidenceDiff = b.confidence - a.confidence;
|
|
779
|
-
if (Math.abs(confidenceDiff) > 1e-3) return confidenceDiff;
|
|
780
|
-
return b.consumedTokens - a.consumedTokens;
|
|
781
|
-
});
|
|
782
|
-
const best = matches[0];
|
|
783
|
-
this.matchPattern(tokens, best.pattern);
|
|
784
|
-
return best;
|
|
785
|
-
}
|
|
786
|
-
/**
|
|
787
|
-
* Match a sequence of pattern tokens against the token stream.
|
|
788
|
-
*
|
|
789
|
-
* Supports bounded single-step backtracking: if an optional role consumes
|
|
790
|
-
* a token and the immediately following pattern token fails, the matcher
|
|
791
|
-
* resets to before the optional role and retries the failed token.
|
|
792
|
-
*/
|
|
793
|
-
matchTokenSequence(tokens, patternTokens, captured, parentStopMarkers) {
|
|
794
|
-
const firstPatternToken = patternTokens[0];
|
|
795
|
-
const patternExpectsConjunction = firstPatternToken?.type === "literal" && (firstPatternToken.value === "and" || firstPatternToken.value === "then" || firstPatternToken.alternatives?.includes("and") || firstPatternToken.alternatives?.includes("then"));
|
|
796
|
-
if (this.currentProfile?.code === "ar" && !patternExpectsConjunction) {
|
|
797
|
-
while (tokens.peek()?.kind === "conjunction") {
|
|
798
|
-
tokens.advance();
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
let prevOptionalMark = null;
|
|
802
|
-
let prevOptionalRole = null;
|
|
803
|
-
for (let i = 0; i < patternTokens.length; i++) {
|
|
804
|
-
const patternToken = patternTokens[i];
|
|
805
|
-
this.logger.debug(" >> Matching pattern token:", JSON.stringify(patternToken, null, 2));
|
|
806
|
-
const currTok = tokens.peek();
|
|
807
|
-
this.logger.debug(
|
|
808
|
-
" >> Current input token:",
|
|
809
|
-
currTok ? JSON.stringify({
|
|
810
|
-
type: currTok.type,
|
|
811
|
-
value: currTok.value,
|
|
812
|
-
kind: currTok.kind
|
|
813
|
-
}) : "EOF"
|
|
814
|
-
);
|
|
815
|
-
if (patternToken.type === "role" && patternToken.greedy) {
|
|
816
|
-
const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
817
|
-
if (parentStopMarkers) {
|
|
818
|
-
for (const m of parentStopMarkers) stopMarkers.add(m);
|
|
819
|
-
}
|
|
820
|
-
const values = [];
|
|
821
|
-
while (!tokens.isAtEnd()) {
|
|
822
|
-
const nextToken = tokens.peek();
|
|
823
|
-
if (!nextToken) break;
|
|
824
|
-
if (this.isStopMarker(nextToken, stopMarkers)) break;
|
|
825
|
-
values.push(nextToken.value);
|
|
826
|
-
tokens.advance();
|
|
827
|
-
}
|
|
828
|
-
if (values.length > 0) {
|
|
829
|
-
captured.set(patternToken.role, { type: "expression", raw: values.join(" ") });
|
|
830
|
-
prevOptionalMark = null;
|
|
831
|
-
prevOptionalRole = null;
|
|
832
|
-
continue;
|
|
833
|
-
} else if (patternToken.optional) {
|
|
834
|
-
continue;
|
|
835
|
-
} else {
|
|
836
|
-
return false;
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
if (patternToken.type === "group") {
|
|
840
|
-
const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
841
|
-
if (parentStopMarkers) {
|
|
842
|
-
for (const m of parentStopMarkers) siblingStopMarkers.add(m);
|
|
843
|
-
}
|
|
844
|
-
const groupMatched = this.matchGroupToken(
|
|
845
|
-
tokens,
|
|
846
|
-
patternToken,
|
|
847
|
-
captured,
|
|
848
|
-
siblingStopMarkers
|
|
849
|
-
);
|
|
850
|
-
if (groupMatched) {
|
|
851
|
-
prevOptionalMark = null;
|
|
852
|
-
prevOptionalRole = null;
|
|
853
|
-
continue;
|
|
854
|
-
}
|
|
855
|
-
if (this.isOptional(patternToken)) {
|
|
856
|
-
continue;
|
|
857
|
-
}
|
|
858
|
-
return false;
|
|
859
|
-
}
|
|
860
|
-
const isOptionalRole = patternToken.type === "role" && patternToken.optional === true;
|
|
861
|
-
const markBefore = isOptionalRole ? tokens.mark() : null;
|
|
862
|
-
const isRoleToken = patternToken.type === "role";
|
|
863
|
-
if (isRoleToken && !isOptionalRole) {
|
|
864
|
-
const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
|
|
865
|
-
if (nextPT?.type === "literal") {
|
|
866
|
-
const currentToken = tokens.peek();
|
|
867
|
-
if (currentToken && this.getMatchType(currentToken, nextPT.value) !== "none") {
|
|
868
|
-
this.logger.debug(
|
|
869
|
-
" >> MARKER-STEAL GUARD: current token matches next literal, skipping role"
|
|
870
|
-
);
|
|
871
|
-
this.logger.debug(" >> Token match FAILED");
|
|
872
|
-
if (this.isOptional(patternToken)) {
|
|
873
|
-
continue;
|
|
874
|
-
}
|
|
875
|
-
if (prevOptionalMark && prevOptionalRole) {
|
|
876
|
-
this.logger.debug(" >> BACKTRACKING: undoing optional role", prevOptionalRole);
|
|
877
|
-
tokens.reset(prevOptionalMark);
|
|
878
|
-
captured.delete(prevOptionalRole);
|
|
879
|
-
prevOptionalMark = null;
|
|
880
|
-
prevOptionalRole = null;
|
|
881
|
-
const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
|
|
882
|
-
this.logger.debug(" >> Backtrack retry result:", retryMatched);
|
|
883
|
-
if (retryMatched) {
|
|
884
|
-
continue;
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
return false;
|
|
888
|
-
}
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
|
-
const matched = this.matchPatternToken(tokens, patternToken, captured);
|
|
892
|
-
this.logger.debug(" >> Match result:", matched);
|
|
893
|
-
if (matched) {
|
|
894
|
-
if (isOptionalRole) {
|
|
895
|
-
prevOptionalMark = markBefore;
|
|
896
|
-
prevOptionalRole = patternToken.role;
|
|
897
|
-
} else {
|
|
898
|
-
prevOptionalMark = null;
|
|
899
|
-
prevOptionalRole = null;
|
|
900
|
-
}
|
|
901
|
-
continue;
|
|
902
|
-
}
|
|
903
|
-
this.logger.debug(" >> Token match FAILED");
|
|
904
|
-
if (this.isOptional(patternToken)) {
|
|
905
|
-
continue;
|
|
906
|
-
}
|
|
907
|
-
if (prevOptionalMark && prevOptionalRole) {
|
|
908
|
-
this.logger.debug(" >> BACKTRACKING: undoing optional role", prevOptionalRole);
|
|
909
|
-
tokens.reset(prevOptionalMark);
|
|
910
|
-
captured.delete(prevOptionalRole);
|
|
911
|
-
prevOptionalMark = null;
|
|
912
|
-
prevOptionalRole = null;
|
|
913
|
-
const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
|
|
914
|
-
this.logger.debug(" >> Backtrack retry result:", retryMatched);
|
|
915
|
-
if (retryMatched) {
|
|
916
|
-
continue;
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
return false;
|
|
920
|
-
}
|
|
921
|
-
return true;
|
|
922
|
-
}
|
|
923
|
-
/**
|
|
924
|
-
* Match a single pattern token against the current position in the stream.
|
|
925
|
-
*/
|
|
926
|
-
matchPatternToken(tokens, patternToken, captured) {
|
|
927
|
-
switch (patternToken.type) {
|
|
928
|
-
case "literal":
|
|
929
|
-
return this.matchLiteralToken(tokens, patternToken);
|
|
930
|
-
case "role":
|
|
931
|
-
return readRoleValue(tokens, patternToken, captured, this.currentProfile);
|
|
932
|
-
case "group":
|
|
933
|
-
return this.matchGroupToken(tokens, patternToken, captured);
|
|
934
|
-
default:
|
|
935
|
-
return false;
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
/**
|
|
939
|
-
* Match a literal pattern token (keyword or particle).
|
|
940
|
-
*/
|
|
941
|
-
matchLiteralToken(tokens, patternToken) {
|
|
942
|
-
const token = tokens.peek();
|
|
943
|
-
this.logger.debug(" >>> matchLiteralToken: expecting", patternToken.value);
|
|
944
|
-
this.logger.debug(
|
|
945
|
-
" >>> matchLiteralToken: got token",
|
|
946
|
-
token ? JSON.stringify({
|
|
947
|
-
type: token.type,
|
|
948
|
-
value: token.value,
|
|
949
|
-
kind: token.kind
|
|
950
|
-
}) : "null"
|
|
951
|
-
);
|
|
952
|
-
if (!token) {
|
|
953
|
-
this.logger.debug(" >>> matchLiteralToken: FAIL - no token");
|
|
954
|
-
return false;
|
|
955
|
-
}
|
|
956
|
-
const matchType = this.getMatchType(token, patternToken.value);
|
|
957
|
-
this.logger.debug(
|
|
958
|
-
" >>> matchType for",
|
|
959
|
-
token.value,
|
|
960
|
-
"vs",
|
|
961
|
-
patternToken.value,
|
|
962
|
-
":",
|
|
963
|
-
matchType
|
|
964
|
-
);
|
|
965
|
-
if (matchType !== "none") {
|
|
966
|
-
this.totalKeywordMatches++;
|
|
967
|
-
if (matchType === "stem") {
|
|
968
|
-
this.stemMatchCount++;
|
|
969
|
-
}
|
|
970
|
-
tokens.advance();
|
|
971
|
-
return true;
|
|
972
|
-
}
|
|
973
|
-
if (patternToken.alternatives) {
|
|
974
|
-
for (const alt of patternToken.alternatives) {
|
|
975
|
-
const altMatchType = this.getMatchType(token, alt);
|
|
976
|
-
if (altMatchType !== "none") {
|
|
977
|
-
this.totalKeywordMatches++;
|
|
978
|
-
if (altMatchType === "stem") {
|
|
979
|
-
this.stemMatchCount++;
|
|
980
|
-
}
|
|
981
|
-
tokens.advance();
|
|
982
|
-
return true;
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
|
-
}
|
|
986
|
-
return false;
|
|
987
|
-
}
|
|
988
|
-
/**
|
|
989
|
-
* Match a group pattern token (optional sequence).
|
|
990
|
-
* When the group's leading marker isn't at the current position, scans ahead
|
|
991
|
-
* up to MAX_MARKER_SCAN tokens to find it. This allows unmarked tokens
|
|
992
|
-
* (e.g., a value like 'hello') to sit between groups without blocking later
|
|
993
|
-
* marker matches.
|
|
994
|
-
*/
|
|
995
|
-
matchGroupToken(tokens, patternToken, captured, parentStopMarkers) {
|
|
996
|
-
const mark = tokens.mark();
|
|
997
|
-
const capturedBefore = new Set(captured.keys());
|
|
998
|
-
const success = this.matchTokenSequence(
|
|
999
|
-
tokens,
|
|
1000
|
-
patternToken.tokens,
|
|
1001
|
-
captured,
|
|
1002
|
-
parentStopMarkers
|
|
1003
|
-
);
|
|
1004
|
-
if (success) return true;
|
|
1005
|
-
tokens.reset(mark);
|
|
1006
|
-
for (const role of captured.keys()) {
|
|
1007
|
-
if (!capturedBefore.has(role)) captured.delete(role);
|
|
1008
|
-
}
|
|
1009
|
-
if (!patternToken.optional) return false;
|
|
1010
|
-
const leadingMarker = this.getGroupLeadingMarker(patternToken);
|
|
1011
|
-
if (leadingMarker) {
|
|
1012
|
-
for (let offset = 1; offset <= _PatternMatcher.MAX_MARKER_SCAN; offset++) {
|
|
1013
|
-
const ahead = tokens.peek(offset);
|
|
1014
|
-
if (!ahead) break;
|
|
1015
|
-
const aheadValue = (ahead.normalized || ahead.value).toLowerCase();
|
|
1016
|
-
if (aheadValue === leadingMarker) {
|
|
1017
|
-
this.logger.debug(
|
|
1018
|
-
" >> MARKER SCAN: found",
|
|
1019
|
-
leadingMarker,
|
|
1020
|
-
"at offset",
|
|
1021
|
-
offset,
|
|
1022
|
-
"- skipping intervening tokens"
|
|
1023
|
-
);
|
|
1024
|
-
for (let s = 0; s < offset; s++) tokens.advance();
|
|
1025
|
-
const retrySuccess = this.matchTokenSequence(
|
|
1026
|
-
tokens,
|
|
1027
|
-
patternToken.tokens,
|
|
1028
|
-
captured,
|
|
1029
|
-
parentStopMarkers
|
|
1030
|
-
);
|
|
1031
|
-
if (retrySuccess) return true;
|
|
1032
|
-
tokens.reset(mark);
|
|
1033
|
-
for (const role of captured.keys()) {
|
|
1034
|
-
if (!capturedBefore.has(role)) captured.delete(role);
|
|
1035
|
-
}
|
|
1036
|
-
break;
|
|
1037
|
-
}
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
return true;
|
|
1041
|
-
}
|
|
1042
|
-
/**
|
|
1043
|
-
* Get the leading marker literal from an optional group.
|
|
1044
|
-
* Returns the lowercase marker value, or null if the group
|
|
1045
|
-
* doesn't start with a literal token (e.g., SOV groups where
|
|
1046
|
-
* the role comes before the marker).
|
|
1047
|
-
*/
|
|
1048
|
-
getGroupLeadingMarker(group) {
|
|
1049
|
-
const first = group.tokens[0];
|
|
1050
|
-
if (first?.type === "literal") return first.value.toLowerCase();
|
|
1051
|
-
return null;
|
|
1052
|
-
}
|
|
1053
|
-
/**
|
|
1054
|
-
* Get the type of match for a token against a value.
|
|
1055
|
-
* Used for confidence calculation.
|
|
1056
|
-
*/
|
|
1057
|
-
getMatchType(token, value) {
|
|
1058
|
-
if (token.value === value) return "exact";
|
|
1059
|
-
if (token.normalized === value) return "normalized";
|
|
1060
|
-
if (token.stem === value && token.stemConfidence !== void 0 && token.stemConfidence >= 0.7) {
|
|
1061
|
-
return "stem";
|
|
1062
|
-
}
|
|
1063
|
-
if (token.kind === "keyword" && safeToLowerCase(token.value) === value.toLowerCase()) {
|
|
1064
|
-
return "case-insensitive";
|
|
1065
|
-
}
|
|
1066
|
-
return "none";
|
|
1067
|
-
}
|
|
1068
|
-
/**
|
|
1069
|
-
* Collect literal values from upcoming pattern tokens that act as stop markers
|
|
1070
|
-
* for greedy role capture. Returns the set of lowercase values.
|
|
1071
|
-
*/
|
|
1072
|
-
collectStopMarkers(patternTokens, startIndex) {
|
|
1073
|
-
const markers = /* @__PURE__ */ new Set();
|
|
1074
|
-
for (let j = startIndex; j < patternTokens.length; j++) {
|
|
1075
|
-
const pt = patternTokens[j];
|
|
1076
|
-
if (pt.type === "literal") {
|
|
1077
|
-
markers.add(pt.value.toLowerCase());
|
|
1078
|
-
if (pt.alternatives) {
|
|
1079
|
-
for (const alt of pt.alternatives) {
|
|
1080
|
-
markers.add(alt.toLowerCase());
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
break;
|
|
1084
|
-
}
|
|
1085
|
-
if (pt.type === "group") {
|
|
1086
|
-
for (const gt of pt.tokens) {
|
|
1087
|
-
if (gt.type === "literal") {
|
|
1088
|
-
markers.add(gt.value.toLowerCase());
|
|
1089
|
-
if (gt.alternatives) {
|
|
1090
|
-
for (const alt of gt.alternatives) {
|
|
1091
|
-
markers.add(alt.toLowerCase());
|
|
1092
|
-
}
|
|
1093
|
-
}
|
|
1094
|
-
break;
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1097
|
-
if (!pt.optional) break;
|
|
1098
|
-
}
|
|
1099
|
-
}
|
|
1100
|
-
return markers;
|
|
1101
|
-
}
|
|
1102
|
-
/**
|
|
1103
|
-
* Check if a token matches any stop marker for greedy capture.
|
|
1104
|
-
*/
|
|
1105
|
-
isStopMarker(token, stopMarkers) {
|
|
1106
|
-
if (stopMarkers.size === 0) return false;
|
|
1107
|
-
if (stopMarkers.has(token.value.toLowerCase())) return true;
|
|
1108
|
-
if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
|
|
1109
|
-
return false;
|
|
1110
|
-
}
|
|
1111
|
-
// ==========================================================================
|
|
1112
|
-
// Depth Limits for Expression Parsing (security hardening)
|
|
1113
|
-
// ==========================================================================
|
|
1114
|
-
/**
|
|
1115
|
-
* Apply extraction rules to fill in static values and defaults for missing roles.
|
|
1116
|
-
*/
|
|
1117
|
-
applyExtractionRules(pattern, captured) {
|
|
1118
|
-
for (const [role, rule] of Object.entries(pattern.extraction)) {
|
|
1119
|
-
if (!captured.has(role)) {
|
|
1120
|
-
if (rule.value !== void 0) {
|
|
1121
|
-
captured.set(role, { type: "literal", value: rule.value });
|
|
1122
|
-
} else if (rule.default) {
|
|
1123
|
-
captured.set(role, rule.default);
|
|
1124
|
-
}
|
|
1125
|
-
}
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
/**
|
|
1129
|
-
* Check if a pattern token is optional.
|
|
1130
|
-
*/
|
|
1131
|
-
isOptional(patternToken) {
|
|
1132
|
-
return patternToken.type !== "literal" && patternToken.optional === true;
|
|
1133
|
-
}
|
|
1134
|
-
// ===========================================================================
|
|
1135
|
-
// English Idiom Support - Noise Word Handling
|
|
1136
|
-
// ===========================================================================
|
|
1137
|
-
/**
|
|
1138
|
-
* Extract event modifiers from the token stream.
|
|
1139
|
-
* Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)
|
|
1140
|
-
* that can appear after event names.
|
|
1141
|
-
*
|
|
1142
|
-
* Returns EventModifiers object or undefined if no modifiers found.
|
|
1143
|
-
*/
|
|
1144
|
-
extractEventModifiers(tokens) {
|
|
1145
|
-
const modifiers = {};
|
|
1146
|
-
let foundModifier = false;
|
|
1147
|
-
while (!tokens.isAtEnd()) {
|
|
1148
|
-
const token = tokens.peek();
|
|
1149
|
-
if (!token || token.kind !== "event-modifier") {
|
|
1150
|
-
break;
|
|
1151
|
-
}
|
|
1152
|
-
const metadata = token.metadata;
|
|
1153
|
-
if (!metadata) {
|
|
1154
|
-
break;
|
|
1155
|
-
}
|
|
1156
|
-
foundModifier = true;
|
|
1157
|
-
switch (metadata.modifierName) {
|
|
1158
|
-
case "once":
|
|
1159
|
-
modifiers.once = true;
|
|
1160
|
-
break;
|
|
1161
|
-
case "debounce":
|
|
1162
|
-
if (typeof metadata.value === "number") {
|
|
1163
|
-
modifiers.debounce = metadata.value;
|
|
1164
|
-
}
|
|
1165
|
-
break;
|
|
1166
|
-
case "throttle":
|
|
1167
|
-
if (typeof metadata.value === "number") {
|
|
1168
|
-
modifiers.throttle = metadata.value;
|
|
1169
|
-
}
|
|
1170
|
-
break;
|
|
1171
|
-
case "queue":
|
|
1172
|
-
if (metadata.value === "first" || metadata.value === "last" || metadata.value === "all" || metadata.value === "none") {
|
|
1173
|
-
modifiers.queue = metadata.value;
|
|
1174
|
-
}
|
|
1175
|
-
break;
|
|
1176
|
-
}
|
|
1177
|
-
tokens.advance();
|
|
1178
|
-
}
|
|
1179
|
-
return foundModifier ? modifiers : void 0;
|
|
1180
|
-
}
|
|
1181
|
-
};
|
|
1182
|
-
/** Maximum tokens to scan ahead when looking for a group's leading marker */
|
|
1183
|
-
_PatternMatcher.MAX_MARKER_SCAN = 3;
|
|
1184
|
-
var PatternMatcher = _PatternMatcher;
|
|
1185
|
-
var patternMatcher = new PatternMatcher();
|
|
1186
|
-
function matchPattern(tokens, pattern) {
|
|
1187
|
-
return patternMatcher.matchPattern(tokens, pattern);
|
|
1188
|
-
}
|
|
1189
|
-
function matchBest(tokens, patterns) {
|
|
1190
|
-
return patternMatcher.matchBest(tokens, patterns);
|
|
1191
|
-
}
|
|
1192
|
-
|
|
1193
|
-
// src/core/tokenization/token-utils.ts
|
|
1194
|
-
var TokenStreamImpl = class {
|
|
1195
|
-
constructor(tokens, language) {
|
|
1196
|
-
this.pos = 0;
|
|
1197
|
-
this.tokens = tokens;
|
|
1198
|
-
this.language = language;
|
|
1199
|
-
}
|
|
1200
|
-
peek(offset = 0) {
|
|
1201
|
-
const index = this.pos + offset;
|
|
1202
|
-
if (index < 0 || index >= this.tokens.length) {
|
|
1203
|
-
return null;
|
|
1204
|
-
}
|
|
1205
|
-
return this.tokens[index];
|
|
1206
|
-
}
|
|
1207
|
-
advance() {
|
|
1208
|
-
if (this.isAtEnd()) {
|
|
1209
|
-
throw new Error("Unexpected end of token stream");
|
|
1210
|
-
}
|
|
1211
|
-
return this.tokens[this.pos++];
|
|
1212
|
-
}
|
|
1213
|
-
isAtEnd() {
|
|
1214
|
-
return this.pos >= this.tokens.length;
|
|
1215
|
-
}
|
|
1216
|
-
mark() {
|
|
1217
|
-
return { position: this.pos };
|
|
1218
|
-
}
|
|
1219
|
-
reset(mark) {
|
|
1220
|
-
this.pos = mark.position;
|
|
1221
|
-
}
|
|
1222
|
-
position() {
|
|
1223
|
-
return this.pos;
|
|
1224
|
-
}
|
|
1225
|
-
/**
|
|
1226
|
-
* Get remaining tokens as an array.
|
|
1227
|
-
*/
|
|
1228
|
-
remaining() {
|
|
1229
|
-
return this.tokens.slice(this.pos);
|
|
1230
|
-
}
|
|
1231
|
-
/**
|
|
1232
|
-
* Consume tokens while predicate is true.
|
|
1233
|
-
*/
|
|
1234
|
-
takeWhile(predicate) {
|
|
1235
|
-
const result = [];
|
|
1236
|
-
while (!this.isAtEnd() && predicate(this.peek())) {
|
|
1237
|
-
result.push(this.advance());
|
|
1238
|
-
}
|
|
1239
|
-
return result;
|
|
1240
|
-
}
|
|
1241
|
-
/**
|
|
1242
|
-
* Skip tokens while predicate is true.
|
|
1243
|
-
*/
|
|
1244
|
-
skipWhile(predicate) {
|
|
1245
|
-
while (!this.isAtEnd() && predicate(this.peek())) {
|
|
1246
|
-
this.advance();
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
};
|
|
1250
|
-
function createPosition(start, end) {
|
|
1251
|
-
return { start, end };
|
|
1252
|
-
}
|
|
1253
|
-
function createToken(valueOrParams, kind, position, normalizedOrOptions) {
|
|
1254
|
-
if (typeof valueOrParams === "object") {
|
|
1255
|
-
const { value: value2, kind: kind2, position: position2, normalized: normalized2, stem, stemConfidence, metadata } = valueOrParams;
|
|
1256
|
-
return {
|
|
1257
|
-
value: value2,
|
|
1258
|
-
kind: kind2,
|
|
1259
|
-
position: position2,
|
|
1260
|
-
...normalized2 !== void 0 && { normalized: normalized2 },
|
|
1261
|
-
...stem !== void 0 && { stem },
|
|
1262
|
-
...stemConfidence !== void 0 && { stemConfidence },
|
|
1263
|
-
...metadata !== void 0 && { metadata }
|
|
1264
|
-
};
|
|
1265
|
-
}
|
|
1266
|
-
const value = valueOrParams;
|
|
1267
|
-
if (!kind || !position) {
|
|
1268
|
-
throw new Error("createToken requires kind and position parameters");
|
|
1269
|
-
}
|
|
1270
|
-
if (typeof normalizedOrOptions === "string") {
|
|
1271
|
-
return { value, kind, position, normalized: normalizedOrOptions };
|
|
1272
|
-
}
|
|
1273
|
-
if (normalizedOrOptions) {
|
|
1274
|
-
const { normalized: normalized2, stem, stemConfidence, metadata } = normalizedOrOptions;
|
|
1275
|
-
return {
|
|
1276
|
-
value,
|
|
1277
|
-
kind,
|
|
1278
|
-
position,
|
|
1279
|
-
...normalized2 !== void 0 && { normalized: normalized2 },
|
|
1280
|
-
...stem !== void 0 && { stem },
|
|
1281
|
-
...stemConfidence !== void 0 && { stemConfidence },
|
|
1282
|
-
...metadata !== void 0 && { metadata }
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
return { value, kind, position };
|
|
1286
|
-
}
|
|
1287
|
-
function isWhitespace(char) {
|
|
1288
|
-
return /\s/.test(char);
|
|
1289
|
-
}
|
|
1290
|
-
function isSelectorStart(char) {
|
|
1291
|
-
return char === "#" || char === "." || char === "[" || char === "@" || char === "*" || char === "<";
|
|
1292
|
-
}
|
|
1293
|
-
function isQuote(char) {
|
|
1294
|
-
return char === '"' || char === "'" || char === "`" || char === "\u300C" || char === "\u300D";
|
|
1295
|
-
}
|
|
1296
|
-
function isDigit(char) {
|
|
1297
|
-
return /\d/.test(char);
|
|
1298
|
-
}
|
|
1299
|
-
function stripOptionalDiacritics(word) {
|
|
1300
|
-
return word.replace(/[\u064b-\u0652\u0670]/g, "");
|
|
1301
|
-
}
|
|
1302
|
-
function isAsciiLetter(char) {
|
|
1303
|
-
return /[a-zA-Z]/.test(char);
|
|
1304
|
-
}
|
|
1305
|
-
function isAsciiIdentifierChar(char) {
|
|
1306
|
-
return /[a-zA-Z0-9_-]/.test(char);
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
// src/core/matching/match-skeleton.ts
|
|
1310
|
-
function groupHasRole(group) {
|
|
1311
|
-
return group.tokens.some((t) => t.type === "role" || t.type === "group" && groupHasRole(t));
|
|
1312
|
-
}
|
|
1313
|
-
var MatchSkeleton = class {
|
|
1314
|
-
/**
|
|
1315
|
-
* @param makeReader - builds the grammar's reader, handed this matcher's view
|
|
1316
|
-
* of the match in progress (one reader per matcher).
|
|
1317
|
-
*/
|
|
1318
|
-
constructor(makeReader, confidenceModel) {
|
|
1319
|
-
/**
|
|
1320
|
-
* The pattern tokens still to match after the one being matched, through
|
|
1321
|
-
* every enclosing group: a value stops at any marker the pattern still owes.
|
|
1322
|
-
*/
|
|
1323
|
-
this.patternRest = [];
|
|
1324
|
-
/**
|
|
1325
|
-
* Whether the role slot being matched directly follows a literal the pattern
|
|
1326
|
-
* matched: its marker (`[por {quantity}]`), or the command's verb.
|
|
1327
|
-
*/
|
|
1328
|
-
this.slotAfterLiteral = false;
|
|
1329
|
-
/**
|
|
1330
|
-
* Selective memoization cache (Phase 6.1).
|
|
1331
|
-
* Caches pattern match results keyed by `tokenPosition:patternId`.
|
|
1332
|
-
* Cleared before each top-level matchBest() call.
|
|
1333
|
-
*/
|
|
1334
|
-
this.matchCache = /* @__PURE__ */ new Map();
|
|
1335
|
-
/**
|
|
1336
|
-
* Stream index where each role's capture began, for the current
|
|
1337
|
-
* `matchPattern` call. Rebuilt per call, written on the same success path as
|
|
1338
|
-
* `captured` — see `PatternMatchResult.roleStarts`.
|
|
1339
|
-
*/
|
|
1340
|
-
this.roleStarts = /* @__PURE__ */ new Map();
|
|
1341
|
-
/**
|
|
1342
|
-
* Track stem matches for confidence calculation.
|
|
1343
|
-
* This is set during matching and read during confidence calculation.
|
|
1344
|
-
*/
|
|
1345
|
-
this.stemMatchCount = 0;
|
|
1346
|
-
this.totalKeywordMatches = 0;
|
|
1347
|
-
this.confidenceModel = confidenceModel;
|
|
1348
|
-
this.reader = makeReader({
|
|
1349
|
-
patternRest: () => this.patternRest,
|
|
1350
|
-
slotAfterLiteral: () => this.slotAfterLiteral,
|
|
1351
|
-
wouldMatch: (pt, token) => this.patternTokenWouldMatch(pt, token)
|
|
1352
|
-
});
|
|
1353
|
-
}
|
|
1354
|
-
/**
|
|
1355
|
-
* Try to match a single pattern against the token stream.
|
|
1356
|
-
* Returns the match result or null if no match.
|
|
1357
|
-
*/
|
|
1358
|
-
matchPattern(tokens, pattern) {
|
|
1359
|
-
const mark = tokens.mark();
|
|
1360
|
-
const captured = /* @__PURE__ */ new Map();
|
|
1361
|
-
this.reader.beginPattern(pattern);
|
|
1362
|
-
this.stemMatchCount = 0;
|
|
1363
|
-
this.totalKeywordMatches = 0;
|
|
1364
|
-
this.roleStarts = /* @__PURE__ */ new Map();
|
|
1365
|
-
this.patternRest = [];
|
|
1366
|
-
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
1367
|
-
if (!success) {
|
|
1368
|
-
tokens.reset(mark);
|
|
1369
|
-
return null;
|
|
1370
|
-
}
|
|
1371
|
-
const confidence = this.confidenceModel.calculate({
|
|
1372
|
-
pattern,
|
|
1373
|
-
captured,
|
|
1374
|
-
stemMatchCount: this.stemMatchCount,
|
|
1375
|
-
totalKeywordMatches: this.totalKeywordMatches
|
|
1376
|
-
});
|
|
1377
|
-
this.applyExtractionRules(pattern, captured);
|
|
1378
|
-
return {
|
|
1379
|
-
pattern,
|
|
1380
|
-
captured,
|
|
1381
|
-
consumedTokens: tokens.position() - mark.position,
|
|
1382
|
-
confidence,
|
|
1383
|
-
roleStarts: this.roleStarts,
|
|
1384
|
-
...this.reader.resultFields?.()
|
|
1385
|
-
};
|
|
1386
|
-
}
|
|
1387
|
-
/**
|
|
1388
|
-
* Try to match multiple patterns, return the best match.
|
|
1389
|
-
*/
|
|
1390
|
-
matchBest(tokens, patterns) {
|
|
1391
|
-
this.matchCache.clear();
|
|
1392
|
-
const matches = [];
|
|
1393
|
-
const startPos = tokens.position();
|
|
1394
|
-
for (const pattern of patterns) {
|
|
1395
|
-
const mark = tokens.mark();
|
|
1396
|
-
const cacheKey = `${startPos}:${pattern.id}`;
|
|
1397
|
-
let result;
|
|
1398
|
-
if (this.matchCache.has(cacheKey)) {
|
|
1399
|
-
result = this.matchCache.get(cacheKey);
|
|
1400
|
-
} else {
|
|
1401
|
-
result = this.matchPattern(tokens, pattern);
|
|
1402
|
-
this.matchCache.set(cacheKey, result);
|
|
1403
|
-
}
|
|
1404
|
-
if (result) {
|
|
1405
|
-
matches.push(result);
|
|
1406
|
-
}
|
|
1407
|
-
tokens.reset(mark);
|
|
1408
|
-
}
|
|
1409
|
-
if (matches.length === 0) {
|
|
1410
|
-
return null;
|
|
1411
|
-
}
|
|
1412
|
-
matches.sort((a, b) => {
|
|
1413
|
-
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
1414
|
-
if (priorityDiff !== 0) return priorityDiff;
|
|
1415
|
-
return b.confidence - a.confidence;
|
|
1416
|
-
});
|
|
1417
|
-
const best = matches[0];
|
|
1418
|
-
this.matchPattern(tokens, best.pattern);
|
|
1419
|
-
return best;
|
|
1420
|
-
}
|
|
1421
|
-
/**
|
|
1422
|
-
* Match a sequence of pattern tokens against the token stream.
|
|
1423
|
-
*/
|
|
1424
|
-
matchTokenSequence(tokens, patternTokens, captured, nextAfterSequence) {
|
|
1425
|
-
this.reader.skipLeading?.(tokens, patternTokens);
|
|
1426
|
-
for (let i = 0; i < patternTokens.length; i++) {
|
|
1427
|
-
const patternToken = patternTokens[i];
|
|
1428
|
-
this.reader.beforePatternToken?.(tokens, patternTokens, i, captured);
|
|
1429
|
-
if (this.reader.maySkipSlot(
|
|
1430
|
-
patternToken,
|
|
1431
|
-
tokens.peek(),
|
|
1432
|
-
patternTokens[i + 1] ?? nextAfterSequence
|
|
1433
|
-
)) {
|
|
1434
|
-
const skipMark = tokens.mark();
|
|
1435
|
-
const speculative = new Map(captured);
|
|
1436
|
-
if (this.matchTokenSequence(
|
|
1437
|
-
tokens,
|
|
1438
|
-
patternTokens.slice(i + 1),
|
|
1439
|
-
speculative,
|
|
1440
|
-
nextAfterSequence
|
|
1441
|
-
) && this.reader.atClauseEnd(tokens)) {
|
|
1442
|
-
captured.clear();
|
|
1443
|
-
for (const [role, value] of speculative) captured.set(role, value);
|
|
1444
|
-
return true;
|
|
1445
|
-
}
|
|
1446
|
-
tokens.reset(skipMark);
|
|
1447
|
-
}
|
|
1448
|
-
const outerRest = this.patternRest;
|
|
1449
|
-
this.patternRest = [...patternTokens.slice(i + 1), ...outerRest];
|
|
1450
|
-
this.slotAfterLiteral = patternToken.type === "role" && patternTokens[i - 1]?.type === "literal";
|
|
1451
|
-
const matched = this.matchPatternToken(
|
|
1452
|
-
tokens,
|
|
1453
|
-
patternToken,
|
|
1454
|
-
captured,
|
|
1455
|
-
patternTokens[i + 1] ?? nextAfterSequence
|
|
1456
|
-
);
|
|
1457
|
-
this.patternRest = outerRest;
|
|
1458
|
-
if (!matched) {
|
|
1459
|
-
if (this.isOptional(patternToken)) {
|
|
1460
|
-
continue;
|
|
1461
|
-
}
|
|
1462
|
-
return false;
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
return true;
|
|
1466
|
-
}
|
|
1467
|
-
/**
|
|
1468
|
-
* Match a single pattern token against the current position in the stream.
|
|
1469
|
-
*/
|
|
1470
|
-
matchPatternToken(tokens, patternToken, captured, nextPatternToken) {
|
|
1471
|
-
switch (patternToken.type) {
|
|
1472
|
-
case "literal":
|
|
1473
|
-
return this.matchLiteralToken(tokens, patternToken);
|
|
1474
|
-
case "role": {
|
|
1475
|
-
const roleStart = tokens.position();
|
|
1476
|
-
const matchedRole = this.reader.readRole(tokens, patternToken, captured, nextPatternToken);
|
|
1477
|
-
if (matchedRole && captured.has(patternToken.role)) {
|
|
1478
|
-
this.roleStarts.set(patternToken.role, roleStart);
|
|
1479
|
-
}
|
|
1480
|
-
return matchedRole;
|
|
1481
|
-
}
|
|
1482
|
-
case "group":
|
|
1483
|
-
return this.matchGroupToken(tokens, patternToken, captured, nextPatternToken);
|
|
1484
|
-
default:
|
|
1485
|
-
return false;
|
|
1486
|
-
}
|
|
1487
|
-
}
|
|
1488
|
-
/**
|
|
1489
|
-
* Match a literal pattern token (keyword or particle).
|
|
1490
|
-
*/
|
|
1491
|
-
matchLiteralToken(tokens, patternToken) {
|
|
1492
|
-
const token = tokens.peek();
|
|
1493
|
-
if (!token) return false;
|
|
1494
|
-
const matchType = this.getMatchType(token, patternToken.value);
|
|
1495
|
-
if (matchType !== "none") {
|
|
1496
|
-
this.totalKeywordMatches++;
|
|
1497
|
-
if (matchType === "stem") {
|
|
1498
|
-
this.stemMatchCount++;
|
|
1499
|
-
}
|
|
1500
|
-
tokens.advance();
|
|
1501
|
-
return true;
|
|
1502
|
-
}
|
|
1503
|
-
if (patternToken.alternatives) {
|
|
1504
|
-
for (const alt of patternToken.alternatives) {
|
|
1505
|
-
const altMatchType = this.getMatchType(token, alt);
|
|
1506
|
-
if (altMatchType !== "none") {
|
|
1507
|
-
this.totalKeywordMatches++;
|
|
1508
|
-
if (altMatchType === "stem") {
|
|
1509
|
-
this.stemMatchCount++;
|
|
1510
|
-
}
|
|
1511
|
-
tokens.advance();
|
|
1512
|
-
return true;
|
|
1513
|
-
}
|
|
1514
|
-
}
|
|
1515
|
-
}
|
|
1516
|
-
for (const value of [patternToken.value, ...patternToken.alternatives ?? []]) {
|
|
1517
|
-
for (const sep of ["-", "_"]) {
|
|
1518
|
-
if (value.includes(sep) && this.tryMatchShatteredCompound(tokens, value, sep)) {
|
|
1519
|
-
this.totalKeywordMatches++;
|
|
1520
|
-
return true;
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
}
|
|
1524
|
-
return false;
|
|
1525
|
-
}
|
|
1526
|
-
/**
|
|
1527
|
-
* Match a compound keyword (`titik-henti`, `по_умолчанию`) against a
|
|
1528
|
-
* shattered token run (`titik` `-` `henti` / `по` `_` `умолчанию`): word
|
|
1529
|
-
* segments must be identifier/keyword/particle tokens equal to the expected
|
|
1530
|
-
* segments (case-insensitive; particle because prepositions like ru `по` /
|
|
1531
|
-
* uk `за` lead some underscore compounds), separators lone `sep` tokens.
|
|
1532
|
-
* Consumes the run on success, resets the stream on failure.
|
|
1533
|
-
*/
|
|
1534
|
-
tryMatchShatteredCompound(tokens, expected, sep) {
|
|
1535
|
-
const segments = expected.split(sep);
|
|
1536
|
-
if (segments.length < 2 || segments.some((s) => s.length === 0)) return false;
|
|
1537
|
-
const mark = tokens.mark();
|
|
1538
|
-
for (let i = 0; i < segments.length; i++) {
|
|
1539
|
-
if (i > 0) {
|
|
1540
|
-
const sepToken = tokens.peek();
|
|
1541
|
-
if (!sepToken || sepToken.value !== sep) {
|
|
1542
|
-
tokens.reset(mark);
|
|
1543
|
-
return false;
|
|
1544
|
-
}
|
|
1545
|
-
tokens.advance();
|
|
1546
|
-
}
|
|
1547
|
-
const word = tokens.peek();
|
|
1548
|
-
if (!word || word.kind !== "identifier" && word.kind !== "keyword" && word.kind !== "particle" || word.value.toLowerCase() !== segments[i].toLowerCase()) {
|
|
1549
|
-
tokens.reset(mark);
|
|
1550
|
-
return false;
|
|
1551
|
-
}
|
|
1552
|
-
tokens.advance();
|
|
1553
|
-
}
|
|
1554
|
-
return true;
|
|
1555
|
-
}
|
|
1556
|
-
/**
|
|
1557
|
-
* Match a group pattern token (optional sequence).
|
|
1558
|
-
*/
|
|
1559
|
-
matchGroupToken(tokens, patternToken, captured, nextPatternToken) {
|
|
1560
|
-
const mark = tokens.mark();
|
|
1561
|
-
const capturedBefore = new Map(captured);
|
|
1562
|
-
let success = this.matchTokenSequence(tokens, patternToken.tokens, captured, nextPatternToken);
|
|
1563
|
-
if (success && tokens.position() > mark.position && groupHasRole(patternToken) && [...captured].every(([role, value]) => capturedBefore.get(role) === value) && captured.size === capturedBefore.size) {
|
|
1564
|
-
success = false;
|
|
1565
|
-
}
|
|
1566
|
-
if (success) {
|
|
1567
|
-
const markerLiterals = [];
|
|
1568
|
-
for (const inner of patternToken.tokens) {
|
|
1569
|
-
if (inner.type === "group" && inner.renderRequired) {
|
|
1570
|
-
for (const t of inner.tokens) {
|
|
1571
|
-
if (t.type === "literal") markerLiterals.push(t.value, ...t.alternatives ?? []);
|
|
1572
|
-
}
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
if (markerLiterals.length > 0) {
|
|
1576
|
-
const addedRole = [...captured.keys()].some((role) => !capturedBefore.has(role));
|
|
1577
|
-
if (addedRole) {
|
|
1578
|
-
const last = tokens.tokens[tokens.position() - 1];
|
|
1579
|
-
const markerSeen = last !== void 0 && tokens.position() > mark.position && markerLiterals.some((lit) => this.getMatchType(last, lit) !== "none");
|
|
1580
|
-
if (!markerSeen) success = false;
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
if (!success) {
|
|
1585
|
-
tokens.reset(mark);
|
|
1586
|
-
for (const role of captured.keys()) {
|
|
1587
|
-
if (!capturedBefore.has(role)) {
|
|
1588
|
-
captured.delete(role);
|
|
1589
|
-
}
|
|
1590
|
-
}
|
|
1591
|
-
for (const [role, value] of capturedBefore) {
|
|
1592
|
-
if (captured.get(role) !== value) {
|
|
1593
|
-
captured.set(role, value);
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
return patternToken.optional || false;
|
|
1597
|
-
}
|
|
1598
|
-
return true;
|
|
1599
|
-
}
|
|
1600
|
-
/**
|
|
1601
|
-
* Whether a pattern token (literal, or a group starting with a literal) would
|
|
1602
|
-
* match the given stream token. Used to keep the event-head source-clause
|
|
1603
|
-
* consumption from stealing a marker the pattern explicitly expects.
|
|
1604
|
-
*/
|
|
1605
|
-
patternTokenWouldMatch(pt, token) {
|
|
1606
|
-
if (!pt) return false;
|
|
1607
|
-
if (pt.type === "literal") {
|
|
1608
|
-
if (this.getMatchType(token, pt.value) !== "none") return true;
|
|
1609
|
-
return !!pt.alternatives?.some((a) => this.getMatchType(token, a) !== "none");
|
|
1610
|
-
}
|
|
1611
|
-
if (pt.type === "group") {
|
|
1612
|
-
return this.patternTokenWouldMatch(pt.tokens?.[0], token);
|
|
1613
|
-
}
|
|
1614
|
-
return false;
|
|
1615
|
-
}
|
|
1616
|
-
/**
|
|
1617
|
-
* Get the type of match for a token against a value.
|
|
1618
|
-
* Used for confidence calculation.
|
|
1619
|
-
*/
|
|
1620
|
-
getMatchType(token, value) {
|
|
1621
|
-
if (token.value === value) return "exact";
|
|
1622
|
-
if (token.normalized === value) return "normalized";
|
|
1623
|
-
if (token.stem === value && token.stemConfidence !== void 0 && token.stemConfidence >= 0.7) {
|
|
1624
|
-
return "stem";
|
|
1625
|
-
}
|
|
1626
|
-
if (token.kind === "keyword" && token.value.toLowerCase() === value.toLowerCase()) {
|
|
1627
|
-
return "case-insensitive";
|
|
1628
|
-
}
|
|
1629
|
-
if (token.kind === "keyword") {
|
|
1630
|
-
const strippedToken = stripOptionalDiacritics(token.value).toLowerCase();
|
|
1631
|
-
if (strippedToken && strippedToken === stripOptionalDiacritics(value).toLowerCase()) {
|
|
1632
|
-
return "case-insensitive";
|
|
898
|
+
return createLiteral(token.normalized || token.value);
|
|
899
|
+
}
|
|
900
|
+
case "identifier": {
|
|
901
|
+
if (typeof token.value === "string" && token.value.startsWith(":")) {
|
|
902
|
+
return createReference(token.value);
|
|
903
|
+
}
|
|
904
|
+
const identLower = safeToLowerCase(token.value);
|
|
905
|
+
if (isValidReference(identLower)) {
|
|
906
|
+
return createReference(identLower);
|
|
1633
907
|
}
|
|
908
|
+
return { type: "expression", raw: token.value };
|
|
1634
909
|
}
|
|
1635
|
-
|
|
910
|
+
case "url":
|
|
911
|
+
return createLiteral(token.value, "string");
|
|
912
|
+
default:
|
|
913
|
+
return null;
|
|
1636
914
|
}
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
}
|
|
1652
|
-
} else if (rule.value !== void 0) {
|
|
1653
|
-
captured.set(
|
|
1654
|
-
role,
|
|
1655
|
-
rule.valueIsExpression ? { type: "expression", raw: rule.value } : { type: "literal", value: rule.value }
|
|
1656
|
-
);
|
|
1657
|
-
} else if (rule.default) {
|
|
1658
|
-
const implicitDefault = {
|
|
1659
|
-
...rule.default,
|
|
1660
|
-
implicit: true
|
|
1661
|
-
};
|
|
1662
|
-
captured.set(role, implicitDefault);
|
|
1663
|
-
}
|
|
915
|
+
}
|
|
916
|
+
function parseLiteralValue(value) {
|
|
917
|
+
if (value.startsWith('"') || value.startsWith("'") || value.startsWith("`") || value.startsWith("\u300C")) {
|
|
918
|
+
const inner = value.slice(1, -1);
|
|
919
|
+
return createLiteral(inner, "string");
|
|
920
|
+
}
|
|
921
|
+
if (value === "true") return createLiteral(true, "boolean");
|
|
922
|
+
if (value === "false") return createLiteral(false, "boolean");
|
|
923
|
+
const durationMatch = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
|
|
924
|
+
if (durationMatch) {
|
|
925
|
+
const num2 = parseFloat(durationMatch[1]);
|
|
926
|
+
const unit = durationMatch[2];
|
|
927
|
+
if (unit) {
|
|
928
|
+
return createLiteral(value, "duration");
|
|
1664
929
|
}
|
|
930
|
+
return createLiteral(num2, "number");
|
|
1665
931
|
}
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
isOptional(patternToken) {
|
|
1670
|
-
return patternToken.type !== "literal" && patternToken.optional === true;
|
|
932
|
+
const num = parseFloat(value);
|
|
933
|
+
if (!isNaN(num)) {
|
|
934
|
+
return createLiteral(num, "number");
|
|
1671
935
|
}
|
|
1672
|
-
|
|
936
|
+
return createLiteral(value, "string");
|
|
937
|
+
}
|
|
938
|
+
var ENGLISH_NOISE_WORDS = /* @__PURE__ */ new Set(["the", "a", "an"]);
|
|
939
|
+
function skipNoiseWords(tokens) {
|
|
940
|
+
const token = tokens.peek();
|
|
941
|
+
if (!token) return;
|
|
942
|
+
const tokenLower = safeToLowerCase(token.value);
|
|
943
|
+
if (ENGLISH_NOISE_WORDS.has(tokenLower)) {
|
|
944
|
+
const mark = tokens.mark();
|
|
945
|
+
tokens.advance();
|
|
946
|
+
const nextToken = tokens.peek();
|
|
947
|
+
if (nextToken && nextToken.kind === "selector") {
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
tokens.reset(mark);
|
|
951
|
+
}
|
|
952
|
+
if (tokenLower === "class") {
|
|
953
|
+
tokens.advance();
|
|
954
|
+
}
|
|
955
|
+
}
|
|
1673
956
|
|
|
1674
957
|
// src/core/matching/stock-reader.ts
|
|
1675
958
|
function greedyStopMarkers(rest) {
|
|
@@ -1706,6 +989,11 @@ function readGreedy(tokens, stopMarkers) {
|
|
|
1706
989
|
}
|
|
1707
990
|
return values.length > 0 ? { type: "expression", raw: values.join(" ") } : null;
|
|
1708
991
|
}
|
|
992
|
+
function isListedWord(token, words) {
|
|
993
|
+
if (!token) return false;
|
|
994
|
+
const forms = [token.value, token.normalized].filter((f) => typeof f === "string");
|
|
995
|
+
return forms.some((form) => words.some((word) => word.toLowerCase() === form.toLowerCase()));
|
|
996
|
+
}
|
|
1709
997
|
var StockValueReader = class {
|
|
1710
998
|
constructor(view, profiles = {}) {
|
|
1711
999
|
this.view = view;
|
|
@@ -1738,6 +1026,11 @@ var StockValueReader = class {
|
|
|
1738
1026
|
const token = tokens.peek();
|
|
1739
1027
|
if (token && this.view.wouldMatch(next, token)) return false;
|
|
1740
1028
|
}
|
|
1029
|
+
if (slot.words && !isListedWord(tokens.peek(), slot.words)) return slot.optional ?? false;
|
|
1030
|
+
if (slot.optional && !slot.greedy) {
|
|
1031
|
+
const token = tokens.peek();
|
|
1032
|
+
if (token && isStopMarker(token, greedyStopMarkers(this.view.patternRest()))) return true;
|
|
1033
|
+
}
|
|
1741
1034
|
if (slot.greedy) {
|
|
1742
1035
|
const value = readGreedy(tokens, greedyStopMarkers(this.view.patternRest()));
|
|
1743
1036
|
if (value) {
|
|
@@ -1761,284 +1054,151 @@ function createStockReader(profiles = {}) {
|
|
|
1761
1054
|
return (view) => new StockValueReader(view, profiles);
|
|
1762
1055
|
}
|
|
1763
1056
|
|
|
1764
|
-
// src/
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
const priority = config.basePriority ?? 100;
|
|
1772
|
-
const keyword = profile.keywords[schema.action];
|
|
1773
|
-
if (!keyword) {
|
|
1774
|
-
throw new Error(`No keyword translation for '${schema.action}' in ${profile.code}`);
|
|
1775
|
-
}
|
|
1776
|
-
const tokens = buildTokens(schema, profile, keyword.primary, keyword.alternatives);
|
|
1777
|
-
const extraction = buildExtractionRules(schema, profile);
|
|
1778
|
-
const format = buildFormatString(schema, profile, keyword.primary);
|
|
1779
|
-
return {
|
|
1780
|
-
id,
|
|
1781
|
-
language: profile.code,
|
|
1782
|
-
command: schema.action,
|
|
1783
|
-
priority,
|
|
1784
|
-
template: {
|
|
1785
|
-
format,
|
|
1786
|
-
tokens
|
|
1787
|
-
},
|
|
1788
|
-
extraction
|
|
1789
|
-
};
|
|
1790
|
-
}
|
|
1791
|
-
function buildTokens(schema, profile, keyword, alternatives) {
|
|
1792
|
-
const tokens = [];
|
|
1793
|
-
const keywordToken = {
|
|
1794
|
-
type: "literal",
|
|
1795
|
-
value: keyword,
|
|
1796
|
-
...alternatives?.length && { alternatives }
|
|
1797
|
-
};
|
|
1798
|
-
const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
|
|
1799
|
-
if (profile.wordOrder === "SVO") {
|
|
1800
|
-
tokens.push(keywordToken);
|
|
1801
|
-
for (const role of sortedRoles) {
|
|
1802
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1803
|
-
}
|
|
1804
|
-
} else if (profile.wordOrder === "SOV") {
|
|
1805
|
-
const preVerb = [];
|
|
1806
|
-
const postVerb = [];
|
|
1807
|
-
for (const role of sortedRoles) {
|
|
1808
|
-
addRoleWithMarker(role.sovSlot === "postVerb" ? postVerb : preVerb, role, profile);
|
|
1809
|
-
}
|
|
1810
|
-
tokens.push(...preVerb, keywordToken, ...postVerb);
|
|
1811
|
-
} else if (profile.wordOrder === "VSO") {
|
|
1812
|
-
tokens.push(keywordToken);
|
|
1813
|
-
for (const role of sortedRoles) {
|
|
1814
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1815
|
-
}
|
|
1816
|
-
} else {
|
|
1817
|
-
tokens.push(keywordToken);
|
|
1818
|
-
for (const role of sortedRoles) {
|
|
1819
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
return tokens;
|
|
1823
|
-
}
|
|
1824
|
-
function addRoleWithMarker(tokens, roleSpec, profile) {
|
|
1825
|
-
const marker = getMarkerForRole(roleSpec, profile);
|
|
1826
|
-
const isOptional = !roleSpec.required;
|
|
1827
|
-
const roleToken = {
|
|
1828
|
-
type: "role",
|
|
1829
|
-
role: roleSpec.role,
|
|
1830
|
-
optional: isOptional,
|
|
1831
|
-
expectedTypes: roleSpec.expectedTypes,
|
|
1832
|
-
...roleSpec.greedy && { greedy: true }
|
|
1057
|
+
// src/core/matching/stock-confidence.ts
|
|
1058
|
+
function scoreParts(ctx) {
|
|
1059
|
+
const { pattern, captured } = ctx;
|
|
1060
|
+
let score = 0;
|
|
1061
|
+
let maxScore = 0;
|
|
1062
|
+
const hasDefault = (role) => {
|
|
1063
|
+
return pattern.extraction?.[role]?.default !== void 0;
|
|
1833
1064
|
};
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
if (position === "before") {
|
|
1840
|
-
tokens.push({
|
|
1841
|
-
type: "group",
|
|
1842
|
-
optional: true,
|
|
1843
|
-
tokens: [{ type: "literal", value: marker }, roleToken]
|
|
1844
|
-
});
|
|
1845
|
-
} else {
|
|
1846
|
-
tokens.push({
|
|
1847
|
-
type: "group",
|
|
1848
|
-
optional: true,
|
|
1849
|
-
tokens: [roleToken, { type: "literal", value: marker }]
|
|
1850
|
-
});
|
|
1065
|
+
for (const token of pattern.template.tokens) {
|
|
1066
|
+
if (token.type === "role") {
|
|
1067
|
+
maxScore += 1;
|
|
1068
|
+
if (captured.has(token.role)) {
|
|
1069
|
+
score += 1;
|
|
1851
1070
|
}
|
|
1852
|
-
} else {
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1071
|
+
} else if (token.type === "group") {
|
|
1072
|
+
for (const subToken of token.tokens) {
|
|
1073
|
+
if (subToken.type === "role") {
|
|
1074
|
+
const roleHasDefault = hasDefault(subToken.role);
|
|
1075
|
+
const weight = 0.8;
|
|
1076
|
+
maxScore += weight;
|
|
1077
|
+
if (captured.has(subToken.role)) {
|
|
1078
|
+
score += weight;
|
|
1079
|
+
} else if (roleHasDefault) {
|
|
1080
|
+
score += weight * 0.6;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1859
1083
|
}
|
|
1860
1084
|
}
|
|
1861
|
-
} else {
|
|
1862
|
-
tokens.push(roleToken);
|
|
1863
|
-
}
|
|
1864
|
-
}
|
|
1865
|
-
function getMarkerForRole(roleSpec, profile) {
|
|
1866
|
-
if (roleSpec.markerOverride?.[profile.code]) {
|
|
1867
|
-
return roleSpec.markerOverride[profile.code];
|
|
1868
|
-
}
|
|
1869
|
-
return profile.roleMarkers?.[roleSpec.role]?.primary;
|
|
1870
|
-
}
|
|
1871
|
-
function sortRolesByWordOrder(roles, wordOrder) {
|
|
1872
|
-
const sorted = [...roles];
|
|
1873
|
-
if (wordOrder === "SVO") {
|
|
1874
|
-
sorted.sort((a, b) => (b.svoPosition ?? 0) - (a.svoPosition ?? 0));
|
|
1875
|
-
} else if (wordOrder === "SOV") {
|
|
1876
|
-
sorted.sort((a, b) => (b.sovPosition ?? 0) - (a.sovPosition ?? 0));
|
|
1877
1085
|
}
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
const rule = {
|
|
1885
|
-
...roleSpec.default !== void 0 && { default: roleSpec.default },
|
|
1886
|
-
...marker && { marker }
|
|
1887
|
-
};
|
|
1888
|
-
rules[roleSpec.role] = rule;
|
|
1086
|
+
const roleCoverage = maxScore > 0 ? score / maxScore : 1;
|
|
1087
|
+
let baseConfidence = roleCoverage;
|
|
1088
|
+
let stemPenalty = 0;
|
|
1089
|
+
if (ctx.stemMatchCount > 0 && ctx.totalKeywordMatches > 0) {
|
|
1090
|
+
stemPenalty = ctx.stemMatchCount / ctx.totalKeywordMatches * 0.15;
|
|
1091
|
+
baseConfidence = Math.max(0.5, baseConfidence - stemPenalty);
|
|
1889
1092
|
}
|
|
1890
|
-
|
|
1093
|
+
const vsoBoost = calculateVSOConfidenceBoost(pattern);
|
|
1094
|
+
baseConfidence = Math.min(1, baseConfidence + vsoBoost);
|
|
1095
|
+
const prepositionAdjustment = arabicPrepositionDisambiguation(pattern, captured);
|
|
1096
|
+
baseConfidence = Math.max(0, Math.min(1, baseConfidence + prepositionAdjustment));
|
|
1097
|
+
return {
|
|
1098
|
+
roleCoverage,
|
|
1099
|
+
stemPenalty,
|
|
1100
|
+
languageBoost: vsoBoost,
|
|
1101
|
+
languageAdjustment: prepositionAdjustment,
|
|
1102
|
+
final: baseConfidence
|
|
1103
|
+
};
|
|
1891
1104
|
}
|
|
1892
|
-
function
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
const postVerb = [];
|
|
1896
|
-
const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
|
|
1897
|
-
for (const role of sortedRoles) {
|
|
1898
|
-
const marker = getMarkerForRole(role, profile);
|
|
1899
|
-
const roleName = `{${role.role}}`;
|
|
1900
|
-
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
1901
|
-
if (marker) {
|
|
1902
|
-
const markerInfo = profile.roleMarkers?.[role.role];
|
|
1903
|
-
if (markerInfo?.position === "after") {
|
|
1904
|
-
bucket.push(`${roleName} ${marker}`);
|
|
1905
|
-
} else {
|
|
1906
|
-
bucket.push(`${marker} ${roleName}`);
|
|
1907
|
-
}
|
|
1908
|
-
} else {
|
|
1909
|
-
bucket.push(roleName);
|
|
1910
|
-
}
|
|
1105
|
+
function calculateVSOConfidenceBoost(pattern) {
|
|
1106
|
+
if (pattern.language !== "ar") {
|
|
1107
|
+
return 0;
|
|
1911
1108
|
}
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
}
|
|
1916
|
-
function generatePatternVariants(schema, profile, config = defaultConfig) {
|
|
1917
|
-
const patterns = [];
|
|
1918
|
-
patterns.push(generatePattern(schema, profile, config));
|
|
1919
|
-
return patterns;
|
|
1920
|
-
}
|
|
1921
|
-
|
|
1922
|
-
// src/generation/renderer.ts
|
|
1923
|
-
function lookupKeyword(keywords, action, language) {
|
|
1924
|
-
return keywords[action]?.[language] ?? action;
|
|
1925
|
-
}
|
|
1926
|
-
function lookupMarker(markers, marker, language) {
|
|
1927
|
-
return markers[marker]?.[language] ?? marker;
|
|
1928
|
-
}
|
|
1929
|
-
function buildPhrase(...parts) {
|
|
1930
|
-
return parts.filter(Boolean).join(" ");
|
|
1931
|
-
}
|
|
1932
|
-
function buildTablesFromProfiles(schemas, profiles) {
|
|
1933
|
-
const keywords = {};
|
|
1934
|
-
const markers = {};
|
|
1935
|
-
for (const profile of profiles) {
|
|
1936
|
-
for (const [action, kw] of Object.entries(profile.keywords)) {
|
|
1937
|
-
if (!keywords[action]) keywords[action] = {};
|
|
1938
|
-
keywords[action][profile.code] = kw.primary;
|
|
1939
|
-
}
|
|
1109
|
+
const firstToken = pattern.template.tokens[0];
|
|
1110
|
+
if (!firstToken || firstToken.type !== "literal") {
|
|
1111
|
+
return 0;
|
|
1940
1112
|
}
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1113
|
+
const ARABIC_VERBS = /* @__PURE__ */ new Set([
|
|
1114
|
+
"\u0628\u062F\u0644",
|
|
1115
|
+
"\u063A\u064A\u0631",
|
|
1116
|
+
"\u0623\u0636\u0641",
|
|
1117
|
+
"\u0623\u0632\u0644",
|
|
1118
|
+
"\u0636\u0639",
|
|
1119
|
+
"\u0627\u062C\u0639\u0644",
|
|
1120
|
+
"\u0639\u064A\u0646",
|
|
1121
|
+
"\u0632\u062F",
|
|
1122
|
+
"\u0627\u0646\u0642\u0635",
|
|
1123
|
+
"\u0633\u062C\u0644",
|
|
1124
|
+
"\u0623\u0638\u0647\u0631",
|
|
1125
|
+
"\u0623\u062E\u0641",
|
|
1126
|
+
"\u0634\u063A\u0644",
|
|
1127
|
+
"\u0623\u0631\u0633\u0644",
|
|
1128
|
+
"\u0631\u0643\u0632",
|
|
1129
|
+
"\u0634\u0648\u0634",
|
|
1130
|
+
"\u062A\u0648\u0642\u0641",
|
|
1131
|
+
"\u0627\u0646\u0633\u062E",
|
|
1132
|
+
"\u0627\u062D\u0630\u0641",
|
|
1133
|
+
"\u0627\u0635\u0646\u0639",
|
|
1134
|
+
"\u0627\u0646\u062A\u0638\u0631",
|
|
1135
|
+
"\u0627\u0646\u062A\u0642\u0627\u0644",
|
|
1136
|
+
"\u0623\u0648"
|
|
1137
|
+
]);
|
|
1138
|
+
if (ARABIC_VERBS.has(firstToken.value)) {
|
|
1139
|
+
return 0.15;
|
|
1951
1140
|
}
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
markers[markerKey][profile.code] = markerDef.primary;
|
|
1141
|
+
if (firstToken.alternatives) {
|
|
1142
|
+
for (const alt of firstToken.alternatives) {
|
|
1143
|
+
if (ARABIC_VERBS.has(alt)) {
|
|
1144
|
+
return 0.15;
|
|
1957
1145
|
}
|
|
1958
1146
|
}
|
|
1959
1147
|
}
|
|
1960
|
-
return
|
|
1148
|
+
return 0;
|
|
1961
1149
|
}
|
|
1962
|
-
function
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
for (const profile of profiles) {
|
|
1966
|
-
if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
|
|
1967
|
-
if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
|
|
1150
|
+
function arabicPrepositionDisambiguation(pattern, captured) {
|
|
1151
|
+
if (pattern.language !== "ar") {
|
|
1152
|
+
return 0;
|
|
1968
1153
|
}
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1154
|
+
let adjustment = 0;
|
|
1155
|
+
const PREFERRED_PREPOSITIONS = {
|
|
1156
|
+
patient: ["\u0639\u0644\u0649"],
|
|
1157
|
+
// element selectors prefer على (on/upon)
|
|
1158
|
+
destination: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
1159
|
+
// destination prefers إلى (to)
|
|
1160
|
+
source: ["\u0645\u0646"],
|
|
1161
|
+
// source prefers من (from)
|
|
1162
|
+
agent: ["\u0645\u0646"],
|
|
1163
|
+
// agent/by prefers من (from/by)
|
|
1164
|
+
manner: ["\u0628"],
|
|
1165
|
+
// manner prefers ب (with/by)
|
|
1166
|
+
style: ["\u0628"],
|
|
1167
|
+
// style prefers ب (with)
|
|
1168
|
+
goal: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
1169
|
+
// target state prefers إلى (to)
|
|
1170
|
+
method: ["\u0628"]
|
|
1171
|
+
// method prefers ب (with/by)
|
|
1987
1172
|
};
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
const markerPositions = {};
|
|
1993
|
-
for (const profile of profiles) {
|
|
1994
|
-
for (const [role, markerDef] of Object.entries(profile.roleMarkers ?? {})) {
|
|
1995
|
-
if (!markerDef.position) continue;
|
|
1996
|
-
markerPositions[role] ?? (markerPositions[role] = {});
|
|
1997
|
-
markerPositions[role][profile.code] = markerDef.position;
|
|
1998
|
-
}
|
|
1999
|
-
}
|
|
2000
|
-
const schemaMap = /* @__PURE__ */ new Map();
|
|
2001
|
-
for (const s of schemas) schemaMap.set(s.action, s);
|
|
2002
|
-
return { keywords, markers, markerPositions, sovLanguages, schemaMap };
|
|
2003
|
-
}
|
|
2004
|
-
function renderFromSchema(node, language, tables) {
|
|
2005
|
-
const schema = tables.schemaMap.get(node.action);
|
|
2006
|
-
if (!schema) return null;
|
|
2007
|
-
const keyword = lookupKeyword(tables.keywords, node.action, language);
|
|
2008
|
-
const isSOV = tables.sovLanguages.has(language);
|
|
2009
|
-
const ordered = sortRolesByWordOrder([...schema.roles], isSOV ? "SOV" : "SVO");
|
|
2010
|
-
const preVerb = [];
|
|
2011
|
-
const postVerb = [];
|
|
2012
|
-
for (const role of ordered) {
|
|
2013
|
-
let value = extractRoleValue(node, role.role);
|
|
2014
|
-
if (!value && role.default !== void 0) {
|
|
2015
|
-
value = String(extractValue(role.default));
|
|
2016
|
-
}
|
|
2017
|
-
if (!value) continue;
|
|
2018
|
-
if (role.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
|
|
2019
|
-
value = `"${value}"`;
|
|
1173
|
+
for (const [role, value] of captured.entries()) {
|
|
1174
|
+
const preferred = PREFERRED_PREPOSITIONS[role];
|
|
1175
|
+
if (!preferred || preferred.length === 0) {
|
|
1176
|
+
continue;
|
|
2020
1177
|
}
|
|
2021
|
-
const
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
bucket.push(value);
|
|
1178
|
+
const metadata = "metadata" in value ? value.metadata : void 0;
|
|
1179
|
+
if (metadata && typeof metadata.prepositionValue === "string") {
|
|
1180
|
+
const usedPreposition = metadata.prepositionValue;
|
|
1181
|
+
if (preferred.includes(usedPreposition)) {
|
|
1182
|
+
adjustment += 0.1;
|
|
1183
|
+
} else {
|
|
1184
|
+
adjustment -= 0.1;
|
|
1185
|
+
}
|
|
2030
1186
|
}
|
|
2031
1187
|
}
|
|
2032
|
-
return
|
|
1188
|
+
return Math.max(-0.1, Math.min(0.1, adjustment));
|
|
2033
1189
|
}
|
|
1190
|
+
var stockConfidenceModel = {
|
|
1191
|
+
calculate: (ctx) => scoreParts(ctx).final,
|
|
1192
|
+
calculateWithBreakdown: (ctx) => scoreParts(ctx)
|
|
1193
|
+
};
|
|
2034
1194
|
|
|
2035
1195
|
// src/generation/schema-patterns.ts
|
|
2036
1196
|
function createSchemaPatternGenerator(grammar) {
|
|
2037
1197
|
return {
|
|
2038
|
-
generatePattern: (schema, profile, config) =>
|
|
1198
|
+
generatePattern: (schema, profile, config) => generatePattern(grammar, schema, profile, config),
|
|
2039
1199
|
generateSimplePattern: (schema, profile, config) => generateSimplePattern(grammar, schema, profile, config),
|
|
2040
1200
|
generateVerbFirstPattern: (schema, profile, config) => generateVerbFirstPattern(schema, profile, config),
|
|
2041
|
-
generatePatternVariants: (schema, profile, config) =>
|
|
1201
|
+
generatePatternVariants: (schema, profile, config) => generatePatternVariants(grammar, schema, profile, config),
|
|
2042
1202
|
buildRoleToken
|
|
2043
1203
|
};
|
|
2044
1204
|
}
|
|
@@ -2088,21 +1248,21 @@ function schemaMarkerAlternatives(roleSpec, languageCode, marker) {
|
|
|
2088
1248
|
const alternatives = [.../* @__PURE__ */ new Set([...legacy, ...variants])].filter((a) => a && a !== marker);
|
|
2089
1249
|
return alternatives.length ? alternatives : void 0;
|
|
2090
1250
|
}
|
|
2091
|
-
var
|
|
1251
|
+
var defaultConfig = {
|
|
2092
1252
|
basePriority: 100,
|
|
2093
1253
|
generateSimpleVariants: true,
|
|
2094
1254
|
generateAlternatives: true
|
|
2095
1255
|
};
|
|
2096
|
-
function
|
|
1256
|
+
function generatePattern(grammar, schema, profile, config = defaultConfig) {
|
|
2097
1257
|
const id = `${schema.action}-${profile.code}-generated`;
|
|
2098
1258
|
const priority = config.basePriority ?? 100;
|
|
2099
1259
|
const keyword = profile.keywords[schema.action];
|
|
2100
1260
|
if (!keyword) {
|
|
2101
1261
|
throw new Error(`No keyword translation for '${schema.action}' in ${profile.code}`);
|
|
2102
1262
|
}
|
|
2103
|
-
const tokens =
|
|
1263
|
+
const tokens = buildTokens(grammar, schema, profile, keyword);
|
|
2104
1264
|
const extraction = buildExtractionRulesWithDefaults(schema, profile);
|
|
2105
|
-
const format =
|
|
1265
|
+
const format = buildFormatString(grammar, schema, profile, keyword);
|
|
2106
1266
|
return {
|
|
2107
1267
|
id,
|
|
2108
1268
|
language: profile.code,
|
|
@@ -2115,7 +1275,7 @@ function generatePattern2(grammar, schema, profile, config = defaultConfig2) {
|
|
|
2115
1275
|
extraction
|
|
2116
1276
|
};
|
|
2117
1277
|
}
|
|
2118
|
-
function generateSimplePattern(grammar, schema, profile, config =
|
|
1278
|
+
function generateSimplePattern(grammar, schema, profile, config = defaultConfig) {
|
|
2119
1279
|
const optionalRoles = schema.roles.filter((r) => !r.required);
|
|
2120
1280
|
if (optionalRoles.length === 0) {
|
|
2121
1281
|
return null;
|
|
@@ -2125,7 +1285,7 @@ function generateSimplePattern(grammar, schema, profile, config = defaultConfig2
|
|
|
2125
1285
|
...schema,
|
|
2126
1286
|
roles: requiredRoles
|
|
2127
1287
|
};
|
|
2128
|
-
const pattern =
|
|
1288
|
+
const pattern = generatePattern(grammar, simpleSchema, profile, config);
|
|
2129
1289
|
return {
|
|
2130
1290
|
...pattern,
|
|
2131
1291
|
id: `${schema.action}-${profile.code}-generated-simple`,
|
|
@@ -2134,7 +1294,7 @@ function generateSimplePattern(grammar, schema, profile, config = defaultConfig2
|
|
|
2134
1294
|
extraction: buildExtractionRulesWithDefaults(schema, profile)
|
|
2135
1295
|
};
|
|
2136
1296
|
}
|
|
2137
|
-
function generateVerbFirstPattern(schema, profile, config =
|
|
1297
|
+
function generateVerbFirstPattern(schema, profile, config = defaultConfig) {
|
|
2138
1298
|
if (profile.wordOrder !== "SOV") return null;
|
|
2139
1299
|
const requiredRoles = schema.roles.filter((r) => r.required);
|
|
2140
1300
|
if (requiredRoles.length !== 1) return null;
|
|
@@ -2143,11 +1303,13 @@ function generateVerbFirstPattern(schema, profile, config = defaultConfig2) {
|
|
|
2143
1303
|
const verbToken = keyword.alternatives ? { type: "literal", value: keyword.primary, alternatives: keyword.alternatives } : { type: "literal", value: keyword.primary };
|
|
2144
1304
|
const roleTokens = requiredRoles.flatMap((r) => {
|
|
2145
1305
|
const prefix = r.valuePrefixLiteral?.[profile.code];
|
|
1306
|
+
const words = wordsFor(r, profile.code);
|
|
2146
1307
|
const roleToken = {
|
|
2147
1308
|
type: "role",
|
|
2148
1309
|
role: r.role,
|
|
2149
1310
|
optional: false,
|
|
2150
|
-
expectedTypes: r.expectedTypes
|
|
1311
|
+
expectedTypes: r.expectedTypes,
|
|
1312
|
+
...words ? { words } : {}
|
|
2151
1313
|
};
|
|
2152
1314
|
return prefix ? [{ type: "literal", value: prefix }, roleToken] : [roleToken];
|
|
2153
1315
|
});
|
|
@@ -2165,11 +1327,11 @@ function generateVerbFirstPattern(schema, profile, config = defaultConfig2) {
|
|
|
2165
1327
|
extraction: buildExtractionRulesWithDefaults(schema, profile)
|
|
2166
1328
|
};
|
|
2167
1329
|
}
|
|
2168
|
-
function
|
|
1330
|
+
function generatePatternVariants(grammar, schema, profile, config = defaultConfig) {
|
|
2169
1331
|
const patterns = [];
|
|
2170
1332
|
const full = schema;
|
|
2171
1333
|
schema = withoutVariantOnlyRoles(schema);
|
|
2172
|
-
patterns.push(
|
|
1334
|
+
patterns.push(generatePattern(grammar, schema, profile, config));
|
|
2173
1335
|
if (config.generateSimpleVariants) {
|
|
2174
1336
|
const simple = generateSimplePattern(grammar, schema, profile, config);
|
|
2175
1337
|
if (simple) {
|
|
@@ -2181,7 +1343,7 @@ function generatePatternVariants2(grammar, schema, profile, config = defaultConf
|
|
|
2181
1343
|
...schema,
|
|
2182
1344
|
roles: schema.roles.filter((r) => r.role !== omitted)
|
|
2183
1345
|
};
|
|
2184
|
-
const variant =
|
|
1346
|
+
const variant = generatePattern(grammar, variantSchema, profile, config);
|
|
2185
1347
|
patterns.push({
|
|
2186
1348
|
...variant,
|
|
2187
1349
|
id: `${schema.action}-${profile.code}-generated-no-${omitted}`,
|
|
@@ -2196,7 +1358,7 @@ function generatePatternVariants2(grammar, schema, profile, config = defaultConf
|
|
|
2196
1358
|
(r) => r.role === widen.role ? { ...r, expectedTypes: [...widen.types] } : r
|
|
2197
1359
|
)
|
|
2198
1360
|
};
|
|
2199
|
-
const main =
|
|
1361
|
+
const main = generatePattern(grammar, cloneSchema, profile, config);
|
|
2200
1362
|
patterns.push({
|
|
2201
1363
|
...main,
|
|
2202
1364
|
id: `${schema.action}-${profile.code}-generated-${widen.idSuffix}`,
|
|
@@ -2211,7 +1373,7 @@ function generatePatternVariants2(grammar, schema, profile, config = defaultConf
|
|
|
2211
1373
|
return r.role === v.widen?.role ? { ...required, expectedTypes: [...v.widen.types] } : required;
|
|
2212
1374
|
})
|
|
2213
1375
|
};
|
|
2214
|
-
const variant =
|
|
1376
|
+
const variant = generatePattern(grammar, variantSchema, profile, config);
|
|
2215
1377
|
patterns.push({
|
|
2216
1378
|
...variant,
|
|
2217
1379
|
id: `${schema.action}-${profile.code}-generated-${v.idSuffix}`,
|
|
@@ -2236,7 +1398,7 @@ function generatePatternVariants2(grammar, schema, profile, config = defaultConf
|
|
|
2236
1398
|
};
|
|
2237
1399
|
const delta = v.priorityDelta ?? 5;
|
|
2238
1400
|
const carrier = v.methodCarrier ? { [v.methodCarrier]: { value: literal } } : {};
|
|
2239
|
-
const main =
|
|
1401
|
+
const main = generatePattern(grammar, cloneSchema, profile, config);
|
|
2240
1402
|
patterns.push({
|
|
2241
1403
|
...main,
|
|
2242
1404
|
id: `${schema.action}-${profile.code}-generated-${v.idSuffix}`,
|
|
@@ -2257,7 +1419,7 @@ function generatePatternVariants2(grammar, schema, profile, config = defaultConf
|
|
|
2257
1419
|
}
|
|
2258
1420
|
return patterns;
|
|
2259
1421
|
}
|
|
2260
|
-
function
|
|
1422
|
+
function buildTokens(grammar, schema, profile, keyword) {
|
|
2261
1423
|
const tokens = [];
|
|
2262
1424
|
const verbToken = keyword.alternatives ? { type: "literal", value: keyword.primary, alternatives: keyword.alternatives } : { type: "literal", value: keyword.primary };
|
|
2263
1425
|
const postVerbRoles = profile.wordOrder === "SOV" ? schema.roles.filter((r) => r.sovSlot === "postVerb") : [];
|
|
@@ -2317,11 +1479,17 @@ function buildRoleTokens(grammar, schema, profile) {
|
|
|
2317
1479
|
}
|
|
2318
1480
|
return tokens;
|
|
2319
1481
|
}
|
|
1482
|
+
function wordsFor(roleSpec, language) {
|
|
1483
|
+
if (!roleSpec.words) return void 0;
|
|
1484
|
+
const words = [...roleSpec.words["*"] ?? [], ...roleSpec.words[language] ?? []];
|
|
1485
|
+
return words.length > 0 ? words : void 0;
|
|
1486
|
+
}
|
|
2320
1487
|
function buildRoleToken(roleSpec, profile) {
|
|
2321
1488
|
const tokens = [];
|
|
2322
1489
|
const overrideMarker = roleSpec.markerOverride?.[profile.code];
|
|
2323
1490
|
const defaultMarker = profile.roleMarkers[roleSpec.role];
|
|
2324
1491
|
const suppressMarker = roleSpec.renderOverride?.[profile.code] === "";
|
|
1492
|
+
const words = wordsFor(roleSpec, profile.code);
|
|
2325
1493
|
const roleValueToken = {
|
|
2326
1494
|
type: "role",
|
|
2327
1495
|
role: roleSpec.role,
|
|
@@ -2333,7 +1501,9 @@ function buildRoleToken(roleSpec, profile) {
|
|
|
2333
1501
|
...roleSpec.valueShape !== void 0 ? { valueShape: roleSpec.valueShape } : {},
|
|
2334
1502
|
// A greedy role reads up to the next marker the pattern owes (a domain's
|
|
2335
1503
|
// SQL condition); its value reader does the reading.
|
|
2336
|
-
...roleSpec.greedy ? { greedy: true } : {}
|
|
1504
|
+
...roleSpec.greedy ? { greedy: true } : {},
|
|
1505
|
+
// A role with a word list reads only one of its words.
|
|
1506
|
+
...words ? { words } : {}
|
|
2337
1507
|
};
|
|
2338
1508
|
const prefixLiteral = roleSpec.valuePrefixLiteral?.[profile.code];
|
|
2339
1509
|
const pushPrefixed = () => {
|
|
@@ -2403,7 +1573,7 @@ function buildRoleToken(roleSpec, profile) {
|
|
|
2403
1573
|
}
|
|
2404
1574
|
return tokens;
|
|
2405
1575
|
}
|
|
2406
|
-
function
|
|
1576
|
+
function buildExtractionRules(schema, profile) {
|
|
2407
1577
|
const rules = {};
|
|
2408
1578
|
for (const roleSpec of schema.roles) {
|
|
2409
1579
|
const overrideMarker = roleSpec.markerOverride?.[profile.code];
|
|
@@ -2433,7 +1603,7 @@ function buildExtractionRules2(schema, profile) {
|
|
|
2433
1603
|
return rules;
|
|
2434
1604
|
}
|
|
2435
1605
|
function buildExtractionRulesWithDefaults(schema, profile) {
|
|
2436
|
-
const baseRules =
|
|
1606
|
+
const baseRules = buildExtractionRules(schema, profile);
|
|
2437
1607
|
const rules = {};
|
|
2438
1608
|
for (const roleSpec of schema.roles) {
|
|
2439
1609
|
const baseRule = baseRules[roleSpec.role] || {};
|
|
@@ -2445,7 +1615,7 @@ function buildExtractionRulesWithDefaults(schema, profile) {
|
|
|
2445
1615
|
}
|
|
2446
1616
|
return rules;
|
|
2447
1617
|
}
|
|
2448
|
-
function
|
|
1618
|
+
function buildFormatString(grammar, schema, profile, keyword) {
|
|
2449
1619
|
const parts = [];
|
|
2450
1620
|
const sortedRoles = sortRolesByPosition(schema.roles, profile.wordOrder, grammar.positionOrder);
|
|
2451
1621
|
const roleParts = sortedRoles.map((roleSpec) => {
|
|
@@ -3889,12 +3059,15 @@ function readingDifference(source, back) {
|
|
|
3889
3059
|
|
|
3890
3060
|
// src/api/create-dsl.ts
|
|
3891
3061
|
var UnreadInputError = class extends Error {
|
|
3892
|
-
constructor(input, language, reading, unread) {
|
|
3893
|
-
super(
|
|
3062
|
+
constructor(input, language, reading, unread, hint) {
|
|
3063
|
+
super(
|
|
3064
|
+
`"${input}" (${language}) reads as ${reading.action} only by leaving "${unread}" unread` + (hint ? `; ${hint}` : "")
|
|
3065
|
+
);
|
|
3894
3066
|
this.input = input;
|
|
3895
3067
|
this.language = language;
|
|
3896
3068
|
this.reading = reading;
|
|
3897
3069
|
this.unread = unread;
|
|
3070
|
+
this.hint = hint;
|
|
3898
3071
|
this.code = "unread-input";
|
|
3899
3072
|
this.name = "UnreadInputError";
|
|
3900
3073
|
}
|
|
@@ -4013,15 +3186,32 @@ var MultilingualDSLImpl = class {
|
|
|
4013
3186
|
}
|
|
4014
3187
|
if (longestPartial) {
|
|
4015
3188
|
const unread = tokens.tokens.slice(longestPartial.consumedTokens).map((t) => t.value).join(" ");
|
|
3189
|
+
const hint = this.quoteHint(longestPartial, tokens.tokens);
|
|
4016
3190
|
throw new UnreadInputError(
|
|
4017
3191
|
input,
|
|
4018
3192
|
language,
|
|
4019
3193
|
this.nodeOf(longestPartial, input, language),
|
|
4020
|
-
unread
|
|
3194
|
+
unread,
|
|
3195
|
+
...hint ? [hint] : []
|
|
4021
3196
|
);
|
|
4022
3197
|
}
|
|
4023
3198
|
throw new Error(`No pattern matched for input: ${input}`);
|
|
4024
3199
|
}
|
|
3200
|
+
/**
|
|
3201
|
+
* `if "buy milk" is one value, quote it`, where the last token a match read is a bare word it took
|
|
3202
|
+
* as a role's value (a role with no word list) and the first it left is a bare word too.
|
|
3203
|
+
*/
|
|
3204
|
+
quoteHint(match, tokens) {
|
|
3205
|
+
const last = tokens[match.consumedTokens - 1];
|
|
3206
|
+
const next = tokens[match.consumedTokens];
|
|
3207
|
+
if (last?.kind !== "identifier" || next?.kind !== "identifier") return void 0;
|
|
3208
|
+
const schema = this.schemaLookup.getSchema(match.pattern.command);
|
|
3209
|
+
const role = [...match.captured].find(
|
|
3210
|
+
([, v]) => v.raw === last.value
|
|
3211
|
+
)?.[0];
|
|
3212
|
+
if (!role || schema?.roles.find((r) => r.role === role)?.words) return void 0;
|
|
3213
|
+
return `if "${last.value} ${next.value}" is one value, quote it`;
|
|
3214
|
+
}
|
|
4025
3215
|
nodeOf(match, input, language) {
|
|
4026
3216
|
return {
|
|
4027
3217
|
kind: "command",
|
|
@@ -4245,21 +3435,14 @@ function createMultilingualDSL(config) {
|
|
|
4245
3435
|
return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider, engine);
|
|
4246
3436
|
}
|
|
4247
3437
|
function engineFor(config) {
|
|
4248
|
-
const name = config.engine ?? "
|
|
4249
|
-
if (name === "classic") return classicEngine();
|
|
3438
|
+
const name = config.engine ?? "core";
|
|
4250
3439
|
if (name === "core") return coreEngine();
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
match: (tokens, pattern, language) => matcher.matchPattern(tokens, pattern, { code: language }),
|
|
4258
|
-
fallbackRenderer: (config) => createDomainRenderer({
|
|
4259
|
-
schemas: config.schemas,
|
|
4260
|
-
profiles: config.languages.map((l) => l.patternProfile)
|
|
4261
|
-
})
|
|
4262
|
-
};
|
|
3440
|
+
if (name === "classic") {
|
|
3441
|
+
throw new Error(
|
|
3442
|
+
'The "classic" DSL engine was removed in @lokascript/framework 5.0. Omit `engine` (or pass "core"): every domain reads on the core.'
|
|
3443
|
+
);
|
|
3444
|
+
}
|
|
3445
|
+
throw new Error(`Unknown DSL engine "${name}" (expected "core")`);
|
|
4263
3446
|
}
|
|
4264
3447
|
function schemaProfileOf(profile) {
|
|
4265
3448
|
const side = profile.wordOrder === "SOV" ? "after" : "before";
|
|
@@ -5947,6 +5130,130 @@ async function registryToAOTBackends(registry) {
|
|
|
5947
5130
|
// src/schema/command-schema.ts
|
|
5948
5131
|
import { defineCommand, defineRole, getRoleSpec } from "@lokascript/intent";
|
|
5949
5132
|
|
|
5133
|
+
// src/generation/profile.ts
|
|
5134
|
+
function sortRolesByWordOrder(roles, wordOrder) {
|
|
5135
|
+
const sorted = [...roles];
|
|
5136
|
+
if (wordOrder === "SVO") {
|
|
5137
|
+
sorted.sort((a, b) => (b.svoPosition ?? 0) - (a.svoPosition ?? 0));
|
|
5138
|
+
} else if (wordOrder === "SOV") {
|
|
5139
|
+
sorted.sort((a, b) => (b.sovPosition ?? 0) - (a.sovPosition ?? 0));
|
|
5140
|
+
}
|
|
5141
|
+
return sorted;
|
|
5142
|
+
}
|
|
5143
|
+
|
|
5144
|
+
// src/generation/renderer.ts
|
|
5145
|
+
function lookupKeyword(keywords, action, language) {
|
|
5146
|
+
return keywords[action]?.[language] ?? action;
|
|
5147
|
+
}
|
|
5148
|
+
function lookupMarker(markers, marker, language) {
|
|
5149
|
+
return markers[marker]?.[language] ?? marker;
|
|
5150
|
+
}
|
|
5151
|
+
function buildPhrase(...parts) {
|
|
5152
|
+
return parts.filter(Boolean).join(" ");
|
|
5153
|
+
}
|
|
5154
|
+
function buildTablesFromProfiles(schemas, profiles) {
|
|
5155
|
+
const keywords = {};
|
|
5156
|
+
const markers = {};
|
|
5157
|
+
for (const profile of profiles) {
|
|
5158
|
+
for (const [action, kw] of Object.entries(profile.keywords)) {
|
|
5159
|
+
if (!keywords[action]) keywords[action] = {};
|
|
5160
|
+
keywords[action][profile.code] = kw.primary;
|
|
5161
|
+
}
|
|
5162
|
+
}
|
|
5163
|
+
for (const schema of schemas) {
|
|
5164
|
+
for (const role of schema.roles) {
|
|
5165
|
+
if (role.markerOverride) {
|
|
5166
|
+
const markerKey = role.role;
|
|
5167
|
+
if (!markers[markerKey]) markers[markerKey] = {};
|
|
5168
|
+
for (const [lang, markerText] of Object.entries(role.markerOverride)) {
|
|
5169
|
+
markers[markerKey][lang] = markerText;
|
|
5170
|
+
}
|
|
5171
|
+
}
|
|
5172
|
+
}
|
|
5173
|
+
}
|
|
5174
|
+
for (const profile of profiles) {
|
|
5175
|
+
if (profile.roleMarkers) {
|
|
5176
|
+
for (const [markerKey, markerDef] of Object.entries(profile.roleMarkers)) {
|
|
5177
|
+
if (!markers[markerKey]) markers[markerKey] = {};
|
|
5178
|
+
markers[markerKey][profile.code] = markerDef.primary;
|
|
5179
|
+
}
|
|
5180
|
+
}
|
|
5181
|
+
}
|
|
5182
|
+
return { keywords, markers };
|
|
5183
|
+
}
|
|
5184
|
+
function detectWordOrders(profiles) {
|
|
5185
|
+
const sovLanguages = /* @__PURE__ */ new Set();
|
|
5186
|
+
const vsoLanguages = /* @__PURE__ */ new Set();
|
|
5187
|
+
for (const profile of profiles) {
|
|
5188
|
+
if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
|
|
5189
|
+
if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
|
|
5190
|
+
}
|
|
5191
|
+
return { sovLanguages, vsoLanguages };
|
|
5192
|
+
}
|
|
5193
|
+
function createSchemaRenderer(schemas, profiles) {
|
|
5194
|
+
const tables = buildRenderTables(schemas, profiles);
|
|
5195
|
+
return {
|
|
5196
|
+
render(node, language) {
|
|
5197
|
+
return renderFromSchema(node, language, tables) ?? node.action;
|
|
5198
|
+
}
|
|
5199
|
+
};
|
|
5200
|
+
}
|
|
5201
|
+
function createDomainRenderer(config) {
|
|
5202
|
+
const { schemas, profiles, overrides } = config;
|
|
5203
|
+
let tables;
|
|
5204
|
+
return (node, language) => {
|
|
5205
|
+
const override = overrides?.[node.action];
|
|
5206
|
+
if (override) return override(node, language);
|
|
5207
|
+
tables ?? (tables = buildRenderTables(schemas, profiles));
|
|
5208
|
+
return renderFromSchema(node, language, tables);
|
|
5209
|
+
};
|
|
5210
|
+
}
|
|
5211
|
+
function buildRenderTables(schemas, profiles) {
|
|
5212
|
+
const { keywords, markers } = buildTablesFromProfiles(schemas, profiles);
|
|
5213
|
+
const { sovLanguages } = detectWordOrders(profiles);
|
|
5214
|
+
const markerPositions = {};
|
|
5215
|
+
for (const profile of profiles) {
|
|
5216
|
+
for (const [role, markerDef] of Object.entries(profile.roleMarkers ?? {})) {
|
|
5217
|
+
if (!markerDef.position) continue;
|
|
5218
|
+
markerPositions[role] ?? (markerPositions[role] = {});
|
|
5219
|
+
markerPositions[role][profile.code] = markerDef.position;
|
|
5220
|
+
}
|
|
5221
|
+
}
|
|
5222
|
+
const schemaMap = /* @__PURE__ */ new Map();
|
|
5223
|
+
for (const s of schemas) schemaMap.set(s.action, s);
|
|
5224
|
+
return { keywords, markers, markerPositions, sovLanguages, schemaMap };
|
|
5225
|
+
}
|
|
5226
|
+
function renderFromSchema(node, language, tables) {
|
|
5227
|
+
const schema = tables.schemaMap.get(node.action);
|
|
5228
|
+
if (!schema) return null;
|
|
5229
|
+
const keyword = lookupKeyword(tables.keywords, node.action, language);
|
|
5230
|
+
const isSOV = tables.sovLanguages.has(language);
|
|
5231
|
+
const ordered = sortRolesByWordOrder([...schema.roles], isSOV ? "SOV" : "SVO");
|
|
5232
|
+
const preVerb = [];
|
|
5233
|
+
const postVerb = [];
|
|
5234
|
+
for (const role of ordered) {
|
|
5235
|
+
let value = extractRoleValue(node, role.role);
|
|
5236
|
+
if (!value && role.default !== void 0) {
|
|
5237
|
+
value = String(extractValue(role.default));
|
|
5238
|
+
}
|
|
5239
|
+
if (!value) continue;
|
|
5240
|
+
if (role.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
|
|
5241
|
+
value = `"${value}"`;
|
|
5242
|
+
}
|
|
5243
|
+
const markerText = role.renderOverride?.[language] ?? role.markerOverride?.[language] ?? role.renderOverride?.["*"] ?? tables.markers[role.role]?.[language];
|
|
5244
|
+
const markerPosition = role.markerPositionOverride?.[language] ?? role.markerPosition ?? tables.markerPositions[role.role]?.[language] ?? (isSOV ? "after" : "before");
|
|
5245
|
+
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
5246
|
+
if (markerText && markerPosition === "before") {
|
|
5247
|
+
bucket.push(markerText, value);
|
|
5248
|
+
} else if (markerText) {
|
|
5249
|
+
bucket.push(value, markerText);
|
|
5250
|
+
} else {
|
|
5251
|
+
bucket.push(value);
|
|
5252
|
+
}
|
|
5253
|
+
}
|
|
5254
|
+
return isSOV ? buildPhrase(...preVerb, keyword, ...postVerb) : buildPhrase(keyword, ...preVerb, ...postVerb);
|
|
5255
|
+
}
|
|
5256
|
+
|
|
5950
5257
|
// src/generation/diagnostics.ts
|
|
5951
5258
|
import { createDiagnosticCollector, fromError, filterBySeverity } from "@lokascript/intent";
|
|
5952
5259
|
|
|
@@ -7800,7 +7107,6 @@ export {
|
|
|
7800
7107
|
NullProfileProvider,
|
|
7801
7108
|
NumberExtractor,
|
|
7802
7109
|
OperatorExtractor,
|
|
7803
|
-
PatternMatcher,
|
|
7804
7110
|
PatternTracker,
|
|
7805
7111
|
PunctuationExtractor,
|
|
7806
7112
|
StockValueReader,
|
|
@@ -7864,8 +7170,6 @@ export {
|
|
|
7864
7170
|
fromInterchangeNode,
|
|
7865
7171
|
fromProtocolJSON,
|
|
7866
7172
|
generateExamples,
|
|
7867
|
-
generatePattern,
|
|
7868
|
-
generatePatternVariants,
|
|
7869
7173
|
generatePrompt,
|
|
7870
7174
|
generateProtocolReference,
|
|
7871
7175
|
getAllPossessiveKeywords,
|
|
@@ -7904,8 +7208,6 @@ export {
|
|
|
7904
7208
|
legacyMarkerAlternatives,
|
|
7905
7209
|
lookupKeyword,
|
|
7906
7210
|
lookupMarker,
|
|
7907
|
-
matchBest,
|
|
7908
|
-
matchPattern,
|
|
7909
7211
|
needsDisambiguation,
|
|
7910
7212
|
noChange,
|
|
7911
7213
|
normalized,
|
|
@@ -7913,7 +7215,6 @@ export {
|
|
|
7913
7215
|
parseDocument,
|
|
7914
7216
|
parseExplicit,
|
|
7915
7217
|
parseJSONL,
|
|
7916
|
-
patternMatcher,
|
|
7917
7218
|
readGreedy,
|
|
7918
7219
|
readRoleValue,
|
|
7919
7220
|
registryToAOTBackends,
|