@lokascript/framework 4.7.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -1
- package/dist/api/create-dsl.d.ts +14 -12
- package/dist/api/create-dsl.d.ts.map +1 -1
- package/dist/api/index.js +888 -1683
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.d.ts +0 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +560 -1133
- package/dist/core/index.js.map +1 -1
- package/dist/core/matching/index.d.ts +2 -0
- package/dist/core/matching/index.d.ts.map +1 -1
- package/dist/core/matching/possessive-keywords.d.ts.map +1 -0
- package/dist/core/matching/stock-confidence.d.ts +3 -4
- package/dist/core/matching/stock-confidence.d.ts.map +1 -1
- package/dist/core/matching/stock-reader.d.ts +8 -1
- package/dist/core/matching/stock-reader.d.ts.map +1 -1
- package/dist/core/matching/stock-reading.d.ts +4 -4
- package/dist/core/{pattern-matching/utils → matching}/type-validation.d.ts +1 -1
- package/dist/core/matching/type-validation.d.ts.map +1 -0
- package/dist/core/types.d.ts +5 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/generation/index.d.ts +1 -1
- package/dist/generation/index.d.ts.map +1 -1
- package/dist/generation/index.js +32 -171
- package/dist/generation/index.js.map +1 -1
- package/dist/generation/profile.d.ts +30 -0
- package/dist/generation/profile.d.ts.map +1 -0
- package/dist/generation/renderer.d.ts +1 -1
- package/dist/generation/renderer.d.ts.map +1 -1
- package/dist/generation/schema-patterns.d.ts +3 -3
- package/dist/index.cjs +969 -1674
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +969 -1668
- package/dist/index.js.map +1 -1
- package/dist/ir/index.js.map +1 -1
- package/dist/multilingual/builders.d.ts +3 -3
- package/dist/multilingual/builders.d.ts.map +1 -1
- package/dist/multilingual/index.js.map +1 -1
- package/dist/parsing/index.d.ts +0 -1
- package/dist/parsing/index.d.ts.map +1 -1
- package/dist/parsing/index.js +0 -1145
- package/dist/parsing/index.js.map +1 -1
- package/dist/testing/gate/index.js +44 -12
- package/dist/testing/gate/index.js.map +1 -1
- package/dist/testing/gate/lint/types.d.ts +1 -1
- package/dist/testing/gate/lint/types.d.ts.map +1 -1
- package/dist/testing/gate/round-trip.d.ts +24 -3
- package/dist/testing/gate/round-trip.d.ts.map +1 -1
- package/package.json +2 -7
- package/src/__test__/test-utils.ts +1 -1
- package/src/api/create-dsl-engine.test.ts +123 -34
- package/src/api/create-dsl.test.ts +7 -0
- package/src/api/create-dsl.ts +56 -39
- package/src/core/index.ts +0 -1
- package/src/core/matching/index.ts +2 -0
- package/src/core/matching/stock-confidence.ts +3 -4
- package/src/core/matching/stock-reader.test.ts +87 -3
- package/src/core/matching/stock-reader.ts +20 -1
- package/src/core/matching/stock-reading.ts +6 -6
- package/src/core/{pattern-matching/utils → matching}/type-validation.test.ts +2 -2
- package/src/core/{pattern-matching/utils → matching}/type-validation.ts +1 -1
- package/src/core/types.ts +5 -0
- package/src/generation/domain-renderer.test.ts +1 -1
- package/src/generation/index.ts +1 -1
- package/src/generation/profile.ts +41 -0
- package/src/generation/renderer.test.ts +1 -1
- package/src/generation/renderer.ts +1 -1
- package/src/generation/schema-patterns.test.ts +41 -1
- package/src/generation/schema-patterns.ts +15 -3
- package/src/multilingual/bridge.test.ts +31 -23
- package/src/multilingual/builders.ts +3 -3
- package/src/parsing/index.ts +0 -1
- package/src/testing/gate/lint/types.ts +1 -1
- package/src/testing/gate/round-trip.test.ts +99 -9
- package/src/testing/gate/round-trip.ts +76 -11
- package/dist/core/pattern-matching/index.d.ts +0 -6
- package/dist/core/pattern-matching/index.d.ts.map +0 -1
- package/dist/core/pattern-matching/index.js +0 -1211
- package/dist/core/pattern-matching/index.js.map +0 -1
- package/dist/core/pattern-matching/pattern-matcher.d.ts +0 -122
- package/dist/core/pattern-matching/pattern-matcher.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/index.d.ts +0 -6
- package/dist/core/pattern-matching/utils/index.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/possessive-keywords.d.ts.map +0 -1
- package/dist/core/pattern-matching/utils/type-validation.d.ts.map +0 -1
- package/dist/generation/pattern-generator.d.ts +0 -55
- package/dist/generation/pattern-generator.d.ts.map +0 -1
- package/src/core/pattern-matching/index.ts +0 -6
- package/src/core/pattern-matching/pattern-matcher.test.ts +0 -1031
- package/src/core/pattern-matching/pattern-matcher.ts +0 -765
- package/src/core/pattern-matching/utils/index.ts +0 -6
- package/src/generation/pattern-generator.test.ts +0 -532
- package/src/generation/pattern-generator.ts +0 -327
- /package/dist/core/{pattern-matching/utils → matching}/possessive-keywords.d.ts +0 -0
- /package/src/core/{pattern-matching/utils → matching}/possessive-keywords.ts +0 -0
package/dist/api/index.js
CHANGED
|
@@ -19,292 +19,595 @@ import {
|
|
|
19
19
|
getRoleValue
|
|
20
20
|
} from "@lokascript/intent";
|
|
21
21
|
|
|
22
|
-
// src/core/
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}
|
|
33
|
-
return false;
|
|
34
|
-
}
|
|
35
|
-
log(level, message, ...args) {
|
|
36
|
-
if (!this.enabled) return;
|
|
37
|
-
const prefix = `[framework:${this.namespace}]`;
|
|
38
|
-
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
39
|
-
switch (level) {
|
|
40
|
-
case "debug":
|
|
41
|
-
console.debug(`${timestamp} ${prefix} DEBUG:`, message, ...args);
|
|
42
|
-
break;
|
|
43
|
-
case "info":
|
|
44
|
-
console.info(`${timestamp} ${prefix} INFO:`, message, ...args);
|
|
45
|
-
break;
|
|
46
|
-
case "warn":
|
|
47
|
-
console.warn(`${timestamp} ${prefix} WARN:`, message, ...args);
|
|
48
|
-
break;
|
|
49
|
-
case "error":
|
|
50
|
-
console.error(`${timestamp} ${prefix} ERROR:`, message, ...args);
|
|
51
|
-
break;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
debug(message, ...args) {
|
|
55
|
-
this.log("debug", message, ...args);
|
|
56
|
-
}
|
|
57
|
-
info(message, ...args) {
|
|
58
|
-
this.log("info", message, ...args);
|
|
59
|
-
}
|
|
60
|
-
warn(message, ...args) {
|
|
61
|
-
this.log("warn", message, ...args);
|
|
62
|
-
}
|
|
63
|
-
error(message, ...args) {
|
|
64
|
-
this.log("error", message, ...args);
|
|
65
|
-
}
|
|
22
|
+
// src/core/tokenization/token-utils.ts
|
|
23
|
+
function stripOptionalDiacritics(word) {
|
|
24
|
+
return word.replace(/[\u064b-\u0652\u0670]/g, "");
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// src/core/matching/match-skeleton.ts
|
|
28
|
+
function groupHasRole(group) {
|
|
29
|
+
return group.tokens.some((t) => t.type === "role" || t.type === "group" && groupHasRole(t));
|
|
30
|
+
}
|
|
31
|
+
var MatchSkeleton = class {
|
|
66
32
|
/**
|
|
67
|
-
*
|
|
33
|
+
* @param makeReader - builds the grammar's reader, handed this matcher's view
|
|
34
|
+
* of the match in progress (one reader per matcher).
|
|
68
35
|
*/
|
|
69
|
-
|
|
70
|
-
|
|
36
|
+
constructor(makeReader, confidenceModel) {
|
|
37
|
+
/**
|
|
38
|
+
* The pattern tokens still to match after the one being matched, through
|
|
39
|
+
* every enclosing group: a value stops at any marker the pattern still owes.
|
|
40
|
+
*/
|
|
41
|
+
this.patternRest = [];
|
|
42
|
+
/**
|
|
43
|
+
* Whether the role slot being matched directly follows a literal the pattern
|
|
44
|
+
* matched: its marker (`[por {quantity}]`), or the command's verb.
|
|
45
|
+
*/
|
|
46
|
+
this.slotAfterLiteral = false;
|
|
47
|
+
/**
|
|
48
|
+
* Selective memoization cache (Phase 6.1).
|
|
49
|
+
* Caches pattern match results keyed by `tokenPosition:patternId`.
|
|
50
|
+
* Cleared before each top-level matchBest() call.
|
|
51
|
+
*/
|
|
52
|
+
this.matchCache = /* @__PURE__ */ new Map();
|
|
53
|
+
/**
|
|
54
|
+
* Stream index where each role's capture began, for the current
|
|
55
|
+
* `matchPattern` call. Rebuilt per call, written on the same success path as
|
|
56
|
+
* `captured` — see `PatternMatchResult.roleStarts`.
|
|
57
|
+
*/
|
|
58
|
+
this.roleStarts = /* @__PURE__ */ new Map();
|
|
59
|
+
/**
|
|
60
|
+
* Track stem matches for confidence calculation.
|
|
61
|
+
* This is set during matching and read during confidence calculation.
|
|
62
|
+
*/
|
|
63
|
+
this.stemMatchCount = 0;
|
|
64
|
+
this.totalKeywordMatches = 0;
|
|
65
|
+
this.confidenceModel = confidenceModel;
|
|
66
|
+
this.reader = makeReader({
|
|
67
|
+
patternRest: () => this.patternRest,
|
|
68
|
+
slotAfterLiteral: () => this.slotAfterLiteral,
|
|
69
|
+
wouldMatch: (pt, token) => this.patternTokenWouldMatch(pt, token)
|
|
70
|
+
});
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
73
|
-
*
|
|
73
|
+
* Try to match a single pattern against the token stream.
|
|
74
|
+
* Returns the match result or null if no match.
|
|
74
75
|
*/
|
|
75
|
-
|
|
76
|
-
|
|
76
|
+
matchPattern(tokens, pattern) {
|
|
77
|
+
const mark = tokens.mark();
|
|
78
|
+
const captured = /* @__PURE__ */ new Map();
|
|
79
|
+
this.reader.beginPattern(pattern);
|
|
80
|
+
this.stemMatchCount = 0;
|
|
81
|
+
this.totalKeywordMatches = 0;
|
|
82
|
+
this.roleStarts = /* @__PURE__ */ new Map();
|
|
83
|
+
this.patternRest = [];
|
|
84
|
+
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
85
|
+
if (!success) {
|
|
86
|
+
tokens.reset(mark);
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
const confidence = this.confidenceModel.calculate({
|
|
90
|
+
pattern,
|
|
91
|
+
captured,
|
|
92
|
+
stemMatchCount: this.stemMatchCount,
|
|
93
|
+
totalKeywordMatches: this.totalKeywordMatches
|
|
94
|
+
});
|
|
95
|
+
this.applyExtractionRules(pattern, captured);
|
|
96
|
+
return {
|
|
97
|
+
pattern,
|
|
98
|
+
captured,
|
|
99
|
+
consumedTokens: tokens.position() - mark.position,
|
|
100
|
+
confidence,
|
|
101
|
+
roleStarts: this.roleStarts,
|
|
102
|
+
...this.reader.resultFields?.()
|
|
103
|
+
};
|
|
77
104
|
}
|
|
78
105
|
/**
|
|
79
|
-
*
|
|
106
|
+
* Try to match multiple patterns, return the best match.
|
|
80
107
|
*/
|
|
81
|
-
|
|
82
|
-
this.
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if (expectedTypes.includes(actualType)) {
|
|
95
|
-
return true;
|
|
96
|
-
}
|
|
97
|
-
if (expectedTypes.includes("expression")) {
|
|
98
|
-
return true;
|
|
99
|
-
}
|
|
100
|
-
if (actualType === "property-path") {
|
|
101
|
-
return expectedTypes.some((t) => ["selector", "reference", "expression"].includes(t));
|
|
102
|
-
}
|
|
103
|
-
return false;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// src/core/pattern-matching/utils/possessive-keywords.ts
|
|
107
|
-
function getPossessiveReference(profile, keyword) {
|
|
108
|
-
return profile?.possessive?.keywords?.[keyword];
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// src/core/matching/stock-reading.ts
|
|
112
|
-
function isValidReference(_value) {
|
|
113
|
-
return false;
|
|
114
|
-
}
|
|
115
|
-
function safeToLowerCase(value) {
|
|
116
|
-
if (typeof value === "string") {
|
|
117
|
-
return value.toLowerCase();
|
|
118
|
-
}
|
|
119
|
-
if (value === null || value === void 0) {
|
|
120
|
-
return "";
|
|
121
|
-
}
|
|
122
|
-
return String(value).toLowerCase();
|
|
123
|
-
}
|
|
124
|
-
function readRoleValue(tokens, patternToken, captured, profile) {
|
|
125
|
-
skipNoiseWords(tokens);
|
|
126
|
-
const token = tokens.peek();
|
|
127
|
-
if (!token) {
|
|
128
|
-
return patternToken.optional || false;
|
|
129
|
-
}
|
|
130
|
-
const possessiveValue = tryMatchPossessiveExpression(tokens, profile);
|
|
131
|
-
if (possessiveValue) {
|
|
132
|
-
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
133
|
-
if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
|
|
134
|
-
return patternToken.optional || false;
|
|
108
|
+
matchBest(tokens, patterns) {
|
|
109
|
+
this.matchCache.clear();
|
|
110
|
+
const matches = [];
|
|
111
|
+
const startPos = tokens.position();
|
|
112
|
+
for (const pattern of patterns) {
|
|
113
|
+
const mark = tokens.mark();
|
|
114
|
+
const cacheKey = `${startPos}:${pattern.id}`;
|
|
115
|
+
let result;
|
|
116
|
+
if (this.matchCache.has(cacheKey)) {
|
|
117
|
+
result = this.matchCache.get(cacheKey);
|
|
118
|
+
} else {
|
|
119
|
+
result = this.matchPattern(tokens, pattern);
|
|
120
|
+
this.matchCache.set(cacheKey, result);
|
|
135
121
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
return true;
|
|
139
|
-
}
|
|
140
|
-
const methodCallValue = tryMatchMethodCallExpression(tokens);
|
|
141
|
-
if (methodCallValue) {
|
|
142
|
-
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
143
|
-
if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
|
|
144
|
-
return patternToken.optional || false;
|
|
122
|
+
if (result) {
|
|
123
|
+
matches.push(result);
|
|
145
124
|
}
|
|
125
|
+
tokens.reset(mark);
|
|
146
126
|
}
|
|
147
|
-
|
|
148
|
-
|
|
127
|
+
if (matches.length === 0) {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
matches.sort((a, b) => {
|
|
131
|
+
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
132
|
+
if (priorityDiff !== 0) return priorityDiff;
|
|
133
|
+
return b.confidence - a.confidence;
|
|
134
|
+
});
|
|
135
|
+
const best = matches[0];
|
|
136
|
+
this.matchPattern(tokens, best.pattern);
|
|
137
|
+
return best;
|
|
149
138
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
139
|
+
/**
|
|
140
|
+
* Match a sequence of pattern tokens against the token stream.
|
|
141
|
+
*/
|
|
142
|
+
matchTokenSequence(tokens, patternTokens, captured, nextAfterSequence) {
|
|
143
|
+
this.reader.skipLeading?.(tokens, patternTokens);
|
|
144
|
+
for (let i = 0; i < patternTokens.length; i++) {
|
|
145
|
+
const patternToken = patternTokens[i];
|
|
146
|
+
this.reader.beforePatternToken?.(tokens, patternTokens, i, captured);
|
|
147
|
+
if (this.reader.maySkipSlot(
|
|
148
|
+
patternToken,
|
|
149
|
+
tokens.peek(),
|
|
150
|
+
patternTokens[i + 1] ?? nextAfterSequence
|
|
151
|
+
)) {
|
|
152
|
+
const skipMark = tokens.mark();
|
|
153
|
+
const speculative = new Map(captured);
|
|
154
|
+
if (this.matchTokenSequence(
|
|
155
|
+
tokens,
|
|
156
|
+
patternTokens.slice(i + 1),
|
|
157
|
+
speculative,
|
|
158
|
+
nextAfterSequence
|
|
159
|
+
) && this.reader.atClauseEnd(tokens)) {
|
|
160
|
+
captured.clear();
|
|
161
|
+
for (const [role, value] of speculative) captured.set(role, value);
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
tokens.reset(skipMark);
|
|
165
|
+
}
|
|
166
|
+
const outerRest = this.patternRest;
|
|
167
|
+
this.patternRest = [...patternTokens.slice(i + 1), ...outerRest];
|
|
168
|
+
this.slotAfterLiteral = patternToken.type === "role" && patternTokens[i - 1]?.type === "literal";
|
|
169
|
+
const matched = this.matchPatternToken(
|
|
170
|
+
tokens,
|
|
171
|
+
patternToken,
|
|
172
|
+
captured,
|
|
173
|
+
patternTokens[i + 1] ?? nextAfterSequence
|
|
174
|
+
);
|
|
175
|
+
this.patternRest = outerRest;
|
|
176
|
+
if (!matched) {
|
|
177
|
+
if (this.isOptional(patternToken)) {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
return false;
|
|
155
181
|
}
|
|
156
182
|
}
|
|
157
|
-
captured.set(patternToken.role, possessiveSelectorValue);
|
|
158
183
|
return true;
|
|
159
184
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
185
|
+
/**
|
|
186
|
+
* Match a single pattern token against the current position in the stream.
|
|
187
|
+
*/
|
|
188
|
+
matchPatternToken(tokens, patternToken, captured, nextPatternToken) {
|
|
189
|
+
switch (patternToken.type) {
|
|
190
|
+
case "literal":
|
|
191
|
+
return this.matchLiteralToken(tokens, patternToken);
|
|
192
|
+
case "role": {
|
|
193
|
+
const roleStart = tokens.position();
|
|
194
|
+
const matchedRole = this.reader.readRole(tokens, patternToken, captured, nextPatternToken);
|
|
195
|
+
if (matchedRole && captured.has(patternToken.role)) {
|
|
196
|
+
this.roleStarts.set(patternToken.role, roleStart);
|
|
197
|
+
}
|
|
198
|
+
return matchedRole;
|
|
165
199
|
}
|
|
200
|
+
case "group":
|
|
201
|
+
return this.matchGroupToken(tokens, patternToken, captured, nextPatternToken);
|
|
202
|
+
default:
|
|
203
|
+
return false;
|
|
166
204
|
}
|
|
167
|
-
captured.set(patternToken.role, propertyAccessValue);
|
|
168
|
-
return true;
|
|
169
205
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
206
|
+
/**
|
|
207
|
+
* Match a literal pattern token (keyword or particle).
|
|
208
|
+
*/
|
|
209
|
+
matchLiteralToken(tokens, patternToken) {
|
|
210
|
+
const token = tokens.peek();
|
|
211
|
+
if (!token) return false;
|
|
212
|
+
const matchType = this.getMatchType(token, patternToken.value);
|
|
213
|
+
if (matchType !== "none") {
|
|
214
|
+
this.totalKeywordMatches++;
|
|
215
|
+
if (matchType === "stem") {
|
|
216
|
+
this.stemMatchCount++;
|
|
175
217
|
}
|
|
218
|
+
tokens.advance();
|
|
219
|
+
return true;
|
|
176
220
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
221
|
+
if (patternToken.alternatives) {
|
|
222
|
+
for (const alt of patternToken.alternatives) {
|
|
223
|
+
const altMatchType = this.getMatchType(token, alt);
|
|
224
|
+
if (altMatchType !== "none") {
|
|
225
|
+
this.totalKeywordMatches++;
|
|
226
|
+
if (altMatchType === "stem") {
|
|
227
|
+
this.stemMatchCount++;
|
|
228
|
+
}
|
|
229
|
+
tokens.advance();
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
for (const value of [patternToken.value, ...patternToken.alternatives ?? []]) {
|
|
235
|
+
for (const sep of ["-", "_"]) {
|
|
236
|
+
if (value.includes(sep) && this.tryMatchShatteredCompound(tokens, value, sep)) {
|
|
237
|
+
this.totalKeywordMatches++;
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
187
241
|
}
|
|
242
|
+
return false;
|
|
188
243
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
244
|
+
/**
|
|
245
|
+
* Match a compound keyword (`titik-henti`, `по_умолчанию`) against a
|
|
246
|
+
* shattered token run (`titik` `-` `henti` / `по` `_` `умолчанию`): word
|
|
247
|
+
* segments must be identifier/keyword/particle tokens equal to the expected
|
|
248
|
+
* segments (case-insensitive; particle because prepositions like ru `по` /
|
|
249
|
+
* uk `за` lead some underscore compounds), separators lone `sep` tokens.
|
|
250
|
+
* Consumes the run on success, resets the stream on failure.
|
|
251
|
+
*/
|
|
252
|
+
tryMatchShatteredCompound(tokens, expected, sep) {
|
|
253
|
+
const segments = expected.split(sep);
|
|
254
|
+
if (segments.length < 2 || segments.some((s) => s.length === 0)) return false;
|
|
255
|
+
const mark = tokens.mark();
|
|
256
|
+
for (let i = 0; i < segments.length; i++) {
|
|
257
|
+
if (i > 0) {
|
|
258
|
+
const sepToken = tokens.peek();
|
|
259
|
+
if (!sepToken || sepToken.value !== sep) {
|
|
260
|
+
tokens.reset(mark);
|
|
261
|
+
return false;
|
|
262
|
+
}
|
|
263
|
+
tokens.advance();
|
|
264
|
+
}
|
|
265
|
+
const word = tokens.peek();
|
|
266
|
+
if (!word || word.kind !== "identifier" && word.kind !== "keyword" && word.kind !== "particle" || word.value.toLowerCase() !== segments[i].toLowerCase()) {
|
|
267
|
+
tokens.reset(mark);
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
tokens.advance();
|
|
271
|
+
}
|
|
272
|
+
return true;
|
|
207
273
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
274
|
+
/**
|
|
275
|
+
* Match a group pattern token (optional sequence).
|
|
276
|
+
*/
|
|
277
|
+
matchGroupToken(tokens, patternToken, captured, nextPatternToken) {
|
|
278
|
+
const mark = tokens.mark();
|
|
279
|
+
const capturedBefore = new Map(captured);
|
|
280
|
+
let success = this.matchTokenSequence(tokens, patternToken.tokens, captured, nextPatternToken);
|
|
281
|
+
if (success && tokens.position() > mark.position && groupHasRole(patternToken) && [...captured].every(([role, value]) => capturedBefore.get(role) === value) && captured.size === capturedBefore.size) {
|
|
282
|
+
success = false;
|
|
213
283
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
284
|
+
if (success) {
|
|
285
|
+
const markerLiterals = [];
|
|
286
|
+
for (const inner of patternToken.tokens) {
|
|
287
|
+
if (inner.type === "group" && inner.renderRequired) {
|
|
288
|
+
for (const t of inner.tokens) {
|
|
289
|
+
if (t.type === "literal") markerLiterals.push(t.value, ...t.alternatives ?? []);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (markerLiterals.length > 0) {
|
|
294
|
+
const addedRole = [...captured.keys()].some((role) => !capturedBefore.has(role));
|
|
295
|
+
if (addedRole) {
|
|
296
|
+
const last = tokens.tokens[tokens.position() - 1];
|
|
297
|
+
const markerSeen = last !== void 0 && tokens.position() > mark.position && markerLiterals.some((lit) => this.getMatchType(last, lit) !== "none");
|
|
298
|
+
if (!markerSeen) success = false;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
218
301
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
302
|
+
if (!success) {
|
|
303
|
+
tokens.reset(mark);
|
|
304
|
+
for (const role of captured.keys()) {
|
|
305
|
+
if (!capturedBefore.has(role)) {
|
|
306
|
+
captured.delete(role);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
for (const [role, value] of capturedBefore) {
|
|
310
|
+
if (captured.get(role) !== value) {
|
|
311
|
+
captured.set(role, value);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return patternToken.optional || false;
|
|
223
315
|
}
|
|
224
|
-
return
|
|
225
|
-
}
|
|
226
|
-
tokens.reset(mark);
|
|
227
|
-
return null;
|
|
228
|
-
}
|
|
229
|
-
function isStructuralKeyword(value) {
|
|
230
|
-
const structural = /* @__PURE__ */ new Set([
|
|
231
|
-
// Prepositions
|
|
232
|
-
"into",
|
|
233
|
-
"in",
|
|
234
|
-
"to",
|
|
235
|
-
"from",
|
|
236
|
-
"at",
|
|
237
|
-
"by",
|
|
238
|
-
"with",
|
|
239
|
-
"without",
|
|
240
|
-
"before",
|
|
241
|
-
"after",
|
|
242
|
-
"of",
|
|
243
|
-
"as",
|
|
244
|
-
"on",
|
|
245
|
-
// Control flow
|
|
246
|
-
"then",
|
|
247
|
-
"end",
|
|
248
|
-
"else",
|
|
249
|
-
"if",
|
|
250
|
-
"repeat",
|
|
251
|
-
"while",
|
|
252
|
-
"for",
|
|
253
|
-
// Commands (shouldn't be property names)
|
|
254
|
-
"toggle",
|
|
255
|
-
"add",
|
|
256
|
-
"remove",
|
|
257
|
-
"put",
|
|
258
|
-
"set",
|
|
259
|
-
"show",
|
|
260
|
-
"hide",
|
|
261
|
-
"increment",
|
|
262
|
-
"decrement",
|
|
263
|
-
"send",
|
|
264
|
-
"trigger",
|
|
265
|
-
"call"
|
|
266
|
-
]);
|
|
267
|
-
return structural.has(value.toLowerCase());
|
|
268
|
-
}
|
|
269
|
-
function tryMatchMethodCallExpression(tokens) {
|
|
270
|
-
const token = tokens.peek();
|
|
271
|
-
if (!token || token.kind !== "selector") return null;
|
|
272
|
-
const mark = tokens.mark();
|
|
273
|
-
tokens.advance();
|
|
274
|
-
const dotToken = tokens.peek();
|
|
275
|
-
if (!dotToken || dotToken.kind !== "operator" || dotToken.value !== ".") {
|
|
276
|
-
tokens.reset(mark);
|
|
277
|
-
return null;
|
|
278
|
-
}
|
|
279
|
-
tokens.advance();
|
|
280
|
-
const methodToken = tokens.peek();
|
|
281
|
-
if (!methodToken || methodToken.kind !== "identifier") {
|
|
282
|
-
tokens.reset(mark);
|
|
283
|
-
return null;
|
|
316
|
+
return true;
|
|
284
317
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
318
|
+
/**
|
|
319
|
+
* Whether a pattern token (literal, or a group starting with a literal) would
|
|
320
|
+
* match the given stream token. Used to keep the event-head source-clause
|
|
321
|
+
* consumption from stealing a marker the pattern explicitly expects.
|
|
322
|
+
*/
|
|
323
|
+
patternTokenWouldMatch(pt, token) {
|
|
324
|
+
if (!pt) return false;
|
|
325
|
+
if (pt.type === "literal") {
|
|
326
|
+
if (this.getMatchType(token, pt.value) !== "none") return true;
|
|
327
|
+
return !!pt.alternatives?.some((a) => this.getMatchType(token, a) !== "none");
|
|
328
|
+
}
|
|
329
|
+
if (pt.type === "group") {
|
|
330
|
+
return this.patternTokenWouldMatch(pt.tokens?.[0], token);
|
|
331
|
+
}
|
|
332
|
+
return false;
|
|
290
333
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
if (
|
|
297
|
-
|
|
298
|
-
|
|
334
|
+
/**
|
|
335
|
+
* Get the type of match for a token against a value.
|
|
336
|
+
* Used for confidence calculation.
|
|
337
|
+
*/
|
|
338
|
+
getMatchType(token, value) {
|
|
339
|
+
if (token.value === value) return "exact";
|
|
340
|
+
if (token.normalized === value) return "normalized";
|
|
341
|
+
if (token.stem === value && token.stemConfidence !== void 0 && token.stemConfidence >= 0.7) {
|
|
342
|
+
return "stem";
|
|
299
343
|
}
|
|
300
|
-
if (
|
|
301
|
-
|
|
302
|
-
continue;
|
|
344
|
+
if (token.kind === "keyword" && token.value.toLowerCase() === value.toLowerCase()) {
|
|
345
|
+
return "case-insensitive";
|
|
303
346
|
}
|
|
304
|
-
|
|
305
|
-
|
|
347
|
+
if (token.kind === "keyword") {
|
|
348
|
+
const strippedToken = stripOptionalDiacritics(token.value).toLowerCase();
|
|
349
|
+
if (strippedToken && strippedToken === stripOptionalDiacritics(value).toLowerCase()) {
|
|
350
|
+
return "case-insensitive";
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return "none";
|
|
306
354
|
}
|
|
307
|
-
|
|
355
|
+
/**
|
|
356
|
+
* Apply extraction rules to fill in static values and defaults for missing
|
|
357
|
+
* roles, and re-shape captured roles that declare a `transform` (the
|
|
358
|
+
* ExtractionRule field was documented but unwired — go-qu-url-dest is the
|
|
359
|
+
* first consumer: it re-types the marker-bound capture to the expression
|
|
360
|
+
* shape the en reference produces).
|
|
361
|
+
*/
|
|
362
|
+
applyExtractionRules(pattern, captured) {
|
|
363
|
+
for (const [role, rule] of Object.entries(pattern.extraction)) {
|
|
364
|
+
const existing = captured.get(role);
|
|
365
|
+
if (existing) {
|
|
366
|
+
if (rule.transform) {
|
|
367
|
+
const raw = existing.type === "literal" ? String(existing.value) : String(existing.raw ?? "");
|
|
368
|
+
captured.set(role, rule.transform(raw));
|
|
369
|
+
}
|
|
370
|
+
} else if (rule.value !== void 0) {
|
|
371
|
+
captured.set(
|
|
372
|
+
role,
|
|
373
|
+
rule.valueIsExpression ? { type: "expression", raw: rule.value } : { type: "literal", value: rule.value }
|
|
374
|
+
);
|
|
375
|
+
} else if (rule.default) {
|
|
376
|
+
const implicitDefault = {
|
|
377
|
+
...rule.default,
|
|
378
|
+
implicit: true
|
|
379
|
+
};
|
|
380
|
+
captured.set(role, implicitDefault);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Check if a pattern token is optional.
|
|
386
|
+
*/
|
|
387
|
+
isOptional(patternToken) {
|
|
388
|
+
return patternToken.type !== "literal" && patternToken.optional === true;
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// src/core/matching/type-validation.ts
|
|
393
|
+
function isTypeCompatible(actualType, expectedTypes) {
|
|
394
|
+
if (!expectedTypes || expectedTypes.length === 0) {
|
|
395
|
+
return true;
|
|
396
|
+
}
|
|
397
|
+
if (expectedTypes.includes(actualType)) {
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
if (expectedTypes.includes("expression")) {
|
|
401
|
+
return true;
|
|
402
|
+
}
|
|
403
|
+
if (actualType === "property-path") {
|
|
404
|
+
return expectedTypes.some((t) => ["selector", "reference", "expression"].includes(t));
|
|
405
|
+
}
|
|
406
|
+
return false;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// src/core/matching/possessive-keywords.ts
|
|
410
|
+
function getPossessiveReference(profile, keyword) {
|
|
411
|
+
return profile?.possessive?.keywords?.[keyword];
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// src/core/matching/stock-reading.ts
|
|
415
|
+
function isValidReference(_value) {
|
|
416
|
+
return false;
|
|
417
|
+
}
|
|
418
|
+
function safeToLowerCase(value) {
|
|
419
|
+
if (typeof value === "string") {
|
|
420
|
+
return value.toLowerCase();
|
|
421
|
+
}
|
|
422
|
+
if (value === null || value === void 0) {
|
|
423
|
+
return "";
|
|
424
|
+
}
|
|
425
|
+
return String(value).toLowerCase();
|
|
426
|
+
}
|
|
427
|
+
function readRoleValue(tokens, patternToken, captured, profile) {
|
|
428
|
+
skipNoiseWords(tokens);
|
|
429
|
+
const token = tokens.peek();
|
|
430
|
+
if (!token) {
|
|
431
|
+
return patternToken.optional || false;
|
|
432
|
+
}
|
|
433
|
+
const possessiveValue = tryMatchPossessiveExpression(tokens, profile);
|
|
434
|
+
if (possessiveValue) {
|
|
435
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
436
|
+
if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
|
|
437
|
+
return patternToken.optional || false;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
captured.set(patternToken.role, possessiveValue);
|
|
441
|
+
return true;
|
|
442
|
+
}
|
|
443
|
+
const methodCallValue = tryMatchMethodCallExpression(tokens);
|
|
444
|
+
if (methodCallValue) {
|
|
445
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
446
|
+
if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
|
|
447
|
+
return patternToken.optional || false;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
captured.set(patternToken.role, methodCallValue);
|
|
451
|
+
return true;
|
|
452
|
+
}
|
|
453
|
+
const possessiveSelectorValue = tryMatchPossessiveSelectorExpression(tokens);
|
|
454
|
+
if (possessiveSelectorValue) {
|
|
455
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
456
|
+
if (!isTypeCompatible(possessiveSelectorValue.type, patternToken.expectedTypes)) {
|
|
457
|
+
return patternToken.optional || false;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
captured.set(patternToken.role, possessiveSelectorValue);
|
|
461
|
+
return true;
|
|
462
|
+
}
|
|
463
|
+
const propertyAccessValue = tryMatchPropertyAccessExpression(tokens);
|
|
464
|
+
if (propertyAccessValue) {
|
|
465
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
466
|
+
if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
|
|
467
|
+
return patternToken.optional || false;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
captured.set(patternToken.role, propertyAccessValue);
|
|
471
|
+
return true;
|
|
472
|
+
}
|
|
473
|
+
const selectorPropertyValue = tryMatchSelectorPropertyExpression(tokens);
|
|
474
|
+
if (selectorPropertyValue) {
|
|
475
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
476
|
+
if (!isTypeCompatible(selectorPropertyValue.type, patternToken.expectedTypes)) {
|
|
477
|
+
return patternToken.optional || false;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
captured.set(patternToken.role, selectorPropertyValue);
|
|
481
|
+
return true;
|
|
482
|
+
}
|
|
483
|
+
const value = tokenToSemanticValue(token);
|
|
484
|
+
if (!value) {
|
|
485
|
+
return patternToken.optional || false;
|
|
486
|
+
}
|
|
487
|
+
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
488
|
+
if (!isTypeCompatible(value.type, patternToken.expectedTypes)) {
|
|
489
|
+
return patternToken.optional || false;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
captured.set(patternToken.role, value);
|
|
493
|
+
tokens.advance();
|
|
494
|
+
return true;
|
|
495
|
+
}
|
|
496
|
+
function tryMatchPossessiveExpression(tokens, profile) {
|
|
497
|
+
const token = tokens.peek();
|
|
498
|
+
if (!token) return null;
|
|
499
|
+
if (!profile) return null;
|
|
500
|
+
const tokenValue = token.normalized || token.value;
|
|
501
|
+
const tokenLower = safeToLowerCase(tokenValue);
|
|
502
|
+
const baseRef = getPossessiveReference(profile, tokenLower);
|
|
503
|
+
if (!baseRef) return null;
|
|
504
|
+
const mark = tokens.mark();
|
|
505
|
+
tokens.advance();
|
|
506
|
+
const propertyToken = tokens.peek();
|
|
507
|
+
if (!propertyToken) {
|
|
508
|
+
tokens.reset(mark);
|
|
509
|
+
return null;
|
|
510
|
+
}
|
|
511
|
+
if (propertyToken.kind === "identifier" || propertyToken.kind === "keyword" && !isStructuralKeyword(propertyToken.value) || propertyToken.kind === "selector" && propertyToken.value.startsWith("*") || propertyToken.kind === "selector" && propertyToken.value.startsWith("@") || propertyToken.kind === "selector" && propertyToken.value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyToken.value)) {
|
|
512
|
+
tokens.advance();
|
|
513
|
+
let propertyName = propertyToken.value;
|
|
514
|
+
if (propertyToken.kind === "selector" && propertyName.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(propertyName)) {
|
|
515
|
+
propertyName = propertyName.substring(1);
|
|
516
|
+
}
|
|
517
|
+
let chainedProps = propertyName;
|
|
518
|
+
while (tokens.peek()?.kind === "selector" && tokens.peek().value.startsWith(".") && /^\.[a-zA-Z_]\w*/.test(tokens.peek().value)) {
|
|
519
|
+
chainedProps += tokens.peek().value;
|
|
520
|
+
tokens.advance();
|
|
521
|
+
}
|
|
522
|
+
const nextPeek = tokens.peek();
|
|
523
|
+
if (nextPeek?.kind === "literal" && nextPeek.value.startsWith("(")) {
|
|
524
|
+
chainedProps += nextPeek.value;
|
|
525
|
+
tokens.advance();
|
|
526
|
+
}
|
|
527
|
+
return createPropertyPath(createReference(baseRef), chainedProps);
|
|
528
|
+
}
|
|
529
|
+
tokens.reset(mark);
|
|
530
|
+
return null;
|
|
531
|
+
}
|
|
532
|
+
function isStructuralKeyword(value) {
|
|
533
|
+
const structural = /* @__PURE__ */ new Set([
|
|
534
|
+
// Prepositions
|
|
535
|
+
"into",
|
|
536
|
+
"in",
|
|
537
|
+
"to",
|
|
538
|
+
"from",
|
|
539
|
+
"at",
|
|
540
|
+
"by",
|
|
541
|
+
"with",
|
|
542
|
+
"without",
|
|
543
|
+
"before",
|
|
544
|
+
"after",
|
|
545
|
+
"of",
|
|
546
|
+
"as",
|
|
547
|
+
"on",
|
|
548
|
+
// Control flow
|
|
549
|
+
"then",
|
|
550
|
+
"end",
|
|
551
|
+
"else",
|
|
552
|
+
"if",
|
|
553
|
+
"repeat",
|
|
554
|
+
"while",
|
|
555
|
+
"for",
|
|
556
|
+
// Commands (shouldn't be property names)
|
|
557
|
+
"toggle",
|
|
558
|
+
"add",
|
|
559
|
+
"remove",
|
|
560
|
+
"put",
|
|
561
|
+
"set",
|
|
562
|
+
"show",
|
|
563
|
+
"hide",
|
|
564
|
+
"increment",
|
|
565
|
+
"decrement",
|
|
566
|
+
"send",
|
|
567
|
+
"trigger",
|
|
568
|
+
"call"
|
|
569
|
+
]);
|
|
570
|
+
return structural.has(value.toLowerCase());
|
|
571
|
+
}
|
|
572
|
+
function tryMatchMethodCallExpression(tokens) {
|
|
573
|
+
const token = tokens.peek();
|
|
574
|
+
if (!token || token.kind !== "selector") return null;
|
|
575
|
+
const mark = tokens.mark();
|
|
576
|
+
tokens.advance();
|
|
577
|
+
const dotToken = tokens.peek();
|
|
578
|
+
if (!dotToken || dotToken.kind !== "operator" || dotToken.value !== ".") {
|
|
579
|
+
tokens.reset(mark);
|
|
580
|
+
return null;
|
|
581
|
+
}
|
|
582
|
+
tokens.advance();
|
|
583
|
+
const methodToken = tokens.peek();
|
|
584
|
+
if (!methodToken || methodToken.kind !== "identifier") {
|
|
585
|
+
tokens.reset(mark);
|
|
586
|
+
return null;
|
|
587
|
+
}
|
|
588
|
+
tokens.advance();
|
|
589
|
+
const openParen = tokens.peek();
|
|
590
|
+
if (!openParen || openParen.kind !== "punctuation" || openParen.value !== "(") {
|
|
591
|
+
tokens.reset(mark);
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
tokens.advance();
|
|
595
|
+
const args = [];
|
|
596
|
+
while (!tokens.isAtEnd() && args.length < MAX_METHOD_ARGS) {
|
|
597
|
+
const argToken = tokens.peek();
|
|
598
|
+
if (!argToken) break;
|
|
599
|
+
if (argToken.kind === "punctuation" && argToken.value === ")") {
|
|
600
|
+
tokens.advance();
|
|
601
|
+
break;
|
|
602
|
+
}
|
|
603
|
+
if (argToken.kind === "punctuation" && argToken.value === ",") {
|
|
604
|
+
tokens.advance();
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
args.push(argToken.value);
|
|
608
|
+
tokens.advance();
|
|
609
|
+
}
|
|
610
|
+
const methodCall = `${token.value}.${methodToken.value}(${args.join(", ")})`;
|
|
308
611
|
return {
|
|
309
612
|
type: "expression",
|
|
310
613
|
raw: methodCall
|
|
@@ -336,1181 +639,167 @@ function tryMatchPropertyAccessExpression(tokens) {
|
|
|
336
639
|
break;
|
|
337
640
|
}
|
|
338
641
|
tokens.advance();
|
|
339
|
-
const nextProp = tokens.peek();
|
|
340
|
-
if (!nextProp || nextProp.kind !== "identifier") {
|
|
341
|
-
break;
|
|
342
|
-
}
|
|
343
|
-
tokens.advance();
|
|
344
|
-
chain += `.${nextProp.value}`;
|
|
345
|
-
depth++;
|
|
346
|
-
}
|
|
347
|
-
const openParen = tokens.peek();
|
|
348
|
-
if (openParen && openParen.kind === "punctuation" && openParen.value === "(") {
|
|
349
|
-
tokens.advance();
|
|
350
|
-
const args = [];
|
|
351
|
-
let argDepth = 0;
|
|
352
|
-
while (!tokens.isAtEnd() && args.length < MAX_METHOD_ARGS) {
|
|
353
|
-
const argToken = tokens.peek();
|
|
354
|
-
if (!argToken) break;
|
|
355
|
-
if (argToken.kind === "punctuation" && argToken.value === ")") {
|
|
356
|
-
if (argDepth === 0) {
|
|
357
|
-
tokens.advance();
|
|
358
|
-
break;
|
|
359
|
-
}
|
|
360
|
-
argDepth--;
|
|
361
|
-
}
|
|
362
|
-
if (argToken.kind === "punctuation" && argToken.value === "(") {
|
|
363
|
-
argDepth++;
|
|
364
|
-
}
|
|
365
|
-
if (argToken.kind === "punctuation" && argToken.value === ",") {
|
|
366
|
-
tokens.advance();
|
|
367
|
-
continue;
|
|
368
|
-
}
|
|
369
|
-
args.push(argToken.value);
|
|
370
|
-
tokens.advance();
|
|
371
|
-
}
|
|
372
|
-
const methodCall = `${chain}(${args.join(", ")})`;
|
|
373
|
-
return {
|
|
374
|
-
type: "expression",
|
|
375
|
-
raw: methodCall
|
|
376
|
-
};
|
|
377
|
-
}
|
|
378
|
-
return {
|
|
379
|
-
type: "expression",
|
|
380
|
-
raw: chain
|
|
381
|
-
};
|
|
382
|
-
}
|
|
383
|
-
function tryMatchPossessiveSelectorExpression(tokens) {
|
|
384
|
-
const token = tokens.peek();
|
|
385
|
-
if (!token || token.kind !== "selector") return null;
|
|
386
|
-
const mark = tokens.mark();
|
|
387
|
-
tokens.advance();
|
|
388
|
-
const possessiveToken = tokens.peek();
|
|
389
|
-
if (!possessiveToken || possessiveToken.kind !== "punctuation" || possessiveToken.value !== "'s") {
|
|
390
|
-
tokens.reset(mark);
|
|
391
|
-
return null;
|
|
392
|
-
}
|
|
393
|
-
tokens.advance();
|
|
394
|
-
const propertyToken = tokens.peek();
|
|
395
|
-
if (!propertyToken) {
|
|
396
|
-
tokens.reset(mark);
|
|
397
|
-
return null;
|
|
398
|
-
}
|
|
399
|
-
if (propertyToken.kind !== "selector" && propertyToken.kind !== "identifier") {
|
|
400
|
-
tokens.reset(mark);
|
|
401
|
-
return null;
|
|
402
|
-
}
|
|
403
|
-
tokens.advance();
|
|
404
|
-
return createPropertyPath(createSelector(token.value), propertyToken.value);
|
|
405
|
-
}
|
|
406
|
-
function tryMatchSelectorPropertyExpression(tokens) {
|
|
407
|
-
const token = tokens.peek();
|
|
408
|
-
if (!token || token.kind !== "selector") return null;
|
|
409
|
-
if (typeof token.value !== "string" || !token.value.startsWith("#")) return null;
|
|
410
|
-
const mark = tokens.mark();
|
|
411
|
-
tokens.advance();
|
|
412
|
-
const propertyToken = tokens.peek();
|
|
413
|
-
if (!propertyToken || propertyToken.kind !== "selector") {
|
|
414
|
-
tokens.reset(mark);
|
|
415
|
-
return null;
|
|
416
|
-
}
|
|
417
|
-
if (!propertyToken.value.startsWith(".")) {
|
|
418
|
-
tokens.reset(mark);
|
|
419
|
-
return null;
|
|
420
|
-
}
|
|
421
|
-
const peek2 = tokens.peek(1);
|
|
422
|
-
if (peek2 && peek2.kind === "selector") {
|
|
423
|
-
}
|
|
424
|
-
tokens.advance();
|
|
425
|
-
const propertyName = propertyToken.value.slice(1);
|
|
426
|
-
return createPropertyPath(createSelector(token.value), propertyName);
|
|
427
|
-
}
|
|
428
|
-
var MAX_PROPERTY_DEPTH = 10;
|
|
429
|
-
var MAX_METHOD_ARGS = 20;
|
|
430
|
-
function tokenToSemanticValue(token) {
|
|
431
|
-
switch (token.kind) {
|
|
432
|
-
case "selector":
|
|
433
|
-
return createSelector(token.value);
|
|
434
|
-
case "literal":
|
|
435
|
-
return parseLiteralValue(token.value);
|
|
436
|
-
case "keyword": {
|
|
437
|
-
const tokenValue = token.normalized || token.value;
|
|
438
|
-
const lower = safeToLowerCase(tokenValue);
|
|
439
|
-
if (isValidReference(lower)) {
|
|
440
|
-
return createReference(lower);
|
|
441
|
-
}
|
|
442
|
-
return createLiteral(token.normalized || token.value);
|
|
443
|
-
}
|
|
444
|
-
case "identifier": {
|
|
445
|
-
if (typeof token.value === "string" && token.value.startsWith(":")) {
|
|
446
|
-
return createReference(token.value);
|
|
447
|
-
}
|
|
448
|
-
const identLower = safeToLowerCase(token.value);
|
|
449
|
-
if (isValidReference(identLower)) {
|
|
450
|
-
return createReference(identLower);
|
|
451
|
-
}
|
|
452
|
-
return { type: "expression", raw: token.value };
|
|
453
|
-
}
|
|
454
|
-
case "url":
|
|
455
|
-
return createLiteral(token.value, "string");
|
|
456
|
-
default:
|
|
457
|
-
return null;
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
function parseLiteralValue(value) {
|
|
461
|
-
if (value.startsWith('"') || value.startsWith("'") || value.startsWith("`") || value.startsWith("\u300C")) {
|
|
462
|
-
const inner = value.slice(1, -1);
|
|
463
|
-
return createLiteral(inner, "string");
|
|
464
|
-
}
|
|
465
|
-
if (value === "true") return createLiteral(true, "boolean");
|
|
466
|
-
if (value === "false") return createLiteral(false, "boolean");
|
|
467
|
-
const durationMatch = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
|
|
468
|
-
if (durationMatch) {
|
|
469
|
-
const num2 = parseFloat(durationMatch[1]);
|
|
470
|
-
const unit = durationMatch[2];
|
|
471
|
-
if (unit) {
|
|
472
|
-
return createLiteral(value, "duration");
|
|
473
|
-
}
|
|
474
|
-
return createLiteral(num2, "number");
|
|
475
|
-
}
|
|
476
|
-
const num = parseFloat(value);
|
|
477
|
-
if (!isNaN(num)) {
|
|
478
|
-
return createLiteral(num, "number");
|
|
479
|
-
}
|
|
480
|
-
return createLiteral(value, "string");
|
|
481
|
-
}
|
|
482
|
-
var ENGLISH_NOISE_WORDS = /* @__PURE__ */ new Set(["the", "a", "an"]);
|
|
483
|
-
function skipNoiseWords(tokens) {
|
|
484
|
-
const token = tokens.peek();
|
|
485
|
-
if (!token) return;
|
|
486
|
-
const tokenLower = safeToLowerCase(token.value);
|
|
487
|
-
if (ENGLISH_NOISE_WORDS.has(tokenLower)) {
|
|
488
|
-
const mark = tokens.mark();
|
|
489
|
-
tokens.advance();
|
|
490
|
-
const nextToken = tokens.peek();
|
|
491
|
-
if (nextToken && nextToken.kind === "selector") {
|
|
492
|
-
return;
|
|
493
|
-
}
|
|
494
|
-
tokens.reset(mark);
|
|
495
|
-
}
|
|
496
|
-
if (tokenLower === "class") {
|
|
497
|
-
tokens.advance();
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
// src/core/matching/stock-confidence.ts
|
|
502
|
-
function scoreParts(ctx) {
|
|
503
|
-
const { pattern, captured } = ctx;
|
|
504
|
-
let score = 0;
|
|
505
|
-
let maxScore = 0;
|
|
506
|
-
const hasDefault = (role) => {
|
|
507
|
-
return pattern.extraction?.[role]?.default !== void 0;
|
|
508
|
-
};
|
|
509
|
-
for (const token of pattern.template.tokens) {
|
|
510
|
-
if (token.type === "role") {
|
|
511
|
-
maxScore += 1;
|
|
512
|
-
if (captured.has(token.role)) {
|
|
513
|
-
score += 1;
|
|
514
|
-
}
|
|
515
|
-
} else if (token.type === "group") {
|
|
516
|
-
for (const subToken of token.tokens) {
|
|
517
|
-
if (subToken.type === "role") {
|
|
518
|
-
const roleHasDefault = hasDefault(subToken.role);
|
|
519
|
-
const weight = 0.8;
|
|
520
|
-
maxScore += weight;
|
|
521
|
-
if (captured.has(subToken.role)) {
|
|
522
|
-
score += weight;
|
|
523
|
-
} else if (roleHasDefault) {
|
|
524
|
-
score += weight * 0.6;
|
|
525
|
-
}
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
const roleCoverage = maxScore > 0 ? score / maxScore : 1;
|
|
531
|
-
let baseConfidence = roleCoverage;
|
|
532
|
-
let stemPenalty = 0;
|
|
533
|
-
if (ctx.stemMatchCount > 0 && ctx.totalKeywordMatches > 0) {
|
|
534
|
-
stemPenalty = ctx.stemMatchCount / ctx.totalKeywordMatches * 0.15;
|
|
535
|
-
baseConfidence = Math.max(0.5, baseConfidence - stemPenalty);
|
|
536
|
-
}
|
|
537
|
-
const vsoBoost = calculateVSOConfidenceBoost(pattern);
|
|
538
|
-
baseConfidence = Math.min(1, baseConfidence + vsoBoost);
|
|
539
|
-
const prepositionAdjustment = arabicPrepositionDisambiguation(pattern, captured);
|
|
540
|
-
baseConfidence = Math.max(0, Math.min(1, baseConfidence + prepositionAdjustment));
|
|
541
|
-
return {
|
|
542
|
-
roleCoverage,
|
|
543
|
-
stemPenalty,
|
|
544
|
-
languageBoost: vsoBoost,
|
|
545
|
-
languageAdjustment: prepositionAdjustment,
|
|
546
|
-
final: baseConfidence
|
|
547
|
-
};
|
|
548
|
-
}
|
|
549
|
-
function calculateVSOConfidenceBoost(pattern) {
|
|
550
|
-
if (pattern.language !== "ar") {
|
|
551
|
-
return 0;
|
|
552
|
-
}
|
|
553
|
-
const firstToken = pattern.template.tokens[0];
|
|
554
|
-
if (!firstToken || firstToken.type !== "literal") {
|
|
555
|
-
return 0;
|
|
556
|
-
}
|
|
557
|
-
const ARABIC_VERBS = /* @__PURE__ */ new Set([
|
|
558
|
-
"\u0628\u062F\u0644",
|
|
559
|
-
"\u063A\u064A\u0631",
|
|
560
|
-
"\u0623\u0636\u0641",
|
|
561
|
-
"\u0623\u0632\u0644",
|
|
562
|
-
"\u0636\u0639",
|
|
563
|
-
"\u0627\u062C\u0639\u0644",
|
|
564
|
-
"\u0639\u064A\u0646",
|
|
565
|
-
"\u0632\u062F",
|
|
566
|
-
"\u0627\u0646\u0642\u0635",
|
|
567
|
-
"\u0633\u062C\u0644",
|
|
568
|
-
"\u0623\u0638\u0647\u0631",
|
|
569
|
-
"\u0623\u062E\u0641",
|
|
570
|
-
"\u0634\u063A\u0644",
|
|
571
|
-
"\u0623\u0631\u0633\u0644",
|
|
572
|
-
"\u0631\u0643\u0632",
|
|
573
|
-
"\u0634\u0648\u0634",
|
|
574
|
-
"\u062A\u0648\u0642\u0641",
|
|
575
|
-
"\u0627\u0646\u0633\u062E",
|
|
576
|
-
"\u0627\u062D\u0630\u0641",
|
|
577
|
-
"\u0627\u0635\u0646\u0639",
|
|
578
|
-
"\u0627\u0646\u062A\u0638\u0631",
|
|
579
|
-
"\u0627\u0646\u062A\u0642\u0627\u0644",
|
|
580
|
-
"\u0623\u0648"
|
|
581
|
-
]);
|
|
582
|
-
if (ARABIC_VERBS.has(firstToken.value)) {
|
|
583
|
-
return 0.15;
|
|
584
|
-
}
|
|
585
|
-
if (firstToken.alternatives) {
|
|
586
|
-
for (const alt of firstToken.alternatives) {
|
|
587
|
-
if (ARABIC_VERBS.has(alt)) {
|
|
588
|
-
return 0.15;
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
return 0;
|
|
593
|
-
}
|
|
594
|
-
function arabicPrepositionDisambiguation(pattern, captured) {
|
|
595
|
-
if (pattern.language !== "ar") {
|
|
596
|
-
return 0;
|
|
597
|
-
}
|
|
598
|
-
let adjustment = 0;
|
|
599
|
-
const PREFERRED_PREPOSITIONS = {
|
|
600
|
-
patient: ["\u0639\u0644\u0649"],
|
|
601
|
-
// element selectors prefer على (on/upon)
|
|
602
|
-
destination: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
603
|
-
// destination prefers إلى (to)
|
|
604
|
-
source: ["\u0645\u0646"],
|
|
605
|
-
// source prefers من (from)
|
|
606
|
-
agent: ["\u0645\u0646"],
|
|
607
|
-
// agent/by prefers من (from/by)
|
|
608
|
-
manner: ["\u0628"],
|
|
609
|
-
// manner prefers ب (with/by)
|
|
610
|
-
style: ["\u0628"],
|
|
611
|
-
// style prefers ب (with)
|
|
612
|
-
goal: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
613
|
-
// target state prefers إلى (to)
|
|
614
|
-
method: ["\u0628"]
|
|
615
|
-
// method prefers ب (with/by)
|
|
616
|
-
};
|
|
617
|
-
for (const [role, value] of captured.entries()) {
|
|
618
|
-
const preferred = PREFERRED_PREPOSITIONS[role];
|
|
619
|
-
if (!preferred || preferred.length === 0) {
|
|
620
|
-
continue;
|
|
621
|
-
}
|
|
622
|
-
const metadata = "metadata" in value ? value.metadata : void 0;
|
|
623
|
-
if (metadata && typeof metadata.prepositionValue === "string") {
|
|
624
|
-
const usedPreposition = metadata.prepositionValue;
|
|
625
|
-
if (preferred.includes(usedPreposition)) {
|
|
626
|
-
adjustment += 0.1;
|
|
627
|
-
} else {
|
|
628
|
-
adjustment -= 0.1;
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
return Math.max(-0.1, Math.min(0.1, adjustment));
|
|
633
|
-
}
|
|
634
|
-
var stockConfidenceModel = {
|
|
635
|
-
calculate: (ctx) => scoreParts(ctx).final,
|
|
636
|
-
calculateWithBreakdown: (ctx) => scoreParts(ctx)
|
|
637
|
-
};
|
|
638
|
-
|
|
639
|
-
// src/core/pattern-matching/pattern-matcher.ts
|
|
640
|
-
var _PatternMatcher = class _PatternMatcher {
|
|
641
|
-
constructor() {
|
|
642
|
-
/** Debug logger */
|
|
643
|
-
this.logger = createLogger("pattern-matcher");
|
|
644
|
-
/**
|
|
645
|
-
* Track stem matches for confidence calculation.
|
|
646
|
-
* This is set during matching and read during confidence calculation.
|
|
647
|
-
*/
|
|
648
|
-
this.stemMatchCount = 0;
|
|
649
|
-
this.totalKeywordMatches = 0;
|
|
650
|
-
}
|
|
651
|
-
/**
|
|
652
|
-
* Try to match a single pattern against the token stream.
|
|
653
|
-
* Returns the match result or null if no match.
|
|
654
|
-
*
|
|
655
|
-
* @param tokens - Token stream to match against
|
|
656
|
-
* @param pattern - Pattern to match
|
|
657
|
-
* @param profile - Optional language profile for possessive handling
|
|
658
|
-
*/
|
|
659
|
-
matchPattern(tokens, pattern, profile) {
|
|
660
|
-
const mark = tokens.mark();
|
|
661
|
-
const captured = /* @__PURE__ */ new Map();
|
|
662
|
-
this.logger.debug("========================================");
|
|
663
|
-
this.logger.debug("matchPattern ENTRY");
|
|
664
|
-
this.logger.debug("Pattern ID:", pattern.id);
|
|
665
|
-
this.logger.debug("Pattern command:", pattern.command);
|
|
666
|
-
this.logger.debug("Pattern language:", pattern.language);
|
|
667
|
-
this.logger.debug("Pattern template:", JSON.stringify(pattern.template, null, 2));
|
|
668
|
-
if (this.logger.isEnabled()) {
|
|
669
|
-
const firstTokens = [];
|
|
670
|
-
for (let i = 0; i < 10; i++) {
|
|
671
|
-
const t = tokens.peek(i);
|
|
672
|
-
if (t)
|
|
673
|
-
firstTokens.push({
|
|
674
|
-
type: t.type,
|
|
675
|
-
value: t.value,
|
|
676
|
-
kind: t.kind
|
|
677
|
-
});
|
|
678
|
-
else break;
|
|
679
|
-
}
|
|
680
|
-
this.logger.debug("Input tokens (first 10):", firstTokens);
|
|
681
|
-
this.logger.debug("Profile code:", profile?.code);
|
|
682
|
-
}
|
|
683
|
-
this.currentProfile = profile;
|
|
684
|
-
this.stemMatchCount = 0;
|
|
685
|
-
this.totalKeywordMatches = 0;
|
|
686
|
-
this.logger.debug("--- Calling matchTokenSequence ---");
|
|
687
|
-
this.logger.debug("Pattern tokens to match:", JSON.stringify(pattern.template.tokens, null, 2));
|
|
688
|
-
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
689
|
-
this.logger.debug("matchTokenSequence returned:", success);
|
|
690
|
-
this.logger.debug(
|
|
691
|
-
"Captured roles:",
|
|
692
|
-
Array.from(captured.entries()).map(([k, v]) => [k, JSON.stringify(v)])
|
|
693
|
-
);
|
|
694
|
-
if (!success) {
|
|
695
|
-
this.logger.debug(">>> MATCH FAILED - resetting token position");
|
|
696
|
-
tokens.reset(mark);
|
|
697
|
-
return null;
|
|
698
|
-
}
|
|
699
|
-
const confidence = stockConfidenceModel.calculate({
|
|
700
|
-
pattern,
|
|
701
|
-
captured,
|
|
702
|
-
stemMatchCount: this.stemMatchCount,
|
|
703
|
-
totalKeywordMatches: this.totalKeywordMatches
|
|
704
|
-
});
|
|
705
|
-
this.applyExtractionRules(pattern, captured);
|
|
706
|
-
return {
|
|
707
|
-
pattern,
|
|
708
|
-
captured,
|
|
709
|
-
consumedTokens: tokens.position() - mark.position,
|
|
710
|
-
confidence
|
|
711
|
-
};
|
|
712
|
-
}
|
|
713
|
-
/**
|
|
714
|
-
* Try to match multiple patterns, return the best match.
|
|
715
|
-
*
|
|
716
|
-
* @param tokens - Token stream to match against
|
|
717
|
-
* @param patterns - Candidate patterns to try
|
|
718
|
-
* @param profile - Optional language profile for possessive handling
|
|
719
|
-
*/
|
|
720
|
-
matchBest(tokens, patterns, profile) {
|
|
721
|
-
const matches = [];
|
|
722
|
-
for (const pattern of patterns) {
|
|
723
|
-
const mark = tokens.mark();
|
|
724
|
-
const result = this.matchPattern(tokens, pattern, profile);
|
|
725
|
-
if (result) {
|
|
726
|
-
matches.push(result);
|
|
727
|
-
}
|
|
728
|
-
tokens.reset(mark);
|
|
729
|
-
}
|
|
730
|
-
if (matches.length === 0) {
|
|
731
|
-
return null;
|
|
732
|
-
}
|
|
733
|
-
matches.sort((a, b) => {
|
|
734
|
-
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
735
|
-
if (priorityDiff !== 0) return priorityDiff;
|
|
736
|
-
const confidenceDiff = b.confidence - a.confidence;
|
|
737
|
-
if (Math.abs(confidenceDiff) > 1e-3) return confidenceDiff;
|
|
738
|
-
return b.consumedTokens - a.consumedTokens;
|
|
739
|
-
});
|
|
740
|
-
const best = matches[0];
|
|
741
|
-
this.matchPattern(tokens, best.pattern);
|
|
742
|
-
return best;
|
|
743
|
-
}
|
|
744
|
-
/**
|
|
745
|
-
* Match a sequence of pattern tokens against the token stream.
|
|
746
|
-
*
|
|
747
|
-
* Supports bounded single-step backtracking: if an optional role consumes
|
|
748
|
-
* a token and the immediately following pattern token fails, the matcher
|
|
749
|
-
* resets to before the optional role and retries the failed token.
|
|
750
|
-
*/
|
|
751
|
-
matchTokenSequence(tokens, patternTokens, captured, parentStopMarkers) {
|
|
752
|
-
const firstPatternToken = patternTokens[0];
|
|
753
|
-
const patternExpectsConjunction = firstPatternToken?.type === "literal" && (firstPatternToken.value === "and" || firstPatternToken.value === "then" || firstPatternToken.alternatives?.includes("and") || firstPatternToken.alternatives?.includes("then"));
|
|
754
|
-
if (this.currentProfile?.code === "ar" && !patternExpectsConjunction) {
|
|
755
|
-
while (tokens.peek()?.kind === "conjunction") {
|
|
756
|
-
tokens.advance();
|
|
757
|
-
}
|
|
758
|
-
}
|
|
759
|
-
let prevOptionalMark = null;
|
|
760
|
-
let prevOptionalRole = null;
|
|
761
|
-
for (let i = 0; i < patternTokens.length; i++) {
|
|
762
|
-
const patternToken = patternTokens[i];
|
|
763
|
-
this.logger.debug(" >> Matching pattern token:", JSON.stringify(patternToken, null, 2));
|
|
764
|
-
const currTok = tokens.peek();
|
|
765
|
-
this.logger.debug(
|
|
766
|
-
" >> Current input token:",
|
|
767
|
-
currTok ? JSON.stringify({
|
|
768
|
-
type: currTok.type,
|
|
769
|
-
value: currTok.value,
|
|
770
|
-
kind: currTok.kind
|
|
771
|
-
}) : "EOF"
|
|
772
|
-
);
|
|
773
|
-
if (patternToken.type === "role" && patternToken.greedy) {
|
|
774
|
-
const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
775
|
-
if (parentStopMarkers) {
|
|
776
|
-
for (const m of parentStopMarkers) stopMarkers.add(m);
|
|
777
|
-
}
|
|
778
|
-
const values = [];
|
|
779
|
-
while (!tokens.isAtEnd()) {
|
|
780
|
-
const nextToken = tokens.peek();
|
|
781
|
-
if (!nextToken) break;
|
|
782
|
-
if (this.isStopMarker(nextToken, stopMarkers)) break;
|
|
783
|
-
values.push(nextToken.value);
|
|
784
|
-
tokens.advance();
|
|
785
|
-
}
|
|
786
|
-
if (values.length > 0) {
|
|
787
|
-
captured.set(patternToken.role, { type: "expression", raw: values.join(" ") });
|
|
788
|
-
prevOptionalMark = null;
|
|
789
|
-
prevOptionalRole = null;
|
|
790
|
-
continue;
|
|
791
|
-
} else if (patternToken.optional) {
|
|
792
|
-
continue;
|
|
793
|
-
} else {
|
|
794
|
-
return false;
|
|
795
|
-
}
|
|
796
|
-
}
|
|
797
|
-
if (patternToken.type === "group") {
|
|
798
|
-
const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
799
|
-
if (parentStopMarkers) {
|
|
800
|
-
for (const m of parentStopMarkers) siblingStopMarkers.add(m);
|
|
801
|
-
}
|
|
802
|
-
const groupMatched = this.matchGroupToken(
|
|
803
|
-
tokens,
|
|
804
|
-
patternToken,
|
|
805
|
-
captured,
|
|
806
|
-
siblingStopMarkers
|
|
807
|
-
);
|
|
808
|
-
if (groupMatched) {
|
|
809
|
-
prevOptionalMark = null;
|
|
810
|
-
prevOptionalRole = null;
|
|
811
|
-
continue;
|
|
812
|
-
}
|
|
813
|
-
if (this.isOptional(patternToken)) {
|
|
814
|
-
continue;
|
|
815
|
-
}
|
|
816
|
-
return false;
|
|
817
|
-
}
|
|
818
|
-
const isOptionalRole = patternToken.type === "role" && patternToken.optional === true;
|
|
819
|
-
const markBefore = isOptionalRole ? tokens.mark() : null;
|
|
820
|
-
const isRoleToken = patternToken.type === "role";
|
|
821
|
-
if (isRoleToken && !isOptionalRole) {
|
|
822
|
-
const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
|
|
823
|
-
if (nextPT?.type === "literal") {
|
|
824
|
-
const currentToken = tokens.peek();
|
|
825
|
-
if (currentToken && this.getMatchType(currentToken, nextPT.value) !== "none") {
|
|
826
|
-
this.logger.debug(
|
|
827
|
-
" >> MARKER-STEAL GUARD: current token matches next literal, skipping role"
|
|
828
|
-
);
|
|
829
|
-
this.logger.debug(" >> Token match FAILED");
|
|
830
|
-
if (this.isOptional(patternToken)) {
|
|
831
|
-
continue;
|
|
832
|
-
}
|
|
833
|
-
if (prevOptionalMark && prevOptionalRole) {
|
|
834
|
-
this.logger.debug(" >> BACKTRACKING: undoing optional role", prevOptionalRole);
|
|
835
|
-
tokens.reset(prevOptionalMark);
|
|
836
|
-
captured.delete(prevOptionalRole);
|
|
837
|
-
prevOptionalMark = null;
|
|
838
|
-
prevOptionalRole = null;
|
|
839
|
-
const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
|
|
840
|
-
this.logger.debug(" >> Backtrack retry result:", retryMatched);
|
|
841
|
-
if (retryMatched) {
|
|
842
|
-
continue;
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
return false;
|
|
846
|
-
}
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
const matched = this.matchPatternToken(tokens, patternToken, captured);
|
|
850
|
-
this.logger.debug(" >> Match result:", matched);
|
|
851
|
-
if (matched) {
|
|
852
|
-
if (isOptionalRole) {
|
|
853
|
-
prevOptionalMark = markBefore;
|
|
854
|
-
prevOptionalRole = patternToken.role;
|
|
855
|
-
} else {
|
|
856
|
-
prevOptionalMark = null;
|
|
857
|
-
prevOptionalRole = null;
|
|
858
|
-
}
|
|
859
|
-
continue;
|
|
860
|
-
}
|
|
861
|
-
this.logger.debug(" >> Token match FAILED");
|
|
862
|
-
if (this.isOptional(patternToken)) {
|
|
863
|
-
continue;
|
|
864
|
-
}
|
|
865
|
-
if (prevOptionalMark && prevOptionalRole) {
|
|
866
|
-
this.logger.debug(" >> BACKTRACKING: undoing optional role", prevOptionalRole);
|
|
867
|
-
tokens.reset(prevOptionalMark);
|
|
868
|
-
captured.delete(prevOptionalRole);
|
|
869
|
-
prevOptionalMark = null;
|
|
870
|
-
prevOptionalRole = null;
|
|
871
|
-
const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
|
|
872
|
-
this.logger.debug(" >> Backtrack retry result:", retryMatched);
|
|
873
|
-
if (retryMatched) {
|
|
874
|
-
continue;
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
return false;
|
|
878
|
-
}
|
|
879
|
-
return true;
|
|
880
|
-
}
|
|
881
|
-
/**
|
|
882
|
-
* Match a single pattern token against the current position in the stream.
|
|
883
|
-
*/
|
|
884
|
-
matchPatternToken(tokens, patternToken, captured) {
|
|
885
|
-
switch (patternToken.type) {
|
|
886
|
-
case "literal":
|
|
887
|
-
return this.matchLiteralToken(tokens, patternToken);
|
|
888
|
-
case "role":
|
|
889
|
-
return readRoleValue(tokens, patternToken, captured, this.currentProfile);
|
|
890
|
-
case "group":
|
|
891
|
-
return this.matchGroupToken(tokens, patternToken, captured);
|
|
892
|
-
default:
|
|
893
|
-
return false;
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
/**
|
|
897
|
-
* Match a literal pattern token (keyword or particle).
|
|
898
|
-
*/
|
|
899
|
-
matchLiteralToken(tokens, patternToken) {
|
|
900
|
-
const token = tokens.peek();
|
|
901
|
-
this.logger.debug(" >>> matchLiteralToken: expecting", patternToken.value);
|
|
902
|
-
this.logger.debug(
|
|
903
|
-
" >>> matchLiteralToken: got token",
|
|
904
|
-
token ? JSON.stringify({
|
|
905
|
-
type: token.type,
|
|
906
|
-
value: token.value,
|
|
907
|
-
kind: token.kind
|
|
908
|
-
}) : "null"
|
|
909
|
-
);
|
|
910
|
-
if (!token) {
|
|
911
|
-
this.logger.debug(" >>> matchLiteralToken: FAIL - no token");
|
|
912
|
-
return false;
|
|
913
|
-
}
|
|
914
|
-
const matchType = this.getMatchType(token, patternToken.value);
|
|
915
|
-
this.logger.debug(
|
|
916
|
-
" >>> matchType for",
|
|
917
|
-
token.value,
|
|
918
|
-
"vs",
|
|
919
|
-
patternToken.value,
|
|
920
|
-
":",
|
|
921
|
-
matchType
|
|
922
|
-
);
|
|
923
|
-
if (matchType !== "none") {
|
|
924
|
-
this.totalKeywordMatches++;
|
|
925
|
-
if (matchType === "stem") {
|
|
926
|
-
this.stemMatchCount++;
|
|
927
|
-
}
|
|
928
|
-
tokens.advance();
|
|
929
|
-
return true;
|
|
930
|
-
}
|
|
931
|
-
if (patternToken.alternatives) {
|
|
932
|
-
for (const alt of patternToken.alternatives) {
|
|
933
|
-
const altMatchType = this.getMatchType(token, alt);
|
|
934
|
-
if (altMatchType !== "none") {
|
|
935
|
-
this.totalKeywordMatches++;
|
|
936
|
-
if (altMatchType === "stem") {
|
|
937
|
-
this.stemMatchCount++;
|
|
938
|
-
}
|
|
939
|
-
tokens.advance();
|
|
940
|
-
return true;
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
}
|
|
944
|
-
return false;
|
|
945
|
-
}
|
|
946
|
-
/**
|
|
947
|
-
* Match a group pattern token (optional sequence).
|
|
948
|
-
* When the group's leading marker isn't at the current position, scans ahead
|
|
949
|
-
* up to MAX_MARKER_SCAN tokens to find it. This allows unmarked tokens
|
|
950
|
-
* (e.g., a value like 'hello') to sit between groups without blocking later
|
|
951
|
-
* marker matches.
|
|
952
|
-
*/
|
|
953
|
-
matchGroupToken(tokens, patternToken, captured, parentStopMarkers) {
|
|
954
|
-
const mark = tokens.mark();
|
|
955
|
-
const capturedBefore = new Set(captured.keys());
|
|
956
|
-
const success = this.matchTokenSequence(
|
|
957
|
-
tokens,
|
|
958
|
-
patternToken.tokens,
|
|
959
|
-
captured,
|
|
960
|
-
parentStopMarkers
|
|
961
|
-
);
|
|
962
|
-
if (success) return true;
|
|
963
|
-
tokens.reset(mark);
|
|
964
|
-
for (const role of captured.keys()) {
|
|
965
|
-
if (!capturedBefore.has(role)) captured.delete(role);
|
|
966
|
-
}
|
|
967
|
-
if (!patternToken.optional) return false;
|
|
968
|
-
const leadingMarker = this.getGroupLeadingMarker(patternToken);
|
|
969
|
-
if (leadingMarker) {
|
|
970
|
-
for (let offset = 1; offset <= _PatternMatcher.MAX_MARKER_SCAN; offset++) {
|
|
971
|
-
const ahead = tokens.peek(offset);
|
|
972
|
-
if (!ahead) break;
|
|
973
|
-
const aheadValue = (ahead.normalized || ahead.value).toLowerCase();
|
|
974
|
-
if (aheadValue === leadingMarker) {
|
|
975
|
-
this.logger.debug(
|
|
976
|
-
" >> MARKER SCAN: found",
|
|
977
|
-
leadingMarker,
|
|
978
|
-
"at offset",
|
|
979
|
-
offset,
|
|
980
|
-
"- skipping intervening tokens"
|
|
981
|
-
);
|
|
982
|
-
for (let s = 0; s < offset; s++) tokens.advance();
|
|
983
|
-
const retrySuccess = this.matchTokenSequence(
|
|
984
|
-
tokens,
|
|
985
|
-
patternToken.tokens,
|
|
986
|
-
captured,
|
|
987
|
-
parentStopMarkers
|
|
988
|
-
);
|
|
989
|
-
if (retrySuccess) return true;
|
|
990
|
-
tokens.reset(mark);
|
|
991
|
-
for (const role of captured.keys()) {
|
|
992
|
-
if (!capturedBefore.has(role)) captured.delete(role);
|
|
993
|
-
}
|
|
994
|
-
break;
|
|
995
|
-
}
|
|
996
|
-
}
|
|
997
|
-
}
|
|
998
|
-
return true;
|
|
999
|
-
}
|
|
1000
|
-
/**
|
|
1001
|
-
* Get the leading marker literal from an optional group.
|
|
1002
|
-
* Returns the lowercase marker value, or null if the group
|
|
1003
|
-
* doesn't start with a literal token (e.g., SOV groups where
|
|
1004
|
-
* the role comes before the marker).
|
|
1005
|
-
*/
|
|
1006
|
-
getGroupLeadingMarker(group) {
|
|
1007
|
-
const first = group.tokens[0];
|
|
1008
|
-
if (first?.type === "literal") return first.value.toLowerCase();
|
|
1009
|
-
return null;
|
|
1010
|
-
}
|
|
1011
|
-
/**
|
|
1012
|
-
* Get the type of match for a token against a value.
|
|
1013
|
-
* Used for confidence calculation.
|
|
1014
|
-
*/
|
|
1015
|
-
getMatchType(token, value) {
|
|
1016
|
-
if (token.value === value) return "exact";
|
|
1017
|
-
if (token.normalized === value) return "normalized";
|
|
1018
|
-
if (token.stem === value && token.stemConfidence !== void 0 && token.stemConfidence >= 0.7) {
|
|
1019
|
-
return "stem";
|
|
1020
|
-
}
|
|
1021
|
-
if (token.kind === "keyword" && safeToLowerCase(token.value) === value.toLowerCase()) {
|
|
1022
|
-
return "case-insensitive";
|
|
1023
|
-
}
|
|
1024
|
-
return "none";
|
|
1025
|
-
}
|
|
1026
|
-
/**
|
|
1027
|
-
* Collect literal values from upcoming pattern tokens that act as stop markers
|
|
1028
|
-
* for greedy role capture. Returns the set of lowercase values.
|
|
1029
|
-
*/
|
|
1030
|
-
collectStopMarkers(patternTokens, startIndex) {
|
|
1031
|
-
const markers = /* @__PURE__ */ new Set();
|
|
1032
|
-
for (let j = startIndex; j < patternTokens.length; j++) {
|
|
1033
|
-
const pt = patternTokens[j];
|
|
1034
|
-
if (pt.type === "literal") {
|
|
1035
|
-
markers.add(pt.value.toLowerCase());
|
|
1036
|
-
if (pt.alternatives) {
|
|
1037
|
-
for (const alt of pt.alternatives) {
|
|
1038
|
-
markers.add(alt.toLowerCase());
|
|
1039
|
-
}
|
|
1040
|
-
}
|
|
1041
|
-
break;
|
|
1042
|
-
}
|
|
1043
|
-
if (pt.type === "group") {
|
|
1044
|
-
for (const gt of pt.tokens) {
|
|
1045
|
-
if (gt.type === "literal") {
|
|
1046
|
-
markers.add(gt.value.toLowerCase());
|
|
1047
|
-
if (gt.alternatives) {
|
|
1048
|
-
for (const alt of gt.alternatives) {
|
|
1049
|
-
markers.add(alt.toLowerCase());
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
|
-
break;
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
if (!pt.optional) break;
|
|
1056
|
-
}
|
|
1057
|
-
}
|
|
1058
|
-
return markers;
|
|
1059
|
-
}
|
|
1060
|
-
/**
|
|
1061
|
-
* Check if a token matches any stop marker for greedy capture.
|
|
1062
|
-
*/
|
|
1063
|
-
isStopMarker(token, stopMarkers) {
|
|
1064
|
-
if (stopMarkers.size === 0) return false;
|
|
1065
|
-
if (stopMarkers.has(token.value.toLowerCase())) return true;
|
|
1066
|
-
if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
|
|
1067
|
-
return false;
|
|
1068
|
-
}
|
|
1069
|
-
// ==========================================================================
|
|
1070
|
-
// Depth Limits for Expression Parsing (security hardening)
|
|
1071
|
-
// ==========================================================================
|
|
1072
|
-
/**
|
|
1073
|
-
* Apply extraction rules to fill in static values and defaults for missing roles.
|
|
1074
|
-
*/
|
|
1075
|
-
applyExtractionRules(pattern, captured) {
|
|
1076
|
-
for (const [role, rule] of Object.entries(pattern.extraction)) {
|
|
1077
|
-
if (!captured.has(role)) {
|
|
1078
|
-
if (rule.value !== void 0) {
|
|
1079
|
-
captured.set(role, { type: "literal", value: rule.value });
|
|
1080
|
-
} else if (rule.default) {
|
|
1081
|
-
captured.set(role, rule.default);
|
|
1082
|
-
}
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
/**
|
|
1087
|
-
* Check if a pattern token is optional.
|
|
1088
|
-
*/
|
|
1089
|
-
isOptional(patternToken) {
|
|
1090
|
-
return patternToken.type !== "literal" && patternToken.optional === true;
|
|
1091
|
-
}
|
|
1092
|
-
// ===========================================================================
|
|
1093
|
-
// English Idiom Support - Noise Word Handling
|
|
1094
|
-
// ===========================================================================
|
|
1095
|
-
/**
|
|
1096
|
-
* Extract event modifiers from the token stream.
|
|
1097
|
-
* Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)
|
|
1098
|
-
* that can appear after event names.
|
|
1099
|
-
*
|
|
1100
|
-
* Returns EventModifiers object or undefined if no modifiers found.
|
|
1101
|
-
*/
|
|
1102
|
-
extractEventModifiers(tokens) {
|
|
1103
|
-
const modifiers = {};
|
|
1104
|
-
let foundModifier = false;
|
|
1105
|
-
while (!tokens.isAtEnd()) {
|
|
1106
|
-
const token = tokens.peek();
|
|
1107
|
-
if (!token || token.kind !== "event-modifier") {
|
|
1108
|
-
break;
|
|
1109
|
-
}
|
|
1110
|
-
const metadata = token.metadata;
|
|
1111
|
-
if (!metadata) {
|
|
1112
|
-
break;
|
|
1113
|
-
}
|
|
1114
|
-
foundModifier = true;
|
|
1115
|
-
switch (metadata.modifierName) {
|
|
1116
|
-
case "once":
|
|
1117
|
-
modifiers.once = true;
|
|
1118
|
-
break;
|
|
1119
|
-
case "debounce":
|
|
1120
|
-
if (typeof metadata.value === "number") {
|
|
1121
|
-
modifiers.debounce = metadata.value;
|
|
1122
|
-
}
|
|
1123
|
-
break;
|
|
1124
|
-
case "throttle":
|
|
1125
|
-
if (typeof metadata.value === "number") {
|
|
1126
|
-
modifiers.throttle = metadata.value;
|
|
1127
|
-
}
|
|
1128
|
-
break;
|
|
1129
|
-
case "queue":
|
|
1130
|
-
if (metadata.value === "first" || metadata.value === "last" || metadata.value === "all" || metadata.value === "none") {
|
|
1131
|
-
modifiers.queue = metadata.value;
|
|
1132
|
-
}
|
|
1133
|
-
break;
|
|
1134
|
-
}
|
|
1135
|
-
tokens.advance();
|
|
1136
|
-
}
|
|
1137
|
-
return foundModifier ? modifiers : void 0;
|
|
1138
|
-
}
|
|
1139
|
-
};
|
|
1140
|
-
/** Maximum tokens to scan ahead when looking for a group's leading marker */
|
|
1141
|
-
_PatternMatcher.MAX_MARKER_SCAN = 3;
|
|
1142
|
-
var PatternMatcher = _PatternMatcher;
|
|
1143
|
-
var patternMatcher = new PatternMatcher();
|
|
1144
|
-
|
|
1145
|
-
// src/core/tokenization/token-utils.ts
|
|
1146
|
-
function stripOptionalDiacritics(word) {
|
|
1147
|
-
return word.replace(/[\u064b-\u0652\u0670]/g, "");
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
// src/core/matching/match-skeleton.ts
|
|
1151
|
-
function groupHasRole(group) {
|
|
1152
|
-
return group.tokens.some((t) => t.type === "role" || t.type === "group" && groupHasRole(t));
|
|
1153
|
-
}
|
|
1154
|
-
var MatchSkeleton = class {
|
|
1155
|
-
/**
|
|
1156
|
-
* @param makeReader - builds the grammar's reader, handed this matcher's view
|
|
1157
|
-
* of the match in progress (one reader per matcher).
|
|
1158
|
-
*/
|
|
1159
|
-
constructor(makeReader, confidenceModel) {
|
|
1160
|
-
/**
|
|
1161
|
-
* The pattern tokens still to match after the one being matched, through
|
|
1162
|
-
* every enclosing group: a value stops at any marker the pattern still owes.
|
|
1163
|
-
*/
|
|
1164
|
-
this.patternRest = [];
|
|
1165
|
-
/**
|
|
1166
|
-
* Whether the role slot being matched directly follows a literal the pattern
|
|
1167
|
-
* matched: its marker (`[por {quantity}]`), or the command's verb.
|
|
1168
|
-
*/
|
|
1169
|
-
this.slotAfterLiteral = false;
|
|
1170
|
-
/**
|
|
1171
|
-
* Selective memoization cache (Phase 6.1).
|
|
1172
|
-
* Caches pattern match results keyed by `tokenPosition:patternId`.
|
|
1173
|
-
* Cleared before each top-level matchBest() call.
|
|
1174
|
-
*/
|
|
1175
|
-
this.matchCache = /* @__PURE__ */ new Map();
|
|
1176
|
-
/**
|
|
1177
|
-
* Stream index where each role's capture began, for the current
|
|
1178
|
-
* `matchPattern` call. Rebuilt per call, written on the same success path as
|
|
1179
|
-
* `captured` — see `PatternMatchResult.roleStarts`.
|
|
1180
|
-
*/
|
|
1181
|
-
this.roleStarts = /* @__PURE__ */ new Map();
|
|
1182
|
-
/**
|
|
1183
|
-
* Track stem matches for confidence calculation.
|
|
1184
|
-
* This is set during matching and read during confidence calculation.
|
|
1185
|
-
*/
|
|
1186
|
-
this.stemMatchCount = 0;
|
|
1187
|
-
this.totalKeywordMatches = 0;
|
|
1188
|
-
this.confidenceModel = confidenceModel;
|
|
1189
|
-
this.reader = makeReader({
|
|
1190
|
-
patternRest: () => this.patternRest,
|
|
1191
|
-
slotAfterLiteral: () => this.slotAfterLiteral,
|
|
1192
|
-
wouldMatch: (pt, token) => this.patternTokenWouldMatch(pt, token)
|
|
1193
|
-
});
|
|
1194
|
-
}
|
|
1195
|
-
/**
|
|
1196
|
-
* Try to match a single pattern against the token stream.
|
|
1197
|
-
* Returns the match result or null if no match.
|
|
1198
|
-
*/
|
|
1199
|
-
matchPattern(tokens, pattern) {
|
|
1200
|
-
const mark = tokens.mark();
|
|
1201
|
-
const captured = /* @__PURE__ */ new Map();
|
|
1202
|
-
this.reader.beginPattern(pattern);
|
|
1203
|
-
this.stemMatchCount = 0;
|
|
1204
|
-
this.totalKeywordMatches = 0;
|
|
1205
|
-
this.roleStarts = /* @__PURE__ */ new Map();
|
|
1206
|
-
this.patternRest = [];
|
|
1207
|
-
const success = this.matchTokenSequence(tokens, pattern.template.tokens, captured);
|
|
1208
|
-
if (!success) {
|
|
1209
|
-
tokens.reset(mark);
|
|
1210
|
-
return null;
|
|
1211
|
-
}
|
|
1212
|
-
const confidence = this.confidenceModel.calculate({
|
|
1213
|
-
pattern,
|
|
1214
|
-
captured,
|
|
1215
|
-
stemMatchCount: this.stemMatchCount,
|
|
1216
|
-
totalKeywordMatches: this.totalKeywordMatches
|
|
1217
|
-
});
|
|
1218
|
-
this.applyExtractionRules(pattern, captured);
|
|
1219
|
-
return {
|
|
1220
|
-
pattern,
|
|
1221
|
-
captured,
|
|
1222
|
-
consumedTokens: tokens.position() - mark.position,
|
|
1223
|
-
confidence,
|
|
1224
|
-
roleStarts: this.roleStarts,
|
|
1225
|
-
...this.reader.resultFields?.()
|
|
1226
|
-
};
|
|
1227
|
-
}
|
|
1228
|
-
/**
|
|
1229
|
-
* Try to match multiple patterns, return the best match.
|
|
1230
|
-
*/
|
|
1231
|
-
matchBest(tokens, patterns) {
|
|
1232
|
-
this.matchCache.clear();
|
|
1233
|
-
const matches = [];
|
|
1234
|
-
const startPos = tokens.position();
|
|
1235
|
-
for (const pattern of patterns) {
|
|
1236
|
-
const mark = tokens.mark();
|
|
1237
|
-
const cacheKey = `${startPos}:${pattern.id}`;
|
|
1238
|
-
let result;
|
|
1239
|
-
if (this.matchCache.has(cacheKey)) {
|
|
1240
|
-
result = this.matchCache.get(cacheKey);
|
|
1241
|
-
} else {
|
|
1242
|
-
result = this.matchPattern(tokens, pattern);
|
|
1243
|
-
this.matchCache.set(cacheKey, result);
|
|
1244
|
-
}
|
|
1245
|
-
if (result) {
|
|
1246
|
-
matches.push(result);
|
|
1247
|
-
}
|
|
1248
|
-
tokens.reset(mark);
|
|
1249
|
-
}
|
|
1250
|
-
if (matches.length === 0) {
|
|
1251
|
-
return null;
|
|
1252
|
-
}
|
|
1253
|
-
matches.sort((a, b) => {
|
|
1254
|
-
const priorityDiff = b.pattern.priority - a.pattern.priority;
|
|
1255
|
-
if (priorityDiff !== 0) return priorityDiff;
|
|
1256
|
-
return b.confidence - a.confidence;
|
|
1257
|
-
});
|
|
1258
|
-
const best = matches[0];
|
|
1259
|
-
this.matchPattern(tokens, best.pattern);
|
|
1260
|
-
return best;
|
|
1261
|
-
}
|
|
1262
|
-
/**
|
|
1263
|
-
* Match a sequence of pattern tokens against the token stream.
|
|
1264
|
-
*/
|
|
1265
|
-
matchTokenSequence(tokens, patternTokens, captured, nextAfterSequence) {
|
|
1266
|
-
this.reader.skipLeading?.(tokens, patternTokens);
|
|
1267
|
-
for (let i = 0; i < patternTokens.length; i++) {
|
|
1268
|
-
const patternToken = patternTokens[i];
|
|
1269
|
-
this.reader.beforePatternToken?.(tokens, patternTokens, i, captured);
|
|
1270
|
-
if (this.reader.maySkipSlot(
|
|
1271
|
-
patternToken,
|
|
1272
|
-
tokens.peek(),
|
|
1273
|
-
patternTokens[i + 1] ?? nextAfterSequence
|
|
1274
|
-
)) {
|
|
1275
|
-
const skipMark = tokens.mark();
|
|
1276
|
-
const speculative = new Map(captured);
|
|
1277
|
-
if (this.matchTokenSequence(
|
|
1278
|
-
tokens,
|
|
1279
|
-
patternTokens.slice(i + 1),
|
|
1280
|
-
speculative,
|
|
1281
|
-
nextAfterSequence
|
|
1282
|
-
) && this.reader.atClauseEnd(tokens)) {
|
|
1283
|
-
captured.clear();
|
|
1284
|
-
for (const [role, value] of speculative) captured.set(role, value);
|
|
1285
|
-
return true;
|
|
1286
|
-
}
|
|
1287
|
-
tokens.reset(skipMark);
|
|
1288
|
-
}
|
|
1289
|
-
const outerRest = this.patternRest;
|
|
1290
|
-
this.patternRest = [...patternTokens.slice(i + 1), ...outerRest];
|
|
1291
|
-
this.slotAfterLiteral = patternToken.type === "role" && patternTokens[i - 1]?.type === "literal";
|
|
1292
|
-
const matched = this.matchPatternToken(
|
|
1293
|
-
tokens,
|
|
1294
|
-
patternToken,
|
|
1295
|
-
captured,
|
|
1296
|
-
patternTokens[i + 1] ?? nextAfterSequence
|
|
1297
|
-
);
|
|
1298
|
-
this.patternRest = outerRest;
|
|
1299
|
-
if (!matched) {
|
|
1300
|
-
if (this.isOptional(patternToken)) {
|
|
1301
|
-
continue;
|
|
1302
|
-
}
|
|
1303
|
-
return false;
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1306
|
-
return true;
|
|
1307
|
-
}
|
|
1308
|
-
/**
|
|
1309
|
-
* Match a single pattern token against the current position in the stream.
|
|
1310
|
-
*/
|
|
1311
|
-
matchPatternToken(tokens, patternToken, captured, nextPatternToken) {
|
|
1312
|
-
switch (patternToken.type) {
|
|
1313
|
-
case "literal":
|
|
1314
|
-
return this.matchLiteralToken(tokens, patternToken);
|
|
1315
|
-
case "role": {
|
|
1316
|
-
const roleStart = tokens.position();
|
|
1317
|
-
const matchedRole = this.reader.readRole(tokens, patternToken, captured, nextPatternToken);
|
|
1318
|
-
if (matchedRole && captured.has(patternToken.role)) {
|
|
1319
|
-
this.roleStarts.set(patternToken.role, roleStart);
|
|
1320
|
-
}
|
|
1321
|
-
return matchedRole;
|
|
1322
|
-
}
|
|
1323
|
-
case "group":
|
|
1324
|
-
return this.matchGroupToken(tokens, patternToken, captured, nextPatternToken);
|
|
1325
|
-
default:
|
|
1326
|
-
return false;
|
|
642
|
+
const nextProp = tokens.peek();
|
|
643
|
+
if (!nextProp || nextProp.kind !== "identifier") {
|
|
644
|
+
break;
|
|
1327
645
|
}
|
|
646
|
+
tokens.advance();
|
|
647
|
+
chain += `.${nextProp.value}`;
|
|
648
|
+
depth++;
|
|
1328
649
|
}
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
this.stemMatchCount++;
|
|
1340
|
-
}
|
|
1341
|
-
tokens.advance();
|
|
1342
|
-
return true;
|
|
1343
|
-
}
|
|
1344
|
-
if (patternToken.alternatives) {
|
|
1345
|
-
for (const alt of patternToken.alternatives) {
|
|
1346
|
-
const altMatchType = this.getMatchType(token, alt);
|
|
1347
|
-
if (altMatchType !== "none") {
|
|
1348
|
-
this.totalKeywordMatches++;
|
|
1349
|
-
if (altMatchType === "stem") {
|
|
1350
|
-
this.stemMatchCount++;
|
|
1351
|
-
}
|
|
650
|
+
const openParen = tokens.peek();
|
|
651
|
+
if (openParen && openParen.kind === "punctuation" && openParen.value === "(") {
|
|
652
|
+
tokens.advance();
|
|
653
|
+
const args = [];
|
|
654
|
+
let argDepth = 0;
|
|
655
|
+
while (!tokens.isAtEnd() && args.length < MAX_METHOD_ARGS) {
|
|
656
|
+
const argToken = tokens.peek();
|
|
657
|
+
if (!argToken) break;
|
|
658
|
+
if (argToken.kind === "punctuation" && argToken.value === ")") {
|
|
659
|
+
if (argDepth === 0) {
|
|
1352
660
|
tokens.advance();
|
|
1353
|
-
|
|
661
|
+
break;
|
|
1354
662
|
}
|
|
663
|
+
argDepth--;
|
|
1355
664
|
}
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
for (const sep of ["-", "_"]) {
|
|
1359
|
-
if (value.includes(sep) && this.tryMatchShatteredCompound(tokens, value, sep)) {
|
|
1360
|
-
this.totalKeywordMatches++;
|
|
1361
|
-
return true;
|
|
1362
|
-
}
|
|
665
|
+
if (argToken.kind === "punctuation" && argToken.value === "(") {
|
|
666
|
+
argDepth++;
|
|
1363
667
|
}
|
|
1364
|
-
|
|
1365
|
-
return false;
|
|
1366
|
-
}
|
|
1367
|
-
/**
|
|
1368
|
-
* Match a compound keyword (`titik-henti`, `по_умолчанию`) against a
|
|
1369
|
-
* shattered token run (`titik` `-` `henti` / `по` `_` `умолчанию`): word
|
|
1370
|
-
* segments must be identifier/keyword/particle tokens equal to the expected
|
|
1371
|
-
* segments (case-insensitive; particle because prepositions like ru `по` /
|
|
1372
|
-
* uk `за` lead some underscore compounds), separators lone `sep` tokens.
|
|
1373
|
-
* Consumes the run on success, resets the stream on failure.
|
|
1374
|
-
*/
|
|
1375
|
-
tryMatchShatteredCompound(tokens, expected, sep) {
|
|
1376
|
-
const segments = expected.split(sep);
|
|
1377
|
-
if (segments.length < 2 || segments.some((s) => s.length === 0)) return false;
|
|
1378
|
-
const mark = tokens.mark();
|
|
1379
|
-
for (let i = 0; i < segments.length; i++) {
|
|
1380
|
-
if (i > 0) {
|
|
1381
|
-
const sepToken = tokens.peek();
|
|
1382
|
-
if (!sepToken || sepToken.value !== sep) {
|
|
1383
|
-
tokens.reset(mark);
|
|
1384
|
-
return false;
|
|
1385
|
-
}
|
|
668
|
+
if (argToken.kind === "punctuation" && argToken.value === ",") {
|
|
1386
669
|
tokens.advance();
|
|
670
|
+
continue;
|
|
1387
671
|
}
|
|
1388
|
-
|
|
1389
|
-
if (!word || word.kind !== "identifier" && word.kind !== "keyword" && word.kind !== "particle" || word.value.toLowerCase() !== segments[i].toLowerCase()) {
|
|
1390
|
-
tokens.reset(mark);
|
|
1391
|
-
return false;
|
|
1392
|
-
}
|
|
672
|
+
args.push(argToken.value);
|
|
1393
673
|
tokens.advance();
|
|
1394
674
|
}
|
|
1395
|
-
|
|
675
|
+
const methodCall = `${chain}(${args.join(", ")})`;
|
|
676
|
+
return {
|
|
677
|
+
type: "expression",
|
|
678
|
+
raw: methodCall
|
|
679
|
+
};
|
|
1396
680
|
}
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
for (const t of inner.tokens) {
|
|
1412
|
-
if (t.type === "literal") markerLiterals.push(t.value, ...t.alternatives ?? []);
|
|
1413
|
-
}
|
|
1414
|
-
}
|
|
1415
|
-
}
|
|
1416
|
-
if (markerLiterals.length > 0) {
|
|
1417
|
-
const addedRole = [...captured.keys()].some((role) => !capturedBefore.has(role));
|
|
1418
|
-
if (addedRole) {
|
|
1419
|
-
const last = tokens.tokens[tokens.position() - 1];
|
|
1420
|
-
const markerSeen = last !== void 0 && tokens.position() > mark.position && markerLiterals.some((lit) => this.getMatchType(last, lit) !== "none");
|
|
1421
|
-
if (!markerSeen) success = false;
|
|
1422
|
-
}
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
if (!success) {
|
|
1426
|
-
tokens.reset(mark);
|
|
1427
|
-
for (const role of captured.keys()) {
|
|
1428
|
-
if (!capturedBefore.has(role)) {
|
|
1429
|
-
captured.delete(role);
|
|
1430
|
-
}
|
|
1431
|
-
}
|
|
1432
|
-
for (const [role, value] of capturedBefore) {
|
|
1433
|
-
if (captured.get(role) !== value) {
|
|
1434
|
-
captured.set(role, value);
|
|
1435
|
-
}
|
|
1436
|
-
}
|
|
1437
|
-
return patternToken.optional || false;
|
|
1438
|
-
}
|
|
1439
|
-
return true;
|
|
681
|
+
return {
|
|
682
|
+
type: "expression",
|
|
683
|
+
raw: chain
|
|
684
|
+
};
|
|
685
|
+
}
|
|
686
|
+
function tryMatchPossessiveSelectorExpression(tokens) {
|
|
687
|
+
const token = tokens.peek();
|
|
688
|
+
if (!token || token.kind !== "selector") return null;
|
|
689
|
+
const mark = tokens.mark();
|
|
690
|
+
tokens.advance();
|
|
691
|
+
const possessiveToken = tokens.peek();
|
|
692
|
+
if (!possessiveToken || possessiveToken.kind !== "punctuation" || possessiveToken.value !== "'s") {
|
|
693
|
+
tokens.reset(mark);
|
|
694
|
+
return null;
|
|
1440
695
|
}
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
patternTokenWouldMatch(pt, token) {
|
|
1447
|
-
if (!pt) return false;
|
|
1448
|
-
if (pt.type === "literal") {
|
|
1449
|
-
if (this.getMatchType(token, pt.value) !== "none") return true;
|
|
1450
|
-
return !!pt.alternatives?.some((a) => this.getMatchType(token, a) !== "none");
|
|
1451
|
-
}
|
|
1452
|
-
if (pt.type === "group") {
|
|
1453
|
-
return this.patternTokenWouldMatch(pt.tokens?.[0], token);
|
|
1454
|
-
}
|
|
1455
|
-
return false;
|
|
696
|
+
tokens.advance();
|
|
697
|
+
const propertyToken = tokens.peek();
|
|
698
|
+
if (!propertyToken) {
|
|
699
|
+
tokens.reset(mark);
|
|
700
|
+
return null;
|
|
1456
701
|
}
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
702
|
+
if (propertyToken.kind !== "selector" && propertyToken.kind !== "identifier") {
|
|
703
|
+
tokens.reset(mark);
|
|
704
|
+
return null;
|
|
705
|
+
}
|
|
706
|
+
tokens.advance();
|
|
707
|
+
return createPropertyPath(createSelector(token.value), propertyToken.value);
|
|
708
|
+
}
|
|
709
|
+
function tryMatchSelectorPropertyExpression(tokens) {
|
|
710
|
+
const token = tokens.peek();
|
|
711
|
+
if (!token || token.kind !== "selector") return null;
|
|
712
|
+
if (typeof token.value !== "string" || !token.value.startsWith("#")) return null;
|
|
713
|
+
const mark = tokens.mark();
|
|
714
|
+
tokens.advance();
|
|
715
|
+
const propertyToken = tokens.peek();
|
|
716
|
+
if (!propertyToken || propertyToken.kind !== "selector") {
|
|
717
|
+
tokens.reset(mark);
|
|
718
|
+
return null;
|
|
719
|
+
}
|
|
720
|
+
if (!propertyToken.value.startsWith(".")) {
|
|
721
|
+
tokens.reset(mark);
|
|
722
|
+
return null;
|
|
723
|
+
}
|
|
724
|
+
const peek2 = tokens.peek(1);
|
|
725
|
+
if (peek2 && peek2.kind === "selector") {
|
|
726
|
+
}
|
|
727
|
+
tokens.advance();
|
|
728
|
+
const propertyName = propertyToken.value.slice(1);
|
|
729
|
+
return createPropertyPath(createSelector(token.value), propertyName);
|
|
730
|
+
}
|
|
731
|
+
var MAX_PROPERTY_DEPTH = 10;
|
|
732
|
+
var MAX_METHOD_ARGS = 20;
|
|
733
|
+
function tokenToSemanticValue(token) {
|
|
734
|
+
switch (token.kind) {
|
|
735
|
+
case "selector":
|
|
736
|
+
return createSelector(token.value);
|
|
737
|
+
case "literal":
|
|
738
|
+
return parseLiteralValue(token.value);
|
|
739
|
+
case "keyword": {
|
|
740
|
+
const tokenValue = token.normalized || token.value;
|
|
741
|
+
const lower = safeToLowerCase(tokenValue);
|
|
742
|
+
if (isValidReference(lower)) {
|
|
743
|
+
return createReference(lower);
|
|
1474
744
|
}
|
|
745
|
+
return createLiteral(token.normalized || token.value);
|
|
1475
746
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
* shape the en reference produces).
|
|
1484
|
-
*/
|
|
1485
|
-
applyExtractionRules(pattern, captured) {
|
|
1486
|
-
for (const [role, rule] of Object.entries(pattern.extraction)) {
|
|
1487
|
-
const existing = captured.get(role);
|
|
1488
|
-
if (existing) {
|
|
1489
|
-
if (rule.transform) {
|
|
1490
|
-
const raw = existing.type === "literal" ? String(existing.value) : String(existing.raw ?? "");
|
|
1491
|
-
captured.set(role, rule.transform(raw));
|
|
1492
|
-
}
|
|
1493
|
-
} else if (rule.value !== void 0) {
|
|
1494
|
-
captured.set(
|
|
1495
|
-
role,
|
|
1496
|
-
rule.valueIsExpression ? { type: "expression", raw: rule.value } : { type: "literal", value: rule.value }
|
|
1497
|
-
);
|
|
1498
|
-
} else if (rule.default) {
|
|
1499
|
-
const implicitDefault = {
|
|
1500
|
-
...rule.default,
|
|
1501
|
-
implicit: true
|
|
1502
|
-
};
|
|
1503
|
-
captured.set(role, implicitDefault);
|
|
747
|
+
case "identifier": {
|
|
748
|
+
if (typeof token.value === "string" && token.value.startsWith(":")) {
|
|
749
|
+
return createReference(token.value);
|
|
750
|
+
}
|
|
751
|
+
const identLower = safeToLowerCase(token.value);
|
|
752
|
+
if (isValidReference(identLower)) {
|
|
753
|
+
return createReference(identLower);
|
|
1504
754
|
}
|
|
755
|
+
return { type: "expression", raw: token.value };
|
|
1505
756
|
}
|
|
757
|
+
case "url":
|
|
758
|
+
return createLiteral(token.value, "string");
|
|
759
|
+
default:
|
|
760
|
+
return null;
|
|
1506
761
|
}
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
return
|
|
762
|
+
}
|
|
763
|
+
function parseLiteralValue(value) {
|
|
764
|
+
if (value.startsWith('"') || value.startsWith("'") || value.startsWith("`") || value.startsWith("\u300C")) {
|
|
765
|
+
const inner = value.slice(1, -1);
|
|
766
|
+
return createLiteral(inner, "string");
|
|
1512
767
|
}
|
|
1513
|
-
|
|
768
|
+
if (value === "true") return createLiteral(true, "boolean");
|
|
769
|
+
if (value === "false") return createLiteral(false, "boolean");
|
|
770
|
+
const durationMatch = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
|
|
771
|
+
if (durationMatch) {
|
|
772
|
+
const num2 = parseFloat(durationMatch[1]);
|
|
773
|
+
const unit = durationMatch[2];
|
|
774
|
+
if (unit) {
|
|
775
|
+
return createLiteral(value, "duration");
|
|
776
|
+
}
|
|
777
|
+
return createLiteral(num2, "number");
|
|
778
|
+
}
|
|
779
|
+
const num = parseFloat(value);
|
|
780
|
+
if (!isNaN(num)) {
|
|
781
|
+
return createLiteral(num, "number");
|
|
782
|
+
}
|
|
783
|
+
return createLiteral(value, "string");
|
|
784
|
+
}
|
|
785
|
+
var ENGLISH_NOISE_WORDS = /* @__PURE__ */ new Set(["the", "a", "an"]);
|
|
786
|
+
function skipNoiseWords(tokens) {
|
|
787
|
+
const token = tokens.peek();
|
|
788
|
+
if (!token) return;
|
|
789
|
+
const tokenLower = safeToLowerCase(token.value);
|
|
790
|
+
if (ENGLISH_NOISE_WORDS.has(tokenLower)) {
|
|
791
|
+
const mark = tokens.mark();
|
|
792
|
+
tokens.advance();
|
|
793
|
+
const nextToken = tokens.peek();
|
|
794
|
+
if (nextToken && nextToken.kind === "selector") {
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
tokens.reset(mark);
|
|
798
|
+
}
|
|
799
|
+
if (tokenLower === "class") {
|
|
800
|
+
tokens.advance();
|
|
801
|
+
}
|
|
802
|
+
}
|
|
1514
803
|
|
|
1515
804
|
// src/core/matching/stock-reader.ts
|
|
1516
805
|
function greedyStopMarkers(rest) {
|
|
@@ -1547,6 +836,11 @@ function readGreedy(tokens, stopMarkers) {
|
|
|
1547
836
|
}
|
|
1548
837
|
return values.length > 0 ? { type: "expression", raw: values.join(" ") } : null;
|
|
1549
838
|
}
|
|
839
|
+
function isListedWord(token, words) {
|
|
840
|
+
if (!token) return false;
|
|
841
|
+
const forms = [token.value, token.normalized].filter((f) => typeof f === "string");
|
|
842
|
+
return forms.some((form) => words.some((word) => word.toLowerCase() === form.toLowerCase()));
|
|
843
|
+
}
|
|
1550
844
|
var StockValueReader = class {
|
|
1551
845
|
constructor(view, profiles = {}) {
|
|
1552
846
|
this.view = view;
|
|
@@ -1579,6 +873,11 @@ var StockValueReader = class {
|
|
|
1579
873
|
const token = tokens.peek();
|
|
1580
874
|
if (token && this.view.wouldMatch(next, token)) return false;
|
|
1581
875
|
}
|
|
876
|
+
if (slot.words && !isListedWord(tokens.peek(), slot.words)) return slot.optional ?? false;
|
|
877
|
+
if (slot.optional && !slot.greedy) {
|
|
878
|
+
const token = tokens.peek();
|
|
879
|
+
if (token && isStopMarker(token, greedyStopMarkers(this.view.patternRest()))) return true;
|
|
880
|
+
}
|
|
1582
881
|
if (slot.greedy) {
|
|
1583
882
|
const value = readGreedy(tokens, greedyStopMarkers(this.view.patternRest()));
|
|
1584
883
|
if (value) {
|
|
@@ -1602,265 +901,148 @@ function createStockReader(profiles = {}) {
|
|
|
1602
901
|
return (view) => new StockValueReader(view, profiles);
|
|
1603
902
|
}
|
|
1604
903
|
|
|
1605
|
-
// src/
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
const priority = config.basePriority ?? 100;
|
|
1613
|
-
const keyword = profile.keywords[schema.action];
|
|
1614
|
-
if (!keyword) {
|
|
1615
|
-
throw new Error(`No keyword translation for '${schema.action}' in ${profile.code}`);
|
|
1616
|
-
}
|
|
1617
|
-
const tokens = buildTokens(schema, profile, keyword.primary, keyword.alternatives);
|
|
1618
|
-
const extraction = buildExtractionRules(schema, profile);
|
|
1619
|
-
const format = buildFormatString(schema, profile, keyword.primary);
|
|
1620
|
-
return {
|
|
1621
|
-
id,
|
|
1622
|
-
language: profile.code,
|
|
1623
|
-
command: schema.action,
|
|
1624
|
-
priority,
|
|
1625
|
-
template: {
|
|
1626
|
-
format,
|
|
1627
|
-
tokens
|
|
1628
|
-
},
|
|
1629
|
-
extraction
|
|
1630
|
-
};
|
|
1631
|
-
}
|
|
1632
|
-
function buildTokens(schema, profile, keyword, alternatives) {
|
|
1633
|
-
const tokens = [];
|
|
1634
|
-
const keywordToken = {
|
|
1635
|
-
type: "literal",
|
|
1636
|
-
value: keyword,
|
|
1637
|
-
...alternatives?.length && { alternatives }
|
|
1638
|
-
};
|
|
1639
|
-
const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
|
|
1640
|
-
if (profile.wordOrder === "SVO") {
|
|
1641
|
-
tokens.push(keywordToken);
|
|
1642
|
-
for (const role of sortedRoles) {
|
|
1643
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1644
|
-
}
|
|
1645
|
-
} else if (profile.wordOrder === "SOV") {
|
|
1646
|
-
const preVerb = [];
|
|
1647
|
-
const postVerb = [];
|
|
1648
|
-
for (const role of sortedRoles) {
|
|
1649
|
-
addRoleWithMarker(role.sovSlot === "postVerb" ? postVerb : preVerb, role, profile);
|
|
1650
|
-
}
|
|
1651
|
-
tokens.push(...preVerb, keywordToken, ...postVerb);
|
|
1652
|
-
} else if (profile.wordOrder === "VSO") {
|
|
1653
|
-
tokens.push(keywordToken);
|
|
1654
|
-
for (const role of sortedRoles) {
|
|
1655
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1656
|
-
}
|
|
1657
|
-
} else {
|
|
1658
|
-
tokens.push(keywordToken);
|
|
1659
|
-
for (const role of sortedRoles) {
|
|
1660
|
-
addRoleWithMarker(tokens, role, profile);
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
return tokens;
|
|
1664
|
-
}
|
|
1665
|
-
function addRoleWithMarker(tokens, roleSpec, profile) {
|
|
1666
|
-
const marker = getMarkerForRole(roleSpec, profile);
|
|
1667
|
-
const isOptional = !roleSpec.required;
|
|
1668
|
-
const roleToken = {
|
|
1669
|
-
type: "role",
|
|
1670
|
-
role: roleSpec.role,
|
|
1671
|
-
optional: isOptional,
|
|
1672
|
-
expectedTypes: roleSpec.expectedTypes,
|
|
1673
|
-
...roleSpec.greedy && { greedy: true }
|
|
904
|
+
// src/core/matching/stock-confidence.ts
|
|
905
|
+
function scoreParts(ctx) {
|
|
906
|
+
const { pattern, captured } = ctx;
|
|
907
|
+
let score = 0;
|
|
908
|
+
let maxScore = 0;
|
|
909
|
+
const hasDefault = (role) => {
|
|
910
|
+
return pattern.extraction?.[role]?.default !== void 0;
|
|
1674
911
|
};
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
if (position === "before") {
|
|
1681
|
-
tokens.push({
|
|
1682
|
-
type: "group",
|
|
1683
|
-
optional: true,
|
|
1684
|
-
tokens: [{ type: "literal", value: marker }, roleToken]
|
|
1685
|
-
});
|
|
1686
|
-
} else {
|
|
1687
|
-
tokens.push({
|
|
1688
|
-
type: "group",
|
|
1689
|
-
optional: true,
|
|
1690
|
-
tokens: [roleToken, { type: "literal", value: marker }]
|
|
1691
|
-
});
|
|
912
|
+
for (const token of pattern.template.tokens) {
|
|
913
|
+
if (token.type === "role") {
|
|
914
|
+
maxScore += 1;
|
|
915
|
+
if (captured.has(token.role)) {
|
|
916
|
+
score += 1;
|
|
1692
917
|
}
|
|
1693
|
-
} else {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
918
|
+
} else if (token.type === "group") {
|
|
919
|
+
for (const subToken of token.tokens) {
|
|
920
|
+
if (subToken.type === "role") {
|
|
921
|
+
const roleHasDefault = hasDefault(subToken.role);
|
|
922
|
+
const weight = 0.8;
|
|
923
|
+
maxScore += weight;
|
|
924
|
+
if (captured.has(subToken.role)) {
|
|
925
|
+
score += weight;
|
|
926
|
+
} else if (roleHasDefault) {
|
|
927
|
+
score += weight * 0.6;
|
|
928
|
+
}
|
|
929
|
+
}
|
|
1700
930
|
}
|
|
1701
931
|
}
|
|
1702
|
-
} else {
|
|
1703
|
-
tokens.push(roleToken);
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1706
|
-
function getMarkerForRole(roleSpec, profile) {
|
|
1707
|
-
if (roleSpec.markerOverride?.[profile.code]) {
|
|
1708
|
-
return roleSpec.markerOverride[profile.code];
|
|
1709
932
|
}
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
} else if (wordOrder === "SOV") {
|
|
1717
|
-
sorted.sort((a, b) => (b.sovPosition ?? 0) - (a.sovPosition ?? 0));
|
|
1718
|
-
}
|
|
1719
|
-
return sorted;
|
|
1720
|
-
}
|
|
1721
|
-
function buildExtractionRules(schema, profile) {
|
|
1722
|
-
const rules = {};
|
|
1723
|
-
for (const roleSpec of schema.roles) {
|
|
1724
|
-
const marker = getMarkerForRole(roleSpec, profile);
|
|
1725
|
-
const rule = {
|
|
1726
|
-
...roleSpec.default !== void 0 && { default: roleSpec.default },
|
|
1727
|
-
...marker && { marker }
|
|
1728
|
-
};
|
|
1729
|
-
rules[roleSpec.role] = rule;
|
|
933
|
+
const roleCoverage = maxScore > 0 ? score / maxScore : 1;
|
|
934
|
+
let baseConfidence = roleCoverage;
|
|
935
|
+
let stemPenalty = 0;
|
|
936
|
+
if (ctx.stemMatchCount > 0 && ctx.totalKeywordMatches > 0) {
|
|
937
|
+
stemPenalty = ctx.stemMatchCount / ctx.totalKeywordMatches * 0.15;
|
|
938
|
+
baseConfidence = Math.max(0.5, baseConfidence - stemPenalty);
|
|
1730
939
|
}
|
|
1731
|
-
|
|
940
|
+
const vsoBoost = calculateVSOConfidenceBoost(pattern);
|
|
941
|
+
baseConfidence = Math.min(1, baseConfidence + vsoBoost);
|
|
942
|
+
const prepositionAdjustment = arabicPrepositionDisambiguation(pattern, captured);
|
|
943
|
+
baseConfidence = Math.max(0, Math.min(1, baseConfidence + prepositionAdjustment));
|
|
944
|
+
return {
|
|
945
|
+
roleCoverage,
|
|
946
|
+
stemPenalty,
|
|
947
|
+
languageBoost: vsoBoost,
|
|
948
|
+
languageAdjustment: prepositionAdjustment,
|
|
949
|
+
final: baseConfidence
|
|
950
|
+
};
|
|
1732
951
|
}
|
|
1733
|
-
function
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
const postVerb = [];
|
|
1737
|
-
const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
|
|
1738
|
-
for (const role of sortedRoles) {
|
|
1739
|
-
const marker = getMarkerForRole(role, profile);
|
|
1740
|
-
const roleName = `{${role.role}}`;
|
|
1741
|
-
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
1742
|
-
if (marker) {
|
|
1743
|
-
const markerInfo = profile.roleMarkers?.[role.role];
|
|
1744
|
-
if (markerInfo?.position === "after") {
|
|
1745
|
-
bucket.push(`${roleName} ${marker}`);
|
|
1746
|
-
} else {
|
|
1747
|
-
bucket.push(`${marker} ${roleName}`);
|
|
1748
|
-
}
|
|
1749
|
-
} else {
|
|
1750
|
-
bucket.push(roleName);
|
|
1751
|
-
}
|
|
952
|
+
function calculateVSOConfidenceBoost(pattern) {
|
|
953
|
+
if (pattern.language !== "ar") {
|
|
954
|
+
return 0;
|
|
1752
955
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
}
|
|
1757
|
-
|
|
1758
|
-
// src/generation/renderer.ts
|
|
1759
|
-
function lookupKeyword(keywords, action, language) {
|
|
1760
|
-
return keywords[action]?.[language] ?? action;
|
|
1761
|
-
}
|
|
1762
|
-
function buildPhrase(...parts) {
|
|
1763
|
-
return parts.filter(Boolean).join(" ");
|
|
1764
|
-
}
|
|
1765
|
-
function buildTablesFromProfiles(schemas, profiles) {
|
|
1766
|
-
const keywords = {};
|
|
1767
|
-
const markers = {};
|
|
1768
|
-
for (const profile of profiles) {
|
|
1769
|
-
for (const [action, kw] of Object.entries(profile.keywords)) {
|
|
1770
|
-
if (!keywords[action]) keywords[action] = {};
|
|
1771
|
-
keywords[action][profile.code] = kw.primary;
|
|
1772
|
-
}
|
|
956
|
+
const firstToken = pattern.template.tokens[0];
|
|
957
|
+
if (!firstToken || firstToken.type !== "literal") {
|
|
958
|
+
return 0;
|
|
1773
959
|
}
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
960
|
+
const ARABIC_VERBS = /* @__PURE__ */ new Set([
|
|
961
|
+
"\u0628\u062F\u0644",
|
|
962
|
+
"\u063A\u064A\u0631",
|
|
963
|
+
"\u0623\u0636\u0641",
|
|
964
|
+
"\u0623\u0632\u0644",
|
|
965
|
+
"\u0636\u0639",
|
|
966
|
+
"\u0627\u062C\u0639\u0644",
|
|
967
|
+
"\u0639\u064A\u0646",
|
|
968
|
+
"\u0632\u062F",
|
|
969
|
+
"\u0627\u0646\u0642\u0635",
|
|
970
|
+
"\u0633\u062C\u0644",
|
|
971
|
+
"\u0623\u0638\u0647\u0631",
|
|
972
|
+
"\u0623\u062E\u0641",
|
|
973
|
+
"\u0634\u063A\u0644",
|
|
974
|
+
"\u0623\u0631\u0633\u0644",
|
|
975
|
+
"\u0631\u0643\u0632",
|
|
976
|
+
"\u0634\u0648\u0634",
|
|
977
|
+
"\u062A\u0648\u0642\u0641",
|
|
978
|
+
"\u0627\u0646\u0633\u062E",
|
|
979
|
+
"\u0627\u062D\u0630\u0641",
|
|
980
|
+
"\u0627\u0635\u0646\u0639",
|
|
981
|
+
"\u0627\u0646\u062A\u0638\u0631",
|
|
982
|
+
"\u0627\u0646\u062A\u0642\u0627\u0644",
|
|
983
|
+
"\u0623\u0648"
|
|
984
|
+
]);
|
|
985
|
+
if (ARABIC_VERBS.has(firstToken.value)) {
|
|
986
|
+
return 0.15;
|
|
1784
987
|
}
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
markers[markerKey][profile.code] = markerDef.primary;
|
|
988
|
+
if (firstToken.alternatives) {
|
|
989
|
+
for (const alt of firstToken.alternatives) {
|
|
990
|
+
if (ARABIC_VERBS.has(alt)) {
|
|
991
|
+
return 0.15;
|
|
1790
992
|
}
|
|
1791
993
|
}
|
|
1792
994
|
}
|
|
1793
|
-
return
|
|
1794
|
-
}
|
|
1795
|
-
function detectWordOrders(profiles) {
|
|
1796
|
-
const sovLanguages = /* @__PURE__ */ new Set();
|
|
1797
|
-
const vsoLanguages = /* @__PURE__ */ new Set();
|
|
1798
|
-
for (const profile of profiles) {
|
|
1799
|
-
if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
|
|
1800
|
-
if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
|
|
1801
|
-
}
|
|
1802
|
-
return { sovLanguages, vsoLanguages };
|
|
995
|
+
return 0;
|
|
1803
996
|
}
|
|
1804
|
-
function
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
997
|
+
function arabicPrepositionDisambiguation(pattern, captured) {
|
|
998
|
+
if (pattern.language !== "ar") {
|
|
999
|
+
return 0;
|
|
1000
|
+
}
|
|
1001
|
+
let adjustment = 0;
|
|
1002
|
+
const PREFERRED_PREPOSITIONS = {
|
|
1003
|
+
patient: ["\u0639\u0644\u0649"],
|
|
1004
|
+
// element selectors prefer على (on/upon)
|
|
1005
|
+
destination: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
1006
|
+
// destination prefers إلى (to)
|
|
1007
|
+
source: ["\u0645\u0646"],
|
|
1008
|
+
// source prefers من (from)
|
|
1009
|
+
agent: ["\u0645\u0646"],
|
|
1010
|
+
// agent/by prefers من (from/by)
|
|
1011
|
+
manner: ["\u0628"],
|
|
1012
|
+
// manner prefers ب (with/by)
|
|
1013
|
+
style: ["\u0628"],
|
|
1014
|
+
// style prefers ب (with)
|
|
1015
|
+
goal: ["\u0625\u0644\u0649", "\u0627\u0644\u0649"],
|
|
1016
|
+
// target state prefers إلى (to)
|
|
1017
|
+
method: ["\u0628"]
|
|
1018
|
+
// method prefers ب (with/by)
|
|
1812
1019
|
};
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
for (const s of schemas) schemaMap.set(s.action, s);
|
|
1827
|
-
return { keywords, markers, markerPositions, sovLanguages, schemaMap };
|
|
1828
|
-
}
|
|
1829
|
-
function renderFromSchema(node, language, tables) {
|
|
1830
|
-
const schema = tables.schemaMap.get(node.action);
|
|
1831
|
-
if (!schema) return null;
|
|
1832
|
-
const keyword = lookupKeyword(tables.keywords, node.action, language);
|
|
1833
|
-
const isSOV = tables.sovLanguages.has(language);
|
|
1834
|
-
const ordered = sortRolesByWordOrder([...schema.roles], isSOV ? "SOV" : "SVO");
|
|
1835
|
-
const preVerb = [];
|
|
1836
|
-
const postVerb = [];
|
|
1837
|
-
for (const role of ordered) {
|
|
1838
|
-
let value = extractRoleValue(node, role.role);
|
|
1839
|
-
if (!value && role.default !== void 0) {
|
|
1840
|
-
value = String(extractValue(role.default));
|
|
1841
|
-
}
|
|
1842
|
-
if (!value) continue;
|
|
1843
|
-
if (role.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
|
|
1844
|
-
value = `"${value}"`;
|
|
1845
|
-
}
|
|
1846
|
-
const markerText = role.renderOverride?.[language] ?? role.markerOverride?.[language] ?? role.renderOverride?.["*"] ?? tables.markers[role.role]?.[language];
|
|
1847
|
-
const markerPosition = role.markerPositionOverride?.[language] ?? role.markerPosition ?? tables.markerPositions[role.role]?.[language] ?? (isSOV ? "after" : "before");
|
|
1848
|
-
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
1849
|
-
if (markerText && markerPosition === "before") {
|
|
1850
|
-
bucket.push(markerText, value);
|
|
1851
|
-
} else if (markerText) {
|
|
1852
|
-
bucket.push(value, markerText);
|
|
1853
|
-
} else {
|
|
1854
|
-
bucket.push(value);
|
|
1020
|
+
for (const [role, value] of captured.entries()) {
|
|
1021
|
+
const preferred = PREFERRED_PREPOSITIONS[role];
|
|
1022
|
+
if (!preferred || preferred.length === 0) {
|
|
1023
|
+
continue;
|
|
1024
|
+
}
|
|
1025
|
+
const metadata = "metadata" in value ? value.metadata : void 0;
|
|
1026
|
+
if (metadata && typeof metadata.prepositionValue === "string") {
|
|
1027
|
+
const usedPreposition = metadata.prepositionValue;
|
|
1028
|
+
if (preferred.includes(usedPreposition)) {
|
|
1029
|
+
adjustment += 0.1;
|
|
1030
|
+
} else {
|
|
1031
|
+
adjustment -= 0.1;
|
|
1032
|
+
}
|
|
1855
1033
|
}
|
|
1856
1034
|
}
|
|
1857
|
-
return
|
|
1035
|
+
return Math.max(-0.1, Math.min(0.1, adjustment));
|
|
1858
1036
|
}
|
|
1037
|
+
var stockConfidenceModel = {
|
|
1038
|
+
calculate: (ctx) => scoreParts(ctx).final,
|
|
1039
|
+
calculateWithBreakdown: (ctx) => scoreParts(ctx)
|
|
1040
|
+
};
|
|
1859
1041
|
|
|
1860
1042
|
// src/generation/schema-patterns.ts
|
|
1861
1043
|
function createSchemaPatternGenerator(grammar) {
|
|
1862
1044
|
return {
|
|
1863
|
-
generatePattern: (schema, profile, config) =>
|
|
1045
|
+
generatePattern: (schema, profile, config) => generatePattern(grammar, schema, profile, config),
|
|
1864
1046
|
generateSimplePattern: (schema, profile, config) => generateSimplePattern(grammar, schema, profile, config),
|
|
1865
1047
|
generateVerbFirstPattern: (schema, profile, config) => generateVerbFirstPattern(schema, profile, config),
|
|
1866
1048
|
generatePatternVariants: (schema, profile, config) => generatePatternVariants(grammar, schema, profile, config),
|
|
@@ -1913,21 +1095,21 @@ function schemaMarkerAlternatives(roleSpec, languageCode, marker) {
|
|
|
1913
1095
|
const alternatives = [.../* @__PURE__ */ new Set([...legacy, ...variants])].filter((a) => a && a !== marker);
|
|
1914
1096
|
return alternatives.length ? alternatives : void 0;
|
|
1915
1097
|
}
|
|
1916
|
-
var
|
|
1098
|
+
var defaultConfig = {
|
|
1917
1099
|
basePriority: 100,
|
|
1918
1100
|
generateSimpleVariants: true,
|
|
1919
1101
|
generateAlternatives: true
|
|
1920
1102
|
};
|
|
1921
|
-
function
|
|
1103
|
+
function generatePattern(grammar, schema, profile, config = defaultConfig) {
|
|
1922
1104
|
const id = `${schema.action}-${profile.code}-generated`;
|
|
1923
1105
|
const priority = config.basePriority ?? 100;
|
|
1924
1106
|
const keyword = profile.keywords[schema.action];
|
|
1925
1107
|
if (!keyword) {
|
|
1926
1108
|
throw new Error(`No keyword translation for '${schema.action}' in ${profile.code}`);
|
|
1927
1109
|
}
|
|
1928
|
-
const tokens =
|
|
1110
|
+
const tokens = buildTokens(grammar, schema, profile, keyword);
|
|
1929
1111
|
const extraction = buildExtractionRulesWithDefaults(schema, profile);
|
|
1930
|
-
const format =
|
|
1112
|
+
const format = buildFormatString(grammar, schema, profile, keyword);
|
|
1931
1113
|
return {
|
|
1932
1114
|
id,
|
|
1933
1115
|
language: profile.code,
|
|
@@ -1940,7 +1122,7 @@ function generatePattern2(grammar, schema, profile, config = defaultConfig2) {
|
|
|
1940
1122
|
extraction
|
|
1941
1123
|
};
|
|
1942
1124
|
}
|
|
1943
|
-
function generateSimplePattern(grammar, schema, profile, config =
|
|
1125
|
+
function generateSimplePattern(grammar, schema, profile, config = defaultConfig) {
|
|
1944
1126
|
const optionalRoles = schema.roles.filter((r) => !r.required);
|
|
1945
1127
|
if (optionalRoles.length === 0) {
|
|
1946
1128
|
return null;
|
|
@@ -1950,7 +1132,7 @@ function generateSimplePattern(grammar, schema, profile, config = defaultConfig2
|
|
|
1950
1132
|
...schema,
|
|
1951
1133
|
roles: requiredRoles
|
|
1952
1134
|
};
|
|
1953
|
-
const pattern =
|
|
1135
|
+
const pattern = generatePattern(grammar, simpleSchema, profile, config);
|
|
1954
1136
|
return {
|
|
1955
1137
|
...pattern,
|
|
1956
1138
|
id: `${schema.action}-${profile.code}-generated-simple`,
|
|
@@ -1959,7 +1141,7 @@ function generateSimplePattern(grammar, schema, profile, config = defaultConfig2
|
|
|
1959
1141
|
extraction: buildExtractionRulesWithDefaults(schema, profile)
|
|
1960
1142
|
};
|
|
1961
1143
|
}
|
|
1962
|
-
function generateVerbFirstPattern(schema, profile, config =
|
|
1144
|
+
function generateVerbFirstPattern(schema, profile, config = defaultConfig) {
|
|
1963
1145
|
if (profile.wordOrder !== "SOV") return null;
|
|
1964
1146
|
const requiredRoles = schema.roles.filter((r) => r.required);
|
|
1965
1147
|
if (requiredRoles.length !== 1) return null;
|
|
@@ -1968,11 +1150,13 @@ function generateVerbFirstPattern(schema, profile, config = defaultConfig2) {
|
|
|
1968
1150
|
const verbToken = keyword.alternatives ? { type: "literal", value: keyword.primary, alternatives: keyword.alternatives } : { type: "literal", value: keyword.primary };
|
|
1969
1151
|
const roleTokens = requiredRoles.flatMap((r) => {
|
|
1970
1152
|
const prefix = r.valuePrefixLiteral?.[profile.code];
|
|
1153
|
+
const words = wordsFor(r, profile.code);
|
|
1971
1154
|
const roleToken = {
|
|
1972
1155
|
type: "role",
|
|
1973
1156
|
role: r.role,
|
|
1974
1157
|
optional: false,
|
|
1975
|
-
expectedTypes: r.expectedTypes
|
|
1158
|
+
expectedTypes: r.expectedTypes,
|
|
1159
|
+
...words ? { words } : {}
|
|
1976
1160
|
};
|
|
1977
1161
|
return prefix ? [{ type: "literal", value: prefix }, roleToken] : [roleToken];
|
|
1978
1162
|
});
|
|
@@ -1990,11 +1174,11 @@ function generateVerbFirstPattern(schema, profile, config = defaultConfig2) {
|
|
|
1990
1174
|
extraction: buildExtractionRulesWithDefaults(schema, profile)
|
|
1991
1175
|
};
|
|
1992
1176
|
}
|
|
1993
|
-
function generatePatternVariants(grammar, schema, profile, config =
|
|
1177
|
+
function generatePatternVariants(grammar, schema, profile, config = defaultConfig) {
|
|
1994
1178
|
const patterns = [];
|
|
1995
1179
|
const full = schema;
|
|
1996
1180
|
schema = withoutVariantOnlyRoles(schema);
|
|
1997
|
-
patterns.push(
|
|
1181
|
+
patterns.push(generatePattern(grammar, schema, profile, config));
|
|
1998
1182
|
if (config.generateSimpleVariants) {
|
|
1999
1183
|
const simple = generateSimplePattern(grammar, schema, profile, config);
|
|
2000
1184
|
if (simple) {
|
|
@@ -2006,7 +1190,7 @@ function generatePatternVariants(grammar, schema, profile, config = defaultConfi
|
|
|
2006
1190
|
...schema,
|
|
2007
1191
|
roles: schema.roles.filter((r) => r.role !== omitted)
|
|
2008
1192
|
};
|
|
2009
|
-
const variant =
|
|
1193
|
+
const variant = generatePattern(grammar, variantSchema, profile, config);
|
|
2010
1194
|
patterns.push({
|
|
2011
1195
|
...variant,
|
|
2012
1196
|
id: `${schema.action}-${profile.code}-generated-no-${omitted}`,
|
|
@@ -2021,7 +1205,7 @@ function generatePatternVariants(grammar, schema, profile, config = defaultConfi
|
|
|
2021
1205
|
(r) => r.role === widen.role ? { ...r, expectedTypes: [...widen.types] } : r
|
|
2022
1206
|
)
|
|
2023
1207
|
};
|
|
2024
|
-
const main =
|
|
1208
|
+
const main = generatePattern(grammar, cloneSchema, profile, config);
|
|
2025
1209
|
patterns.push({
|
|
2026
1210
|
...main,
|
|
2027
1211
|
id: `${schema.action}-${profile.code}-generated-${widen.idSuffix}`,
|
|
@@ -2036,7 +1220,7 @@ function generatePatternVariants(grammar, schema, profile, config = defaultConfi
|
|
|
2036
1220
|
return r.role === v.widen?.role ? { ...required, expectedTypes: [...v.widen.types] } : required;
|
|
2037
1221
|
})
|
|
2038
1222
|
};
|
|
2039
|
-
const variant =
|
|
1223
|
+
const variant = generatePattern(grammar, variantSchema, profile, config);
|
|
2040
1224
|
patterns.push({
|
|
2041
1225
|
...variant,
|
|
2042
1226
|
id: `${schema.action}-${profile.code}-generated-${v.idSuffix}`,
|
|
@@ -2061,7 +1245,7 @@ function generatePatternVariants(grammar, schema, profile, config = defaultConfi
|
|
|
2061
1245
|
};
|
|
2062
1246
|
const delta = v.priorityDelta ?? 5;
|
|
2063
1247
|
const carrier = v.methodCarrier ? { [v.methodCarrier]: { value: literal } } : {};
|
|
2064
|
-
const main =
|
|
1248
|
+
const main = generatePattern(grammar, cloneSchema, profile, config);
|
|
2065
1249
|
patterns.push({
|
|
2066
1250
|
...main,
|
|
2067
1251
|
id: `${schema.action}-${profile.code}-generated-${v.idSuffix}`,
|
|
@@ -2082,7 +1266,7 @@ function generatePatternVariants(grammar, schema, profile, config = defaultConfi
|
|
|
2082
1266
|
}
|
|
2083
1267
|
return patterns;
|
|
2084
1268
|
}
|
|
2085
|
-
function
|
|
1269
|
+
function buildTokens(grammar, schema, profile, keyword) {
|
|
2086
1270
|
const tokens = [];
|
|
2087
1271
|
const verbToken = keyword.alternatives ? { type: "literal", value: keyword.primary, alternatives: keyword.alternatives } : { type: "literal", value: keyword.primary };
|
|
2088
1272
|
const postVerbRoles = profile.wordOrder === "SOV" ? schema.roles.filter((r) => r.sovSlot === "postVerb") : [];
|
|
@@ -2142,11 +1326,17 @@ function buildRoleTokens(grammar, schema, profile) {
|
|
|
2142
1326
|
}
|
|
2143
1327
|
return tokens;
|
|
2144
1328
|
}
|
|
1329
|
+
function wordsFor(roleSpec, language) {
|
|
1330
|
+
if (!roleSpec.words) return void 0;
|
|
1331
|
+
const words = [...roleSpec.words["*"] ?? [], ...roleSpec.words[language] ?? []];
|
|
1332
|
+
return words.length > 0 ? words : void 0;
|
|
1333
|
+
}
|
|
2145
1334
|
function buildRoleToken(roleSpec, profile) {
|
|
2146
1335
|
const tokens = [];
|
|
2147
1336
|
const overrideMarker = roleSpec.markerOverride?.[profile.code];
|
|
2148
1337
|
const defaultMarker = profile.roleMarkers[roleSpec.role];
|
|
2149
1338
|
const suppressMarker = roleSpec.renderOverride?.[profile.code] === "";
|
|
1339
|
+
const words = wordsFor(roleSpec, profile.code);
|
|
2150
1340
|
const roleValueToken = {
|
|
2151
1341
|
type: "role",
|
|
2152
1342
|
role: roleSpec.role,
|
|
@@ -2158,7 +1348,9 @@ function buildRoleToken(roleSpec, profile) {
|
|
|
2158
1348
|
...roleSpec.valueShape !== void 0 ? { valueShape: roleSpec.valueShape } : {},
|
|
2159
1349
|
// A greedy role reads up to the next marker the pattern owes (a domain's
|
|
2160
1350
|
// SQL condition); its value reader does the reading.
|
|
2161
|
-
...roleSpec.greedy ? { greedy: true } : {}
|
|
1351
|
+
...roleSpec.greedy ? { greedy: true } : {},
|
|
1352
|
+
// A role with a word list reads only one of its words.
|
|
1353
|
+
...words ? { words } : {}
|
|
2162
1354
|
};
|
|
2163
1355
|
const prefixLiteral = roleSpec.valuePrefixLiteral?.[profile.code];
|
|
2164
1356
|
const pushPrefixed = () => {
|
|
@@ -2228,7 +1420,7 @@ function buildRoleToken(roleSpec, profile) {
|
|
|
2228
1420
|
}
|
|
2229
1421
|
return tokens;
|
|
2230
1422
|
}
|
|
2231
|
-
function
|
|
1423
|
+
function buildExtractionRules(schema, profile) {
|
|
2232
1424
|
const rules = {};
|
|
2233
1425
|
for (const roleSpec of schema.roles) {
|
|
2234
1426
|
const overrideMarker = roleSpec.markerOverride?.[profile.code];
|
|
@@ -2258,7 +1450,7 @@ function buildExtractionRules2(schema, profile) {
|
|
|
2258
1450
|
return rules;
|
|
2259
1451
|
}
|
|
2260
1452
|
function buildExtractionRulesWithDefaults(schema, profile) {
|
|
2261
|
-
const baseRules =
|
|
1453
|
+
const baseRules = buildExtractionRules(schema, profile);
|
|
2262
1454
|
const rules = {};
|
|
2263
1455
|
for (const roleSpec of schema.roles) {
|
|
2264
1456
|
const baseRule = baseRules[roleSpec.role] || {};
|
|
@@ -2270,7 +1462,7 @@ function buildExtractionRulesWithDefaults(schema, profile) {
|
|
|
2270
1462
|
}
|
|
2271
1463
|
return rules;
|
|
2272
1464
|
}
|
|
2273
|
-
function
|
|
1465
|
+
function buildFormatString(grammar, schema, profile, keyword) {
|
|
2274
1466
|
const parts = [];
|
|
2275
1467
|
const sortedRoles = sortRolesByPosition(schema.roles, profile.wordOrder, grammar.positionOrder);
|
|
2276
1468
|
const roleParts = sortedRoles.map((roleSpec) => {
|
|
@@ -2683,12 +1875,15 @@ function readingDifference(source, back) {
|
|
|
2683
1875
|
|
|
2684
1876
|
// src/api/create-dsl.ts
|
|
2685
1877
|
var UnreadInputError = class extends Error {
|
|
2686
|
-
constructor(input, language, reading, unread) {
|
|
2687
|
-
super(
|
|
1878
|
+
constructor(input, language, reading, unread, hint) {
|
|
1879
|
+
super(
|
|
1880
|
+
`"${input}" (${language}) reads as ${reading.action} only by leaving "${unread}" unread` + (hint ? `; ${hint}` : "")
|
|
1881
|
+
);
|
|
2688
1882
|
this.input = input;
|
|
2689
1883
|
this.language = language;
|
|
2690
1884
|
this.reading = reading;
|
|
2691
1885
|
this.unread = unread;
|
|
1886
|
+
this.hint = hint;
|
|
2692
1887
|
this.code = "unread-input";
|
|
2693
1888
|
this.name = "UnreadInputError";
|
|
2694
1889
|
}
|
|
@@ -2807,15 +2002,32 @@ var MultilingualDSLImpl = class {
|
|
|
2807
2002
|
}
|
|
2808
2003
|
if (longestPartial) {
|
|
2809
2004
|
const unread = tokens.tokens.slice(longestPartial.consumedTokens).map((t) => t.value).join(" ");
|
|
2005
|
+
const hint = this.quoteHint(longestPartial, tokens.tokens);
|
|
2810
2006
|
throw new UnreadInputError(
|
|
2811
2007
|
input,
|
|
2812
2008
|
language,
|
|
2813
2009
|
this.nodeOf(longestPartial, input, language),
|
|
2814
|
-
unread
|
|
2010
|
+
unread,
|
|
2011
|
+
...hint ? [hint] : []
|
|
2815
2012
|
);
|
|
2816
2013
|
}
|
|
2817
2014
|
throw new Error(`No pattern matched for input: ${input}`);
|
|
2818
2015
|
}
|
|
2016
|
+
/**
|
|
2017
|
+
* `if "buy milk" is one value, quote it`, where the last token a match read is a bare word it took
|
|
2018
|
+
* as a role's value (a role with no word list) and the first it left is a bare word too.
|
|
2019
|
+
*/
|
|
2020
|
+
quoteHint(match, tokens) {
|
|
2021
|
+
const last = tokens[match.consumedTokens - 1];
|
|
2022
|
+
const next = tokens[match.consumedTokens];
|
|
2023
|
+
if (last?.kind !== "identifier" || next?.kind !== "identifier") return void 0;
|
|
2024
|
+
const schema = this.schemaLookup.getSchema(match.pattern.command);
|
|
2025
|
+
const role = [...match.captured].find(
|
|
2026
|
+
([, v]) => v.raw === last.value
|
|
2027
|
+
)?.[0];
|
|
2028
|
+
if (!role || schema?.roles.find((r) => r.role === role)?.words) return void 0;
|
|
2029
|
+
return `if "${last.value} ${next.value}" is one value, quote it`;
|
|
2030
|
+
}
|
|
2819
2031
|
nodeOf(match, input, language) {
|
|
2820
2032
|
return {
|
|
2821
2033
|
kind: "command",
|
|
@@ -3039,21 +2251,14 @@ function createMultilingualDSL(config) {
|
|
|
3039
2251
|
return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider, engine);
|
|
3040
2252
|
}
|
|
3041
2253
|
function engineFor(config) {
|
|
3042
|
-
const name = config.engine ?? "
|
|
3043
|
-
if (name === "classic") return classicEngine();
|
|
2254
|
+
const name = config.engine ?? "core";
|
|
3044
2255
|
if (name === "core") return coreEngine();
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
match: (tokens, pattern, language) => matcher.matchPattern(tokens, pattern, { code: language }),
|
|
3052
|
-
fallbackRenderer: (config) => createDomainRenderer({
|
|
3053
|
-
schemas: config.schemas,
|
|
3054
|
-
profiles: config.languages.map((l) => l.patternProfile)
|
|
3055
|
-
})
|
|
3056
|
-
};
|
|
2256
|
+
if (name === "classic") {
|
|
2257
|
+
throw new Error(
|
|
2258
|
+
'The "classic" DSL engine was removed in @lokascript/framework 5.0. Omit `engine` (or pass "core"): every domain reads on the core.'
|
|
2259
|
+
);
|
|
2260
|
+
}
|
|
2261
|
+
throw new Error(`Unknown DSL engine "${name}" (expected "core")`);
|
|
3057
2262
|
}
|
|
3058
2263
|
function schemaProfileOf(profile) {
|
|
3059
2264
|
const side = profile.wordOrder === "SOV" ? "after" : "before";
|