@lokascript/framework 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aot/index.d.ts +1 -0
- package/dist/aot/index.d.ts.map +1 -1
- package/dist/aot/registry-bridge.d.ts +27 -0
- package/dist/aot/registry-bridge.d.ts.map +1 -0
- package/dist/api/create-dsl.d.ts.map +1 -1
- package/dist/api/dispatcher.d.ts +22 -0
- package/dist/api/dispatcher.d.ts.map +1 -1
- package/dist/api/domain-registry.d.ts +41 -0
- package/dist/api/domain-registry.d.ts.map +1 -1
- package/dist/api/index.js +1351 -21
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.js +114 -10
- package/dist/core/index.js.map +1 -1
- package/dist/core/pattern-matching/index.js +74 -10
- package/dist/core/pattern-matching/index.js.map +1 -1
- package/dist/core/pattern-matching/pattern-matcher.d.ts.map +1 -1
- package/dist/core/types.d.ts +80 -2
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +40 -0
- package/dist/core/types.js.map +1 -1
- package/dist/feedback/confidence-gate.d.ts +28 -0
- package/dist/feedback/confidence-gate.d.ts.map +1 -0
- package/dist/feedback/feedback-formatter.d.ts +21 -0
- package/dist/feedback/feedback-formatter.d.ts.map +1 -0
- package/dist/feedback/index.d.ts +11 -0
- package/dist/feedback/index.d.ts.map +1 -0
- package/dist/feedback/pattern-tracker.d.ts +47 -0
- package/dist/feedback/pattern-tracker.d.ts.map +1 -0
- package/dist/feedback/types.d.ts +113 -0
- package/dist/feedback/types.d.ts.map +1 -0
- package/dist/generation/index.js +2 -1
- package/dist/generation/index.js.map +1 -1
- package/dist/index.cjs +2685 -71
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2652 -71
- package/dist/index.js.map +1 -1
- package/dist/ir/explicit-parser.d.ts +31 -0
- package/dist/ir/explicit-parser.d.ts.map +1 -0
- package/dist/ir/explicit-renderer.d.ts +17 -0
- package/dist/ir/explicit-renderer.d.ts.map +1 -0
- package/dist/ir/from-interchange.d.ts +51 -0
- package/dist/ir/from-interchange.d.ts.map +1 -0
- package/dist/ir/index.d.ts +29 -0
- package/dist/ir/index.d.ts.map +1 -0
- package/dist/ir/index.js +1557 -0
- package/dist/ir/index.js.map +1 -0
- package/dist/ir/json-schema.d.ts +40 -0
- package/dist/ir/json-schema.d.ts.map +1 -0
- package/dist/ir/protocol-json.d.ts +73 -0
- package/dist/ir/protocol-json.d.ts.map +1 -0
- package/dist/ir/references.d.ts +19 -0
- package/dist/ir/references.d.ts.map +1 -0
- package/dist/ir/types.d.ts +141 -0
- package/dist/ir/types.d.ts.map +1 -0
- package/dist/parsing/index.js +74 -10
- package/dist/parsing/index.js.map +1 -1
- package/dist/prompts/index.d.ts +9 -0
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/prompt-generator.d.ts +49 -0
- package/dist/prompts/prompt-generator.d.ts.map +1 -0
- package/dist/prompts/types.d.ts +64 -0
- package/dist/prompts/types.d.ts.map +1 -0
- package/dist/schema/command-schema.d.ts +17 -0
- package/dist/schema/command-schema.d.ts.map +1 -1
- package/dist/schema/index.js.map +1 -1
- package/dist/testing/index.js +3 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/training/index.d.ts +11 -0
- package/dist/training/index.d.ts.map +1 -0
- package/dist/training/jsonl-writer.d.ts +42 -0
- package/dist/training/jsonl-writer.d.ts.map +1 -0
- package/dist/training/schema-synthesizer.d.ts +20 -0
- package/dist/training/schema-synthesizer.d.ts.map +1 -0
- package/dist/training/types.d.ts +73 -0
- package/dist/training/types.d.ts.map +1 -0
- package/package.json +5 -1
- package/src/aot/index.ts +1 -0
- package/src/aot/registry-bridge.test.ts +104 -0
- package/src/aot/registry-bridge.ts +66 -0
- package/src/api/create-dsl.test.ts +306 -0
- package/src/api/create-dsl.ts +21 -0
- package/src/api/dispatcher.test.ts +66 -0
- package/src/api/dispatcher.ts +77 -9
- package/src/api/domain-registry.ts +120 -4
- package/src/core/pattern-matching/pattern-matcher.test.ts +131 -0
- package/src/core/pattern-matching/pattern-matcher.ts +102 -19
- package/src/core/types.ts +145 -2
- package/src/feedback/confidence-gate.test.ts +111 -0
- package/src/feedback/confidence-gate.ts +70 -0
- package/src/feedback/feedback-formatter.test.ts +198 -0
- package/src/feedback/feedback-formatter.ts +298 -0
- package/src/feedback/index.ts +24 -0
- package/src/feedback/pattern-tracker.test.ts +198 -0
- package/src/feedback/pattern-tracker.ts +140 -0
- package/src/feedback/types.ts +149 -0
- package/src/generation/pattern-generator.ts +2 -2
- package/src/index.ts +22 -0
- package/src/integration/llm-lse-integration.test.ts +457 -0
- package/src/ir/explicit-parser.test.ts +306 -0
- package/src/ir/explicit-parser.ts +419 -0
- package/src/ir/explicit-renderer.test.ts +144 -0
- package/src/ir/explicit-renderer.ts +146 -0
- package/src/ir/from-interchange.test.ts +853 -0
- package/src/ir/from-interchange.ts +594 -0
- package/src/ir/index.ts +66 -0
- package/src/ir/json-schema.test.ts +281 -0
- package/src/ir/json-schema.ts +299 -0
- package/src/ir/protocol-json.test.ts +1185 -0
- package/src/ir/protocol-json.ts +745 -0
- package/src/ir/references.test.ts +38 -0
- package/src/ir/references.ts +33 -0
- package/src/ir/round-trip.test.ts +557 -0
- package/src/ir/types.ts +181 -0
- package/src/prompts/index.ts +15 -0
- package/src/prompts/prompt-generator.test.ts +386 -0
- package/src/prompts/prompt-generator.ts +531 -0
- package/src/prompts/types.ts +89 -0
- package/src/schema/command-schema.ts +19 -0
- package/src/training/index.ts +13 -0
- package/src/training/jsonl-writer.test.ts +138 -0
- package/src/training/jsonl-writer.ts +67 -0
- package/src/training/schema-synthesizer.test.ts +250 -0
- package/src/training/schema-synthesizer.ts +306 -0
- package/src/training/types.ts +103 -0
package/dist/core/index.js
CHANGED
|
@@ -14,6 +14,9 @@ function createPropertyPath(object, property) {
|
|
|
14
14
|
function createExpression(raw) {
|
|
15
15
|
return { type: "expression", raw };
|
|
16
16
|
}
|
|
17
|
+
function createFlag(name, enabled = true) {
|
|
18
|
+
return { type: "flag", name, enabled };
|
|
19
|
+
}
|
|
17
20
|
function createCommandNode(action, roles, metadata) {
|
|
18
21
|
const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
|
|
19
22
|
const node = {
|
|
@@ -65,6 +68,7 @@ function createCompoundNode(statements, chainType = "sequential", metadata) {
|
|
|
65
68
|
return metadata ? { ...base, metadata } : base;
|
|
66
69
|
}
|
|
67
70
|
function extractValue(value) {
|
|
71
|
+
if (value.type === "flag") return value.name;
|
|
68
72
|
if ("raw" in value && value.raw !== void 0) return String(value.raw);
|
|
69
73
|
if ("value" in value && value.value !== void 0) return String(value.value);
|
|
70
74
|
if (value.type === "property-path") return `${extractValue(value.object)}.${value.property}`;
|
|
@@ -91,6 +95,38 @@ function createLoopNode(action, roles, loopVariant, body, loopVariable, indexVar
|
|
|
91
95
|
...metadata && { metadata }
|
|
92
96
|
};
|
|
93
97
|
}
|
|
98
|
+
function createTryNode(body, catchBranch, finallyBranch, metadata) {
|
|
99
|
+
return {
|
|
100
|
+
kind: "command",
|
|
101
|
+
action: "try",
|
|
102
|
+
roles: /* @__PURE__ */ new Map(),
|
|
103
|
+
body,
|
|
104
|
+
...catchBranch && catchBranch.length > 0 ? { catchBranch } : {},
|
|
105
|
+
...finallyBranch && finallyBranch.length > 0 ? { finallyBranch } : {},
|
|
106
|
+
...metadata ? { metadata } : {}
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
function createAsyncNode(variant, asyncBody, metadata) {
|
|
110
|
+
return {
|
|
111
|
+
kind: "command",
|
|
112
|
+
action: variant,
|
|
113
|
+
roles: /* @__PURE__ */ new Map(),
|
|
114
|
+
asyncVariant: variant,
|
|
115
|
+
asyncBody,
|
|
116
|
+
...metadata ? { metadata } : {}
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function createMatchNode(roles, arms, defaultArm, metadata) {
|
|
120
|
+
const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
|
|
121
|
+
return {
|
|
122
|
+
kind: "command",
|
|
123
|
+
action: "match",
|
|
124
|
+
roles: rolesMap,
|
|
125
|
+
arms,
|
|
126
|
+
...defaultArm && defaultArm.length > 0 ? { defaultArm } : {},
|
|
127
|
+
...metadata ? { metadata } : {}
|
|
128
|
+
};
|
|
129
|
+
}
|
|
94
130
|
|
|
95
131
|
// src/core/tokenization/token-utils.ts
|
|
96
132
|
var TokenStreamImpl = class {
|
|
@@ -1661,7 +1697,7 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1661
1697
|
* a token and the immediately following pattern token fails, the matcher
|
|
1662
1698
|
* resets to before the optional role and retries the failed token.
|
|
1663
1699
|
*/
|
|
1664
|
-
matchTokenSequence(tokens, patternTokens, captured) {
|
|
1700
|
+
matchTokenSequence(tokens, patternTokens, captured, parentStopMarkers) {
|
|
1665
1701
|
const firstPatternToken = patternTokens[0];
|
|
1666
1702
|
const patternExpectsConjunction = firstPatternToken?.type === "literal" && (firstPatternToken.value === "and" || firstPatternToken.value === "then" || firstPatternToken.alternatives?.includes("and") || firstPatternToken.alternatives?.includes("then"));
|
|
1667
1703
|
if (this.currentProfile?.code === "ar" && !patternExpectsConjunction) {
|
|
@@ -1685,6 +1721,9 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1685
1721
|
);
|
|
1686
1722
|
if (patternToken.type === "role" && patternToken.greedy) {
|
|
1687
1723
|
const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
1724
|
+
if (parentStopMarkers) {
|
|
1725
|
+
for (const m of parentStopMarkers) stopMarkers.add(m);
|
|
1726
|
+
}
|
|
1688
1727
|
const values = [];
|
|
1689
1728
|
while (!tokens.isAtEnd()) {
|
|
1690
1729
|
const nextToken = tokens.peek();
|
|
@@ -1704,8 +1743,58 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1704
1743
|
return false;
|
|
1705
1744
|
}
|
|
1706
1745
|
}
|
|
1746
|
+
if (patternToken.type === "group") {
|
|
1747
|
+
const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
|
|
1748
|
+
if (parentStopMarkers) {
|
|
1749
|
+
for (const m of parentStopMarkers) siblingStopMarkers.add(m);
|
|
1750
|
+
}
|
|
1751
|
+
const groupMatched = this.matchGroupToken(
|
|
1752
|
+
tokens,
|
|
1753
|
+
patternToken,
|
|
1754
|
+
captured,
|
|
1755
|
+
siblingStopMarkers
|
|
1756
|
+
);
|
|
1757
|
+
if (groupMatched) {
|
|
1758
|
+
prevOptionalMark = null;
|
|
1759
|
+
prevOptionalRole = null;
|
|
1760
|
+
continue;
|
|
1761
|
+
}
|
|
1762
|
+
if (this.isOptional(patternToken)) {
|
|
1763
|
+
continue;
|
|
1764
|
+
}
|
|
1765
|
+
return false;
|
|
1766
|
+
}
|
|
1707
1767
|
const isOptionalRole = patternToken.type === "role" && patternToken.optional === true;
|
|
1708
1768
|
const markBefore = isOptionalRole ? tokens.mark() : null;
|
|
1769
|
+
const isRoleToken = patternToken.type === "role";
|
|
1770
|
+
if (isRoleToken && !isOptionalRole) {
|
|
1771
|
+
const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
|
|
1772
|
+
if (nextPT?.type === "literal") {
|
|
1773
|
+
const currentToken = tokens.peek();
|
|
1774
|
+
if (currentToken && this.getMatchType(currentToken, nextPT.value) !== "none") {
|
|
1775
|
+
this.logger.debug(
|
|
1776
|
+
" >> MARKER-STEAL GUARD: current token matches next literal, skipping role"
|
|
1777
|
+
);
|
|
1778
|
+
this.logger.debug(" >> Token match FAILED");
|
|
1779
|
+
if (this.isOptional(patternToken)) {
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
if (prevOptionalMark && prevOptionalRole) {
|
|
1783
|
+
this.logger.debug(" >> BACKTRACKING: undoing optional role", prevOptionalRole);
|
|
1784
|
+
tokens.reset(prevOptionalMark);
|
|
1785
|
+
captured.delete(prevOptionalRole);
|
|
1786
|
+
prevOptionalMark = null;
|
|
1787
|
+
prevOptionalRole = null;
|
|
1788
|
+
const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
|
|
1789
|
+
this.logger.debug(" >> Backtrack retry result:", retryMatched);
|
|
1790
|
+
if (retryMatched) {
|
|
1791
|
+
continue;
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
return false;
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1709
1798
|
const matched = this.matchPatternToken(tokens, patternToken, captured);
|
|
1710
1799
|
this.logger.debug(" >> Match result:", matched);
|
|
1711
1800
|
if (matched) {
|
|
@@ -1826,7 +1915,7 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1826
1915
|
const possessiveValue = this.tryMatchPossessiveExpression(tokens);
|
|
1827
1916
|
if (possessiveValue) {
|
|
1828
1917
|
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
1829
|
-
if (!
|
|
1918
|
+
if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
|
|
1830
1919
|
return patternToken.optional || false;
|
|
1831
1920
|
}
|
|
1832
1921
|
}
|
|
@@ -1836,7 +1925,7 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1836
1925
|
const methodCallValue = this.tryMatchMethodCallExpression(tokens);
|
|
1837
1926
|
if (methodCallValue) {
|
|
1838
1927
|
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
1839
|
-
if (!
|
|
1928
|
+
if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
|
|
1840
1929
|
return patternToken.optional || false;
|
|
1841
1930
|
}
|
|
1842
1931
|
}
|
|
@@ -1856,7 +1945,7 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
1856
1945
|
const propertyAccessValue = this.tryMatchPropertyAccessExpression(tokens);
|
|
1857
1946
|
if (propertyAccessValue) {
|
|
1858
1947
|
if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
|
|
1859
|
-
if (!
|
|
1948
|
+
if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
|
|
1860
1949
|
return patternToken.optional || false;
|
|
1861
1950
|
}
|
|
1862
1951
|
}
|
|
@@ -2176,10 +2265,15 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
2176
2265
|
* (e.g., a value like 'hello') to sit between groups without blocking later
|
|
2177
2266
|
* marker matches.
|
|
2178
2267
|
*/
|
|
2179
|
-
matchGroupToken(tokens, patternToken, captured) {
|
|
2268
|
+
matchGroupToken(tokens, patternToken, captured, parentStopMarkers) {
|
|
2180
2269
|
const mark = tokens.mark();
|
|
2181
2270
|
const capturedBefore = new Set(captured.keys());
|
|
2182
|
-
const success = this.matchTokenSequence(
|
|
2271
|
+
const success = this.matchTokenSequence(
|
|
2272
|
+
tokens,
|
|
2273
|
+
patternToken.tokens,
|
|
2274
|
+
captured,
|
|
2275
|
+
parentStopMarkers
|
|
2276
|
+
);
|
|
2183
2277
|
if (success) return true;
|
|
2184
2278
|
tokens.reset(mark);
|
|
2185
2279
|
for (const role of captured.keys()) {
|
|
@@ -2201,7 +2295,12 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
2201
2295
|
"- skipping intervening tokens"
|
|
2202
2296
|
);
|
|
2203
2297
|
for (let s = 0; s < offset; s++) tokens.advance();
|
|
2204
|
-
const retrySuccess = this.matchTokenSequence(
|
|
2298
|
+
const retrySuccess = this.matchTokenSequence(
|
|
2299
|
+
tokens,
|
|
2300
|
+
patternToken.tokens,
|
|
2301
|
+
captured,
|
|
2302
|
+
parentStopMarkers
|
|
2303
|
+
);
|
|
2205
2304
|
if (retrySuccess) return true;
|
|
2206
2305
|
tokens.reset(mark);
|
|
2207
2306
|
for (const role of captured.keys()) {
|
|
@@ -2268,7 +2367,7 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
2268
2367
|
break;
|
|
2269
2368
|
}
|
|
2270
2369
|
}
|
|
2271
|
-
break;
|
|
2370
|
+
if (!pt.optional) break;
|
|
2272
2371
|
}
|
|
2273
2372
|
}
|
|
2274
2373
|
return markers;
|
|
@@ -2278,8 +2377,9 @@ var _PatternMatcher = class _PatternMatcher {
|
|
|
2278
2377
|
*/
|
|
2279
2378
|
isStopMarker(token, stopMarkers) {
|
|
2280
2379
|
if (stopMarkers.size === 0) return false;
|
|
2281
|
-
|
|
2282
|
-
|
|
2380
|
+
if (stopMarkers.has(token.value.toLowerCase())) return true;
|
|
2381
|
+
if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
|
|
2382
|
+
return false;
|
|
2283
2383
|
}
|
|
2284
2384
|
/**
|
|
2285
2385
|
* Convert a language token to a semantic value.
|
|
@@ -2621,20 +2721,24 @@ export {
|
|
|
2621
2721
|
PunctuationExtractor,
|
|
2622
2722
|
TokenStreamImpl,
|
|
2623
2723
|
combineClassifiers,
|
|
2724
|
+
createAsyncNode,
|
|
2624
2725
|
createCommandNode,
|
|
2625
2726
|
createCompoundNode,
|
|
2626
2727
|
createConditionalNode,
|
|
2627
2728
|
createEventHandlerNode,
|
|
2628
2729
|
createExpression,
|
|
2730
|
+
createFlag,
|
|
2629
2731
|
createLatinCharClassifiers,
|
|
2630
2732
|
createLiteral,
|
|
2631
2733
|
createLoopNode,
|
|
2734
|
+
createMatchNode,
|
|
2632
2735
|
createPosition,
|
|
2633
2736
|
createPropertyPath,
|
|
2634
2737
|
createReference,
|
|
2635
2738
|
createSelector,
|
|
2636
2739
|
createSimpleTokenizer,
|
|
2637
2740
|
createToken,
|
|
2741
|
+
createTryNode,
|
|
2638
2742
|
createUnicodeRangeClassifier,
|
|
2639
2743
|
extractCssSelector,
|
|
2640
2744
|
extractNumber,
|