@lokascript/framework 2.7.2 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +393 -0
- package/dist/api/create-dsl.d.ts +93 -1
- package/dist/api/create-dsl.d.ts.map +1 -1
- package/dist/api/domain-registry.d.ts +5 -3
- package/dist/api/domain-registry.d.ts.map +1 -1
- package/dist/api/index.js +238 -20
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.js +67 -7
- package/dist/core/index.js.map +1 -1
- package/dist/core/tokenization/base-tokenizer.d.ts +39 -3
- package/dist/core/tokenization/base-tokenizer.d.ts.map +1 -1
- package/dist/core/tokenization/extractors.d.ts +6 -0
- package/dist/core/tokenization/extractors.d.ts.map +1 -1
- package/dist/core/tokenization/index.js +67 -7
- package/dist/core/tokenization/index.js.map +1 -1
- package/dist/core/tokenization/token-utils.d.ts +15 -0
- package/dist/core/tokenization/token-utils.d.ts.map +1 -1
- package/dist/generation/index.js +75 -48
- package/dist/generation/index.js.map +1 -1
- package/dist/generation/pattern-generator.d.ts +8 -1
- package/dist/generation/pattern-generator.d.ts.map +1 -1
- package/dist/generation/renderer.d.ts +53 -1
- package/dist/generation/renderer.d.ts.map +1 -1
- package/dist/index.cjs +349 -118
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +345 -118
- package/dist/index.js.map +1 -1
- package/dist/interfaces/value-extractor.d.ts +5 -0
- package/dist/interfaces/value-extractor.d.ts.map +1 -1
- package/dist/multilingual/index.js +66 -7
- package/dist/multilingual/index.js.map +1 -1
- package/dist/testing/index.js +4 -12
- package/dist/testing/index.js.map +1 -1
- package/package.json +4 -3
- package/src/api/create-dsl.test.ts +11 -0
- package/src/api/create-dsl.ts +278 -9
- package/src/api/domain-registry.ts +15 -10
- package/src/api/extensions.test.ts +322 -0
- package/src/core/tokenization/base-tokenizer.ts +78 -9
- package/src/core/tokenization/colon-qualifier.test.ts +129 -0
- package/src/core/tokenization/css-selector-extractor.test.ts +67 -0
- package/src/core/tokenization/extractors.ts +6 -0
- package/src/core/tokenization/token-utils.ts +18 -0
- package/src/generation/domain-renderer.test.ts +172 -0
- package/src/generation/pattern-generator.test.ts +102 -0
- package/src/generation/pattern-generator.ts +32 -20
- package/src/generation/renderer.test.ts +243 -4
- package/src/generation/renderer.ts +188 -45
- package/src/index.ts +9 -1
- package/src/interfaces/value-extractor.ts +50 -0
- package/src/ir/protocol-json.test.ts +21 -0
- package/src/ir/references.test.ts +5 -2
- package/src/prompts/prompt-generator.ts +4 -1
package/dist/index.d.ts
CHANGED
|
@@ -33,12 +33,12 @@ export type { GrammarProfileSlice, DomainVocabulary, DomainKeywordTranslation, D
|
|
|
33
33
|
export * from './core';
|
|
34
34
|
export type { ActionType, SemanticRole as SemanticRoleType, SemanticValue, SemanticNode, CommandSemanticNode, EventHandlerSemanticNode, ConditionalSemanticNode, CompoundSemanticNode, LoopSemanticNode, LiteralValue, SelectorValue, ReferenceValue, PropertyPathValue, ExpressionValue, SemanticMetadata, SourcePosition, LanguageToken, TokenStream, LanguageTokenizer, LanguagePattern, Annotation, ProtocolDiagnostic, AsyncVariant, MatchArm, LSEEnvelope, } from './core/types';
|
|
35
35
|
export type { CommandSchema, RoleSpec } from './schema';
|
|
36
|
-
export type { NaturalLanguageRenderer, KeywordTable, MarkerTable, RendererConfig, } from './generation/renderer';
|
|
37
|
-
export { lookupKeyword, lookupMarker, buildPhrase, buildTablesFromProfiles, detectWordOrders, createSchemaRenderer, } from './generation/renderer';
|
|
36
|
+
export type { NaturalLanguageRenderer, KeywordTable, MarkerTable, RendererConfig, DomainRenderFn, DomainRendererConfig, } from './generation/renderer';
|
|
37
|
+
export { lookupKeyword, lookupMarker, buildPhrase, buildTablesFromProfiles, detectWordOrders, createSchemaRenderer, createDomainRenderer, } from './generation/renderer';
|
|
38
38
|
export type { DiagnosticSeverity, Diagnostic, DiagnosticResult, DiagnosticSummary, DiagnosticCollector, DiagnosticOptions, } from './generation/diagnostics';
|
|
39
39
|
export { createDiagnosticCollector, fromError, filterBySeverity } from './generation/diagnostics';
|
|
40
|
-
export type { LanguageConfig, DSLConfig, MultilingualDSL, CodeGenerator, ValidationResult, CompileResult, DomainDescriptor, MCPToolDefinition, MCPToolResponse, DispatchResult, CompositeParseResult, CompositeStatement, CompositeError, DispatcherOptions, } from './api';
|
|
41
|
-
export { DomainRegistry, CrossDomainDispatcher } from './api';
|
|
40
|
+
export type { LanguageConfig, DSLConfig, MultilingualDSL, CodeGenerator, DomainExtension, ExtensionVocabulary, ValidationResult, CompileResult, DomainDescriptor, MCPToolDefinition, MCPToolResponse, DispatchResult, CompositeParseResult, CompositeStatement, CompositeError, DispatcherOptions, } from './api';
|
|
41
|
+
export { createMultilingualDSL, DomainRegistry, CrossDomainDispatcher } from './api';
|
|
42
42
|
export { createLiteral, createSelector, createReference, createPropertyPath, createExpression, createCommandNode, createEventHandlerNode, createConditionalNode, createCompoundNode, createLoopNode, extractValue, extractRoleValue, createTryNode, createAsyncNode, createMatchNode, } from './core/types';
|
|
43
43
|
export { defineCommand, defineRole } from './schema';
|
|
44
44
|
export { createSimpleTokenizer } from './core/tokenization/base-tokenizer';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,cAAc,OAAO,CAAC;AAGtB,cAAc,OAAO,CAAC;AAGtB,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAG7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EACV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,iBAAiB,EACjB,cAAc,GACf,MAAM,iBAAiB,CAAC;AAIzB,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,EACjB,SAAS,EACT,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAGxB,cAAc,QAAQ,CAAC;AAGvB,YAAY,EACV,UAAU,EACV,YAAY,IAAI,gBAAgB,EAChC,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,eAAe,EAEf,UAAU,EACV,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGxD,YAAY,EACV,uBAAuB,EACvB,YAAY,EACZ,WAAW,EACX,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,cAAc,OAAO,CAAC;AAGtB,cAAc,OAAO,CAAC;AAGtB,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAG7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EACV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,iBAAiB,EACjB,cAAc,GACf,MAAM,iBAAiB,CAAC;AAIzB,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,EACjB,SAAS,EACT,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAGxB,cAAc,QAAQ,CAAC;AAGvB,YAAY,EACV,UAAU,EACV,YAAY,IAAI,gBAAgB,EAChC,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,eAAe,EAEf,UAAU,EACV,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGxD,YAAY,EACV,uBAAuB,EACvB,YAAY,EACZ,WAAW,EACX,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAG/B,YAAY,EACV,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,yBAAyB,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAElG,YAAY,EACV,cAAc,EACd,SAAS,EACT,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,iBAAiB,GAClB,MAAM,OAAO,CAAC;AAKf,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAGrF,OAAO,EACL,aAAa,EACb,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAEhB,aAAa,EACb,eAAe,EACf,eAAe,GAChB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,YAAY,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAGhF,OAAO,EAAE,2BAA2B,EAAE,MAAM,gDAAgD,CAAC;AAC7F,YAAY,EACV,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,gDAAgD,CAAC;AAGxD,cAAc,MAAM,CAAC;AAGrB,cAAc,WAAW,CAAC;AAG1B,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AACzF,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,cAAc,EACd,WAAW,EACX,WAAW,EACX,UAAU,EACV,aAAa,GACd,MAAM,2BAA2B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1330,10 +1330,12 @@ function buildTokens(schema, profile, keyword, alternatives) {
|
|
|
1330
1330
|
addRoleWithMarker(tokens, role, profile);
|
|
1331
1331
|
}
|
|
1332
1332
|
} else if (profile.wordOrder === "SOV") {
|
|
1333
|
+
const preVerb = [];
|
|
1334
|
+
const postVerb = [];
|
|
1333
1335
|
for (const role of sortedRoles) {
|
|
1334
|
-
addRoleWithMarker(
|
|
1336
|
+
addRoleWithMarker(role.sovSlot === "postVerb" ? postVerb : preVerb, role, profile);
|
|
1335
1337
|
}
|
|
1336
|
-
tokens.push(keywordToken);
|
|
1338
|
+
tokens.push(...preVerb, keywordToken, ...postVerb);
|
|
1337
1339
|
} else if (profile.wordOrder === "VSO") {
|
|
1338
1340
|
tokens.push(keywordToken);
|
|
1339
1341
|
for (const role of sortedRoles) {
|
|
@@ -1416,28 +1418,28 @@ function buildExtractionRules(schema, profile) {
|
|
|
1416
1418
|
return rules;
|
|
1417
1419
|
}
|
|
1418
1420
|
function buildFormatString(schema, profile, keyword) {
|
|
1419
|
-
const
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
for (const role of
|
|
1421
|
+
const isSOV = profile.wordOrder === "SOV";
|
|
1422
|
+
const preVerb = [];
|
|
1423
|
+
const postVerb = [];
|
|
1424
|
+
const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
|
|
1425
|
+
for (const role of sortedRoles) {
|
|
1424
1426
|
const marker = getMarkerForRole(role, profile);
|
|
1425
1427
|
const roleName = `{${role.role}}`;
|
|
1428
|
+
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
1426
1429
|
if (marker) {
|
|
1427
1430
|
const markerInfo = profile.roleMarkers?.[role.role];
|
|
1428
1431
|
if (markerInfo?.position === "after") {
|
|
1429
|
-
|
|
1432
|
+
bucket.push(`${roleName} ${marker}`);
|
|
1430
1433
|
} else {
|
|
1431
|
-
|
|
1434
|
+
bucket.push(`${marker} ${roleName}`);
|
|
1432
1435
|
}
|
|
1433
1436
|
} else {
|
|
1434
|
-
|
|
1437
|
+
bucket.push(roleName);
|
|
1435
1438
|
}
|
|
1436
1439
|
}
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
return parts.join(" ");
|
|
1440
|
+
return (isSOV ? [...preVerb, keyword, ...postVerb] : [keyword, ...preVerb, ...postVerb]).join(
|
|
1441
|
+
" "
|
|
1442
|
+
);
|
|
1441
1443
|
}
|
|
1442
1444
|
function generatePatternVariants(schema, profile, config = defaultConfig) {
|
|
1443
1445
|
const patterns = [];
|
|
@@ -1445,6 +1447,119 @@ function generatePatternVariants(schema, profile, config = defaultConfig) {
|
|
|
1445
1447
|
return patterns;
|
|
1446
1448
|
}
|
|
1447
1449
|
|
|
1450
|
+
// src/generation/renderer.ts
|
|
1451
|
+
function lookupKeyword(keywords, action, language) {
|
|
1452
|
+
return keywords[action]?.[language] ?? action;
|
|
1453
|
+
}
|
|
1454
|
+
function lookupMarker(markers, marker, language) {
|
|
1455
|
+
return markers[marker]?.[language] ?? marker;
|
|
1456
|
+
}
|
|
1457
|
+
function buildPhrase(...parts) {
|
|
1458
|
+
return parts.filter(Boolean).join(" ");
|
|
1459
|
+
}
|
|
1460
|
+
function buildTablesFromProfiles(schemas, profiles) {
|
|
1461
|
+
const keywords = {};
|
|
1462
|
+
const markers = {};
|
|
1463
|
+
for (const profile of profiles) {
|
|
1464
|
+
for (const [action, kw] of Object.entries(profile.keywords)) {
|
|
1465
|
+
if (!keywords[action]) keywords[action] = {};
|
|
1466
|
+
keywords[action][profile.code] = kw.primary;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
for (const schema of schemas) {
|
|
1470
|
+
for (const role of schema.roles) {
|
|
1471
|
+
if (role.markerOverride) {
|
|
1472
|
+
const markerKey = role.role;
|
|
1473
|
+
if (!markers[markerKey]) markers[markerKey] = {};
|
|
1474
|
+
for (const [lang, markerText] of Object.entries(role.markerOverride)) {
|
|
1475
|
+
markers[markerKey][lang] = markerText;
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
for (const profile of profiles) {
|
|
1481
|
+
if (profile.roleMarkers) {
|
|
1482
|
+
for (const [markerKey, markerDef] of Object.entries(profile.roleMarkers)) {
|
|
1483
|
+
if (!markers[markerKey]) markers[markerKey] = {};
|
|
1484
|
+
markers[markerKey][profile.code] = markerDef.primary;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
return { keywords, markers };
|
|
1489
|
+
}
|
|
1490
|
+
function detectWordOrders(profiles) {
|
|
1491
|
+
const sovLanguages = /* @__PURE__ */ new Set();
|
|
1492
|
+
const vsoLanguages = /* @__PURE__ */ new Set();
|
|
1493
|
+
for (const profile of profiles) {
|
|
1494
|
+
if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
|
|
1495
|
+
if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
|
|
1496
|
+
}
|
|
1497
|
+
return { sovLanguages, vsoLanguages };
|
|
1498
|
+
}
|
|
1499
|
+
function createSchemaRenderer(schemas, profiles) {
|
|
1500
|
+
const tables = buildRenderTables(schemas, profiles);
|
|
1501
|
+
return {
|
|
1502
|
+
render(node, language) {
|
|
1503
|
+
return renderFromSchema(node, language, tables) ?? node.action;
|
|
1504
|
+
}
|
|
1505
|
+
};
|
|
1506
|
+
}
|
|
1507
|
+
function createDomainRenderer(config) {
|
|
1508
|
+
const { schemas, profiles, overrides } = config;
|
|
1509
|
+
let tables;
|
|
1510
|
+
return (node, language) => {
|
|
1511
|
+
const override = overrides?.[node.action];
|
|
1512
|
+
if (override) return override(node, language);
|
|
1513
|
+
tables ?? (tables = buildRenderTables(schemas, profiles));
|
|
1514
|
+
return renderFromSchema(node, language, tables);
|
|
1515
|
+
};
|
|
1516
|
+
}
|
|
1517
|
+
function buildRenderTables(schemas, profiles) {
|
|
1518
|
+
const { keywords, markers } = buildTablesFromProfiles(schemas, profiles);
|
|
1519
|
+
const { sovLanguages } = detectWordOrders(profiles);
|
|
1520
|
+
const markerPositions = {};
|
|
1521
|
+
for (const profile of profiles) {
|
|
1522
|
+
for (const [role, markerDef] of Object.entries(profile.roleMarkers ?? {})) {
|
|
1523
|
+
if (!markerDef.position) continue;
|
|
1524
|
+
markerPositions[role] ?? (markerPositions[role] = {});
|
|
1525
|
+
markerPositions[role][profile.code] = markerDef.position;
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
const schemaMap = /* @__PURE__ */ new Map();
|
|
1529
|
+
for (const s of schemas) schemaMap.set(s.action, s);
|
|
1530
|
+
return { keywords, markers, markerPositions, sovLanguages, schemaMap };
|
|
1531
|
+
}
|
|
1532
|
+
function renderFromSchema(node, language, tables) {
|
|
1533
|
+
const schema = tables.schemaMap.get(node.action);
|
|
1534
|
+
if (!schema) return null;
|
|
1535
|
+
const keyword = lookupKeyword(tables.keywords, node.action, language);
|
|
1536
|
+
const isSOV = tables.sovLanguages.has(language);
|
|
1537
|
+
const ordered = sortRolesByWordOrder([...schema.roles], isSOV ? "SOV" : "SVO");
|
|
1538
|
+
const preVerb = [];
|
|
1539
|
+
const postVerb = [];
|
|
1540
|
+
for (const role of ordered) {
|
|
1541
|
+
let value = extractRoleValue(node, role.role);
|
|
1542
|
+
if (!value && role.default !== void 0) {
|
|
1543
|
+
value = String(extractValue(role.default));
|
|
1544
|
+
}
|
|
1545
|
+
if (!value) continue;
|
|
1546
|
+
if (role.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
|
|
1547
|
+
value = `"${value}"`;
|
|
1548
|
+
}
|
|
1549
|
+
const markerText = role.renderOverride?.[language] ?? role.markerOverride?.[language] ?? role.renderOverride?.["*"] ?? tables.markers[role.role]?.[language];
|
|
1550
|
+
const markerPosition = role.markerPositionOverride?.[language] ?? role.markerPosition ?? tables.markerPositions[role.role]?.[language] ?? (isSOV ? "after" : "before");
|
|
1551
|
+
const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
|
|
1552
|
+
if (markerText && markerPosition === "before") {
|
|
1553
|
+
bucket.push(markerText, value);
|
|
1554
|
+
} else if (markerText) {
|
|
1555
|
+
bucket.push(value, markerText);
|
|
1556
|
+
} else {
|
|
1557
|
+
bucket.push(value);
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
return isSOV ? buildPhrase(...preVerb, keyword, ...postVerb) : buildPhrase(keyword, ...preVerb, ...postVerb);
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1448
1563
|
// src/grammar/types.ts
|
|
1449
1564
|
function reorderRoles(roles, targetOrder) {
|
|
1450
1565
|
const result = [];
|
|
@@ -1881,6 +1996,31 @@ var LatinExtendedIdentifierExtractor = class {
|
|
|
1881
1996
|
return { value: input.slice(position, end), length: end - position };
|
|
1882
1997
|
}
|
|
1883
1998
|
};
|
|
1999
|
+
var SELECTOR_BODY_CHAR = /[\p{L}\p{N}_-]/u;
|
|
2000
|
+
var PARTICLE_SCRIPT_CHAR = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}]/u;
|
|
2001
|
+
function isSelectorBodyChar(char) {
|
|
2002
|
+
return SELECTOR_BODY_CHAR.test(char) && !PARTICLE_SCRIPT_CHAR.test(char);
|
|
2003
|
+
}
|
|
2004
|
+
var CssSelectorExtractor = class {
|
|
2005
|
+
constructor() {
|
|
2006
|
+
this.name = "css-selector";
|
|
2007
|
+
}
|
|
2008
|
+
canExtract(input, position) {
|
|
2009
|
+
const char = input[position];
|
|
2010
|
+
if (char !== "#" && char !== ".") return false;
|
|
2011
|
+
const next = input[position + 1];
|
|
2012
|
+
if (next === void 0) return false;
|
|
2013
|
+
return next === "_" || next === "-" || /\p{L}/u.test(next) && !PARTICLE_SCRIPT_CHAR.test(next);
|
|
2014
|
+
}
|
|
2015
|
+
extract(input, position) {
|
|
2016
|
+
let end = position + 1;
|
|
2017
|
+
while (end < input.length && isSelectorBodyChar(input[end])) {
|
|
2018
|
+
end++;
|
|
2019
|
+
}
|
|
2020
|
+
if (end === position + 1) return null;
|
|
2021
|
+
return { value: input.slice(position, end), length: end - position };
|
|
2022
|
+
}
|
|
2023
|
+
};
|
|
1884
2024
|
var WhitespaceExtractor = class {
|
|
1885
2025
|
constructor() {
|
|
1886
2026
|
this.name = "whitespace";
|
|
@@ -2635,10 +2775,17 @@ function semanticValueToAST(value) {
|
|
|
2635
2775
|
|
|
2636
2776
|
// src/api/create-dsl.ts
|
|
2637
2777
|
var DSLRegistry = class {
|
|
2638
|
-
|
|
2778
|
+
/**
|
|
2779
|
+
* @param partiallyTranslatedActions Actions permitted to lack a keyword in
|
|
2780
|
+
* some configured languages — extension commands, which may be added for a
|
|
2781
|
+
* subset of the DSL's languages. A built-in command missing a keyword stays
|
|
2782
|
+
* an error, since that is a domain-authoring mistake.
|
|
2783
|
+
*/
|
|
2784
|
+
constructor(config, partiallyTranslatedActions = /* @__PURE__ */ new Set()) {
|
|
2639
2785
|
this.patterns = /* @__PURE__ */ new Map();
|
|
2640
2786
|
this.tokenizers = /* @__PURE__ */ new Map();
|
|
2641
2787
|
this.schemas = config.schemas;
|
|
2788
|
+
this.partiallyTranslatedActions = partiallyTranslatedActions;
|
|
2642
2789
|
for (const lang of config.languages) {
|
|
2643
2790
|
this.registerLanguage(lang);
|
|
2644
2791
|
}
|
|
@@ -2647,6 +2794,9 @@ var DSLRegistry = class {
|
|
|
2647
2794
|
this.tokenizers.set(lang.code, lang.tokenizer);
|
|
2648
2795
|
const patterns = [];
|
|
2649
2796
|
for (const schema of this.schemas) {
|
|
2797
|
+
if (this.partiallyTranslatedActions.has(schema.action) && !lang.patternProfile.keywords[schema.action]) {
|
|
2798
|
+
continue;
|
|
2799
|
+
}
|
|
2650
2800
|
const pattern = generatePattern(schema, lang.patternProfile);
|
|
2651
2801
|
patterns.push(pattern);
|
|
2652
2802
|
}
|
|
@@ -2663,13 +2813,25 @@ var DSLRegistry = class {
|
|
|
2663
2813
|
}
|
|
2664
2814
|
};
|
|
2665
2815
|
var MultilingualDSLImpl = class {
|
|
2666
|
-
constructor(config, registry, transformer) {
|
|
2816
|
+
constructor(config, registry, transformer, profileProvider) {
|
|
2667
2817
|
this.registry = registry;
|
|
2668
2818
|
this.matcher = new PatternMatcher();
|
|
2669
2819
|
this.transformer = transformer;
|
|
2820
|
+
this.profileProvider = profileProvider;
|
|
2670
2821
|
if (config.codeGenerator) {
|
|
2671
2822
|
this.codeGenerator = config.codeGenerator;
|
|
2672
2823
|
}
|
|
2824
|
+
if (config.renderer) {
|
|
2825
|
+
this.domainRenderer = config.renderer;
|
|
2826
|
+
}
|
|
2827
|
+
this.extensionRenderers = /* @__PURE__ */ new Map();
|
|
2828
|
+
for (const extension of config.extensions ?? []) {
|
|
2829
|
+
if (extension.render) this.extensionRenderers.set(extension.schema.action, extension.render);
|
|
2830
|
+
}
|
|
2831
|
+
this.schemaFallbackRenderer = createDomainRenderer({
|
|
2832
|
+
schemas: config.schemas,
|
|
2833
|
+
profiles: config.languages.map((l) => l.patternProfile)
|
|
2834
|
+
});
|
|
2673
2835
|
const schemaMap = new Map(config.schemas.map((s) => [s.action, s]));
|
|
2674
2836
|
this.schemaLookup = {
|
|
2675
2837
|
getSchema: (action) => schemaMap.get(action)
|
|
@@ -2731,6 +2893,13 @@ var MultilingualDSLImpl = class {
|
|
|
2731
2893
|
}
|
|
2732
2894
|
translate(input, fromLanguage, toLanguage) {
|
|
2733
2895
|
if (isExplicitSyntax(input)) return input;
|
|
2896
|
+
for (const language of [fromLanguage, toLanguage]) {
|
|
2897
|
+
if (!this.profileProvider.getProfile(language)) {
|
|
2898
|
+
throw new Error(
|
|
2899
|
+
`translate() requires a grammar profile for language "${language}", but none is configured. Set 'grammarProfile' on the LanguageConfig for "${language}" in createMultilingualDSL() (parse/validate/compile do not need it), or inject a custom 'profileProvider'.`
|
|
2900
|
+
);
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2734
2903
|
return this.transformer.transform(input, fromLanguage, toLanguage);
|
|
2735
2904
|
}
|
|
2736
2905
|
compile(input, language) {
|
|
@@ -2759,6 +2928,18 @@ var MultilingualDSLImpl = class {
|
|
|
2759
2928
|
};
|
|
2760
2929
|
}
|
|
2761
2930
|
}
|
|
2931
|
+
render(node, language) {
|
|
2932
|
+
const extensionRenderer = this.extensionRenderers.get(node.action);
|
|
2933
|
+
if (extensionRenderer) {
|
|
2934
|
+
const rendered = extensionRenderer(node, language);
|
|
2935
|
+
if (rendered != null) return rendered;
|
|
2936
|
+
}
|
|
2937
|
+
if (this.domainRenderer) {
|
|
2938
|
+
const rendered = this.domainRenderer(node, language);
|
|
2939
|
+
if (rendered != null) return rendered;
|
|
2940
|
+
}
|
|
2941
|
+
return this.schemaFallbackRenderer(node, language);
|
|
2942
|
+
}
|
|
2762
2943
|
getSupportedLanguages() {
|
|
2763
2944
|
return this.registry.getSupportedLanguages();
|
|
2764
2945
|
}
|
|
@@ -2783,15 +2964,80 @@ function createDefaultProfileProvider(config) {
|
|
|
2783
2964
|
}
|
|
2784
2965
|
return new InMemoryProfileProvider(profiles);
|
|
2785
2966
|
}
|
|
2967
|
+
function applyExtensions(config) {
|
|
2968
|
+
const extensions = config.extensions;
|
|
2969
|
+
if (!extensions || extensions.length === 0) return config;
|
|
2970
|
+
const configuredLanguages = new Set(config.languages.map((l) => l.code));
|
|
2971
|
+
const actions = new Set(config.schemas.map((s) => s.action));
|
|
2972
|
+
for (const extension of extensions) {
|
|
2973
|
+
const { action } = extension.schema;
|
|
2974
|
+
if (actions.has(action)) {
|
|
2975
|
+
throw new Error(
|
|
2976
|
+
`Extension command "${action}" collides with a command this DSL already defines. Pick a different action name.`
|
|
2977
|
+
);
|
|
2978
|
+
}
|
|
2979
|
+
actions.add(action);
|
|
2980
|
+
for (const code of Object.keys(extension.vocabulary)) {
|
|
2981
|
+
if (!configuredLanguages.has(code)) {
|
|
2982
|
+
throw new Error(
|
|
2983
|
+
`Extension command "${action}" supplies vocabulary for language "${code}", which is not configured on this DSL. Configured languages: ${[...configuredLanguages].join(", ")}.`
|
|
2984
|
+
);
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
}
|
|
2988
|
+
const schemas = [...config.schemas, ...extensions.map((e) => e.schema)];
|
|
2989
|
+
const languages = config.languages.map((lang) => {
|
|
2990
|
+
const keywords = { ...lang.patternProfile.keywords };
|
|
2991
|
+
let roleMarkers = lang.patternProfile.roleMarkers;
|
|
2992
|
+
for (const extension of extensions) {
|
|
2993
|
+
const vocabulary = extension.vocabulary[lang.code];
|
|
2994
|
+
if (!vocabulary) continue;
|
|
2995
|
+
keywords[extension.schema.action] = { ...vocabulary.keyword };
|
|
2996
|
+
if (vocabulary.roleMarkers) {
|
|
2997
|
+
roleMarkers = { ...roleMarkers, ...vocabulary.roleMarkers };
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
return {
|
|
3001
|
+
...lang,
|
|
3002
|
+
patternProfile: {
|
|
3003
|
+
...lang.patternProfile,
|
|
3004
|
+
keywords,
|
|
3005
|
+
...roleMarkers !== void 0 && { roleMarkers }
|
|
3006
|
+
}
|
|
3007
|
+
};
|
|
3008
|
+
});
|
|
3009
|
+
const extensionGenerators = new Map(
|
|
3010
|
+
extensions.filter((e) => e.generate).map((e) => [e.schema.action, e.generate])
|
|
3011
|
+
);
|
|
3012
|
+
const baseGenerator = config.codeGenerator;
|
|
3013
|
+
const codeGenerator = extensionGenerators.size > 0 ? {
|
|
3014
|
+
generate(node) {
|
|
3015
|
+
const generate = extensionGenerators.get(node.action);
|
|
3016
|
+
if (generate) return generate(node);
|
|
3017
|
+
if (!baseGenerator) {
|
|
3018
|
+
throw new Error(`No code generator for action "${node.action}"`);
|
|
3019
|
+
}
|
|
3020
|
+
return baseGenerator.generate(node);
|
|
3021
|
+
}
|
|
3022
|
+
} : baseGenerator;
|
|
3023
|
+
return {
|
|
3024
|
+
...config,
|
|
3025
|
+
schemas,
|
|
3026
|
+
languages,
|
|
3027
|
+
...codeGenerator !== void 0 && { codeGenerator }
|
|
3028
|
+
};
|
|
3029
|
+
}
|
|
2786
3030
|
function createMultilingualDSL(config) {
|
|
2787
|
-
const
|
|
2788
|
-
const
|
|
3031
|
+
const effectiveConfig = applyExtensions(config);
|
|
3032
|
+
const extensionActions = new Set((config.extensions ?? []).map((e) => e.schema.action));
|
|
3033
|
+
const dictionary = effectiveConfig.dictionary ?? createDefaultDictionary(effectiveConfig);
|
|
3034
|
+
const profileProvider = effectiveConfig.profileProvider ?? createDefaultProfileProvider(effectiveConfig);
|
|
2789
3035
|
const transformer = new GrammarTransformer({
|
|
2790
3036
|
dictionary,
|
|
2791
3037
|
profileProvider
|
|
2792
3038
|
});
|
|
2793
|
-
const registry = new DSLRegistry(
|
|
2794
|
-
return new MultilingualDSLImpl(
|
|
3039
|
+
const registry = new DSLRegistry(effectiveConfig, extensionActions);
|
|
3040
|
+
return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider);
|
|
2795
3041
|
}
|
|
2796
3042
|
|
|
2797
3043
|
// src/prompts/prompt-generator.ts
|
|
@@ -2878,7 +3124,9 @@ function buildProtocolSection() {
|
|
|
2878
3124
|
3. No spaces around the colon in role:value pairs
|
|
2879
3125
|
4. Strings with spaces must be quoted: \`patient:"hello world"\`
|
|
2880
3126
|
5. Selectors start with \`#\`, \`.\`, \`[\`, \`@\`, or \`*\`
|
|
2881
|
-
6.
|
|
3127
|
+
6. A selector containing a space, a combinator (\`>\` \`+\` \`~\`), or a comma must use a selector literal: \`patient:<ul > li/>\`, \`patient:<.a, .b/>\`
|
|
3128
|
+
7. Inside a structural role (\`body\`, \`then\`, \`else\`, \`condition\`, \`loop-body\`, \`variable\`, \`catch\`, \`finally\`) a \`[...]\` value is always a nested command. Write an attribute selector there as \`condition:<[data-active]/>\`
|
|
3129
|
+
8. Output must be valid bracket syntax: \`[action role:value ...]\``;
|
|
2882
3130
|
return {
|
|
2883
3131
|
id: "protocol",
|
|
2884
3132
|
title: "LSE Protocol",
|
|
@@ -2891,6 +3139,7 @@ function buildValueTypeSection() {
|
|
|
2891
3139
|
|
|
2892
3140
|
| Type | Syntax | Example |
|
|
2893
3141
|
|------|--------|---------|
|
|
3142
|
+
| Selector literal | Delimited by \`<\` and \`/>\` | \`<ul > li/>\`, \`<.a, .b/>\`, \`<[data-id]/>\` |
|
|
2894
3143
|
| Selector | Starts with \`#\` \`.\` \`[\` \`@\` \`*\` | \`#button\`, \`.active\`, \`[data-id]\` |
|
|
2895
3144
|
| String | Quoted with \`"\` or \`'\` | \`"hello world"\`, \`'json'\` |
|
|
2896
3145
|
| Boolean | Exact: \`true\` / \`false\` | \`visible:true\` |
|
|
@@ -3759,6 +4008,11 @@ var DomainRegistry = class {
|
|
|
3759
4008
|
rendered = typeof renderer === "function" ? renderer(node, to) : renderer.render(node, to);
|
|
3760
4009
|
} catch {
|
|
3761
4010
|
}
|
|
4011
|
+
} else if (dsl.render) {
|
|
4012
|
+
try {
|
|
4013
|
+
rendered = dsl.render(node, to);
|
|
4014
|
+
} catch {
|
|
4015
|
+
}
|
|
3762
4016
|
}
|
|
3763
4017
|
const roles = {};
|
|
3764
4018
|
for (const [key, value] of node.roles) {
|
|
@@ -4316,96 +4570,6 @@ async function registryToAOTBackends(registry) {
|
|
|
4316
4570
|
// src/schema/command-schema.ts
|
|
4317
4571
|
import { defineCommand, defineRole, getRoleSpec } from "@lokascript/intent";
|
|
4318
4572
|
|
|
4319
|
-
// src/generation/renderer.ts
|
|
4320
|
-
function lookupKeyword(keywords, action, language) {
|
|
4321
|
-
return keywords[action]?.[language] ?? action;
|
|
4322
|
-
}
|
|
4323
|
-
function lookupMarker(markers, marker, language) {
|
|
4324
|
-
return markers[marker]?.[language] ?? marker;
|
|
4325
|
-
}
|
|
4326
|
-
function buildPhrase(...parts) {
|
|
4327
|
-
return parts.filter(Boolean).join(" ");
|
|
4328
|
-
}
|
|
4329
|
-
function buildTablesFromProfiles(schemas, profiles) {
|
|
4330
|
-
const keywords = {};
|
|
4331
|
-
const markers = {};
|
|
4332
|
-
for (const profile of profiles) {
|
|
4333
|
-
for (const [action, kw] of Object.entries(profile.keywords)) {
|
|
4334
|
-
if (!keywords[action]) keywords[action] = {};
|
|
4335
|
-
keywords[action][profile.code] = kw.primary;
|
|
4336
|
-
}
|
|
4337
|
-
}
|
|
4338
|
-
for (const schema of schemas) {
|
|
4339
|
-
for (const role of schema.roles) {
|
|
4340
|
-
if (role.markerOverride) {
|
|
4341
|
-
const markerKey = role.role;
|
|
4342
|
-
if (!markers[markerKey]) markers[markerKey] = {};
|
|
4343
|
-
for (const [lang, markerText] of Object.entries(role.markerOverride)) {
|
|
4344
|
-
markers[markerKey][lang] = markerText;
|
|
4345
|
-
}
|
|
4346
|
-
}
|
|
4347
|
-
}
|
|
4348
|
-
}
|
|
4349
|
-
for (const profile of profiles) {
|
|
4350
|
-
if (profile.roleMarkers) {
|
|
4351
|
-
for (const [markerKey, markerDef] of Object.entries(profile.roleMarkers)) {
|
|
4352
|
-
if (!markers[markerKey]) markers[markerKey] = {};
|
|
4353
|
-
markers[markerKey][profile.code] = markerDef.primary;
|
|
4354
|
-
}
|
|
4355
|
-
}
|
|
4356
|
-
}
|
|
4357
|
-
return { keywords, markers };
|
|
4358
|
-
}
|
|
4359
|
-
function detectWordOrders(profiles) {
|
|
4360
|
-
const sovLanguages = /* @__PURE__ */ new Set();
|
|
4361
|
-
const vsoLanguages = /* @__PURE__ */ new Set();
|
|
4362
|
-
for (const profile of profiles) {
|
|
4363
|
-
if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
|
|
4364
|
-
if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
|
|
4365
|
-
}
|
|
4366
|
-
return { sovLanguages, vsoLanguages };
|
|
4367
|
-
}
|
|
4368
|
-
function createSchemaRenderer(schemas, profiles) {
|
|
4369
|
-
const { keywords, markers } = buildTablesFromProfiles(schemas, profiles);
|
|
4370
|
-
const { sovLanguages } = detectWordOrders(profiles);
|
|
4371
|
-
const schemaMap = /* @__PURE__ */ new Map();
|
|
4372
|
-
for (const s of schemas) schemaMap.set(s.action, s);
|
|
4373
|
-
return {
|
|
4374
|
-
render(node, language) {
|
|
4375
|
-
const schema = schemaMap.get(node.action);
|
|
4376
|
-
if (!schema) return node.action;
|
|
4377
|
-
const keyword = lookupKeyword(keywords, node.action, language);
|
|
4378
|
-
const isSOV = sovLanguages.has(language);
|
|
4379
|
-
const roleParts = [];
|
|
4380
|
-
for (const role of schema.roles) {
|
|
4381
|
-
const value = extractRoleValue(node, role.role);
|
|
4382
|
-
if (!value && !role.required) continue;
|
|
4383
|
-
const markerText = role.markerOverride?.[language] ?? markers[role.role]?.[language] ?? void 0;
|
|
4384
|
-
roleParts.push({
|
|
4385
|
-
...markerText != null && { marker: markerText },
|
|
4386
|
-
value: value || "",
|
|
4387
|
-
role
|
|
4388
|
-
});
|
|
4389
|
-
}
|
|
4390
|
-
const parts = [];
|
|
4391
|
-
if (isSOV) {
|
|
4392
|
-
for (const rp of roleParts) {
|
|
4393
|
-
if (rp.value) parts.push(rp.value);
|
|
4394
|
-
if (rp.marker) parts.push(rp.marker);
|
|
4395
|
-
}
|
|
4396
|
-
parts.push(keyword);
|
|
4397
|
-
} else {
|
|
4398
|
-
parts.push(keyword);
|
|
4399
|
-
for (const rp of roleParts) {
|
|
4400
|
-
if (rp.marker) parts.push(rp.marker);
|
|
4401
|
-
if (rp.value) parts.push(rp.value);
|
|
4402
|
-
}
|
|
4403
|
-
}
|
|
4404
|
-
return buildPhrase(...parts);
|
|
4405
|
-
}
|
|
4406
|
-
};
|
|
4407
|
-
}
|
|
4408
|
-
|
|
4409
4573
|
// src/generation/diagnostics.ts
|
|
4410
4574
|
import { createDiagnosticCollector, fromError, filterBySeverity } from "@lokascript/intent";
|
|
4411
4575
|
|
|
@@ -4515,6 +4679,9 @@ function isQuote(char) {
|
|
|
4515
4679
|
function isDigit(char) {
|
|
4516
4680
|
return /\d/.test(char);
|
|
4517
4681
|
}
|
|
4682
|
+
function stripOptionalDiacritics(word) {
|
|
4683
|
+
return word.replace(/[ً-ْٰ]/g, "");
|
|
4684
|
+
}
|
|
4518
4685
|
function isAsciiLetter(char) {
|
|
4519
4686
|
return /[a-zA-Z]/.test(char);
|
|
4520
4687
|
}
|
|
@@ -5086,7 +5253,39 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
5086
5253
|
pos++;
|
|
5087
5254
|
}
|
|
5088
5255
|
}
|
|
5089
|
-
return new TokenStreamImpl(tokens, this.language);
|
|
5256
|
+
return new TokenStreamImpl(this.mergeColonQualifiedNames(tokens), this.language);
|
|
5257
|
+
}
|
|
5258
|
+
/**
|
|
5259
|
+
* Fuse `name` + `:qualifier` into ONE identifier (`draggable:start`).
|
|
5260
|
+
*
|
|
5261
|
+
* `:name` is hyperscript's local-variable sigil, but a colon IMMEDIATELY
|
|
5262
|
+
* preceded by an identifier is a qualifier (custom event namespace), not a
|
|
5263
|
+
* sigil. The English tokenizer already merges these inside
|
|
5264
|
+
* EnglishKeywordExtractor; this post-pass gives the other 23 languages the
|
|
5265
|
+
* same stream. Strict position adjacency is the discriminator: whitespace
|
|
5266
|
+
* between the tokens (`trigger :start`) breaks `end === start`, so a spaced
|
|
5267
|
+
* local-variable reference survives untouched.
|
|
5268
|
+
*
|
|
5269
|
+
* Self-gating for non-hyperscript tokenizers (domain DSLs): their extractor
|
|
5270
|
+
* sets tokenize `:` as bare punctuation (length 1), which never matches
|
|
5271
|
+
* COLON_QUALIFIER, so this pass is a no-op for them.
|
|
5272
|
+
*/
|
|
5273
|
+
mergeColonQualifiedNames(tokens) {
|
|
5274
|
+
const out = [];
|
|
5275
|
+
for (const tok of tokens) {
|
|
5276
|
+
const prev = out[out.length - 1];
|
|
5277
|
+
if (prev && _BaseTokenizer.ASCII_WORD.test(prev.value) && _BaseTokenizer.COLON_QUALIFIER.test(tok.value) && prev.position.end === tok.position.start) {
|
|
5278
|
+
const merged = prev.value + tok.value;
|
|
5279
|
+
out[out.length - 1] = createToken(
|
|
5280
|
+
merged,
|
|
5281
|
+
this.classifyToken(merged),
|
|
5282
|
+
createPosition(prev.position.start, tok.position.end)
|
|
5283
|
+
);
|
|
5284
|
+
continue;
|
|
5285
|
+
}
|
|
5286
|
+
out.push(tok);
|
|
5287
|
+
}
|
|
5288
|
+
return out;
|
|
5090
5289
|
}
|
|
5091
5290
|
/**
|
|
5092
5291
|
* Classify an unknown character when no extractor matches.
|
|
@@ -5219,7 +5418,7 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
5219
5418
|
* @returns Word without diacritics
|
|
5220
5419
|
*/
|
|
5221
5420
|
removeDiacritics(word) {
|
|
5222
|
-
return word
|
|
5421
|
+
return stripOptionalDiacritics(word);
|
|
5223
5422
|
}
|
|
5224
5423
|
/**
|
|
5225
5424
|
* Try to match a keyword from profile at the current position.
|
|
@@ -5310,24 +5509,40 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
5310
5509
|
});
|
|
5311
5510
|
}
|
|
5312
5511
|
/**
|
|
5313
|
-
* Look up a keyword by native word (case-insensitive).
|
|
5512
|
+
* Look up a keyword by native word (case-insensitive, diacritic-insensitive).
|
|
5314
5513
|
* O(1) lookup using the keyword map.
|
|
5315
5514
|
*
|
|
5515
|
+
* The map is INDEXED both with and without diacritics (see
|
|
5516
|
+
* `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
|
|
5517
|
+
* that: it lets a surface form carrying harakat the profile does not happen to
|
|
5518
|
+
* spell still find its entry. Only consulted after the exact lookup misses, so
|
|
5519
|
+
* every previously-matching word resolves byte-identically.
|
|
5520
|
+
*
|
|
5521
|
+
* Half-implementing this — indexing stripped but querying exact — is what made
|
|
5522
|
+
* diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
|
|
5523
|
+
* `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
|
|
5524
|
+
* exists to prevent exactly that handed the word on, and the single-char `ب`
|
|
5525
|
+
* bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
|
|
5526
|
+
*
|
|
5316
5527
|
* @param native - Native word to look up
|
|
5317
5528
|
* @returns KeywordEntry if found, undefined otherwise
|
|
5318
5529
|
*/
|
|
5319
5530
|
lookupKeyword(native) {
|
|
5320
|
-
|
|
5531
|
+
const exact = this.profileKeywordMap.get(native.toLowerCase());
|
|
5532
|
+
if (exact) return exact;
|
|
5533
|
+
const stripped = this.removeDiacritics(native);
|
|
5534
|
+
if (stripped === native) return void 0;
|
|
5535
|
+
return this.profileKeywordMap.get(stripped.toLowerCase());
|
|
5321
5536
|
}
|
|
5322
5537
|
/**
|
|
5323
|
-
* Check if a word is a known keyword (case-insensitive).
|
|
5324
|
-
* O(1) lookup using the keyword map.
|
|
5538
|
+
* Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
|
|
5539
|
+
* O(1) lookup using the keyword map. See {@link lookupKeyword}.
|
|
5325
5540
|
*
|
|
5326
5541
|
* @param native - Native word to check
|
|
5327
5542
|
* @returns true if the word is a keyword
|
|
5328
5543
|
*/
|
|
5329
5544
|
isKeyword(native) {
|
|
5330
|
-
return this.
|
|
5545
|
+
return this.lookupKeyword(native) !== void 0;
|
|
5331
5546
|
}
|
|
5332
5547
|
/**
|
|
5333
5548
|
* Set the morphological normalizer for this tokenizer.
|
|
@@ -5592,6 +5807,14 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
5592
5807
|
return null;
|
|
5593
5808
|
}
|
|
5594
5809
|
};
|
|
5810
|
+
/**
|
|
5811
|
+
* ASCII word of the shape the English word-walker produces. Excludes `:`, so a
|
|
5812
|
+
* token that already carries a qualifier never merges again — `a:b:c` yields
|
|
5813
|
+
* `a:b` + `:c`, byte-matching the English extractor's single-segment merge.
|
|
5814
|
+
*/
|
|
5815
|
+
_BaseTokenizer.ASCII_WORD = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
5816
|
+
/** `:name` — only a variable-ref-style extractor ever emits this token shape. */
|
|
5817
|
+
_BaseTokenizer.COLON_QUALIFIER = /^:[A-Za-z_][A-Za-z0-9_]*$/;
|
|
5595
5818
|
/**
|
|
5596
5819
|
* Configuration for native language time units.
|
|
5597
5820
|
* Maps patterns to their standard suffix (ms, s, m, h).
|
|
@@ -6296,6 +6519,7 @@ export {
|
|
|
6296
6519
|
BaseMorphologicalNormalizer,
|
|
6297
6520
|
BaseTokenizer,
|
|
6298
6521
|
CrossDomainDispatcher,
|
|
6522
|
+
CssSelectorExtractor,
|
|
6299
6523
|
DEFAULT_OPERATORS,
|
|
6300
6524
|
DEFAULT_PUNCTUATION,
|
|
6301
6525
|
DEFAULT_REFERENCES,
|
|
@@ -6332,6 +6556,7 @@ export {
|
|
|
6332
6556
|
createCompoundNode,
|
|
6333
6557
|
createConditionalNode,
|
|
6334
6558
|
createDiagnosticCollector,
|
|
6559
|
+
createDomainRenderer,
|
|
6335
6560
|
createEventHandlerNode,
|
|
6336
6561
|
createExpression,
|
|
6337
6562
|
createFlag,
|
|
@@ -6423,6 +6648,8 @@ export {
|
|
|
6423
6648
|
semanticNodeToJSON,
|
|
6424
6649
|
semanticNodeToRuntimeAST,
|
|
6425
6650
|
semanticValueToAST,
|
|
6651
|
+
sortRolesByWordOrder,
|
|
6652
|
+
stripOptionalDiacritics,
|
|
6426
6653
|
synthesizeFromSchemas,
|
|
6427
6654
|
toEnvelopeJSON,
|
|
6428
6655
|
toJSONL,
|