@lokascript/framework 2.0.0 → 2.2.0

package/dist/core/index.js

@@ -14,6 +14,9 @@ function createPropertyPath(object, property) {
 function createExpression(raw) {
   return { type: "expression", raw };
 }
+function createFlag(name, enabled = true) {
+  return { type: "flag", name, enabled };
+}
 function createCommandNode(action, roles, metadata) {
   const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
   const node = {
@@ -65,6 +68,7 @@ function createCompoundNode(statements, chainType = "sequential", metadata) {
   return metadata ? { ...base, metadata } : base;
 }
 function extractValue(value) {
+  if (value.type === "flag") return value.name;
   if ("raw" in value && value.raw !== void 0) return String(value.raw);
   if ("value" in value && value.value !== void 0) return String(value.value);
   if (value.type === "property-path") return `${extractValue(value.object)}.${value.property}`;
@@ -91,6 +95,38 @@ function createLoopNode(action, roles, loopVariant, body, loopVariable, indexVar
     ...metadata && { metadata }
   };
 }
+function createTryNode(body, catchBranch, finallyBranch, metadata) {
+  return {
+    kind: "command",
+    action: "try",
+    roles: /* @__PURE__ */ new Map(),
+    body,
+    ...catchBranch && catchBranch.length > 0 ? { catchBranch } : {},
+    ...finallyBranch && finallyBranch.length > 0 ? { finallyBranch } : {},
+    ...metadata ? { metadata } : {}
+  };
+}
+function createAsyncNode(variant, asyncBody, metadata) {
+  return {
+    kind: "command",
+    action: variant,
+    roles: /* @__PURE__ */ new Map(),
+    asyncVariant: variant,
+    asyncBody,
+    ...metadata ? { metadata } : {}
+  };
+}
+function createMatchNode(roles, arms, defaultArm, metadata) {
+  const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
+  return {
+    kind: "command",
+    action: "match",
+    roles: rolesMap,
+    arms,
+    ...defaultArm && defaultArm.length > 0 ? { defaultArm } : {},
+    ...metadata ? { metadata } : {}
+  };
+}
 
 // src/core/tokenization/token-utils.ts
 var TokenStreamImpl = class {
@@ -1407,6 +1443,138 @@ function normalized(stem, confidence, metadata) {
   return { stem, confidence };
 }
 
+// src/core/tokenization/morphology/base-normalizer.ts
+var BaseMorphologicalNormalizer = class {
+  constructor(config) {
+    this.language = config.language;
+    this.config = {
+      minWordLength: 3,
+      minStemLength: 2,
+      ...config
+    };
+  }
+  /**
+   * Standard normalization pipeline. Override for custom behavior.
+   */
+  normalize(word) {
+    const lower = word.toLowerCase();
+    if (this.isAlreadyNormalized(lower)) {
+      return noChange(word);
+    }
+    if (this.config.reflexiveSuffixes) {
+      const reflexive = this.tryReflexiveNormalization(lower);
+      if (reflexive) return reflexive;
+    }
+    if (this.config.endings) {
+      const conjugation = this.tryConjugationEndings(lower);
+      if (conjugation) return conjugation;
+    }
+    if (this.config.suffixRules) {
+      const suffix = this.trySuffixRules(lower);
+      if (suffix) return suffix;
+    }
+    if (this.config.prefixRules) {
+      const prefix = this.tryPrefixRules(lower);
+      if (prefix) return prefix;
+    }
+    return noChange(word);
+  }
+  /**
+   * Check if word is already in dictionary form (e.g., ends in -ar/-er/-ir).
+   * Override for language-specific checks.
+   */
+  isAlreadyNormalized(word) {
+    if (this.config.infinitiveEndings) {
+      return this.config.infinitiveEndings.some((e) => word.endsWith(e));
+    }
+    return false;
+  }
+  /**
+   * Try to strip reflexive suffixes and normalize the remainder.
+   * Common in Romance languages (Spanish, Portuguese, French).
+   */
+  tryReflexiveNormalization(word) {
+    const suffixes = this.config.reflexiveSuffixes;
+    if (!suffixes) return null;
+    for (const suffix of suffixes) {
+      if (!word.endsWith(suffix)) continue;
+      const remainder = word.slice(0, -suffix.length);
+      if (this.isAlreadyNormalized(remainder)) {
+        return normalized(remainder, 0.88, {
+          removedSuffixes: [suffix],
+          conjugationType: "reflexive"
+        });
+      }
+      const inner = this.tryConjugationEndings(remainder) || this.trySuffixRules(remainder);
+      if (inner && inner.stem !== remainder) {
+        return normalized(inner.stem, inner.confidence * 0.95, {
+          removedSuffixes: [suffix, ...inner.metadata?.removedSuffixes || []],
+          conjugationType: "reflexive"
+        });
+      }
+    }
+    return null;
+  }
+  /**
+   * Try conjugation endings (verb class endings like -ar/-er/-ir patterns).
+   * Endings must be pre-sorted longest-first.
+   */
+  tryConjugationEndings(word) {
+    const endings = this.config.endings;
+    if (!endings) return null;
+    const minStem = this.config.minStemLength ?? 2;
+    for (const rule of endings) {
+      if (!word.endsWith(rule.ending)) continue;
+      const stemBase = word.slice(0, -rule.ending.length);
+      if (stemBase.length < minStem) continue;
+      const infinitive = stemBase + rule.stem;
+      return normalized(infinitive, rule.confidence, {
+        removedSuffixes: [rule.ending],
+        conjugationType: rule.type
+      });
+    }
+    return null;
+  }
+  /**
+   * Try SuffixRule-style normalization.
+   * Rules must be pre-sorted longest-first.
+   */
+  trySuffixRules(word) {
+    const rules = this.config.suffixRules;
+    if (!rules) return null;
+    const defaultMinStem = this.config.minStemLength ?? 2;
+    for (const rule of rules) {
+      if (!word.endsWith(rule.pattern)) continue;
+      const stem = word.slice(0, -rule.pattern.length);
+      const minStem = rule.minStemLength ?? defaultMinStem;
+      if (stem.length < minStem) continue;
+      const result = stem + (rule.replacement || "");
+      return normalized(result, rule.confidence, {
+        removedSuffixes: [rule.pattern],
+        ...rule.conjugationType && { conjugationType: rule.conjugationType }
+      });
+    }
+    return null;
+  }
+  /**
+   * Try PrefixRule-style normalization.
+   */
+  tryPrefixRules(word) {
+    const rules = this.config.prefixRules;
+    if (!rules) return null;
+    for (const rule of rules) {
+      if (!word.startsWith(rule.pattern)) continue;
+      const remainder = word.slice(rule.pattern.length);
+      const minRemaining = rule.minRemaining ?? this.config.minStemLength ?? 2;
+      if (remainder.length < minRemaining) continue;
+      return normalized(remainder, 1 - rule.confidencePenalty, {
+        removedPrefixes: [rule.pattern]
+      });
+    }
+    return null;
+  }
+};
+
 // src/core/logger.ts
 var DebugLogger = class {
   constructor(namespace) {
@@ -1661,7 +1829,7 @@ var _PatternMatcher = class _PatternMatcher {
    * a token and the immediately following pattern token fails, the matcher
    * resets to before the optional role and retries the failed token.
    */
-  matchTokenSequence(tokens, patternTokens, captured) {
+  matchTokenSequence(tokens, patternTokens, captured, parentStopMarkers) {
     const firstPatternToken = patternTokens[0];
     const patternExpectsConjunction = firstPatternToken?.type === "literal" && (firstPatternToken.value === "and" || firstPatternToken.value === "then" || firstPatternToken.alternatives?.includes("and") || firstPatternToken.alternatives?.includes("then"));
     if (this.currentProfile?.code === "ar" && !patternExpectsConjunction) {
@@ -1685,6 +1853,9 @@ var _PatternMatcher = class _PatternMatcher {
       );
       if (patternToken.type === "role" && patternToken.greedy) {
         const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) stopMarkers.add(m);
+        }
         const values = [];
         while (!tokens.isAtEnd()) {
           const nextToken = tokens.peek();
@@ -1704,8 +1875,58 @@ var _PatternMatcher = class _PatternMatcher {
           return false;
         }
       }
+      if (patternToken.type === "group") {
+        const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) siblingStopMarkers.add(m);
+        }
+        const groupMatched = this.matchGroupToken(
+          tokens,
+          patternToken,
+          captured,
+          siblingStopMarkers
+        );
+        if (groupMatched) {
+          prevOptionalMark = null;
+          prevOptionalRole = null;
+          continue;
+        }
+        if (this.isOptional(patternToken)) {
+          continue;
+        }
+        return false;
+      }
       const isOptionalRole = patternToken.type === "role" && patternToken.optional === true;
       const markBefore = isOptionalRole ? tokens.mark() : null;
+      const isRoleToken = patternToken.type === "role";
+      if (isRoleToken && !isOptionalRole) {
+        const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
+        if (nextPT?.type === "literal") {
+          const currentToken = tokens.peek();
+          if (currentToken && this.getMatchType(currentToken, nextPT.value) !== "none") {
+            this.logger.debug(
+              "  >> MARKER-STEAL GUARD: current token matches next literal, skipping role"
+            );
+            this.logger.debug("  >> Token match FAILED");
+            if (this.isOptional(patternToken)) {
+              continue;
+            }
+            if (prevOptionalMark && prevOptionalRole) {
+              this.logger.debug("  >> BACKTRACKING: undoing optional role", prevOptionalRole);
+              tokens.reset(prevOptionalMark);
+              captured.delete(prevOptionalRole);
+              prevOptionalMark = null;
+              prevOptionalRole = null;
+              const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
+              this.logger.debug("  >> Backtrack retry result:", retryMatched);
+              if (retryMatched) {
+                continue;
+              }
+            }
+            return false;
+          }
+        }
+      }
       const matched = this.matchPatternToken(tokens, patternToken, captured);
       this.logger.debug("  >> Match result:", matched);
       if (matched) {
@@ -1826,7 +2047,7 @@ var _PatternMatcher = class _PatternMatcher {
     const possessiveValue = this.tryMatchPossessiveExpression(tokens);
     if (possessiveValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (!patternToken.expectedTypes.includes(possessiveValue.type) && !patternToken.expectedTypes.includes("expression")) {
+        if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -1836,7 +2057,7 @@ var _PatternMatcher = class _PatternMatcher {
     const methodCallValue = this.tryMatchMethodCallExpression(tokens);
     if (methodCallValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (!patternToken.expectedTypes.includes(methodCallValue.type) && !patternToken.expectedTypes.includes("expression")) {
+        if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -1856,7 +2077,7 @@ var _PatternMatcher = class _PatternMatcher {
     const propertyAccessValue = this.tryMatchPropertyAccessExpression(tokens);
     if (propertyAccessValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (!patternToken.expectedTypes.includes(propertyAccessValue.type) && !patternToken.expectedTypes.includes("expression")) {
+        if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -2176,10 +2397,15 @@ var _PatternMatcher = class _PatternMatcher {
    * (e.g., a value like 'hello') to sit between groups without blocking later
    * marker matches.
    */
-  matchGroupToken(tokens, patternToken, captured) {
+  matchGroupToken(tokens, patternToken, captured, parentStopMarkers) {
     const mark = tokens.mark();
     const capturedBefore = new Set(captured.keys());
-    const success = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+    const success = this.matchTokenSequence(
+      tokens,
+      patternToken.tokens,
+      captured,
+      parentStopMarkers
+    );
     if (success) return true;
     tokens.reset(mark);
     for (const role of captured.keys()) {
@@ -2201,7 +2427,12 @@ var _PatternMatcher = class _PatternMatcher {
             "- skipping intervening tokens"
           );
           for (let s = 0; s < offset; s++) tokens.advance();
-          const retrySuccess = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+          const retrySuccess = this.matchTokenSequence(
+            tokens,
+            patternToken.tokens,
+            captured,
+            parentStopMarkers
+          );
           if (retrySuccess) return true;
           tokens.reset(mark);
           for (const role of captured.keys()) {
@@ -2268,7 +2499,7 @@ var _PatternMatcher = class _PatternMatcher {
             break;
           }
         }
-        break;
+        if (!pt.optional) break;
       }
     }
     return markers;
@@ -2278,8 +2509,9 @@ var _PatternMatcher = class _PatternMatcher {
    */
   isStopMarker(token, stopMarkers) {
     if (stopMarkers.size === 0) return false;
-    const value = (token.normalized || token.value).toLowerCase();
-    return stopMarkers.has(value);
+    if (stopMarkers.has(token.value.toLowerCase())) return true;
+    if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
+    return false;
   }
   /**
    * Convert a language token to a semantic value.
@@ -2613,6 +2845,7 @@ function matchBest(tokens, patterns) {
   return patternMatcher.matchBest(tokens, patterns);
 }
 export {
+  BaseMorphologicalNormalizer,
   BaseTokenizer,
   DEFAULT_OPERATORS,
   DEFAULT_PUNCTUATION,
@@ -2621,20 +2854,24 @@ export {
   PunctuationExtractor,
   TokenStreamImpl,
   combineClassifiers,
+  createAsyncNode,
   createCommandNode,
   createCompoundNode,
   createConditionalNode,
   createEventHandlerNode,
   createExpression,
+  createFlag,
   createLatinCharClassifiers,
   createLiteral,
   createLoopNode,
+  createMatchNode,
   createPosition,
   createPropertyPath,
   createReference,
   createSelector,
   createSimpleTokenizer,
   createToken,
+  createTryNode,
   createUnicodeRangeClassifier,
   extractCssSelector,
   extractNumber,