@lokascript/semantic 1.0.0 → 1.1.0

package/src/registry.ts

@@ -86,6 +86,94 @@ const externalSources = new Map<string, ExternalPatternsSource>();
 // Pattern generator function - set by patterns module to avoid circular deps
 let patternGenerator: ((profile: LanguageProfile) => LanguagePattern[]) | null = null;
 
+// =============================================================================
+// Profile Inheritance
+// =============================================================================
+
+/**
+ * Deep merge two objects, with variant values overriding base values.
+ * Arrays are replaced, not merged.
+ */
+function deepMerge<T extends object>(base: T, variant: Partial<T>): T {
+  const result = { ...base } as T;
+
+  for (const key of Object.keys(variant) as (keyof T)[]) {
+    const variantValue = variant[key];
+    const baseValue = base[key];
+
+    if (variantValue === undefined) {
+      continue;
+    }
+
+    // If both are objects (but not arrays), merge recursively
+    if (
+      typeof variantValue === 'object' &&
+      variantValue !== null &&
+      !Array.isArray(variantValue) &&
+      typeof baseValue === 'object' &&
+      baseValue !== null &&
+      !Array.isArray(baseValue)
+    ) {
+      result[key] = deepMerge(
+        baseValue as object,
+        variantValue as Partial<typeof baseValue>
+      ) as T[keyof T];
+    } else {
+      // Replace value (including arrays)
+      result[key] = variantValue as T[keyof T];
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Merge a variant profile with its base profile.
+ * The variant's fields override the base, with deep merging for nested objects.
+ *
+ * @example
+ * ```typescript
+ * const esMX = mergeProfiles(spanishProfile, {
+ *   code: 'es-MX',
+ *   name: 'Spanish (Mexico)',
+ *   keywords: {
+ *     toggle: { primary: 'alternar', alternatives: ['dale', 'cambiar'] },
+ *   },
+ * });
+ * ```
+ */
+export function mergeProfiles(
+  base: LanguageProfile,
+  variant: Partial<LanguageProfile>
+): LanguageProfile {
+  return deepMerge(base, variant);
+}
+
+/**
+ * Resolve a profile, applying inheritance if the profile has an `extends` field.
+ * Returns the merged profile with base language properties inherited.
+ */
+export function resolveProfile(profile: LanguageProfile): LanguageProfile {
+  if (!profile.extends) {
+    return profile;
+  }
+
+  const baseProfile = profiles.get(profile.extends);
+  if (!baseProfile) {
+    console.warn(
+      `[Registry] Profile '${profile.code}' extends '${profile.extends}' but base is not registered. ` +
+        `Make sure to import the base language before the variant.`
+    );
+    return profile;
+  }
+
+  // Recursively resolve base profile (in case it also extends something)
+  const resolvedBase = resolveProfile(baseProfile);
+
+  // Merge, with variant overriding base
+  return mergeProfiles(resolvedBase, profile);
+}
+
 // =============================================================================
 // Registration Functions
 // =============================================================================
@@ -93,6 +181,7 @@ let patternGenerator: ((profile: LanguageProfile) => LanguagePattern[]) | null =
 /**
  * Register a language with its tokenizer and profile.
  * Called automatically by language modules when imported.
+ * If the profile has an `extends` field, it will inherit from the base profile.
  */
 export function registerLanguage(
   code: string,
@@ -100,6 +189,7 @@ export function registerLanguage(
   profile: LanguageProfile
 ): void {
   tokenizers.set(code, tokenizer);
+  // Store the original profile (inheritance is resolved at query time)
   profiles.set(code, profile);
   // Clear pattern cache for this language if it was previously cached
   patternCache.delete(code);
@@ -263,16 +353,45 @@ export async function queryExternalPatternsForCommand(
   return allPatterns.sort((a, b) => b.confidence - a.confidence);
 }
 
+// =============================================================================
+// Language Code Utilities
+// =============================================================================
+
+/**
+ * Extract the base language code from a BCP 47 tag.
+ * Examples: 'es-MX' → 'es', 'pt-BR' → 'pt', 'en' → 'en'
+ */
+export function getBaseLanguageCode(code: string): string {
+  return code.split('-')[0];
+}
+
+/**
+ * Check if a code is a language variant (has region subtag).
+ * Examples: 'es-MX' → true, 'pt' → false
+ */
+export function isLanguageVariant(code: string): boolean {
+  return code.includes('-');
+}
+
 // =============================================================================
 // Query Functions
 // =============================================================================
 
 /**
  * Get a tokenizer for the specified language.
- * @throws Error if language is not registered
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ * @throws Error if neither the variant nor base language is registered
  */
 export function getTokenizer(code: string): LanguageTokenizer {
-  const tokenizer = tokenizers.get(code);
+  // Try exact match first
+  let tokenizer = tokenizers.get(code);
+
+  // Fallback: es-MX → es
+  if (!tokenizer && isLanguageVariant(code)) {
+    const baseCode = getBaseLanguageCode(code);
+    tokenizer = tokenizers.get(baseCode);
+  }
+
   if (!tokenizer) {
     const registered = Array.from(tokenizers.keys()).join(', ');
     throw new Error(
@@ -286,10 +405,19 @@ export function getTokenizer(code: string): LanguageTokenizer {
 
 /**
  * Get a profile for the specified language.
- * @throws Error if language is not registered
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ * @throws Error if neither the variant nor base language is registered
  */
 export function getProfile(code: string): LanguageProfile {
-  const profile = profiles.get(code);
+  // Try exact match first
+  let profile = profiles.get(code);
+
+  // Fallback: es-MX → es
+  if (!profile && isLanguageVariant(code)) {
+    const baseCode = getBaseLanguageCode(code);
+    profile = profiles.get(baseCode);
+  }
+
   if (!profile) {
     const registered = Array.from(profiles.keys()).join(', ');
     throw new Error(
@@ -298,21 +426,34 @@ export function getProfile(code: string): LanguageProfile {
         `Import the language module first: import '@lokascript/semantic/languages/${code}';`
     );
   }
-  return profile;
+
+  // Resolve inheritance if profile extends another
+  return resolveProfile(profile);
 }
 
 /**
  * Try to get a tokenizer, returning undefined if not registered.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
  */
 export function tryGetTokenizer(code: string): LanguageTokenizer | undefined {
-  return tokenizers.get(code);
+  let tokenizer = tokenizers.get(code);
+  if (!tokenizer && isLanguageVariant(code)) {
+    tokenizer = tokenizers.get(getBaseLanguageCode(code));
+  }
+  return tokenizer;
 }
 
 /**
  * Try to get a profile, returning undefined if not registered.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
  */
 export function tryGetProfile(code: string): LanguageProfile | undefined {
-  return profiles.get(code);
+  let profile = profiles.get(code);
+  if (!profile && isLanguageVariant(code)) {
+    profile = profiles.get(getBaseLanguageCode(code));
+  }
+  // Resolve inheritance if profile extends another
+  return profile ? resolveProfile(profile) : undefined;
 }
 
 /**
@@ -323,18 +464,33 @@ export function getRegisteredLanguages(): string[] {
 }
 
 /**
- * Check if a language is registered.
+ * Check if a language is registered (exact match or base language fallback).
  */
 export function isLanguageRegistered(code: string): boolean {
-  return tokenizers.has(code) && profiles.has(code);
+  if (tokenizers.has(code) && profiles.has(code)) {
+    return true;
+  }
+  // Check fallback for variants
+  if (isLanguageVariant(code)) {
+    const baseCode = getBaseLanguageCode(code);
+    return tokenizers.has(baseCode) && profiles.has(baseCode);
+  }
+  return false;
 }
 
 /**
- * Check if a language is supported (alias for isLanguageRegistered).
+ * Check if a language is supported (exact match or base language fallback).
  * For backwards compatibility with tokenizers API.
  */
 export function isLanguageSupported(code: string): boolean {
-  return tokenizers.has(code);
+  if (tokenizers.has(code)) {
+    return true;
+  }
+  // Check fallback for variants
+  if (isLanguageVariant(code)) {
+    return tokenizers.has(getBaseLanguageCode(code));
+  }
+  return false;
 }
 
 // =============================================================================
@@ -358,17 +514,25 @@ export function tokenize(input: string, language: string): TokenStream {
  * Get patterns for a specific language.
  * First checks for directly registered patterns (for tree-shaking),
  * then falls back to pattern generator.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
  * @throws Error if language is not registered
  */
 export function getPatternsForLanguage(code: string): LanguagePattern[] {
-  // Check cache first
-  const cached = patternCache.get(code);
+  // Check cache first (try exact, then base language)
+  let cached = patternCache.get(code);
+  if (!cached && isLanguageVariant(code)) {
+    cached = patternCache.get(getBaseLanguageCode(code));
+  }
   if (cached) {
     return cached;
   }
 
   // Check for directly registered patterns (tree-shakeable path)
-  const registered = registeredPatterns.get(code);
+  // Try exact match, then base language fallback
+  let registered = registeredPatterns.get(code);
+  if (!registered && isLanguageVariant(code)) {
+    registered = registeredPatterns.get(getBaseLanguageCode(code));
+  }
   if (registered) {
     patternCache.set(code, registered);
     return registered;
@@ -382,7 +546,7 @@ export function getPatternsForLanguage(code: string): LanguagePattern[] {
     );
   }
 
-  // Get profile (throws if not registered)
+  // Get profile (throws if not registered) - has built-in fallback
   const profile = getProfile(code);
   const patterns = patternGenerator(profile);
   patternCache.set(code, patterns);

package/src/tokenizers/arabic.ts

@@ -195,13 +195,16 @@ const PREPOSITIONS = new Set([
 // =============================================================================
 
 /**
- * Extra keywords not covered by the profile:
+ * Extra keywords not covered by the profile.
+ *
+ * SIMPLIFIED: Following the Tagalog/Hindi model of minimal EXTRAS.
+ * Command synonyms and spelling variants should be in profile alternatives,
+ * not duplicated here. Only includes:
  * - Literals (true, false, null, undefined)
  * - Positional words
  * - Event names
  * - Time units
- * - Temporal conjunctions
- * - Additional synonyms and spelling variants
+ * - References not in profile
  */
 const ARABIC_EXTRAS: KeywordEntry[] = [
   // Values/Literals
@@ -239,13 +242,8 @@ const ARABIC_EXTRAS: KeywordEntry[] = [
   { native: 'تحميل', normalized: 'load' },
   { native: 'تمرير', normalized: 'scroll' },
 
-  // References
-  { native: 'أنا', normalized: 'me' },
-  { native: 'هو', normalized: 'it' },
+  // References (feminine "it" not in profile)
   { native: 'هي', normalized: 'it' },
-  { native: 'النتيجة', normalized: 'result' },
-  { native: 'الحدث', normalized: 'event' },
-  { native: 'الهدف', normalized: 'target' },
 
   // Time units
   { native: 'ثانية', normalized: 's' },
@@ -258,43 +256,12 @@ const ARABIC_EXTRAS: KeywordEntry[] = [
 
   // Note: Temporal markers (عندما, حينما, etc.) are in TEMPORAL_MARKERS map
   // with formality metadata, not in ARABIC_EXTRAS
-
-  // Additional spelling variants (without diacritics)
-  { native: 'بدل', normalized: 'toggle' },
-  { native: 'غير', normalized: 'toggle' },
-  { native: 'اضف', normalized: 'add' },
-  { native: 'ازل', normalized: 'remove' },
-  { native: 'اضع', normalized: 'put' },
-  { native: 'يضع', normalized: 'put' },
-  { native: 'اجعل', normalized: 'put' },
-  { native: 'عين', normalized: 'set' },
-  { native: 'زد', normalized: 'increment' },
-  { native: 'ارفع', normalized: 'increment' },
-  { native: 'انقص', normalized: 'decrement' },
-  { native: 'قلل', normalized: 'decrement' },
-  { native: 'سجل', normalized: 'log' },
-  { native: 'اظهر', normalized: 'show' },
-  { native: 'اعرض', normalized: 'show' },
-  { native: 'اخف', normalized: 'hide' },
-  { native: 'اخفي', normalized: 'hide' },
-  { native: 'شغل', normalized: 'trigger' },
-  { native: 'ارسل', normalized: 'send' },
-  { native: 'ركز', normalized: 'focus' },
-  { native: 'شوش', normalized: 'blur' },
-  { native: 'اذا', normalized: 'if' },
-  { native: 'لو', normalized: 'if' },
-  { native: 'والا', normalized: 'else' },
-  { native: 'توقف', normalized: 'halt' },
-  { native: 'انسخ', normalized: 'clone' },
-
-  // Control flow helpers
-  { native: 'إذن', normalized: 'then' },
-  { native: 'فإن', normalized: 'then' },
-  { native: 'نهاية', normalized: 'end' },
-
-  // Modifiers
-  { native: 'قبل', normalized: 'before' },
-  { native: 'بعد', normalized: 'after' },
+  //
+  // Command spelling variants are now in the profile alternatives:
+  // - toggle: بدل, غيّر, غير (in profile)
+  // - add: اضف, زِد (in profile)
+  // - remove: أزل, امسح (in profile)
+  // - etc.
 ];
 
 // =============================================================================

package/src/tokenizers/bengali.ts

@@ -176,22 +176,8 @@ export class BengaliTokenizer extends BaseTokenizer {
         const startPos = pos;
         let word = '';
 
-        while (pos < input.length && (isBengali(input[pos]) || input[pos] === ' ')) {
-          if (input[pos] === ' ') {
-            // Check if next char is Bengali (compound word)
-            if (pos + 1 < input.length && isBengali(input[pos + 1])) {
-              const rest = input.slice(pos);
-              const compound = [' করুন', ' ফেলুন', ' দিন', ' না হলে', ' যে যান'].find(c =>
-                rest.startsWith(c)
-              );
-              if (compound) {
-                word += compound;
-                pos += compound.length;
-                continue;
-              }
-            }
-            break;
-          }
+        // Extract word without including spaces (let parser handle multi-word patterns)
+        while (pos < input.length && isBengali(input[pos])) {
           word += input[pos];
           pos++;
         }