@lokascript/semantic 1.0.0

package/src/generators/pattern-generator.ts

@@ -0,0 +1,537 @@
+/**
+ * Pattern Generator
+ *
+ * Generates LanguagePattern objects from CommandSchema + LanguageProfile.
+ * This solves the pattern explosion problem by deriving patterns from
+ * high-level definitions rather than hand-writing each one.
+ */
+
+import type { LanguagePattern, PatternToken, ExtractionRule } from '../types';
+import type { LanguageProfile } from './language-profiles';
+import type { CommandSchema, RoleSpec } from './command-schemas';
+import { getDefinedSchemas } from './command-schemas';
+
+// Import shared utilities
+import { sortRolesByWordOrder } from '../parser/utils/role-positioning';
+import { resolveMarkerForRole } from '../parser/utils/marker-resolution';
+
+// Note: languageProfiles is no longer imported here.
+// Pattern generation for specific languages uses the registry instead.
+
+// Import registry functions - this is safe because:
+// 1. Registry doesn't import pattern-generator
+// 2. The circular path is: pattern-generator -> registry -> pattern-generator
+//    But registry only uses setPatternGenerator which is exported, not module-level code
+import {
+  getRegisteredLanguages as registryGetLanguages,
+  tryGetProfile as registryTryGetProfile,
+} from '../registry';
+
+function getAllRegisteredProfiles(): LanguageProfile[] {
+  const languages = registryGetLanguages();
+  return languages
+    .map((lang: string) => registryTryGetProfile(lang))
+    .filter((p): p is LanguageProfile => p !== undefined);
+}
+
+// =============================================================================
+// Pattern Generator
+// =============================================================================
+
+/**
+ * Configuration for pattern generation.
+ */
+export interface GeneratorConfig {
+  /** Base priority for generated patterns (higher = checked first) */
+  basePriority?: number;
+  /** Whether to generate simple patterns (without optional roles) */
+  generateSimpleVariants?: boolean;
+  /** Whether to generate alternative keyword patterns */
+  generateAlternatives?: boolean;
+}
+
+const defaultConfig: GeneratorConfig = {
+  basePriority: 100,
+  generateSimpleVariants: true,
+  generateAlternatives: true,
+};
+
+/**
+ * Generate a pattern for a command in a specific language.
+ */
+export function generatePattern(
+  schema: CommandSchema,
+  profile: LanguageProfile,
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern {
+  const id = `${schema.action}-${profile.code}-generated`;
+  const priority = config.basePriority ?? 100;
+
+  // Get keyword translation
+  const keyword = profile.keywords[schema.action];
+  if (!keyword) {
+    throw new Error(`No keyword translation for '${schema.action}' in ${profile.code}`);
+  }
+
+  // Build tokens based on word order
+  const tokens = buildTokens(schema, profile, keyword);
+
+  // Build extraction rules with defaults for optional roles
+  // This ensures defaults are applied even when optional groups don't match
+  const extraction = buildExtractionRulesWithDefaults(schema, profile);
+
+  // Build template format string (for documentation)
+  const format = buildFormatString(schema, profile, keyword);
+
+  return {
+    id,
+    language: profile.code,
+    command: schema.action,
+    priority,
+    template: {
+      format,
+      tokens,
+    },
+    extraction,
+  };
+}
+
+/**
+ * Generate a simple variant pattern (without optional roles).
+ */
+export function generateSimplePattern(
+  schema: CommandSchema,
+  profile: LanguageProfile,
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern | null {
+  // Only generate simple variant if there are optional roles
+  const optionalRoles = schema.roles.filter(r => !r.required);
+  if (optionalRoles.length === 0) {
+    return null;
+  }
+
+  const requiredRoles = schema.roles.filter(r => r.required);
+  const simpleSchema: CommandSchema = {
+    ...schema,
+    roles: requiredRoles,
+  };
+
+  const pattern = generatePattern(simpleSchema, profile, config);
+
+  // Adjust for simple variant
+  return {
+    ...pattern,
+    id: `${schema.action}-${profile.code}-simple`,
+    priority: (config.basePriority ?? 100) - 5, // Lower priority than full pattern (was -10)
+    extraction: buildExtractionRulesWithDefaults(schema, profile),
+  };
+}
+
+/**
+ * Generate all pattern variants for a command in a language.
+ */
+export function generatePatternVariants(
+  schema: CommandSchema,
+  profile: LanguageProfile,
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern[] {
+  const patterns: LanguagePattern[] = [];
+
+  // Main pattern
+  patterns.push(generatePattern(schema, profile, config));
+
+  // Simple variant (without optional roles)
+  if (config.generateSimpleVariants) {
+    const simple = generateSimplePattern(schema, profile, config);
+    if (simple) {
+      patterns.push(simple);
+    }
+  }
+
+  return patterns;
+}
+
+/**
+ * Generate patterns for all commands in a specific language.
+ */
+export function generatePatternsForLanguage(
+  profile: LanguageProfile,
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern[] {
+  const patterns: LanguagePattern[] = [];
+  const schemas = getDefinedSchemas();
+
+  for (const schema of schemas) {
+    // Skip if no keyword translation exists
+    if (!profile.keywords[schema.action]) {
+      continue;
+    }
+
+    const variants = generatePatternVariants(schema, profile, config);
+    patterns.push(...variants);
+  }
+
+  return patterns;
+}
+
+/**
+ * Generate patterns for a command across specified profiles.
+ *
+ * @param schema Command schema to generate patterns for
+ * @param profiles Array of language profiles to generate patterns for (defaults to all registered)
+ * @param config Generator configuration
+ */
+export function generatePatternsForCommand(
+  schema: CommandSchema,
+  profiles?: LanguageProfile[],
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern[] {
+  const patterns: LanguagePattern[] = [];
+
+  // If no profiles provided, use all registered profiles
+  const profilesToUse = profiles ?? getAllRegisteredProfiles();
+
+  for (const profile of profilesToUse) {
+    // Skip if no keyword translation exists
+    if (!profile.keywords[schema.action]) {
+      continue;
+    }
+
+    const variants = generatePatternVariants(schema, profile, config);
+    patterns.push(...variants);
+  }
+
+  return patterns;
+}
+
+/**
+ * Generate all patterns for all commands across specified profiles.
+ *
+ * @param profiles Array of language profiles to generate patterns for (defaults to all registered)
+ * @param config Generator configuration
+ */
+export function generateAllPatterns(
+  profiles?: LanguageProfile[],
+  config: GeneratorConfig = defaultConfig
+): LanguagePattern[] {
+  const patterns: LanguagePattern[] = [];
+
+  // If no profiles provided, use all registered profiles
+  const profilesToUse = profiles ?? getAllRegisteredProfiles();
+
+  for (const profile of profilesToUse) {
+    const langPatterns = generatePatternsForLanguage(profile, config);
+    patterns.push(...langPatterns);
+  }
+
+  return patterns;
+}
+
+// =============================================================================
+// Token Building
+// =============================================================================
+
+/**
+ * Build pattern tokens based on word order.
+ */
+function buildTokens(
+  schema: CommandSchema,
+  profile: LanguageProfile,
+  keyword: { primary: string; alternatives?: string[] }
+): PatternToken[] {
+  const tokens: PatternToken[] = [];
+
+  // Get verb token
+  const verbToken: PatternToken = keyword.alternatives
+    ? { type: 'literal', value: keyword.primary, alternatives: keyword.alternatives }
+    : { type: 'literal', value: keyword.primary };
+
+  // Get role tokens sorted by position
+  const roleTokens = buildRoleTokens(schema, profile);
+
+  // Arrange based on word order
+  switch (profile.wordOrder) {
+    case 'SVO':
+      // Verb first, then roles in order
+      tokens.push(verbToken);
+      tokens.push(...roleTokens);
+      break;
+
+    case 'SOV':
+      // Roles first (reversed for SOV), then verb
+      tokens.push(...roleTokens);
+      tokens.push(verbToken);
+      break;
+
+    case 'VSO':
+      // Verb first, then subject, then object
+      tokens.push(verbToken);
+      tokens.push(...roleTokens);
+      break;
+
+    default:
+      // Default to SVO
+      tokens.push(verbToken);
+      tokens.push(...roleTokens);
+  }
+
+  return tokens;
+}
+
+/**
+ * Build tokens for roles.
+ */
+function buildRoleTokens(schema: CommandSchema, profile: LanguageProfile): PatternToken[] {
+  const tokens: PatternToken[] = [];
+
+  // Sort roles by position using shared utility
+  const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
+
+  for (const roleSpec of sortedRoles) {
+    const roleToken = buildRoleToken(roleSpec, profile);
+
+    if (!roleSpec.required) {
+      // Wrap optional roles in a group
+      tokens.push({
+        type: 'group',
+        optional: true,
+        tokens: roleToken,
+      });
+    } else {
+      tokens.push(...roleToken);
+    }
+  }
+
+  return tokens;
+}
+
+/**
+ * Build token(s) for a single role.
+ */
+function buildRoleToken(roleSpec: RoleSpec, profile: LanguageProfile): PatternToken[] {
+  const tokens: PatternToken[] = [];
+
+  // Check for command-specific marker override first
+  const overrideMarker = roleSpec.markerOverride?.[profile.code];
+  const defaultMarker = profile.roleMarkers[roleSpec.role];
+
+  // Role value token
+  const roleValueToken: PatternToken = {
+    type: 'role',
+    role: roleSpec.role,
+    optional: !roleSpec.required,
+    expectedTypes: roleSpec.expectedTypes as any,
+  };
+
+  // Use override marker if available, otherwise use default
+  if (overrideMarker !== undefined) {
+    // Command-specific marker override
+    const position = defaultMarker?.position ?? 'before';
+    if (position === 'before') {
+      if (overrideMarker) {
+        tokens.push({ type: 'literal', value: overrideMarker });
+      }
+      tokens.push(roleValueToken);
+    } else {
+      tokens.push(roleValueToken);
+      if (overrideMarker) {
+        tokens.push({ type: 'literal', value: overrideMarker });
+      }
+    }
+  } else if (defaultMarker) {
+    if (defaultMarker.position === 'before') {
+      // Preposition: "on #button"
+      if (defaultMarker.primary) {
+        const markerToken: PatternToken = defaultMarker.alternatives
+          ? {
+              type: 'literal',
+              value: defaultMarker.primary,
+              alternatives: defaultMarker.alternatives,
+            }
+          : { type: 'literal', value: defaultMarker.primary };
+        tokens.push(markerToken);
+      }
+      tokens.push(roleValueToken);
+    } else {
+      // Postposition/particle: "#button に"
+      tokens.push(roleValueToken);
+      const markerToken: PatternToken = defaultMarker.alternatives
+        ? {
+            type: 'literal',
+            value: defaultMarker.primary,
+            alternatives: defaultMarker.alternatives,
+          }
+        : { type: 'literal', value: defaultMarker.primary };
+      tokens.push(markerToken);
+    }
+  } else {
+    // No marker, just the role value
+    tokens.push(roleValueToken);
+  }
+
+  return tokens;
+}
+
+// =============================================================================
+// Extraction Rules Building
+// =============================================================================
+
+/**
+ * Build extraction rules for a pattern.
+ */
+function buildExtractionRules(
+  schema: CommandSchema,
+  profile: LanguageProfile
+): Record<string, ExtractionRule> {
+  const rules: Record<string, ExtractionRule> = {};
+
+  for (const roleSpec of schema.roles) {
+    // Check for command-specific marker override first
+    const overrideMarker = roleSpec.markerOverride?.[profile.code];
+    const defaultMarker = profile.roleMarkers[roleSpec.role];
+
+    if (overrideMarker !== undefined) {
+      // Use the override marker
+      rules[roleSpec.role] = overrideMarker ? { marker: overrideMarker } : {};
+    } else if (defaultMarker && defaultMarker.primary) {
+      rules[roleSpec.role] = defaultMarker.alternatives
+        ? { marker: defaultMarker.primary, markerAlternatives: defaultMarker.alternatives }
+        : { marker: defaultMarker.primary };
+    } else {
+      rules[roleSpec.role] = {};
+    }
+  }
+
+  return rules;
+}
+
+/**
+ * Build extraction rules with defaults for optional roles.
+ */
+function buildExtractionRulesWithDefaults(
+  schema: CommandSchema,
+  profile: LanguageProfile
+): Record<string, ExtractionRule> {
+  const baseRules = buildExtractionRules(schema, profile);
+  const rules: Record<string, ExtractionRule> = {};
+
+  // Copy base rules and add defaults for optional roles
+  for (const roleSpec of schema.roles) {
+    const baseRule = baseRules[roleSpec.role] || {};
+
+    if (!roleSpec.required && roleSpec.default) {
+      rules[roleSpec.role] = { ...baseRule, default: roleSpec.default };
+    } else {
+      rules[roleSpec.role] = baseRule;
+    }
+  }
+
+  return rules;
+}
+
+// =============================================================================
+// Format String Building
+// =============================================================================
+
+/**
+ * Build a human-readable format string for documentation.
+ */
+function buildFormatString(
+  schema: CommandSchema,
+  profile: LanguageProfile,
+  keyword: { primary: string }
+): string {
+  const parts: string[] = [];
+
+  // Sort roles by position using shared utility
+  const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
+
+  // Build role parts
+  const roleParts = sortedRoles.map(roleSpec => {
+    // Use shared marker resolution utility
+    const resolved = resolveMarkerForRole(roleSpec, profile);
+    let part = '';
+
+    if (resolved && resolved.primary) {
+      // Has a marker
+      if (resolved.position === 'before') {
+        part = `${resolved.primary} {${roleSpec.role}}`;
+      } else {
+        part = `{${roleSpec.role}} ${resolved.primary}`;
+      }
+    } else {
+      // No marker
+      part = `{${roleSpec.role}}`;
+    }
+
+    return roleSpec.required ? part : `[${part}]`;
+  });
+
+  // Arrange based on word order
+  switch (profile.wordOrder) {
+    case 'SVO':
+    case 'VSO':
+      parts.push(keyword.primary, ...roleParts);
+      break;
+    case 'SOV':
+      parts.push(...roleParts, keyword.primary);
+      break;
+    default:
+      parts.push(keyword.primary, ...roleParts);
+  }
+
+  return parts.join(' ');
+}
+
+// =============================================================================
+// Utility Functions
+// =============================================================================
+
+/**
+ * Get a summary of what patterns can be generated.
+ * Note: This requires the registry to have languages registered.
+ */
+export function getGeneratorSummary(): {
+  languages: string[];
+  commands: string[];
+  totalPatterns: number;
+} {
+  const languages = registryGetLanguages();
+  const commands = getDefinedSchemas().map(s => s.action);
+
+  // Estimate total patterns (2 variants per command per language)
+  let totalPatterns = 0;
+  for (const lang of languages) {
+    const profile = registryTryGetProfile(lang);
+    if (profile) {
+      for (const schema of getDefinedSchemas()) {
+        if (profile.keywords[schema.action]) {
+          totalPatterns += 2; // Full + simple variant
+        }
+      }
+    }
+  }
+
+  return { languages, commands, totalPatterns };
+}
+
+/**
+ * Validate that all required keywords exist for a language.
+ */
+export function validateLanguageKeywords(
+  profile: LanguageProfile,
+  schemas: CommandSchema[] = getDefinedSchemas()
+): { missing: string[]; available: string[] } {
+  const missing: string[] = [];
+  const available: string[] = [];
+
+  for (const schema of schemas) {
+    if (profile.keywords[schema.action]) {
+      available.push(schema.action);
+    } else {
+      missing.push(schema.action);
+    }
+  }
+
+  return { missing, available };
+}

package/src/generators/profiles/arabic.ts

@@ -0,0 +1,131 @@
+/**
+ * Arabic Language Profile
+ *
+ * VSO word order, prepositions, RTL (right-to-left), space-separated.
+ * Features root-based morphology and rich verb conjugation.
+ */
+
+import type { LanguageProfile } from './types';
+
+export const arabicProfile: LanguageProfile = {
+  code: 'ar',
+  name: 'Arabic',
+  nativeName: 'العربية',
+  direction: 'rtl',
+  wordOrder: 'VSO',
+  markingStrategy: 'preposition',
+  usesSpaces: true,
+  verb: {
+    position: 'start',
+    subjectDrop: true,
+  },
+  references: {
+    me: 'أنا', // "I/me" - first person
+    it: 'هو', // "it" (masculine)
+    you: 'أنت', // "you"
+    result: 'النتيجة',
+    event: 'الحدث',
+    target: 'الهدف',
+    body: 'الجسم',
+  },
+  possessive: {
+    marker: '', // No explicit marker - uses possessive pronouns
+    markerPosition: 'after-object',
+    usePossessiveAdjectives: true,
+    specialForms: {
+      // Arabic: "value لي" (value for-me) - possessive pronoun follows property
+      me: 'لي', // "for me" / "mine"
+      it: 'له', // "for it" / "its"
+      you: 'لك', // "for you" / "yours"
+    },
+    keywords: {
+      // "my" / "mine"
+      لي: 'me',
+      // "your" / "yours"
+      لك: 'you',
+      // "its/his/her"
+      له: 'it', // his/its (masculine)
+      لها: 'it', // her/its (feminine)
+    },
+  },
+  roleMarkers: {
+    destination: { primary: 'على', alternatives: ['في', 'إلى', 'ب'], position: 'before' },
+    source: { primary: 'من', position: 'before' },
+    patient: { primary: '', position: 'before' },
+    style: { primary: 'بـ', alternatives: ['باستخدام'], position: 'before' },
+  },
+  keywords: {
+    // Class/Attribute operations
+    toggle: { primary: 'بدّل', alternatives: ['بدل', 'غيّر', 'غير'], normalized: 'toggle' },
+    add: { primary: 'أضف', alternatives: ['اضف', 'زِد'], normalized: 'add' },
+    remove: { primary: 'احذف', alternatives: ['أزل', 'امسح'], normalized: 'remove' },
+    // Content operations
+    put: { primary: 'ضع', alternatives: ['اجعل'], normalized: 'put' },
+    append: { primary: 'ألحق', normalized: 'append' },
+    prepend: { primary: 'سبق', normalized: 'prepend' },
+    take: { primary: 'خذ', alternatives: ['احصل'], normalized: 'take' },
+    make: { primary: 'اصنع', alternatives: ['أنشئ'], normalized: 'make' },
+    clone: { primary: 'استنسخ', alternatives: ['انسخ'], normalized: 'clone' },
+    swap: { primary: 'استبدل', alternatives: ['تبادل'], normalized: 'swap' },
+    morph: { primary: 'حوّل', alternatives: ['غيّر'], normalized: 'morph' },
+    // Variable operations
+    set: { primary: 'اضبط', alternatives: ['عيّن', 'حدد'], normalized: 'set' },
+    get: { primary: 'احصل', alternatives: ['خذ'], normalized: 'get' },
+    increment: { primary: 'زِد', alternatives: ['زد', 'ارفع'], normalized: 'increment' },
+    decrement: { primary: 'أنقص', alternatives: ['انقص', 'قلل'], normalized: 'decrement' },
+    log: { primary: 'سجل', normalized: 'log' },
+    // Visibility
+    show: { primary: 'اظهر', alternatives: ['أظهر', 'اعرض'], normalized: 'show' },
+    hide: { primary: 'اخف', alternatives: ['أخفِ', 'اخفي'], normalized: 'hide' },
+    transition: { primary: 'انتقال', alternatives: ['انتقل'], normalized: 'transition' },
+    // Events
+    on: { primary: 'على', alternatives: ['عند', 'لدى', 'حين'], normalized: 'on' },
+    trigger: { primary: 'تشغيل', alternatives: ['أطلق', 'فعّل'], normalized: 'trigger' },
+    send: { primary: 'أرسل', normalized: 'send' },
+    // DOM focus
+    focus: { primary: 'تركيز', alternatives: ['ركز'], normalized: 'focus' },
+    blur: { primary: 'ضبابية', alternatives: ['شوش'], normalized: 'blur' },
+    // Navigation
+    go: { primary: 'اذهب', alternatives: ['انتقل'], normalized: 'go' },
+    // Async
+    wait: { primary: 'انتظر', normalized: 'wait' },
+    fetch: { primary: 'احضر', alternatives: ['جلب'], normalized: 'fetch' },
+    settle: { primary: 'استقر', normalized: 'settle' },
+    // Control flow
+    if: { primary: 'إذا', normalized: 'if' },
+    when: { primary: 'عندما', normalized: 'when' },
+    where: { primary: 'أين', normalized: 'where' },
+    else: { primary: 'وإلا', alternatives: ['خلاف ذلك'], normalized: 'else' },
+    repeat: { primary: 'كرر', normalized: 'repeat' },
+    for: { primary: 'لكل', normalized: 'for' },
+    while: { primary: 'بينما', normalized: 'while' },
+    continue: { primary: 'واصل', normalized: 'continue' },
+    halt: { primary: 'أوقف', alternatives: ['توقف'], normalized: 'halt' },
+    throw: { primary: 'ارم', alternatives: ['ارمِ'], normalized: 'throw' },
+    call: { primary: 'استدع', alternatives: ['نادِ'], normalized: 'call' },
+    return: { primary: 'ارجع', alternatives: ['عُد'], normalized: 'return' },
+    then: { primary: 'ثم', alternatives: ['بعدها', 'ثمّ'], normalized: 'then' },
+    and: { primary: 'وأيضاً', alternatives: ['أيضاً'], normalized: 'and' },
+    end: { primary: 'نهاية', alternatives: ['انتهى', 'آخر'], normalized: 'end' },
+    // Advanced
+    js: { primary: 'جافاسكربت', alternatives: ['js'], normalized: 'js' },
+    async: { primary: 'متزامن', normalized: 'async' },
+    tell: { primary: 'أخبر', normalized: 'tell' },
+    default: { primary: 'افتراضي', normalized: 'default' },
+    init: { primary: 'تهيئة', alternatives: ['بدء'], normalized: 'init' },
+    behavior: { primary: 'سلوك', normalized: 'behavior' },
+    install: { primary: 'تثبيت', alternatives: ['ثبّت'], normalized: 'install' },
+    measure: { primary: 'قياس', alternatives: ['قِس'], normalized: 'measure' },
+    // Modifiers
+    into: { primary: 'في', alternatives: ['إلى'], normalized: 'into' },
+    before: { primary: 'قبل', normalized: 'before' },
+    after: { primary: 'بعد', normalized: 'after' },
+    // Event modifiers (for repeat until event)
+    until: { primary: 'حتى', normalized: 'until' },
+    event: { primary: 'حدث', normalized: 'event' },
+    from: { primary: 'من', normalized: 'from' },
+  },
+  tokenization: {
+    prefixes: ['ال', 'و', 'ف', 'ب', 'ك', 'ل'],
+  },
+};