@lokascript/semantic 1.3.0 → 2.0.0

package/dist/public-api.d.ts

@@ -0,0 +1,190 @@
+/**
+ * Public API Surface
+ *
+ * This file defines the stable public API for @lokascript/semantic.
+ * All exports here are considered stable and follow semver guarantees:
+ * - Major version: Breaking changes to public API
+ * - Minor version: New features, backward compatible
+ * - Patch version: Bug fixes only
+ *
+ * Internal modules (tokenizers, generators, patterns) may change without notice.
+ *
+ * @module @lokascript/semantic
+ * @version 1.0.0
+ */
+export { 
+/**
+ * Parse hyperscript code from any supported language into a SemanticNode.
+ * Throws if parsing fails.
+ *
+ * @param input - The hyperscript code to parse
+ * @param language - ISO 639-1 language code (e.g., 'en', 'ja', 'ko')
+ * @returns SemanticNode representing the parsed command
+ * @throws Error if parsing fails
+ *
+ * @example
+ * const node = parse('toggle .active', 'en');
+ * const jaNode = parse('.active を トグル', 'ja');
+ */
+parse, 
+/**
+ * Check if input can be successfully parsed.
+ * Returns false instead of throwing on parse failure.
+ *
+ * @param input - The hyperscript code to check
+ * @param language - ISO 639-1 language code
+ * @returns true if parseable, false otherwise
+ */
+canParse, } from './parser/semantic-parser';
+export { 
+/**
+ * Convert a SemanticNode to a runtime-compatible AST.
+ * Use this to get AST for execution by the hyperfixi runtime.
+ *
+ * @param node - The SemanticNode from parse()
+ * @returns Object with ast and warnings array
+ *
+ * @example
+ * const semanticNode = parse('toggle .active', 'en');
+ * const { ast, warnings } = buildAST(semanticNode);
+ * if (warnings.length > 0) {
+ *   console.warn('AST build warnings:', warnings);
+ * }
+ * await runtime.execute(ast, context);
+ */
+buildAST, 
+/**
+ * The ASTBuilder class for advanced use cases.
+ * Usually you should use buildAST() instead.
+ */
+ASTBuilder, } from './ast-builder';
+export type { 
+/**
+ * Result type returned by buildAST().
+ * Contains the compiled AST and any warnings from the build process.
+ */
+BuildASTResult, } from './ast-builder';
+export { 
+/**
+ * Translate hyperscript between languages.
+ *
+ * @param input - Source hyperscript code
+ * @param sourceLang - Source language code
+ * @param targetLang - Target language code
+ * @returns Translated hyperscript in target language
+ *
+ * @example
+ * const japanese = translate('toggle .active', 'en', 'ja');
+ * // Returns: '.active を トグル'
+ */
+translate, 
+/**
+ * Get translations for all supported languages.
+ *
+ * @param input - Source hyperscript code
+ * @param sourceLang - Source language code
+ * @returns Object mapping language codes to translations
+ *
+ * @example
+ * const all = getAllTranslations('toggle .active', 'en');
+ * // Returns: { en: 'toggle .active', ja: '.active を トグル', ... }
+ */
+getAllTranslations, } from './parser/semantic-parser';
+export { 
+/**
+ * Create a semantic analyzer for parsing with confidence scores.
+ * Use this when you need to check parse confidence before execution.
+ *
+ * @returns SemanticAnalyzer instance
+ *
+ * @example
+ * const analyzer = createSemanticAnalyzer();
+ * const result = analyzer.analyze('toggle .active', 'en');
+ * if (result.confidence >= 0.5) {
+ *   const ast = buildAST(result.node);
+ * }
+ */
+createSemanticAnalyzer, } from './parser/semantic-parser';
+export { 
+/**
+ * Get list of supported language codes.
+ *
+ * @returns Array of ISO 639-1 language codes
+ *
+ * @example
+ * const langs = getSupportedLanguages();
+ * // Returns: ['en', 'ja', 'ar', 'es', 'ko', 'zh', 'tr', 'pt', 'fr', 'de', 'id', 'qu', 'sw']
+ */
+getSupportedLanguages, } from './parser/semantic-parser';
+export { 
+/**
+ * Default confidence threshold for semantic parsing (0.5).
+ * Results with confidence >= this threshold are considered reliable.
+ *
+ * @example
+ * const result = analyzer.analyze('toggle .active', 'en');
+ * if (result.confidence >= DEFAULT_CONFIDENCE_THRESHOLD) {
+ *   // Use semantic result
+ * }
+ */
+DEFAULT_CONFIDENCE_THRESHOLD, 
+/**
+ * High confidence threshold (0.8) for very certain matches.
+ * Use this when you need stricter validation.
+ */
+HIGH_CONFIDENCE_THRESHOLD, } from './core-bridge';
+export { 
+/**
+ * Render a SemanticNode back to hyperscript in a specific language.
+ *
+ * @param node - The SemanticNode to render
+ * @param language - Target language code
+ * @returns Hyperscript string in target language
+ */
+render, 
+/**
+ * Render a SemanticNode in explicit syntax format.
+ * Explicit syntax is a language-agnostic intermediate representation.
+ *
+ * @param node - The SemanticNode to render
+ * @returns Explicit syntax string
+ *
+ * @example
+ * const explicit = renderExplicit(node);
+ * // Returns: '[toggle patient:.active destination:#button]'
+ */
+renderExplicit, } from './parser/semantic-parser';
+export { 
+/**
+ * Tokenize input for a specific language.
+ * This is a lower-level function for advanced use cases.
+ *
+ * @param input - The hyperscript code to tokenize
+ * @param language - ISO 639-1 language code
+ * @returns Tokenization result with tokens array
+ */
+tokenize, } from './parser/semantic-parser';
+export { 
+/**
+ * Parse explicit syntax format.
+ * Explicit syntax is language-agnostic and unambiguous.
+ *
+ * @param input - Explicit syntax string
+ * @returns SemanticNode
+ */
+parseExplicit, 
+/**
+ * Convert natural language to explicit syntax.
+ */
+toExplicit, 
+/**
+ * Convert explicit syntax to natural language.
+ */
+fromExplicit, 
+/**
+ * Round-trip conversion for testing.
+ */
+roundTrip, } from './parser/semantic-parser';
+export type { ActionType, SemanticNode, CommandSemanticNode, EventHandlerSemanticNode, ConditionalSemanticNode, CompoundSemanticNode, LoopSemanticNode, LoopVariant, SemanticValue, LiteralValue, SelectorValue, ReferenceValue, PropertyPathValue, ExpressionValue, SemanticMetadata, SourcePosition, EventModifiers, } from './types';
+export type { SemanticRole } from './types';
+//# sourceMappingURL=public-api.d.ts.map

package/dist/public-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"public-api.d.ts","sourceRoot":"","sources":["../src/public-api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO;AACL;;;;;;;;;;;;GAYG;AACH,KAAK;AAEL;;;;;;;GAOG;AACH,QAAQ,GACT,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;;;;;;;;;GAcG;AACH,QAAQ;AAER;;;GAGG;AACH,UAAU,GACX,MAAM,eAAe,CAAC;AAEvB,YAAY;AACV;;;GAGG;AACH,cAAc,GACf,MAAM,eAAe,CAAC;AAMvB,OAAO;AACL;;;;;;;;;;;GAWG;AACH,SAAS;AAET;;;;;;;;;;GAUG;AACH,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;;;;;;;GAYG;AACH,sBAAsB,GACvB,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;;;GAQG;AACH,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;;;;GASG;AACH,4BAA4B;AAE5B;;;GAGG;AACH,yBAAyB,GAC1B,MAAM,eAAe,CAAC;AAMvB,OAAO;AACL;;;;;;GAMG;AACH,MAAM;AAEN;;;;;;;;;;GAUG;AACH,cAAc,GACf,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;;GAOG;AACH,QAAQ,GACT,MAAM,0BAA0B,CAAC;AAMlC,OAAO;AACL;;;;;;GAMG;AACH,aAAa;AAEb;;GAEG;AACH,UAAU;AAEV;;GAEG;AACH,YAAY;AAEZ;;GAEG;AACH,SAAS,GACV,MAAM,0BAA0B,CAAC;AAMlC,YAAY,EAEV,UAAU,EACV,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EAGX,aAAa,EACb,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EAGf,gBAAgB,EAChB,cAAc,EACd,cAAc,GACf,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC"}

package/dist/registry.d.ts

@@ -0,0 +1,225 @@
+/**
+ * Language Registry
+ *
+ * Central registration point for language support in the semantic parser.
+ * Languages self-register when their modules are imported, enabling
+ * tree-shaking for minimal bundles.
+ *
+ * @example
+ * ```typescript
+ * // Import only the languages you need
+ * import '@lokascript/semantic/languages/en';
+ * import '@lokascript/semantic/languages/es';
+ *
+ * // Now parse works for registered languages
+ * import { parse } from '@lokascript/semantic';
+ * parse('toggle .active', 'en');     // Works
+ * parse('alternar .activo', 'es');   // Works
+ * parse('切り替え .active', 'ja');    // Error: Language not registered
+ * ```
+ */
+import type { LanguageTokenizer, LanguagePattern, TokenStream } from './types';
+export type { LanguageProfile, WordOrder, MarkingStrategy, RoleMarker, VerbConfig, PossessiveConfig, KeywordTranslation, TokenizationConfig, } from './generators/language-profiles';
+import type { LanguageProfile } from './generators/language-profiles';
+/**
+ * Interface for external pattern sources (e.g., @lokascript/patterns-reference database).
+ * External sources can provide additional patterns at runtime.
+ */
+export interface ExternalPatternsSource {
+    /** Unique identifier for the source */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Get patterns for a specific language */
+    getPatternsForLanguage(language: string): Promise<ExternalPatternEntry[]>;
+    /** Get patterns for a specific command */
+    getPatternsForCommand(command: string, language?: string): Promise<ExternalPatternEntry[]>;
+    /** Check if source has patterns for a language */
+    hasPatterns(language: string): Promise<boolean>;
+    /** Get all supported languages */
+    getSupportedLanguages(): Promise<string[]>;
+}
+/**
+ * Pattern entry from external source.
+ */
+export interface ExternalPatternEntry {
+    id: string;
+    code: string;
+    command: string | null;
+    language: string;
+    confidence: number;
+    verified: boolean;
+    title?: string;
+    category?: string;
+}
+/**
+ * Merge a variant profile with its base profile.
+ * The variant's fields override the base, with deep merging for nested objects.
+ *
+ * @example
+ * ```typescript
+ * const esMX = mergeProfiles(spanishProfile, {
+ *   code: 'es-MX',
+ *   name: 'Spanish (Mexico)',
+ *   keywords: {
+ *     toggle: { primary: 'alternar', alternatives: ['dale', 'cambiar'] },
+ *   },
+ * });
+ * ```
+ */
+export declare function mergeProfiles(base: LanguageProfile, variant: Partial<LanguageProfile>): LanguageProfile;
+/**
+ * Resolve a profile, applying inheritance if the profile has an `extends` field.
+ * Returns the merged profile with base language properties inherited.
+ */
+export declare function resolveProfile(profile: LanguageProfile): LanguageProfile;
+/**
+ * Register a language with its tokenizer and profile.
+ * Called automatically by language modules when imported.
+ * If the profile has an `extends` field, it will inherit from the base profile.
+ */
+export declare function registerLanguage(code: string, tokenizer: LanguageTokenizer, profile: LanguageProfile): void;
+/**
+ * Register only a tokenizer (for backwards compatibility).
+ */
+export declare function registerTokenizer(tokenizer: LanguageTokenizer): void;
+/**
+ * Register only a profile (for backwards compatibility).
+ */
+export declare function registerProfile(profile: LanguageProfile): void;
+/**
+ * Set the pattern generator function.
+ * Called by patterns module to inject the generator without circular deps.
+ */
+export declare function setPatternGenerator(generator: (profile: LanguageProfile) => LanguagePattern[]): void;
+/**
+ * Register patterns directly for a language.
+ * This enables tree-shaking by allowing each language module to register
+ * only its own patterns.
+ */
+export declare function registerPatterns(code: string, patterns: LanguagePattern[]): void;
+/**
+ * Check if patterns are directly registered for a language.
+ */
+export declare function hasRegisteredPatterns(code: string): boolean;
+/**
+ * Get directly registered patterns for a language.
+ */
+export declare function getRegisteredPatterns(code: string): LanguagePattern[] | undefined;
+/**
+ * Register an external pattern source.
+ * External sources (like @lokascript/patterns-reference) can provide
+ * additional patterns at runtime.
+ *
+ * @example
+ * ```typescript
+ * import { registerPatternsSource } from '@lokascript/semantic';
+ * import { createPatternsProvider } from '@lokascript/patterns-reference';
+ *
+ * const provider = createPatternsProvider();
+ * registerPatternsSource(provider);
+ * ```
+ */
+export declare function registerPatternsSource(source: ExternalPatternsSource): void;
+/**
+ * Unregister an external pattern source.
+ */
+export declare function unregisterPatternsSource(sourceId: string): boolean;
+/**
+ * Get a registered external pattern source.
+ */
+export declare function getPatternsSource(sourceId: string): ExternalPatternsSource | undefined;
+/**
+ * Get all registered external pattern sources.
+ */
+export declare function getAllPatternsSources(): ExternalPatternsSource[];
+/**
+ * Check if any external pattern sources are registered.
+ */
+export declare function hasExternalSources(): boolean;
+/**
+ * Query patterns from all external sources for a language.
+ * Returns patterns sorted by confidence.
+ */
+export declare function queryExternalPatterns(language: string): Promise<ExternalPatternEntry[]>;
+/**
+ * Query patterns from all external sources for a command.
+ */
+export declare function queryExternalPatternsForCommand(command: string, language?: string): Promise<ExternalPatternEntry[]>;
+/**
+ * Extract the base language code from a BCP 47 tag.
+ * Examples: 'es-MX' → 'es', 'pt-BR' → 'pt', 'en' → 'en'
+ */
+export declare function getBaseLanguageCode(code: string): string;
+/**
+ * Check if a code is a language variant (has region subtag).
+ * Examples: 'es-MX' → true, 'pt' → false
+ */
+export declare function isLanguageVariant(code: string): boolean;
+/**
+ * Get a tokenizer for the specified language.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ * @throws Error if neither the variant nor base language is registered
+ */
+export declare function getTokenizer(code: string): LanguageTokenizer;
+/**
+ * Get a profile for the specified language.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ * @throws Error if neither the variant nor base language is registered
+ */
+export declare function getProfile(code: string): LanguageProfile;
+/**
+ * Try to get a tokenizer, returning undefined if not registered.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ */
+export declare function tryGetTokenizer(code: string): LanguageTokenizer | undefined;
+/**
+ * Try to get a profile, returning undefined if not registered.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ */
+export declare function tryGetProfile(code: string): LanguageProfile | undefined;
+/**
+ * Get all registered language codes.
+ */
+export declare function getRegisteredLanguages(): string[];
+/**
+ * Check if a language is registered (exact match or base language fallback).
+ */
+export declare function isLanguageRegistered(code: string): boolean;
+/**
+ * Check if a language is supported (exact match or base language fallback).
+ * For backwards compatibility with tokenizers API.
+ */
+export declare function isLanguageSupported(code: string): boolean;
+/**
+ * Tokenize input in the specified language.
+ * @throws Error if language is not registered
+ */
+export declare function tokenize(input: string, language: string): TokenStream;
+/**
+ * Get patterns for a specific language.
+ * First checks for directly registered patterns (for tree-shaking),
+ * then falls back to pattern generator.
+ * Supports fallback: if 'es-MX' is not registered, falls back to 'es'.
+ * @throws Error if language is not registered
+ */
+export declare function getPatternsForLanguage(code: string): LanguagePattern[];
+/**
+ * Get patterns for a specific language and command.
+ */
+export declare function getPatternsForLanguageAndCommand(language: string, command: string): LanguagePattern[];
+/**
+ * Clear the pattern cache for a language (useful for testing).
+ */
+export declare function clearPatternCache(code?: string): void;
+/**
+ * Get all profiles as a record (for backwards compatibility).
+ * Note: Only returns registered profiles.
+ */
+export declare function getAllProfiles(): Record<string, LanguageProfile>;
+/**
+ * Get all tokenizers as a record (for backwards compatibility).
+ * Note: Only returns registered tokenizers.
+ */
+export declare function getAllTokenizers(): Record<string, LanguageTokenizer>;
+//# sourceMappingURL=registry.d.ts.map

package/dist/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAG/E,YAAY,EACV,eAAe,EACf,SAAS,EACT,eAAe,EACf,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAMtE;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACrC,uCAAuC;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAC1E,0CAA0C;IAC1C,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAC3F,kDAAkD;IAClD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,kCAAkC;IAClC,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAyDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,OAAO,CAAC,eAAe,CAAC,GAChC,eAAe,CAEjB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAmBxE;AAMD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,iBAAiB,EAC5B,OAAO,EAAE,eAAe,GACvB,IAAI,CAMN;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,GAAG,IAAI,CAEpE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAG9D;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,eAAe,EAAE,GACzD,IAAI,CAEN;AAKD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,IAAI,CAIhF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE3D;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,EAAE,GAAG,SAAS,CAEjF;AAMD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI,CAE3E;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAElE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,sBAAsB,GAAG,SAAS,CAEtF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,sBAAsB,EAAE,CAEhE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,OAAO,CAE5C;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAqB7F;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CACnD,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAoBjC;AAMD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEvD;AAMD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAmB5D;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAqBxD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAM3E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAOvE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,EAAE,CAEjD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAU1D;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CASzD;AAMD;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,CAGrE;AAMD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,EAAE,CAkCtE;AAED;;GAEG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,eAAe,EAAE,CAInB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAMrD;AAMD;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAMhE;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAMpE"}

package/dist/tokenizers/arabic.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Arabic Tokenizer
+ *
+ * Tokenizes Arabic hyperscript input.
+ * Arabic is challenging because:
+ * - Right-to-left (RTL) text direction
+ * - Prefix prepositions that attach to words (بـ, لـ, كـ)
+ * - Root-pattern morphology
+ * - CSS selectors are LTR islands within RTL text
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class ArabicTokenizer extends BaseTokenizer {
+    readonly language = "ar";
+    readonly direction: "rtl";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Try to match an Arabic preposition.
+     * Attaches prepositionValue metadata for disambiguation in pattern matching.
+     */
+    private tryPreposition;
+    /**
+     * Try to extract a proclitic (conjunction or preposition) that's attached to the following word.
+     *
+     * Arabic proclitics attach directly to words without space:
+     * - والنقر → و + النقر (and + the-click)
+     * - فالتبديل → ف + التبديل (then + the-toggle)
+     * - بالنقر → ب + النقر (with + the-click)
+     * - ولالنقر → و + ل + النقر (and + to + the-click)
+     *
+     * This enables:
+     * - Polysyndetic coordination: A وB وC
+     * - Attached prepositions: بالنقر (with-the-click)
+     * - Multi-proclitic sequences: ولالنقر (and-to-the-click)
+     *
+     * Returns null if:
+     * - Not a proclitic character/sequence
+     * - Proclitic is standalone (followed by space)
+     * - Remaining word is too short (< 2 chars, to avoid false positives)
+     * - Full word is a recognized keyword (e.g., بدل should NOT be split to ب + دل)
+     *
+     * @see NATIVE_REVIEW_NEEDED.md for implementation rationale
+     */
+    private tryProclitic;
+    /**
+     * Extract an Arabic word.
+     * Uses morphological normalization to handle prefix/suffix variations.
+     * Attaches metadata for temporal markers (formality, confidence).
+     */
+    private extractArabicWord;
+    /**
+     * Extract an ASCII word.
+     */
+    private extractAsciiWord;
+    /**
+     * Extract a number, including Arabic time unit suffixes.
+     * Arabic allows space between number and unit.
+     */
+    private extractArabicNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const arabicTokenizer: ArabicTokenizer;
+//# sourceMappingURL=arabic.d.ts.map

package/dist/tokenizers/arabic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arabic.d.ts","sourceRoot":"","sources":["../../src/tokenizers/arabic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAad,MAAM,QAAQ,CAAC;AAwQhB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IASpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAqHpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC;;;OAGG;IACH,OAAO,CAAC,cAAc;IAuBtB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,YAAY;IAsHpB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA+DzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;CAM5B;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,iBAAwB,CAAC"}