@lokascript/semantic 1.3.0 → 2.0.0

package/src/tokenizers/tl.ts

@@ -95,6 +95,12 @@ export class TagalogTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -187,7 +193,14 @@ export class TagalogTokenizer extends BaseTokenizer {
   classifyToken(token: string): TokenKind {
     // O(1) Map lookup instead of O(n) array search
     if (this.isKeyword(token)) return 'keyword';
-    if (token.startsWith('.') || token.startsWith('#') || token.startsWith('[')) return 'selector';
+    if (
+      token.startsWith('.') ||
+      token.startsWith('#') ||
+      token.startsWith('[') ||
+      token.startsWith('*') ||
+      token.startsWith('<')
+    )
+      return 'selector';
     if (token.startsWith(':')) return 'identifier';
     if (token.startsWith('"') || token.startsWith("'")) return 'literal';
     if (/^-?\d/.test(token)) return 'literal';

package/src/tokenizers/token-utils.ts

@@ -0,0 +1,205 @@
+/**
+ * Token Utilities
+ *
+ * Core token creation, stream implementation, and character classification.
+ * These are the foundational building blocks used by all tokenizers.
+ */
+
+import type { LanguageToken, TokenKind, TokenStream, StreamMark, SourcePosition } from '../types';
+
+// =============================================================================
+// Time Unit Configuration
+// =============================================================================
+
+/**
+ * Configuration for a native language time unit pattern.
+ * Used by tryNumberWithTimeUnits() to match language-specific time units.
+ */
+export interface TimeUnitMapping {
+  /** The pattern to match (e.g., 'segundos', 'ミリ秒') */
+  readonly pattern: string;
+  /** The standard suffix to use (ms, s, m, h) */
+  readonly suffix: string;
+  /** Length of the pattern (for optimization) */
+  readonly length: number;
+  /** Whether to check for word boundary after the pattern */
+  readonly checkBoundary?: boolean;
+  /** Character that cannot follow the pattern (e.g., 's' for 'm' to avoid 'ms') */
+  readonly notFollowedBy?: string;
+  /** Whether to do case-insensitive matching */
+  readonly caseInsensitive?: boolean;
+}
+
+// =============================================================================
+// Token Stream Implementation
+// =============================================================================
+
+/**
+ * Concrete implementation of TokenStream.
+ */
+export class TokenStreamImpl implements TokenStream {
+  readonly tokens: readonly LanguageToken[];
+  readonly language: string;
+  private pos: number = 0;
+
+  constructor(tokens: LanguageToken[], language: string) {
+    this.tokens = tokens;
+    this.language = language;
+  }
+
+  peek(offset: number = 0): LanguageToken | null {
+    const index = this.pos + offset;
+    if (index < 0 || index >= this.tokens.length) {
+      return null;
+    }
+    return this.tokens[index];
+  }
+
+  advance(): LanguageToken {
+    if (this.isAtEnd()) {
+      throw new Error('Unexpected end of token stream');
+    }
+    return this.tokens[this.pos++];
+  }
+
+  isAtEnd(): boolean {
+    return this.pos >= this.tokens.length;
+  }
+
+  mark(): StreamMark {
+    return { position: this.pos };
+  }
+
+  reset(mark: StreamMark): void {
+    this.pos = mark.position;
+  }
+
+  position(): number {
+    return this.pos;
+  }
+
+  /**
+   * Get remaining tokens as an array.
+   */
+  remaining(): LanguageToken[] {
+    return this.tokens.slice(this.pos);
+  }
+
+  /**
+   * Consume tokens while predicate is true.
+   */
+  takeWhile(predicate: (token: LanguageToken) => boolean): LanguageToken[] {
+    const result: LanguageToken[] = [];
+    while (!this.isAtEnd() && predicate(this.peek()!)) {
+      result.push(this.advance());
+    }
+    return result;
+  }
+
+  /**
+   * Skip tokens while predicate is true.
+   */
+  skipWhile(predicate: (token: LanguageToken) => boolean): void {
+    while (!this.isAtEnd() && predicate(this.peek()!)) {
+      this.advance();
+    }
+  }
+}
+
+// =============================================================================
+// Shared Tokenization Utilities
+// =============================================================================
+
+/**
+ * Create a source position from start and end offsets.
+ */
+export function createPosition(start: number, end: number): SourcePosition {
+  return { start, end };
+}
+
+/**
+ * Options for creating a token with optional morphological data.
+ */
+export interface CreateTokenOptions {
+  /** Explicitly normalized form from keyword map */
+  normalized?: string;
+  /** Morphologically normalized stem */
+  stem?: string;
+  /** Confidence in the stem (0.0-1.0) */
+  stemConfidence?: number;
+}
+
+/**
+ * Create a language token.
+ */
+export function createToken(
+  value: string,
+  kind: TokenKind,
+  position: SourcePosition,
+  normalizedOrOptions?: string | CreateTokenOptions
+): LanguageToken {
+  // Handle legacy string argument for backward compatibility
+  if (typeof normalizedOrOptions === 'string') {
+    return { value, kind, position, normalized: normalizedOrOptions };
+  }
+
+  // Handle options object
+  if (normalizedOrOptions) {
+    const { normalized, stem, stemConfidence } = normalizedOrOptions;
+    return {
+      value,
+      kind,
+      position,
+      ...(normalized !== undefined && { normalized }),
+      ...(stem !== undefined && { stem }),
+      ...(stemConfidence !== undefined && { stemConfidence }),
+    };
+  }
+
+  return { value, kind, position };
+}
+
+/**
+ * Check if a character is whitespace.
+ */
+export function isWhitespace(char: string): boolean {
+  return /\s/.test(char);
+}
+
+/**
+ * Check if a string starts with a CSS selector prefix.
+ * Includes JSX-style element selectors: <form />, <div>
+ */
+export function isSelectorStart(char: string): boolean {
+  return (
+    char === '#' || char === '.' || char === '[' || char === '@' || char === '*' || char === '<'
+  );
+}
+
+/**
+ * Check if a character is a quote (string delimiter).
+ */
+export function isQuote(char: string): boolean {
+  return char === '"' || char === "'" || char === '`' || char === '「' || char === '」';
+}
+
+/**
+ * Check if a character is a digit.
+ */
+export function isDigit(char: string): boolean {
+  return /\d/.test(char);
+}
+
+/**
+ * Check if a character is an ASCII letter.
+ */
+export function isAsciiLetter(char: string): boolean {
+  return /[a-zA-Z]/.test(char);
+}
+
+/**
+ * Check if a character is part of an ASCII identifier.
+ */
+export function isAsciiIdentifierChar(char: string): boolean {
+  return /[a-zA-Z0-9_-]/.test(char);
+}

package/src/tokenizers/turkish.ts

@@ -46,7 +46,7 @@ const POSTPOSITIONS = new Set([
   'için', // for
   'kadar', // until, as far as
   'gibi', // like
-  'sonra', // after
+  // 'sonra' removed — classified as keyword('then') for hyperscript clause chaining
   'önce', // before
   'üzerinde', // on, above
   'altında', // under
@@ -163,6 +163,12 @@ const TURKISH_EXTRAS: KeywordEntry[] = [
   { native: 'dakika', normalized: 'm' },
   { native: 'saat', normalized: 'h' },
 
+  // Then/conjunction (clause chaining)
+  { native: 'sonra', normalized: 'then' },
+  { native: 'ardından', normalized: 'then' },
+  { native: 'ardindan', normalized: 'then' },
+  { native: 'daha sonra', normalized: 'then' },
+
   // Logical
   { native: 've', normalized: 'and' },
   { native: 'veya', normalized: 'or' },
@@ -227,6 +233,12 @@ export class TurkishTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -304,7 +316,14 @@ export class TurkishTokenizer extends BaseTokenizer {
     if (CASE_SUFFIXES.has(lower)) return 'particle';
     // O(1) Map lookup instead of O(n) array search
     if (this.isKeyword(lower)) return 'keyword';
-    if (token.startsWith('#') || token.startsWith('.') || token.startsWith('[')) return 'selector';
+    if (
+      token.startsWith('#') ||
+      token.startsWith('.') ||
+      token.startsWith('[') ||
+      token.startsWith('*') ||
+      token.startsWith('<')
+    )
+      return 'selector';
     if (token.startsWith('"') || token.startsWith("'")) return 'literal';
     if (/^\d/.test(token)) return 'literal';
 

package/src/tokenizers/ukrainian.ts

@@ -26,6 +26,7 @@ import {
   type KeywordEntry,
 } from './base';
 import { ukrainianProfile } from '../generators/profiles/ukrainian';
+import { UkrainianMorphologicalNormalizer } from './morphology/ukrainian-normalizer';
 
 // =============================================================================
 // Ukrainian Character Classification
@@ -158,6 +159,7 @@ export class UkrainianTokenizer extends BaseTokenizer {
     super();
     // Initialize keywords from profile + extras (single source of truth)
     this.initializeKeywordsFromProfile(ukrainianProfile, UKRAINIAN_EXTRAS);
+    this.normalizer = new UkrainianMorphologicalNormalizer();
   }
 
   tokenize(input: string): TokenStream {
@@ -179,6 +181,12 @@ export class UkrainianTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -251,7 +259,14 @@ export class UkrainianTokenizer extends BaseTokenizer {
     if (PREPOSITIONS.has(lower)) return 'particle';
     // O(1) Map lookup instead of O(n) array search
     if (this.isKeyword(lower)) return 'keyword';
-    if (token.startsWith('#') || token.startsWith('.') || token.startsWith('[')) return 'selector';
+    if (
+      token.startsWith('#') ||
+      token.startsWith('.') ||
+      token.startsWith('[') ||
+      token.startsWith('*') ||
+      token.startsWith('<')
+    )
+      return 'selector';
     if (token.startsWith('"') || token.startsWith("'")) return 'literal';
     if (/^\d/.test(token)) return 'literal';
     return 'identifier';

package/src/tokenizers/vietnamese.ts

@@ -185,6 +185,12 @@ export class VietnameseTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -273,6 +279,7 @@ export class VietnameseTokenizer extends BaseTokenizer {
       token.startsWith('#') ||
       token.startsWith('.') ||
       token.startsWith('[') ||
+      token.startsWith('*') ||
       token.startsWith('<')
     )
       return 'selector';

package/src/types.ts

@@ -46,6 +46,9 @@ export type ActionType =
   | 'increment'
   | 'decrement'
   | 'log'
+  | 'copy'
+  | 'pick'
+  | 'beep'
   // Visibility
   | 'show'
   | 'hide'
@@ -75,10 +78,14 @@ export type ActionType =
   | 'for'
   | 'while'
   | 'continue'
+  | 'break'
+  | 'exit'
   | 'halt'
   | 'throw'
   | 'call'
   | 'return'
+  // Templates
+  | 'render'
   // Advanced
   | 'js'
   | 'async'
@@ -104,6 +111,12 @@ export type SemanticValue =
   | PropertyPathValue
   | ExpressionValue;
 
+/**
+ * Expected value types for role tokens.
+ * Shared between RoleSpec (command-schemas) and RolePatternToken.
+ */
+export type ExpectedType = SemanticValue['type'];
+
 export interface LiteralValue {
   readonly type: 'literal';
   readonly value: string | number | boolean;
@@ -190,6 +203,11 @@ export interface EventHandlerSemanticNode extends SemanticNode {
   readonly action: 'on';
   readonly body: SemanticNode[];
   readonly eventModifiers?: EventModifiers;
+  /**
+   * Additional events for multi-event handlers (e.g., "on click or keydown").
+   * The primary event is in the 'event' role; these are the additional "or" events.
+   */
+  readonly additionalEvents?: readonly SemanticValue[];
   /**
    * Event parameter names for destructuring.
    * E.g., for "on click(clientX, clientY)", this would be ['clientX', 'clientY']
@@ -318,7 +336,7 @@ export interface RolePatternToken {
   readonly role: SemanticRole;
   readonly optional?: boolean;
   /** Expected value types (for validation) */
-  readonly expectedTypes?: Array<SemanticValue['type']>;
+  readonly expectedTypes?: Array<ExpectedType>;
 }
 
 export interface GroupPatternToken {
@@ -547,6 +565,23 @@ export function createLiteral(
   return result;
 }
 
+/**
+ * Check if a string is a valid reference keyword.
+ */
+const VALID_REFERENCES: ReadonlySet<string> = new Set<ReferenceValue['value']>([
+  'me',
+  'you',
+  'it',
+  'result',
+  'event',
+  'target',
+  'body',
+]);
+
+export function isValidReference(value: string): value is ReferenceValue['value'] {
+  return VALID_REFERENCES.has(value);
+}
+
 /**
  * Create a reference semantic value.
  */
@@ -588,7 +623,8 @@ export function createEventHandler(
   body: SemanticNode[],
   modifiers?: EventModifiers,
   metadata?: SemanticMetadata,
-  parameterNames?: string[]
+  parameterNames?: string[],
+  additionalEvents?: SemanticValue[]
 ): EventHandlerSemanticNode {
   const roles = new Map<SemanticRole, SemanticValue>();
   roles.set('event', event);
@@ -609,6 +645,9 @@ export function createEventHandler(
   if (parameterNames !== undefined && parameterNames.length > 0) {
     (node as { parameterNames?: readonly string[] }).parameterNames = parameterNames;
   }
+  if (additionalEvents !== undefined && additionalEvents.length > 0) {
+    (node as { additionalEvents?: readonly SemanticValue[] }).additionalEvents = additionalEvents;
+  }
 
   return node;
 }

package/src/validators/command-validator.ts

@@ -11,29 +11,20 @@
  * - Supports confidence scoring for ambiguous parses
  *
  * Integration with core validators:
- * - Uses patterns from @lokascript/core's lightweight-validators where applicable
+ * - Uses patterns from @hyperfixi/core's lightweight-validators where applicable
  * - Type validation follows same patterns as runtime validation
  * - Can be extended with custom validators using the same API
  */
 
-import type { SemanticParseResult, SemanticValue, SemanticRole, ActionType } from '../types';
+import type {
+  SemanticParseResult,
+  SemanticValue,
+  SemanticRole,
+  ActionType,
+  ExpectedType,
+} from '../types';
 import type { CommandSchema } from '../generators/command-schemas';
-import {
-  toggleSchema,
-  addSchema,
-  removeSchema,
-  putSchema,
-  setSchema,
-  showSchema,
-  hideSchema,
-  onSchema,
-  triggerSchema,
-  waitSchema,
-  fetchSchema,
-  incrementSchema,
-  decrementSchema,
-  appendSchema,
-} from '../generators/command-schemas';
+import { commandSchemas } from '../generators/command-schemas';
 
 // =============================================================================
 // Types
@@ -79,23 +70,12 @@ export interface ValidationResult {
 
 /**
  * Maps action types to their schemas.
+ * Initialized from the full commandSchemas record (54 schemas).
+ * Custom schemas can be added via registerSchema().
  */
-const schemaRegistry = new Map<ActionType, CommandSchema>([
-  ['toggle', toggleSchema],
-  ['add', addSchema],
-  ['remove', removeSchema],
-  ['put', putSchema],
-  ['set', setSchema],
-  ['show', showSchema],
-  ['hide', hideSchema],
-  ['on', onSchema],
-  ['trigger', triggerSchema],
-  ['wait', waitSchema],
-  ['fetch', fetchSchema],
-  ['increment', incrementSchema],
-  ['decrement', decrementSchema],
-  ['append', appendSchema],
-]);
+const schemaRegistry = new Map<ActionType, CommandSchema>(
+  Object.entries(commandSchemas) as [ActionType, CommandSchema][]
+);
 
 /**
  * Get schema for an action type.
@@ -135,10 +115,7 @@ function getStringValue(value: SemanticValue): string | undefined {
 /**
  * Check if a semantic value matches expected types.
  */
-function valueMatchesType(
-  value: SemanticValue,
-  expectedTypes: Array<'selector' | 'literal' | 'reference' | 'expression'>
-): boolean {
+function valueMatchesType(value: SemanticValue, expectedTypes: Array<ExpectedType>): boolean {
   // Handle null/undefined
   if (value === null || value === undefined) {
     return false;
@@ -355,12 +332,12 @@ export function validateAndAdjustConfidence(
 //
 // This semantic validator is distinct from but complementary to core validators:
 //
-// 1. @lokascript/core/validation/lightweight-validators.ts
+// 1. @hyperfixi/core/validation/lightweight-validators.ts
 //    - Runtime type validation (v.string(), v.object(), etc.)
 //    - Zod-replacement for validating runtime input shapes
 //    - Used: command inputs, API payloads, configuration
 //
-// 2. @lokascript/core/validation/command-pattern-validator.ts
+// 2. @hyperfixi/core/validation/command-pattern-validator.ts
 //    - Class structure validation
 //    - Checks command classes follow TypeScript patterns
 //    - Used: CI/CD, development tooling