@lokascript/framework 2.0.0

package/src/core/tokenization/char-classifiers.ts

@@ -0,0 +1,79 @@
+/**
+ * Character Classifiers
+ *
+ * Unicode range classification and Latin character classifier factories.
+ * Used by language-specific tokenizers to define character sets.
+ */
+
+// =============================================================================
+// Unicode Range Classification
+// =============================================================================
+
+/**
+ * Unicode range tuple: [start, end] (inclusive).
+ */
+export type UnicodeRange = readonly [number, number];
+
+/**
+ * Create a character classifier for Unicode ranges.
+ * Returns a function that checks if a character's code point falls within any of the ranges.
+ *
+ * @example
+ * // Japanese Hiragana
+ * const isHiragana = createUnicodeRangeClassifier([[0x3040, 0x309f]]);
+ *
+ * // Korean (Hangul syllables + Jamo)
+ * const isKorean = createUnicodeRangeClassifier([
+ *   [0xac00, 0xd7a3],  // Hangul syllables
+ *   [0x1100, 0x11ff],  // Hangul Jamo
+ *   [0x3130, 0x318f],  // Hangul Compatibility Jamo
+ * ]);
+ */
+export function createUnicodeRangeClassifier(
+  ranges: readonly UnicodeRange[]
+): (char: string) => boolean {
+  return (char: string): boolean => {
+    const code = char.charCodeAt(0);
+    return ranges.some(([start, end]) => code >= start && code <= end);
+  };
+}
+
+/**
+ * Combine multiple character classifiers into one.
+ * Returns true if any of the classifiers return true.
+ *
+ * @example
+ * const isJapanese = combineClassifiers(isHiragana, isKatakana, isKanji);
+ */
+export function combineClassifiers(
+  ...classifiers: Array<(char: string) => boolean>
+): (char: string) => boolean {
+  return (char: string): boolean => classifiers.some(fn => fn(char));
+}
+
+/**
+ * Character classifiers for a Latin-based language.
+ */
+export interface LatinCharClassifiers {
+  /** Check if character is a letter in this language (including accented chars). */
+  isLetter: (char: string) => boolean;
+  /** Check if character is part of an identifier (letter, digit, underscore, hyphen). */
+  isIdentifierChar: (char: string) => boolean;
+}
+
+/**
+ * Create character classifiers for a Latin-based language.
+ * Returns isLetter and isIdentifierChar functions based on the provided regex.
+ *
+ * @example
+ * // Spanish letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ]/);
+ *
+ * // German letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZäöüÄÖÜß]/);
+ */
+export function createLatinCharClassifiers(letterPattern: RegExp): LatinCharClassifiers {
+  const isLetter = (char: string): boolean => letterPattern.test(char);
+  const isIdentifierChar = (char: string): boolean => isLetter(char) || /[0-9_-]/.test(char);
+  return { isLetter, isIdentifierChar };
+}

package/src/core/tokenization/create-simple-tokenizer.test.ts

@@ -0,0 +1,260 @@
+/**
+ * Unit tests for createSimpleTokenizer factory function.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { createSimpleTokenizer } from './base-tokenizer';
+import type { SimpleTokenizerConfig } from './base-tokenizer';
+import type { ValueExtractor, ExtractionResult } from '../../interfaces/value-extractor';
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+function makeTokenizer(overrides: Partial<SimpleTokenizerConfig> = {}) {
+  return createSimpleTokenizer({
+    language: 'en',
+    keywords: ['select', 'from', 'where'],
+    ...overrides,
+  });
+}
+
+/** Collect all tokens from a tokenizer run as [value, kind] pairs */
+function tokenize(input: string, config?: Partial<SimpleTokenizerConfig>) {
+  const t = makeTokenizer(config);
+  const stream = t.tokenize(input);
+  const tokens: Array<{ value: string; kind: string }> = [];
+  while (!stream.isAtEnd()) {
+    const tok = stream.advance();
+    tokens.push({ value: tok.value, kind: tok.kind });
+  }
+  return tokens;
+}
+
+// =============================================================================
+// Tests
+// =============================================================================
+
+describe('createSimpleTokenizer', () => {
+  // ---------------------------------------------------------------------------
+  // Keyword classification
+  // ---------------------------------------------------------------------------
+  describe('keyword classification', () => {
+    it('recognizes keywords case-insensitively by default', () => {
+      const t = makeTokenizer();
+      expect(t.classifyToken('select')).toBe('keyword');
+      expect(t.classifyToken('SELECT')).toBe('keyword');
+      expect(t.classifyToken('Select')).toBe('keyword');
+    });
+
+    it('recognizes keywords case-sensitively when caseInsensitive: false', () => {
+      const t = makeTokenizer({ caseInsensitive: false });
+      expect(t.classifyToken('select')).toBe('keyword');
+      expect(t.classifyToken('SELECT')).toBe('identifier');
+    });
+
+    it('classifies non-keywords as identifier', () => {
+      const t = makeTokenizer();
+      expect(t.classifyToken('users')).toBe('identifier');
+      expect(t.classifyToken('name')).toBe('identifier');
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Operator classification
+  // ---------------------------------------------------------------------------
+  describe('operator classification', () => {
+    it('classifies operators when includeOperators: true', () => {
+      const t = makeTokenizer({ includeOperators: true });
+      expect(t.classifyToken('=')).toBe('operator');
+      expect(t.classifyToken('+')).toBe('operator');
+      expect(t.classifyToken('>')).toBe('operator');
+    });
+
+    it('classifies operators as identifier when includeOperators: false', () => {
+      const t = makeTokenizer({ includeOperators: false });
+      expect(t.classifyToken('=')).toBe('identifier');
+      expect(t.classifyToken('+')).toBe('identifier');
+    });
+
+    it('classifies multi-char operators', () => {
+      const t = makeTokenizer({ includeOperators: true });
+      expect(t.classifyToken('>=')).toBe('operator');
+      expect(t.classifyToken('!=')).toBe('operator');
+      expect(t.classifyToken('==')).toBe('operator');
+    });
+
+    it('classifies broader operators from DEFAULT_OPERATORS', () => {
+      const t = makeTokenizer({ includeOperators: true });
+      expect(t.classifyToken('&&')).toBe('operator');
+      expect(t.classifyToken('||')).toBe('operator');
+      expect(t.classifyToken('%')).toBe('operator');
+    });
+
+    it('defaults includeOperators to false', () => {
+      const t = makeTokenizer();
+      expect(t.classifyToken('+')).toBe('identifier');
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Literal classification
+  // ---------------------------------------------------------------------------
+  describe('literal classification', () => {
+    it('classifies digit-starting tokens as literal', () => {
+      const t = makeTokenizer();
+      expect(t.classifyToken('42')).toBe('literal');
+      expect(t.classifyToken('3.14')).toBe('literal');
+    });
+
+    it('classifies quote-starting tokens as literal', () => {
+      const t = makeTokenizer();
+      expect(t.classifyToken("'hello'")).toBe('literal');
+      expect(t.classifyToken('"world"')).toBe('literal');
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Full tokenization
+  // ---------------------------------------------------------------------------
+  describe('tokenization', () => {
+    it('tokenizes simple input into keyword and identifier tokens', () => {
+      const tokens = tokenize('select name from users');
+      expect(tokens).toEqual([
+        { value: 'select', kind: 'keyword' },
+        { value: 'name', kind: 'identifier' },
+        { value: 'from', kind: 'keyword' },
+        { value: 'users', kind: 'identifier' },
+      ]);
+    });
+
+    it('skips whitespace between tokens', () => {
+      const tokens = tokenize('select   name');
+      expect(tokens.map(t => t.value)).toEqual(['select', 'name']);
+    });
+
+    it('tokenizes string literals', () => {
+      const tokens = tokenize("where 'hello'");
+      const kinds = tokens.map(t => t.kind);
+      expect(kinds).toContain('keyword'); // where
+      expect(kinds).toContain('literal'); // 'hello'
+    });
+
+    it('tokenizes numbers', () => {
+      const tokens = tokenize('select 42');
+      expect(tokens[1]).toMatchObject({ value: '42', kind: 'literal' });
+    });
+
+    it('produces tokens with positions', () => {
+      const t = makeTokenizer();
+      const stream = t.tokenize('select name');
+      const first = stream.advance();
+      expect(first).toBeDefined();
+      expect(first.position.start).toBe(0);
+      expect(first.position.end).toBe(6);
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Profile-based keywords (non-Latin)
+  // ---------------------------------------------------------------------------
+  describe('profile-based keywords', () => {
+    const jaConfig: Partial<SimpleTokenizerConfig> = {
+      language: 'ja',
+      keywords: ['選択', 'から', '条件'],
+      keywordExtras: [
+        { native: '選択', normalized: 'select' },
+        { native: 'から', normalized: 'from' },
+        { native: '条件', normalized: 'where' },
+      ],
+      keywordProfile: {
+        keywords: {
+          select: { primary: '選択' },
+          from: { primary: 'から' },
+          where: { primary: '条件' },
+        },
+      },
+      caseInsensitive: false,
+    };
+
+    it('recognizes non-Latin keywords from config.keywords', () => {
+      const t = makeTokenizer(jaConfig);
+      expect(t.classifyToken('選択')).toBe('keyword');
+      expect(t.classifyToken('から')).toBe('keyword');
+    });
+
+    it('recognizes keywords via profile path (isKeyword)', () => {
+      // Create a tokenizer where a keyword is ONLY in the profile, not in keywords[]
+      const t = makeTokenizer({
+        language: 'ja',
+        keywords: [], // empty — no fast-path keywords
+        keywordProfile: {
+          keywords: {
+            select: { primary: '選択' },
+          },
+        },
+        caseInsensitive: false,
+      });
+      expect(t.classifyToken('選択')).toBe('keyword');
+    });
+
+    it('classifies unknown tokens as identifier', () => {
+      const t = makeTokenizer(jaConfig);
+      expect(t.classifyToken('ユーザー')).toBe('identifier');
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Custom extractors
+  // ---------------------------------------------------------------------------
+  describe('custom extractors', () => {
+    it('custom extractors take priority over defaults', () => {
+      // Custom extractor that captures @-mentions
+      const mentionExtractor: ValueExtractor = {
+        name: 'mention',
+        canExtract(input: string, pos: number) {
+          return input[pos] === '@';
+        },
+        extract(input: string, pos: number): ExtractionResult | null {
+          let end = pos + 1;
+          while (end < input.length && /\w/.test(input[end])) end++;
+          if (end === pos + 1) return null;
+          return { value: input.slice(pos, end), length: end - pos };
+        },
+      };
+
+      const tokens = tokenize('select @user', {
+        customExtractors: [mentionExtractor],
+      });
+
+      expect(tokens[0]).toMatchObject({ value: 'select', kind: 'keyword' });
+      // @user extracted by custom extractor before default identifier extractor
+      expect(tokens[1]).toMatchObject({ value: '@user' });
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Direction
+  // ---------------------------------------------------------------------------
+  describe('direction', () => {
+    it('defaults to ltr', () => {
+      const t = makeTokenizer();
+      expect(t.direction).toBe('ltr');
+    });
+
+    it('can be set to rtl', () => {
+      const t = makeTokenizer({ direction: 'rtl' });
+      expect(t.direction).toBe('rtl');
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Language property
+  // ---------------------------------------------------------------------------
+  describe('language', () => {
+    it('exposes the configured language', () => {
+      const t = makeTokenizer({ language: 'es' });
+      expect(t.language).toBe('es');
+    });
+  });
+});

package/src/core/tokenization/default-extractors.ts

@@ -0,0 +1,69 @@
+/**
+ * Default Extractor Sets
+ *
+ * Provides pre-configured sets of extractors for common use cases.
+ * DSLs can use these as a starting point and add domain-specific extractors.
+ */
+
+import type { ValueExtractor } from '../../interfaces/value-extractor';
+import {
+  StringLiteralExtractor,
+  NumberExtractor,
+  IdentifierExtractor,
+  UnicodeIdentifierExtractor,
+} from '../../interfaces/value-extractor';
+import { OperatorExtractor, PunctuationExtractor } from './extractors/index';
+
+/**
+ * Get default extractors for generic programming-language-style DSLs.
+ * These work for most DSLs (SQL, config files, scripts, etc.).
+ *
+ * Included extractors:
+ * - String literals: "double", 'single', `backtick`
+ * - Numbers: 123, 45.67
+ * - Operators: +, -, *, /, =, ==, !=, >=, <=, etc.
+ * - Punctuation: ( ) [ ] { } , : ;
+ * - Identifiers: variable_names, functionNames
+ * - Unicode identifiers: CJK, Arabic, Cyrillic, etc.
+ *
+ * @returns Array of default extractors
+ *
+ * @example
+ * ```typescript
+ * class MyDSLTokenizer extends BaseTokenizer {
+ *   constructor() {
+ *     super();
+ *     this.registerExtractors(getDefaultExtractors());
+ *   }
+ * }
+ * ```
+ */
+export function getDefaultExtractors(): ValueExtractor[] {
+  return [
+    new StringLiteralExtractor(), // "strings", 'strings', `strings`
+    new NumberExtractor(), // 123, 45.67
+    new OperatorExtractor(), // +, -, *, /, =, >, <, etc.
+    new PunctuationExtractor(), // ( ) [ ] { } , : ;
+    new IdentifierExtractor(), // variable_names, functionNames (ASCII)
+    new UnicodeIdentifierExtractor(), // CJK, Arabic, Cyrillic, etc.
+  ];
+}
+
+/**
+ * Auto-register default extractors in a tokenizer.
+ * Convenience helper for chaining.
+ *
+ * @param tokenizer - Tokenizer to configure
+ * @returns The same tokenizer (for chaining)
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = withDefaultExtractors(new MyTokenizer());
+ * ```
+ */
+export function withDefaultExtractors<
+  T extends { registerExtractors(extractors: ValueExtractor[]): void },
+>(tokenizer: T): T {
+  tokenizer.registerExtractors(getDefaultExtractors());
+  return tokenizer;
+}

package/src/core/tokenization/extractors/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Generic Value Extractors
+ *
+ * These extractors work for most programming-language-style DSLs.
+ * DSLs can use these as-is or provide custom extractors for domain-specific syntax.
+ */
+
+export { OperatorExtractor, DEFAULT_OPERATORS } from './operator';
+export { PunctuationExtractor, DEFAULT_PUNCTUATION } from './punctuation';

package/src/core/tokenization/extractors/operator.ts

@@ -0,0 +1,75 @@
+/**
+ * Operator Extractor - Handles programming language operators
+ *
+ * Extracts operators like +, -, *, /, =, >, <, >=, <=, !=, ===, etc.
+ * Supports multi-character operators with longest-match priority.
+ */
+
+import type { ValueExtractor, ExtractionResult } from '../../../interfaces/value-extractor';
+
+/**
+ * Default operators for most programming languages.
+ * Sorted longest-first for greedy matching.
+ */
+export const DEFAULT_OPERATORS = [
+  // Three-character operators
+  '===',
+  '!==',
+  '->',
+  // Two-character operators
+  '==',
+  '!=',
+  '<=',
+  '>=',
+  '&&',
+  '||',
+  '**',
+  '+=',
+  '-=',
+  '*=',
+  '/=',
+  // Single-character operators
+  '+',
+  '-',
+  '*',
+  '/',
+  '=',
+  '>',
+  '<',
+  '!',
+  '&',
+  '|',
+  '%',
+  '^',
+  '~',
+];
+
+/**
+ * OperatorExtractor - Extracts programming language operators.
+ */
+export class OperatorExtractor implements ValueExtractor {
+  readonly name = 'operator';
+
+  constructor(private operators: string[] = DEFAULT_OPERATORS) {
+    // Sort operators longest-first for greedy matching
+    this.operators = [...operators].sort((a, b) => b.length - a.length);
+  }
+
+  canExtract(input: string, position: number): boolean {
+    return this.operators.some(op => input.startsWith(op, position));
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    // Find longest matching operator
+    for (const op of this.operators) {
+      if (input.startsWith(op, position)) {
+        return {
+          value: op,
+          length: op.length,
+        };
+      }
+    }
+
+    return null;
+  }
+}

package/src/core/tokenization/extractors/punctuation.ts

@@ -0,0 +1,39 @@
+/**
+ * Punctuation Extractor - Handles punctuation characters
+ *
+ * Extracts punctuation like parentheses, brackets, braces, commas, colons, semicolons.
+ * Each character is extracted individually (no multi-character punctuation).
+ */
+
+import type { ValueExtractor, ExtractionResult } from '../../../interfaces/value-extractor';
+
+/**
+ * Default punctuation characters for most programming languages.
+ */
+export const DEFAULT_PUNCTUATION = '()[]{},:;';
+
+/**
+ * PunctuationExtractor - Extracts punctuation characters.
+ */
+export class PunctuationExtractor implements ValueExtractor {
+  readonly name = 'punctuation';
+
+  constructor(private punctuation: string = DEFAULT_PUNCTUATION) {}
+
+  canExtract(input: string, position: number): boolean {
+    return this.punctuation.includes(input[position]);
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    const char = input[position];
+
+    if (this.punctuation.includes(char)) {
+      return {
+        value: char,
+        length: 1,
+      };
+    }
+
+    return null;
+  }
+}