@digitaldefiance/i18n-lib 3.7.5 → 3.8.1

package/src/icu/parser.ts

@@ -0,0 +1,242 @@
+/**
+ * ICU MessageFormat Parser
+ */
+
+import { Tokenizer, Token, TokenType } from './tokenizer';
+import {
+  MessageNode,
+  LiteralNode,
+  ArgumentNode,
+  PluralNode,
+  SelectNode,
+  SelectOrdinalNode,
+  NodeType,
+  ASTNode,
+} from './ast';
+
+export class ParseError extends Error {
+  constructor(message: string, public position: number) {
+    super(`Parse error at position ${position}: ${message}`);
+    this.name = 'ParseError';
+  }
+}
+
+export class Parser {
+  private tokens: Token[];
+  private position: number = 0;
+  private depth: number = 0;
+  private readonly MAX_DEPTH = 10;
+
+  constructor(input: string) {
+    if (input.length > 10000) {
+      throw new ParseError('Input exceeds maximum length', 0);
+    }
+    this.tokens = new Tokenizer(input).tokenize();
+  }
+
+  parse(): MessageNode {
+    return this.parseMessage();
+  }
+
+  private parseMessage(): MessageNode {
+    if (++this.depth > this.MAX_DEPTH) {
+      throw new ParseError(`Maximum nesting depth of ${this.MAX_DEPTH} exceeded`, this.position);
+    }
+    
+    const elements: ASTNode[] = [];
+
+    while (!this.isAtEnd() && !this.check(TokenType.CLOSE_BRACE)) {
+      if (this.check(TokenType.TEXT)) {
+        elements.push(this.parseLiteral());
+      } else if (this.check(TokenType.OPEN_BRACE)) {
+        elements.push(this.parseArgument());
+      } else if (this.check(TokenType.HASH)) {
+        // # is treated as literal in message context
+        this.advance();
+        elements.push({ type: NodeType.LITERAL, value: '#' });
+      } else {
+        this.advance(); // Skip unexpected tokens
+      }
+    }
+
+    this.depth--;
+    return { type: NodeType.MESSAGE, elements };
+  }
+
+  private parseLiteral(): LiteralNode {
+    const token = this.advance();
+    return { type: NodeType.LITERAL, value: token.value };
+  }
+
+  private parseArgument(): ArgumentNode | PluralNode | SelectNode | SelectOrdinalNode {
+    this.consume(TokenType.OPEN_BRACE, 'Expected {');
+
+    const name = this.consume(TokenType.IDENTIFIER, 'Expected argument name').value;
+
+    // Simple argument: {name}
+    if (this.check(TokenType.CLOSE_BRACE)) {
+      this.advance();
+      return { type: NodeType.ARGUMENT, name };
+    }
+
+    // Formatted argument: {name, type} or {name, type, style}
+    this.consume(TokenType.COMMA, 'Expected ,');
+
+    const format = this.consume(TokenType.IDENTIFIER, 'Expected format type').value;
+
+    // Check for plural/select/selectordinal
+    if (format === 'plural') {
+      return this.parsePlural(name);
+    }
+    if (format === 'select') {
+      return this.parseSelect(name);
+    }
+    if (format === 'selectordinal') {
+      return this.parseSelectOrdinal(name);
+    }
+
+    // Regular formatted argument: {name, type, style}
+    let style: string | undefined;
+    if (this.check(TokenType.COMMA)) {
+      this.advance();
+      style = this.consume(TokenType.IDENTIFIER, 'Expected style').value;
+    }
+
+    this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+
+    return { type: NodeType.ARGUMENT, name, format, style };
+  }
+
+  private parsePlural(name: string): PluralNode {
+    this.consume(TokenType.COMMA, 'Expected ,');
+    this.skipWhitespace();
+
+    const cases: Record<string, MessageNode> = {};
+    let offset: number | undefined;
+
+    // Parse offset if present
+    if (this.check(TokenType.IDENTIFIER) && this.peek().value === 'offset') {
+      this.advance(); // offset
+      this.skipWhitespace();
+      this.consume(TokenType.IDENTIFIER, 'Expected offset value'); // number
+      offset = 0; // Simplified for now
+      this.skipWhitespace();
+    }
+
+    // Parse cases
+    while (!this.check(TokenType.CLOSE_BRACE) && !this.isAtEnd()) {
+      this.skipWhitespace();
+      if (this.check(TokenType.CLOSE_BRACE)) break;
+      
+      const caseKey = this.consume(TokenType.IDENTIFIER, 'Expected case key').value;
+      this.skipWhitespace();
+      this.consume(TokenType.OPEN_BRACE, 'Expected {');
+      const caseMessage = this.parseMessage();
+      this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+      cases[caseKey] = caseMessage;
+    }
+
+    if (Object.keys(cases).length === 0) {
+      throw new ParseError('Plural must have at least one case', this.peek().position);
+    }
+
+    this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+
+    return { type: NodeType.PLURAL, name, offset, cases };
+  }
+
+  private parseSelect(name: string): SelectNode {
+    this.consume(TokenType.COMMA, 'Expected ,');
+    this.skipWhitespace();
+
+    const cases: Record<string, MessageNode> = {};
+
+    // Parse cases
+    while (!this.check(TokenType.CLOSE_BRACE) && !this.isAtEnd()) {
+      this.skipWhitespace();
+      if (this.check(TokenType.CLOSE_BRACE)) break;
+      
+      const caseKey = this.consume(TokenType.IDENTIFIER, 'Expected case key').value;
+      this.skipWhitespace();
+      this.consume(TokenType.OPEN_BRACE, 'Expected {');
+      const caseMessage = this.parseMessage();
+      this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+      cases[caseKey] = caseMessage;
+    }
+
+    this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+
+    return { type: NodeType.SELECT, name, cases };
+  }
+
+  private parseSelectOrdinal(name: string): SelectOrdinalNode {
+    this.consume(TokenType.COMMA, 'Expected ,');
+    this.skipWhitespace();
+
+    const cases: Record<string, MessageNode> = {};
+    let offset: number | undefined;
+
+    // Parse offset if present
+    if (this.check(TokenType.IDENTIFIER) && this.peek().value === 'offset') {
+      this.advance(); // offset
+      this.skipWhitespace();
+      this.consume(TokenType.IDENTIFIER, 'Expected offset value'); // number
+      offset = 0; // Simplified for now
+      this.skipWhitespace();
+    }
+
+    // Parse cases
+    while (!this.check(TokenType.CLOSE_BRACE) && !this.isAtEnd()) {
+      this.skipWhitespace();
+      if (this.check(TokenType.CLOSE_BRACE)) break;
+      
+      const caseKey = this.consume(TokenType.IDENTIFIER, 'Expected case key').value;
+      this.skipWhitespace();
+      this.consume(TokenType.OPEN_BRACE, 'Expected {');
+      const caseMessage = this.parseMessage();
+      this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+      cases[caseKey] = caseMessage;
+    }
+
+    this.consume(TokenType.CLOSE_BRACE, 'Expected }');
+
+    return { type: NodeType.SELECTORDINAL, name, offset, cases };
+  }
+
+  private check(type: TokenType): boolean {
+    if (this.isAtEnd()) return false;
+    return this.peek().type === type;
+  }
+
+  private advance(): Token {
+    if (!this.isAtEnd()) this.position++;
+    return this.previous();
+  }
+
+  private isAtEnd(): boolean {
+    return this.peek().type === TokenType.EOF;
+  }
+
+  private peek(): Token {
+    return this.tokens[this.position];
+  }
+
+  private previous(): Token {
+    return this.tokens[this.position - 1];
+  }
+
+  private consume(type: TokenType, message: string): Token {
+    if (this.check(type)) return this.advance();
+    throw new ParseError(message, this.peek().position);
+  }
+
+  private skipWhitespace(): void {
+    while (this.check(TokenType.TEXT) && this.peek().value.trim() === '') {
+      this.advance();
+    }
+  }
+}
+
+export function parse(input: string): MessageNode {
+  return new Parser(input).parse();
+}

package/src/icu/runtime.ts

@@ -0,0 +1,37 @@
+import { parse } from './parser';
+import { validate } from './validator';
+import { Compiler, CompiledMessage } from './compiler';
+import { FormatterRegistry } from './formatter-registry';
+import { FormatterContext } from './formatters/base-formatter';
+import { LRUCache } from '../utils/lru-cache';
+
+export class Runtime {
+  private compiler: Compiler;
+  private cache = new LRUCache<string, CompiledMessage>(1000);
+
+  constructor(registry?: FormatterRegistry) {
+    this.compiler = new Compiler(registry);
+  }
+
+  format(message: string, values: Record<string, any>, context: FormatterContext): string {
+    if (message.length > 10000) {
+      throw new Error('Message exceeds maximum length');
+    }
+    
+    const cacheKey = `${context.locale}:${message}`;
+    let compiled = this.cache.get(cacheKey);
+    
+    if (!compiled) {
+      const ast = parse(message);
+      validate(ast);
+      compiled = this.compiler.compile(ast);
+      this.cache.set(cacheKey, compiled);
+    }
+
+    return compiled(values, context);
+  }
+
+  clearCache(): void {
+    this.cache.clear();
+  }
+}

package/src/icu/tokenizer.ts

@@ -0,0 +1,212 @@
+/**
+ * ICU MessageFormat Tokenizer
+ */
+
+export enum TokenType {
+  TEXT = 'TEXT',
+  OPEN_BRACE = 'OPEN_BRACE',
+  CLOSE_BRACE = 'CLOSE_BRACE',
+  COMMA = 'COMMA',
+  HASH = 'HASH',
+  IDENTIFIER = 'IDENTIFIER',
+  EOF = 'EOF',
+}
+
+export interface Token {
+  type: TokenType;
+  value: string;
+  position: number;
+}
+
+export class Tokenizer {
+  private input: string;
+  private position: number = 0;
+  private length: number;
+  private braceDepth: number = 0;
+  private lastTokenType: TokenType | null = null;
+  private formatDepths: Set<number> = new Set(); // Tracks which depths are format specs
+  private lastIdentifierWasAtFormatDepth: boolean = false;
+
+  constructor(input: string) {
+    this.input = input;
+    this.length = input.length;
+  }
+
+  tokenize(): Token[] {
+    const tokens: Token[] = [];
+    while (this.position < this.length) {
+      const token = this.nextToken();
+      if (token.type !== TokenType.EOF) {
+        tokens.push(token);
+      }
+    }
+    tokens.push({ type: TokenType.EOF, value: '', position: this.position });
+    return tokens;
+  }
+
+  private nextToken(): Token {
+    if (this.position >= this.length) {
+      return { type: TokenType.EOF, value: '', position: this.position };
+    }
+
+    const char = this.input[this.position];
+
+    // Handle escape sequences - quoted braces
+    if (char === "'") {
+      const next = this.peek();
+      if (next === '{' || next === '}') {
+        const pos = this.position;
+        this.position++; // Skip quote
+        this.position++; // Skip brace
+        return { type: TokenType.TEXT, value: next, position: pos };
+      }
+    }
+
+    // Special characters
+    if (char === '{') {
+      const prevDepth = this.braceDepth;
+      this.braceDepth++;
+      // Add to format depths if:
+      // 1. Start of message (null)
+      // 2. After TEXT - new format in message (unless after case name)
+      // 3. After CLOSE_BRACE at depth 0 - consecutive formats
+      // 4. After OPEN_BRACE - nested format (unless after case name)
+      // NOT after IDENTIFIER at format depth - that's a case name, next brace is message content
+      if (this.lastTokenType === null || 
+          (this.lastTokenType === TokenType.TEXT && !this.lastIdentifierWasAtFormatDepth) ||
+          (this.lastTokenType === TokenType.CLOSE_BRACE && prevDepth === 0) ||
+          (this.lastTokenType === TokenType.OPEN_BRACE && !this.lastIdentifierWasAtFormatDepth)) {
+        this.formatDepths.add(this.braceDepth);
+      }
+      this.lastIdentifierWasAtFormatDepth = false;
+      const token = { type: TokenType.OPEN_BRACE, value: char, position: this.position++ };
+      this.lastTokenType = TokenType.OPEN_BRACE;
+      return token;
+    }
+    if (char === '}') {
+      this.formatDepths.delete(this.braceDepth);
+      this.braceDepth--;
+      this.lastIdentifierWasAtFormatDepth = false; // Reset after closing
+      const token = { type: TokenType.CLOSE_BRACE, value: char, position: this.position++ };
+      this.lastTokenType = TokenType.CLOSE_BRACE;
+      return token;
+    }
+    if (char === ',') {
+      // Comma at depth 1+ is format separator, at depth 0 is text
+      if (this.braceDepth >= 1) {
+        const token = { type: TokenType.COMMA, value: char, position: this.position++ };
+        this.lastTokenType = TokenType.COMMA;
+        return token;
+      }
+      // At depth 0, comma is just text
+      return this.readText();
+    }
+    if (char === '#') {
+      return { type: TokenType.HASH, value: char, position: this.position++ };
+    }
+
+    // Whitespace - skip at format level OR after identifier at depth 1
+    if (this.isWhitespace(char) && (this.isAtFormatLevel() || (this.braceDepth === 1 && this.lastTokenType === TokenType.IDENTIFIER))) {
+      this.position++;
+      return this.nextToken();
+    }
+
+    // Identifier (variable names, keywords) - at format level
+    if (this.isIdentifierStart(char)) {
+      if (this.isAtFormatLevel()) {
+        return this.readIdentifier();
+      }
+    }
+
+    // Text (everything else)
+    return this.readText();
+  }
+
+  private readIdentifier(): Token {
+    const start = this.position;
+    while (this.position < this.length && this.isIdentifierChar(this.input[this.position])) {
+      this.position++;
+    }
+    const token = {
+      type: TokenType.IDENTIFIER,
+      value: this.input.substring(start, this.position),
+      position: start,
+    };
+    this.lastTokenType = TokenType.IDENTIFIER;
+    this.lastIdentifierWasAtFormatDepth = this.formatDepths.has(this.braceDepth);
+    return token;
+  }
+
+  private readText(): Token {
+    const start = this.position;
+    let text = '';
+    
+    while (this.position < this.length) {
+      const char = this.input[this.position];
+      
+      // Handle escaped quotes - stop before them
+      if (char === "'" && (this.peek() === '{' || this.peek() === '}')) {
+        break;
+      }
+      
+      // Stop at special characters (but not if they're escaped)
+      if (char === '{' || char === '#') {
+        break;
+      }
+      
+      // For }, only stop if we're inside braces
+      if (char === '}' && this.braceDepth > 0) {
+        break;
+      }
+      
+      // Stop at comma if we're at format level (depth >= 1)
+      if (char === ',' && this.braceDepth >= 1) {
+        break;
+      }
+      
+      text += char;
+      this.position++;
+    }
+    
+    const token = {
+      type: TokenType.TEXT,
+      value: text,
+      position: start,
+    };
+    this.lastTokenType = TokenType.TEXT;
+    // Don't reset the identifier flag for whitespace-only text
+    if (text.trim() !== '') {
+      this.lastIdentifierWasAtFormatDepth = false;
+    }
+    return token;
+  }
+
+  private peek(): string {
+    return this.position + 1 < this.length ? this.input[this.position + 1] : '';
+  }
+
+  private isIdentifierStart(char: string): boolean {
+    return /[a-zA-Z_]/.test(char);
+  }
+
+  private isIdentifierChar(char: string): boolean {
+    return /[a-zA-Z0-9_]/.test(char);
+  }
+
+  private isWhitespace(char: string): boolean {
+    return /\s/.test(char);
+  }
+
+  private isAtFormatLevel(): boolean {
+    // We're at format level if current depth is marked as format AND:
+    // 1. After OPEN_BRACE (expecting variable name)
+    // 2. After COMMA (expecting format type or case name)
+    // 3. After CLOSE_BRACE (just finished a case message, expecting another case name)
+    if (!this.formatDepths.has(this.braceDepth)) {
+      return false;
+    }
+    return this.lastTokenType === TokenType.OPEN_BRACE || 
+           this.lastTokenType === TokenType.COMMA ||
+           this.lastTokenType === TokenType.CLOSE_BRACE;
+  }
+}

package/src/icu/validator.ts

@@ -0,0 +1,163 @@
+import { MessageNode, NodeType, ArgumentNode, PluralNode, SelectNode, SelectOrdinalNode } from './ast';
+
+export class ValidationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ValidationError';
+  }
+}
+
+export interface ValidationOptions {
+  requireOtherCase?: boolean;
+  allowCircularReferences?: boolean;
+  maxDepth?: number;
+}
+
+const DEFAULT_OPTIONS: ValidationOptions = {
+  requireOtherCase: true,
+  allowCircularReferences: false,
+  maxDepth: 10,
+};
+
+export function validate(ast: MessageNode, options: ValidationOptions = {}): void {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const context = new ValidationContext(opts);
+  validateMessage(ast, context);
+}
+
+class ValidationContext {
+  private argumentNames = new Set<string>();
+  private currentDepth = 0;
+
+  constructor(public options: ValidationOptions) {}
+
+  addArgument(name: string): void {
+    this.argumentNames.add(name);
+  }
+
+  hasArgument(name: string): boolean {
+    return this.argumentNames.has(name);
+  }
+
+  enterDepth(): void {
+    this.currentDepth++;
+    if (this.currentDepth > this.options.maxDepth!) {
+      throw new ValidationError(`Maximum nesting depth of ${this.options.maxDepth} exceeded`);
+    }
+  }
+
+  exitDepth(): void {
+    this.currentDepth--;
+  }
+}
+
+function validateMessage(node: MessageNode, context: ValidationContext): void {
+  for (const element of node.elements) {
+    validateNode(element, context);
+  }
+}
+
+function validateNode(node: any, context: ValidationContext): void {
+  switch (node.type) {
+    case NodeType.LITERAL:
+      break;
+    case NodeType.ARGUMENT:
+      validateArgument(node, context);
+      break;
+    case NodeType.PLURAL:
+      validatePlural(node, context);
+      break;
+    case NodeType.SELECT:
+      validateSelect(node, context);
+      break;
+    case NodeType.SELECTORDINAL:
+      validateSelectOrdinal(node, context);
+      break;
+    default:
+      throw new ValidationError(`Unknown node type: ${node.type}`);
+  }
+}
+
+function validateArgument(node: ArgumentNode, context: ValidationContext): void {
+  if (!node.name || node.name.trim() === '') {
+    throw new ValidationError('Argument name cannot be empty');
+  }
+  context.addArgument(node.name);
+}
+
+function validatePlural(node: PluralNode, context: ValidationContext): void {
+  if (!node.name || node.name.trim() === '') {
+    throw new ValidationError('Plural variable name cannot be empty');
+  }
+  context.addArgument(node.name);
+
+  if (!node.cases || Object.keys(node.cases).length === 0) {
+    throw new ValidationError(`Plural '${node.name}' must have at least one case`);
+  }
+
+  if (context.options.requireOtherCase && !node.cases.other) {
+    throw new ValidationError(`Plural '${node.name}' must have an 'other' case`);
+  }
+
+  const validCases = ['zero', 'one', 'two', 'few', 'many', 'other'];
+  for (const caseName of Object.keys(node.cases)) {
+    if (!validCases.includes(caseName)) {
+      throw new ValidationError(`Invalid plural case '${caseName}' in '${node.name}'`);
+    }
+  }
+
+  context.enterDepth();
+  for (const caseMessage of Object.values(node.cases)) {
+    validateMessage(caseMessage, context);
+  }
+  context.exitDepth();
+}
+
+function validateSelect(node: SelectNode, context: ValidationContext): void {
+  if (!node.name || node.name.trim() === '') {
+    throw new ValidationError('Select variable name cannot be empty');
+  }
+  context.addArgument(node.name);
+
+  if (!node.cases || Object.keys(node.cases).length === 0) {
+    throw new ValidationError(`Select '${node.name}' must have at least one case`);
+  }
+
+  if (context.options.requireOtherCase && !node.cases.other) {
+    throw new ValidationError(`Select '${node.name}' must have an 'other' case`);
+  }
+
+  context.enterDepth();
+  for (const caseMessage of Object.values(node.cases)) {
+    validateMessage(caseMessage, context);
+  }
+  context.exitDepth();
+}
+
+function validateSelectOrdinal(node: SelectOrdinalNode, context: ValidationContext): void {
+  if (!node.name || node.name.trim() === '') {
+    throw new ValidationError('SelectOrdinal variable name cannot be empty');
+  }
+  context.addArgument(node.name);
+
+  if (!node.cases || Object.keys(node.cases).length === 0) {
+    throw new ValidationError(`SelectOrdinal '${node.name}' must have at least one case`);
+  }
+
+  if (context.options.requireOtherCase && !node.cases.other) {
+    throw new ValidationError(`SelectOrdinal '${node.name}' must have an 'other' case`);
+  }
+
+  const validCases = ['zero', 'one', 'two', 'few', 'many', 'other'];
+  for (const caseName of Object.keys(node.cases)) {
+    if (!validCases.includes(caseName)) {
+      throw new ValidationError(`Invalid selectordinal case '${caseName}' in '${node.name}'`);
+    }
+  }
+
+  context.enterDepth();
+  for (const caseMessage of Object.values(node.cases)) {
+    validateMessage(caseMessage, context);
+  }
+  context.exitDepth();
+}