@lokascript/framework 2.1.0 → 2.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lokascript/framework",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
   "type": "module",
   "main": "dist/index.cjs",

package/src/core/tokenization/morphology/base-normalizer.ts

@@ -0,0 +1,249 @@
+/**
+ * BaseMorphologicalNormalizer — Shared base class for language normalizers.
+ *
+ * Provides the common suffix/prefix stripping loop, reflexive verb handling,
+ * and normalize() pipeline. Language-specific normalizers extend this class
+ * and provide their conjugation rules.
+ *
+ * Phase 3.1 of parser-ecosystem-plan-v3.
+ */
+
+import type {
+  MorphologicalNormalizer,
+  NormalizationResult,
+  ConjugationType,
+  SuffixRule,
+  PrefixRule,
+} from './types';
+import { noChange, normalized } from './types';
+
+/**
+ * Conjugation ending rule for verb classes (Romance languages etc.)
+ * Broader than SuffixRule — includes the replacement stem (e.g., strip -ando, add -ar).
+ */
+export interface ConjugationEnding {
+  readonly ending: string;
+  readonly stem: string;
+  readonly confidence: number;
+  readonly type: ConjugationType;
+}
+
+/**
+ * Configuration for BaseMorphologicalNormalizer.
+ * Subclasses provide this in their constructor.
+ */
+export interface NormalizerConfig {
+  /** Language code */
+  readonly language: string;
+
+  /** Minimum word length to attempt normalization */
+  readonly minWordLength?: number;
+
+  /** Minimum stem length after stripping (default: 2) */
+  readonly minStemLength?: number;
+
+  /** Conjugation endings sorted longest-first */
+  readonly endings?: readonly ConjugationEnding[];
+
+  /** Suffix rules (for SuffixRule-style normalizers) */
+  readonly suffixRules?: readonly SuffixRule[];
+
+  /** Prefix rules */
+  readonly prefixRules?: readonly PrefixRule[];
+
+  /** Reflexive suffixes (for Romance languages) */
+  readonly reflexiveSuffixes?: readonly string[];
+
+  /** Infinitive endings (for checking if already normalized) */
+  readonly infinitiveEndings?: readonly string[];
+}
+
+/**
+ * Abstract base class for morphological normalizers.
+ *
+ * Subclasses must implement `isNormalizable()` and can override any
+ * normalization step. The default `normalize()` pipeline is:
+ *
+ * 1. Check if already in dictionary form → noChange
+ * 2. Try reflexive normalization (if reflexiveSuffixes configured)
+ * 3. Try conjugation endings (if endings configured)
+ * 4. Try suffix rules (if suffixRules configured)
+ * 5. Try prefix rules (if prefixRules configured)
+ * 6. Return noChange
+ */
+export abstract class BaseMorphologicalNormalizer implements MorphologicalNormalizer {
+  readonly language: string;
+  protected readonly config: NormalizerConfig;
+
+  constructor(config: NormalizerConfig) {
+    this.language = config.language;
+    this.config = {
+      minWordLength: 3,
+      minStemLength: 2,
+      ...config,
+    };
+  }
+
+  /**
+   * Check if a word can be normalized. Subclasses must implement this
+   * with language-specific script/character detection.
+   */
+  abstract isNormalizable(word: string): boolean;
+
+  /**
+   * Standard normalization pipeline. Override for custom behavior.
+   */
+  normalize(word: string): NormalizationResult {
+    const lower = word.toLowerCase();
+
+    // Check if already in dictionary form
+    if (this.isAlreadyNormalized(lower)) {
+      return noChange(word);
+    }
+
+    // Try reflexive normalization (Romance languages)
+    if (this.config.reflexiveSuffixes) {
+      const reflexive = this.tryReflexiveNormalization(lower);
+      if (reflexive) return reflexive;
+    }
+
+    // Try conjugation endings
+    if (this.config.endings) {
+      const conjugation = this.tryConjugationEndings(lower);
+      if (conjugation) return conjugation;
+    }
+
+    // Try suffix rules
+    if (this.config.suffixRules) {
+      const suffix = this.trySuffixRules(lower);
+      if (suffix) return suffix;
+    }
+
+    // Try prefix rules
+    if (this.config.prefixRules) {
+      const prefix = this.tryPrefixRules(lower);
+      if (prefix) return prefix;
+    }
+
+    return noChange(word);
+  }
+
+  /**
+   * Check if word is already in dictionary form (e.g., ends in -ar/-er/-ir).
+   * Override for language-specific checks.
+   */
+  protected isAlreadyNormalized(word: string): boolean {
+    if (this.config.infinitiveEndings) {
+      return this.config.infinitiveEndings.some(e => word.endsWith(e));
+    }
+    return false;
+  }
+
+  /**
+   * Try to strip reflexive suffixes and normalize the remainder.
+   * Common in Romance languages (Spanish, Portuguese, French).
+   */
+  protected tryReflexiveNormalization(word: string): NormalizationResult | null {
+    const suffixes = this.config.reflexiveSuffixes;
+    if (!suffixes) return null;
+
+    for (const suffix of suffixes) {
+      if (!word.endsWith(suffix)) continue;
+      const remainder = word.slice(0, -suffix.length);
+
+      // Check if remainder is already an infinitive
+      if (this.isAlreadyNormalized(remainder)) {
+        return normalized(remainder, 0.88, {
+          removedSuffixes: [suffix],
+          conjugationType: 'reflexive',
+        });
+      }
+
+      // Try to normalize the remainder
+      const inner = this.tryConjugationEndings(remainder) || this.trySuffixRules(remainder);
+      if (inner && inner.stem !== remainder) {
+        return normalized(inner.stem, inner.confidence * 0.95, {
+          removedSuffixes: [suffix, ...(inner.metadata?.removedSuffixes || [])],
+          conjugationType: 'reflexive',
+        });
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Try conjugation endings (verb class endings like -ar/-er/-ir patterns).
+   * Endings must be pre-sorted longest-first.
+   */
+  protected tryConjugationEndings(word: string): NormalizationResult | null {
+    const endings = this.config.endings;
+    if (!endings) return null;
+
+    const minStem = this.config.minStemLength ?? 2;
+
+    for (const rule of endings) {
+      if (!word.endsWith(rule.ending)) continue;
+
+      const stemBase = word.slice(0, -rule.ending.length);
+      if (stemBase.length < minStem) continue;
+
+      const infinitive = stemBase + rule.stem;
+      return normalized(infinitive, rule.confidence, {
+        removedSuffixes: [rule.ending],
+        conjugationType: rule.type,
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * Try SuffixRule-style normalization.
+   * Rules must be pre-sorted longest-first.
+   */
+  protected trySuffixRules(word: string): NormalizationResult | null {
+    const rules = this.config.suffixRules;
+    if (!rules) return null;
+
+    const defaultMinStem = this.config.minStemLength ?? 2;
+
+    for (const rule of rules) {
+      if (!word.endsWith(rule.pattern)) continue;
+
+      const stem = word.slice(0, -rule.pattern.length);
+      const minStem = rule.minStemLength ?? defaultMinStem;
+      if (stem.length < minStem) continue;
+
+      const result = stem + (rule.replacement || '');
+      return normalized(result, rule.confidence, {
+        removedSuffixes: [rule.pattern],
+        ...(rule.conjugationType && { conjugationType: rule.conjugationType }),
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * Try PrefixRule-style normalization.
+   */
+  protected tryPrefixRules(word: string): NormalizationResult | null {
+    const rules = this.config.prefixRules;
+    if (!rules) return null;
+
+    for (const rule of rules) {
+      if (!word.startsWith(rule.pattern)) continue;
+
+      const remainder = word.slice(rule.pattern.length);
+      const minRemaining = rule.minRemaining ?? this.config.minStemLength ?? 2;
+      if (remainder.length < minRemaining) continue;
+
+      return normalized(remainder, 1.0 - rule.confidencePenalty, {
+        removedPrefixes: [rule.pattern],
+      });
+    }
+
+    return null;
+  }
+}

package/src/core/tokenization/morphology/index.ts

@@ -1,5 +1,7 @@
 /**
- * Morphological normalization types and interfaces
+ * Morphological normalization types, interfaces, and base class
  */
 
 export * from './types';
+export { BaseMorphologicalNormalizer } from './base-normalizer';
+export type { ConjugationEnding, NormalizerConfig } from './base-normalizer';

package/src/core/types.ts

@@ -8,6 +8,8 @@
  * These types are domain-agnostic - they work for any DSL (SQL, animations, etc.)
  */
 
+import type { Diagnostic } from '../generation/diagnostics';
+
 // =============================================================================
 // Action Types (Generic)
 // =============================================================================
@@ -152,8 +154,8 @@ export interface SemanticNode {
   readonly metadata?: SemanticMetadata;
   /** Metadata annotations (v1.2). */
   readonly annotations?: readonly Annotation[];
-  /** Type constraint diagnostics (v1.2). */
-  readonly diagnostics?: readonly ProtocolDiagnostic[];
+  /** Diagnostics from parsing, validation, and schema checks (v1.2, extended v1.2.1). */
+  readonly diagnostics?: readonly Diagnostic[];
 }
 
 /**

package/src/index.ts

@@ -151,6 +151,13 @@ export { defineCommand, defineRole } from './schema';
 export { createSimpleTokenizer } from './core/tokenization/base-tokenizer';
 export type { SimpleTokenizerConfig } from './core/tokenization/base-tokenizer';
 
+// Morphological normalization base class and types
+export { BaseMorphologicalNormalizer } from './core/tokenization/morphology/base-normalizer';
+export type {
+  ConjugationEnding,
+  NormalizerConfig,
+} from './core/tokenization/morphology/base-normalizer';
+
 // IR (Intermediate Representation) — explicit syntax, JSON conversion, reference validation
 export * from './ir';
 

package/src/ir/conformance.test.ts

@@ -0,0 +1,120 @@
+/**
+ * Conformance Fixture Integration Tests
+ *
+ * Drives the protocol test fixtures through the bracket-syntax
+ * parseCompound/renderExplicit round-trip pipeline.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { parseCompound, parseDocument } from './explicit-parser';
+import { renderExplicit, renderDocument } from './explicit-renderer';
+import { fromProtocolJSON, toProtocolJSON } from './protocol-json';
+import type { CompoundSemanticNode } from '../core/types';
+
+import compoundFixtures from '../../../../protocol/test-fixtures/compound.json';
+import annotationFixtures from '../../../../protocol/test-fixtures/annotations.json';
+import versionFixtures from '../../../../protocol/test-fixtures/version-envelope.json';
+
+// =============================================================================
+// Compound Fixtures
+// =============================================================================
+
+describe('conformance: compound.json', () => {
+  for (const fixture of compoundFixtures) {
+    it(`${fixture.id}: ${fixture.description}`, () => {
+      // Verify rendering matches expected output
+      const node = fromProtocolJSON(fixture.node as any);
+      const rendered = renderExplicit(node);
+      expect(rendered).toBe(fixture.rendered);
+
+      // Now verify we can PARSE it back (previously renderOnly)
+      const reparsed = parseCompound(fixture.rendered);
+      expect(reparsed.kind).toBe('compound');
+      const compound = reparsed as CompoundSemanticNode;
+      expect(compound.chainType).toBe(fixture.node.chainType);
+      expect(compound.statements).toHaveLength(fixture.node.statements!.length);
+
+      for (let i = 0; i < compound.statements.length; i++) {
+        expect(compound.statements[i].action).toBe(fixture.node.statements![i].action);
+      }
+    });
+  }
+});
+
+// =============================================================================
+// Annotation Fixtures
+// =============================================================================
+
+describe('conformance: annotations.json', () => {
+  for (const fixture of annotationFixtures) {
+    it(`${fixture.id}: ${fixture.description}`, () => {
+      // Convert JSON → SemanticNode → render → parse → compare
+      const node = fromProtocolJSON(fixture.jsonInput as any);
+      const rendered = renderExplicit(node);
+      const reparsed = parseCompound(rendered);
+
+      // Verify action preserved
+      expect(reparsed.action).toBe(fixture.jsonInput.action);
+
+      // Verify annotations round-trip
+      if (fixture.jsonInput.annotations && fixture.jsonInput.annotations.length > 0) {
+        expect(reparsed.annotations).toHaveLength(fixture.jsonInput.annotations.length);
+        for (let i = 0; i < fixture.jsonInput.annotations.length; i++) {
+          expect(reparsed.annotations![i].name).toBe(fixture.jsonInput.annotations[i].name);
+          if (fixture.jsonInput.annotations[i].value !== undefined) {
+            expect(reparsed.annotations![i].value).toBe(fixture.jsonInput.annotations[i].value);
+          }
+        }
+      } else {
+        // ann-006: no annotations → should be undefined after parse
+        expect(reparsed.annotations).toBeUndefined();
+      }
+
+      // Verify re-render produces same output
+      const rerendered = renderExplicit(reparsed);
+      expect(rerendered).toBe(rendered);
+    });
+  }
+});
+
+// =============================================================================
+// Version Envelope Fixtures (streaming format)
+// =============================================================================
+
+describe('conformance: version-envelope.json (streaming)', () => {
+  for (const fixture of versionFixtures) {
+    const f = fixture as any;
+    if (!f.streamingInput) continue;
+
+    // ver-007 describes v1.0/v1.1 fallback behavior — our parser is v1.2,
+    // so we parse the version header correctly instead of treating it as a comment.
+    if (f.v1Fallback) {
+      it(`${f.id}: v1.2 parser parses version header — ${f.description}`, () => {
+        const envelope = parseDocument(f.streamingInput);
+        // v1.2 parser extracts the version (v1.0 parsers would ignore it)
+        expect(envelope.lseVersion).toBe('1.2');
+        expect(envelope.nodes).toHaveLength(f.v1Fallback.expectedNodeCount);
+      });
+      continue;
+    }
+
+    it(`${f.id}: ${f.description}`, () => {
+      const envelope = parseDocument(f.streamingInput);
+      expect(envelope.lseVersion).toBe(f.expectedVersion);
+      expect(envelope.nodes).toHaveLength(f.expectedNodeCount);
+    });
+  }
+
+  // Also test JSON-based envelope round-trips
+  for (const fixture of versionFixtures) {
+    const f = fixture as any;
+    if (f.streamingInput) continue;
+    if (!f.expectedRoundTrip) continue;
+
+    it(`${f.id}: JSON envelope round-trip — ${f.description}`, () => {
+      // For JSON envelopes: verify node count and version
+      expect(f.jsonInput.lseVersion).toBe(f.expectedVersion);
+      expect(f.jsonInput.nodes).toHaveLength(f.expectedNodeCount);
+    });
+  }
+});