@lokascript/framework 4.6.0 → 4.8.0

package/src/testing/gate/index.ts

@@ -0,0 +1,8 @@
+/**
+ * @lokascript/framework/testing/gate — the round-trip gate kit and the grammar linter
+ * (domains rethink Phase 4.6). No test runner needed: a grammar's gate test or regen
+ * tool calls `runRoundTrip` and `compare`, its lint test `lintDomain`.
+ */
+
+export * from './round-trip';
+export * from './lint';

package/src/testing/gate/lint/format.ts

@@ -0,0 +1,23 @@
+/**
+ * Pretty-print lint results for test output.
+ */
+
+import type { LintFinding, LintResult, Severity } from './types';
+
+const SEVERITY_GLYPH: Record<Severity, string> = {
+  error: '✗',
+  warning: '⚠',
+  note: 'ℹ',
+};
+
+export function formatFinding(finding: LintFinding): string {
+  const glyph = SEVERITY_GLYPH[finding.severity];
+  return `  ${glyph} [${finding.rule}] ${finding.message}`;
+}
+
+export function formatResult(result: LintResult): string {
+  const header = `domain-toolkit: lint(${result.domain}) — ${result.errorCount} error(s), ${result.warningCount} warning(s)`;
+  if (result.findings.length === 0) return `${header}\n  (clean)`;
+  const body = result.findings.map(formatFinding).join('\n');
+  return `${header}\n${body}`;
+}

package/src/testing/gate/lint/index.ts

@@ -0,0 +1,73 @@
+/**
+ * The domain linter (domains rethink Phase 4.6): moved from lokascript-domains'
+ * `@lokascript/domain-toolkit`. Checks schema structure and cross-file
+ * invariants (profile ↔ tokenizer ↔ schema ↔ renderer agreement); a grammar
+ * runs it from one lint test.
+ *
+ * Implemented rules:
+ *   - R1/R2 (schema-structure): delegates to the validator the input
+ *     brings (`validateSchema`: the domains pass @lokascript/semantic's).
+ *   - R5 (keyword-classification): profile keywords/markers classify as
+ *     'keyword' in the language's tokenizer.
+ *   - R6 (keyword-coverage): every schema command has profile keywords for
+ *     all declared languages.
+ *   - R7 (marker-tokenization): every marker/keyword tokenizes as a single
+ *     token.
+ *   - R8 (extractor-coverage): Latin-script tokenizers handle diacritic
+ *     identifiers (warning).
+ *   - R9 (position-ordering): schema role positions are unique per
+ *     word-order (warning).
+ *   - R10 (renderer-coherence): renderer table keywords match
+ *     profile/schema translations (opt-in via `renderer`).
+ */
+
+import { runRules } from './rules';
+import type { DomainLintInput, LintFinding, LintResult, LintWaiver } from './types';
+
+export type {
+  DomainLintInput,
+  LintResult,
+  LintFinding,
+  LintRule,
+  LintWaiver,
+  Severity,
+  RendererTables,
+  SchemaValidationFinding,
+  SchemaValidator,
+} from './types';
+export { formatResult, formatFinding } from './format';
+
+function matchesWaiver(finding: LintFinding, waiver: LintWaiver): boolean {
+  if (finding.rule !== waiver.rule) return false;
+  if (!waiver.matches) return true;
+  const ctx = finding.context ?? {};
+  for (const [key, expected] of Object.entries(waiver.matches)) {
+    if (ctx[key] !== expected) return false;
+  }
+  return true;
+}
+
+function applyWaivers(
+  findings: LintFinding[],
+  waivers: readonly LintWaiver[] | undefined
+): LintFinding[] {
+  if (!waivers || waivers.length === 0) return findings;
+  return findings.filter(f => !waivers.some(w => matchesWaiver(f, w)));
+}
+
+/**
+ * Lint a domain against all enabled rules and return a structured result.
+ */
+export function lintDomain(input: DomainLintInput): LintResult {
+  const raw = runRules(input);
+  const findings = applyWaivers(raw, input.waivers);
+  const errorCount = findings.filter(f => f.severity === 'error').length;
+  const warningCount = findings.filter(f => f.severity === 'warning').length;
+
+  return {
+    domain: input.name,
+    findings,
+    errorCount,
+    warningCount,
+  };
+}

package/src/testing/gate/lint/lint.test.ts

@@ -0,0 +1,131 @@
+/**
+ * The linter's own logic: waiver matching, count derivation, the format helpers,
+ * and the schema-structure rule's validator input (moved with the linter from
+ * lokascript-domains' domain-toolkit). A grammar's lint test runs the rules on its
+ * real tables; these fixtures are minimal on purpose (keyword-coverage needs only
+ * `schema.action` + `profile.keywords`), and assertions filter by rule id.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { lintDomain, formatResult, formatFinding } from './index';
+import type { DomainLintInput } from './types';
+
+function emptyInput(): DomainLintInput {
+  return { name: 'smoke', schemas: [], profiles: [], tokenizers: {} };
+}
+
+/** One schema + a profile with no keywords → exactly one R6 error finding. */
+function uncoveredInput(): DomainLintInput {
+  return {
+    name: 'smoke',
+    schemas: [
+      {
+        action: 'toggle',
+        description: 'fixture command',
+        roles: [],
+        primaryRole: 'patient',
+        category: 'test',
+      },
+    ],
+    profiles: [{ code: 'xx', wordOrder: 'SVO', keywords: {} }],
+    tokenizers: {},
+  };
+}
+
+describe('lintDomain', () => {
+  it('returns a clean structured result for an empty domain', () => {
+    const result = lintDomain(emptyInput());
+    expect(result.domain).toBe('smoke');
+    expect(result.findings).toEqual([]);
+    expect(result.errorCount).toBe(0);
+    expect(result.warningCount).toBe(0);
+  });
+
+  it('reports keyword-coverage when a profile lacks a schema action keyword', () => {
+    const result = lintDomain(uncoveredInput());
+    const r6 = result.findings.filter(f => f.rule === 'keyword-coverage');
+    expect(r6).toHaveLength(1);
+    expect(r6[0]).toMatchObject({
+      severity: 'error',
+      context: { domain: 'smoke', lang: 'xx', action: 'toggle' },
+    });
+    expect(result.errorCount).toBeGreaterThanOrEqual(1);
+  });
+});
+
+describe('waivers', () => {
+  it('drops a finding whose rule and context both match', () => {
+    const result = lintDomain({
+      ...uncoveredInput(),
+      waivers: [{ rule: 'keyword-coverage', reason: 'fixture waiver', matches: { lang: 'xx' } }],
+    });
+    expect(result.findings.filter(f => f.rule === 'keyword-coverage')).toEqual([]);
+  });
+
+  it('keeps a finding when the waiver context does not match', () => {
+    const result = lintDomain({
+      ...uncoveredInput(),
+      waivers: [{ rule: 'keyword-coverage', reason: 'wrong language', matches: { lang: 'yy' } }],
+    });
+    expect(result.findings.filter(f => f.rule === 'keyword-coverage')).toHaveLength(1);
+  });
+
+  it('a matches-less waiver drops every finding of its rule', () => {
+    const result = lintDomain({
+      ...uncoveredInput(),
+      waivers: [{ rule: 'keyword-coverage', reason: 'blanket' }],
+    });
+    expect(result.findings.filter(f => f.rule === 'keyword-coverage')).toEqual([]);
+  });
+});
+
+describe('format helpers', () => {
+  it('formatFinding renders the severity glyph and rule id', () => {
+    const line = formatFinding({
+      rule: 'keyword-coverage',
+      severity: 'error',
+      message: 'profile [xx] has no keyword',
+    });
+    expect(line).toContain('✗');
+    expect(line).toContain('[keyword-coverage]');
+  });
+
+  it('formatResult renders "(clean)" for an empty result', () => {
+    const text = formatResult(lintDomain(emptyInput()));
+    expect(text).toContain('0 error(s), 0 warning(s)');
+    expect(text).toContain('(clean)');
+  });
+
+  it('formatResult renders one line per finding otherwise', () => {
+    const text = formatResult(lintDomain(uncoveredInput()));
+    expect(text).toContain("no keyword for action 'toggle'");
+  });
+});
+
+describe('schema-structure (R1/R2)', () => {
+  it("reports the input's validator findings, notes dropped", () => {
+    const result = lintDomain({
+      ...uncoveredInput(),
+      validateSchema: schema => ({
+        items: [
+          { code: 'NO_REQUIRED_ROLES', severity: 'warning', message: `${schema.action} has none` },
+          { code: 'A_NOTE', severity: 'note', message: 'dropped' },
+        ],
+      }),
+    });
+    const r1 = result.findings.filter(f => f.rule.startsWith('schema-structure'));
+    expect(r1).toEqual([
+      {
+        rule: 'schema-structure:NO_REQUIRED_ROLES',
+        severity: 'warning',
+        message: '[toggle] toggle has none',
+        context: { domain: 'smoke', action: 'toggle', code: 'NO_REQUIRED_ROLES' },
+      },
+    ]);
+  });
+
+  it('finds nothing without a validator', () => {
+    const result = lintDomain(uncoveredInput());
+    expect(result.findings.filter(f => f.rule.startsWith('schema-structure'))).toEqual([]);
+  });
+});

package/src/testing/gate/lint/rules/extractor-coverage.ts

@@ -0,0 +1,63 @@
+/**
+ * R8: Latin-script tokenizers should extract diacritic identifiers as single tokens.
+ *
+ * The default `IdentifierExtractor` matches `[a-zA-Z0-9_]*` and stops at any
+ * diacritic. Words like `añadir` split into `["a", "ñadir"]` unless the
+ * tokenizer also registers `LatinExtendedIdentifierExtractor`. This rule
+ * probes the behavior by tokenizing a canonical diacritic-containing string
+ * for each language and warning if it doesn't come back as a single token.
+ *
+ * Warning (not error) because:
+ *   - English tokenizers legitimately don't need the extended extractor.
+ *   - A domain might deliberately split compounds for its own reasons.
+ * The warning prompts the author to audit; an explicit waiver can add an
+ * opt-out once we have a mechanism for it.
+ */
+
+import type { LintFinding, LintRule } from '../types';
+
+const RULE_ID = 'extractor-coverage';
+
+/**
+ * Canonical diacritic samples per language. Each must be a single logical
+ * word in the target language. Chosen to exercise the specific characters
+ * that tend to split under the default ASCII extractor.
+ */
+const DIACRITIC_SAMPLES: Record<string, string> = {
+  es: 'añadir', // ñ
+  fr: 'déjà', // é, à
+  pt: 'coração', // ç, ã
+  de: 'größer', // ö, ß
+  tr: 'değiştir', // ğ, ş
+  it: 'però', // ò
+  pl: 'kochać', // ć
+  cs: 'příliš', // ř, í
+  // Not run for en, ja, ar, ko, zh, ru, hi, etc. — either pure ASCII
+  // (en) or non-Latin scripts where the Unicode extractor handles them.
+};
+
+export const extractorCoverageRule: LintRule = input => {
+  const findings: LintFinding[] = [];
+
+  for (const [lang, sample] of Object.entries(DIACRITIC_SAMPLES)) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) continue; // language not registered in this domain
+
+    const stream = tokenizer.tokenize(sample);
+    if (stream.tokens.length !== 1) {
+      findings.push({
+        rule: RULE_ID,
+        severity: 'warning',
+        message: `[${lang}] diacritic probe "${sample}" tokenizes as ${stream.tokens.length} tokens — tokenizer likely missing LatinExtendedIdentifierExtractor`,
+        context: {
+          domain: input.name,
+          lang,
+          sample,
+          tokenCount: stream.tokens.length,
+        },
+      });
+    }
+  }
+
+  return findings;
+};

package/src/testing/gate/lint/rules/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Rule registry — runs all enabled rules and aggregates findings.
+ */
+
+import type { DomainLintInput, LintFinding, LintRule } from '../types';
+import { schemaStructureRule } from './schema-rules';
+import { keywordCoverageRule } from './keyword-coverage';
+import { keywordClassificationRule } from './keyword-classification';
+import { markerTokenizationRule } from './marker-tokenization';
+import { extractorCoverageRule } from './extractor-coverage';
+import { positionOrderingRule } from './position-ordering';
+import { rendererCoherenceRule } from './renderer-coherence';
+
+/**
+ * All rules in execution order.
+ *   Phase 1: schema structure (delegates to @lokascript/semantic)
+ *   Phase 2: keyword coverage + keyword classification + marker tokenization
+ *   Phase 3: extractor coverage + position ordering (warnings only)
+ *   Phase 3b: renderer coherence (opt-in via DomainLintInput.renderer)
+ */
+export const ALL_RULES: readonly LintRule[] = [
+  schemaStructureRule,
+  keywordCoverageRule,
+  keywordClassificationRule,
+  markerTokenizationRule,
+  extractorCoverageRule,
+  positionOrderingRule,
+  rendererCoherenceRule,
+];
+
+export function runRules(input: DomainLintInput): LintFinding[] {
+  const findings: LintFinding[] = [];
+  for (const rule of ALL_RULES) {
+    findings.push(...rule(input));
+  }
+  return findings;
+}

package/src/testing/gate/lint/rules/keyword-classification.ts

@@ -0,0 +1,52 @@
+/**
+ * R5: Keyword classification — profile keywords ⊆ tokenizer keyword lists.
+ *
+ * Every lexical item the parser must recognize as a keyword — profile
+ * keyword primaries/alternatives, profile role markers, and schema
+ * `markerOverride` words — must actually *classify* as `'keyword'` in that
+ * language's tokenizer. This is stricter than R7 (marker-tokenization):
+ * a word can tokenize as a single token yet come back as `'identifier'`
+ * when the tokenizer's keyword list doesn't contain it.
+ *
+ * Under the bridge (`buildDomainTokenizer`) profile and tokenizer derive
+ * from the same vocab file, so this rule is vacuously green there. It
+ * exists for hand-authored tokenizers (e.g. domain-learn) and custom
+ * tokenizer overrides, where a vocab/profile edit can silently go stale
+ * against the tokenizer's keyword list.
+ */
+
+import type { LintFinding, LintRule } from '../types';
+import { collectLexicalItems } from './marker-tokenization';
+
+const RULE_ID = 'keyword-classification';
+
+export const keywordClassificationRule: LintRule = input => {
+  const findings: LintFinding[] = [];
+
+  for (const { lang, word, source } of collectLexicalItems(input)) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) continue; // R6 catches unregistered languages.
+
+    // Same opt-outs as R7: '' means "no marker in this language"; multi-word
+    // items can't be a single keyword token and are out of scope.
+    if (word === '' || /\s/.test(word)) continue;
+
+    const kind = tokenizer.classifyToken(word);
+    if (kind !== 'keyword' && kind !== 'particle') {
+      findings.push({
+        rule: RULE_ID,
+        severity: 'error',
+        message: `"${word}" classifies as '${kind}' (not 'keyword') in ${lang} (${source})`,
+        context: {
+          domain: input.name,
+          lang,
+          word,
+          source,
+          kind,
+        },
+      });
+    }
+  }
+
+  return findings;
+};

package/src/testing/gate/lint/rules/keyword-coverage.ts

@@ -0,0 +1,38 @@
+/**
+ * R6: Keyword coverage across profiles.
+ *
+ * For every schema action + every registered profile, the profile must have
+ * a `keywords[action]` entry. Without it, `createMultilingualDSL()` throws
+ * at pattern generation time ("No keyword translation for 'X' in 'Y'").
+ *
+ * This rule catches the "I added a new schema but only wired it into the
+ * English profile" class of drift.
+ */
+
+import type { LintFinding, LintRule } from '../types';
+
+const RULE_ID = 'keyword-coverage';
+
+export const keywordCoverageRule: LintRule = input => {
+  const findings: LintFinding[] = [];
+
+  for (const profile of input.profiles) {
+    for (const schema of input.schemas) {
+      const action = schema.action;
+      if (!profile.keywords[action]) {
+        findings.push({
+          rule: RULE_ID,
+          severity: 'error',
+          message: `profile [${profile.code}] has no keyword for action '${String(action)}'`,
+          context: {
+            domain: input.name,
+            lang: profile.code,
+            action: String(action),
+          },
+        });
+      }
+    }
+  }
+
+  return findings;
+};

package/src/testing/gate/lint/rules/marker-tokenization.ts

@@ -0,0 +1,129 @@
+/**
+ * R7: Marker and keyword tokenization.
+ *
+ * Every lexical item the parser needs to match atomically — marker words,
+ * profile keyword primaries, and profile keyword alternatives — must
+ * tokenize as a single token in its target language. A word that splits
+ * across multiple tokens (e.g., `añadir` → `["a", "ñadir"]` when the
+ * tokenizer lacks `LatinExtendedIdentifierExtractor`) will never match a
+ * pattern's literal token.
+ *
+ * This is the exact gap that bit us on the Spanish tokenizer mid-session.
+ */
+
+import type { LintFinding, LintRule, DomainLintInput } from '../types';
+import type { LanguageTokenizer } from '../../../../core/types';
+
+const RULE_ID = 'marker-tokenization';
+
+function tokenCount(tokenizer: LanguageTokenizer, word: string): number {
+  const stream = tokenizer.tokenize(word);
+  return stream.tokens.length;
+}
+
+export function collectLexicalItems(input: DomainLintInput): Array<{
+  lang: string;
+  word: string;
+  source: string; // "schema marker <action>.<role>" | "profile keyword <action>.primary" | ...
+}> {
+  const items: Array<{ lang: string; word: string; source: string }> = [];
+
+  // Schema marker overrides (per language)
+  for (const schema of input.schemas) {
+    for (const role of schema.roles) {
+      if (!role.markerOverride) continue;
+      for (const [lang, word] of Object.entries(role.markerOverride)) {
+        items.push({
+          lang,
+          word,
+          source: `schema markerOverride ${String(schema.action)}.${role.role}`,
+        });
+      }
+    }
+  }
+
+  // Profile keywords (primary + alternatives)
+  for (const profile of input.profiles) {
+    for (const [action, entry] of Object.entries(profile.keywords)) {
+      if (entry.primary) {
+        items.push({
+          lang: profile.code,
+          word: entry.primary,
+          source: `profile keyword ${action}.primary`,
+        });
+      }
+      for (const alt of entry.alternatives ?? []) {
+        items.push({
+          lang: profile.code,
+          word: alt,
+          source: `profile keyword ${action}.alternatives`,
+        });
+      }
+    }
+  }
+
+  // Profile role markers (primary + alternatives)
+  for (const profile of input.profiles) {
+    for (const [role, marker] of Object.entries(profile.roleMarkers ?? {})) {
+      if (marker.primary) {
+        items.push({
+          lang: profile.code,
+          word: marker.primary,
+          source: `profile roleMarker ${role}.primary`,
+        });
+      }
+      for (const alt of marker.alternatives ?? []) {
+        items.push({
+          lang: profile.code,
+          word: alt,
+          source: `profile roleMarker ${role}.alternatives`,
+        });
+      }
+    }
+  }
+
+  return items;
+}
+
+export const markerTokenizationRule: LintRule = input => {
+  const findings: LintFinding[] = [];
+  const items = collectLexicalItems(input);
+
+  for (const { lang, word, source } of items) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) {
+      // Tokenizer for this language isn't registered with the linter.
+      // R6 (keyword-coverage) catches the schema side; nothing to do here.
+      continue;
+    }
+
+    // Empty-string markers mean "no marker in this language" (an explicit
+    // opt-out pattern used by domain-learn to signal that a role is
+    // positional in some languages but marker-bearing in others).
+    if (word === '') continue;
+
+    // Whitespace-containing markers (e.g., 'mettre-à-jour' is one word, but
+    // 'ask X to Y' wouldn't be) — tokenize the raw word and expect 1 token.
+    // Multi-word markers with explicit spaces are out of scope for this rule;
+    // skip them rather than flag false positives.
+    if (/\s/.test(word)) continue;
+
+    const count = tokenCount(tokenizer, word);
+    if (count !== 1) {
+      findings.push({
+        rule: RULE_ID,
+        severity: 'error',
+        message: `"${word}" tokenizes as ${count} tokens in ${lang} (${source})`,
+        context: {
+          domain: input.name,
+          lang,
+          word,
+          source,
+          tokenCount: count,
+        },
+      });
+    }
+  }
+
+  return findings;
+};

package/src/testing/gate/lint/rules/position-ordering.ts

@@ -0,0 +1,87 @@
+/**
+ * R9: Role position uniqueness within a schema.
+ *
+ * `sortRolesByWordOrder` in the pattern generator sorts roles descending by
+ * svoPosition or sovPosition (higher value = earlier in surface form). If two
+ * roles share the same position value, their relative order is undefined
+ * (falls through to array stability) — a latent footgun because token
+ * ordering silently changes if someone reorders the roles array.
+ *
+ * This rule warns when a schema has multiple roles AND any two share an
+ * explicit position for the same word order.
+ *
+ * Warning (not error) because:
+ *   - Single-role schemas have nothing to order.
+ *   - A schema might intentionally tie two roles when only one is present
+ *     at a time (e.g., mutually-exclusive alternatives). In practice we
+ *     haven't seen this, but the rule should prompt a review, not block.
+ *
+ * The `sovPosition: 0` default assigned when omitted is NOT flagged — only
+ * explicit non-zero collisions count. Otherwise most schemas with 3+
+ * optional roles would warn spuriously.
+ */
+
+import type { LintFinding, LintRule } from '../types';
+
+const RULE_ID = 'position-ordering';
+
+function findCollisions(
+  values: Array<{ role: string; position: number }>
+): Array<{ position: number; roles: string[] }> {
+  const byPosition = new Map<number, string[]>();
+  for (const { role, position } of values) {
+    const existing = byPosition.get(position) ?? [];
+    existing.push(role);
+    byPosition.set(position, existing);
+  }
+  return Array.from(byPosition.entries())
+    .filter(([, roles]) => roles.length > 1)
+    .map(([position, roles]) => ({ position, roles }));
+}
+
+export const positionOrderingRule: LintRule = input => {
+  const findings: LintFinding[] = [];
+
+  for (const schema of input.schemas) {
+    if (schema.roles.length < 2) continue;
+
+    const svoValues = schema.roles
+      .filter(r => typeof r.svoPosition === 'number' && r.svoPosition !== 0)
+      .map(r => ({ role: r.role, position: r.svoPosition! }));
+    const sovValues = schema.roles
+      .filter(r => typeof r.sovPosition === 'number' && r.sovPosition !== 0)
+      .map(r => ({ role: r.role, position: r.sovPosition! }));
+
+    for (const { position, roles } of findCollisions(svoValues)) {
+      findings.push({
+        rule: RULE_ID,
+        severity: 'warning',
+        message: `[${String(schema.action)}] roles ${roles.join(', ')} share svoPosition=${position} — order is undefined`,
+        context: {
+          domain: input.name,
+          action: String(schema.action),
+          wordOrder: 'SVO',
+          position,
+          roles: roles.join(','),
+        },
+      });
+    }
+
+    for (const { position, roles } of findCollisions(sovValues)) {
+      findings.push({
+        rule: RULE_ID,
+        severity: 'warning',
+        message: `[${String(schema.action)}] roles ${roles.join(', ')} share sovPosition=${position} — order is undefined`,
+        context: {
+          domain: input.name,
+          action: String(schema.action),
+          wordOrder: 'SOV',
+          position,
+          roles: roles.join(','),
+        },
+      });
+    }
+  }
+
+  return findings;
+};