@lokascript/framework 4.5.0 → 4.7.0

package/dist/testing/gate/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @lokascript/framework/testing/gate — the round-trip gate kit and the grammar linter
+ * (domains rethink Phase 4.6). No test runner needed: a grammar's gate test or regen
+ * tool calls `runRoundTrip` and `compare`, its lint test `lintDomain`.
+ */
+export * from './round-trip';
+export * from './lint';
+//# sourceMappingURL=index.d.ts.map

package/dist/testing/gate/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/testing/gate/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAC;AAC7B,cAAc,QAAQ,CAAC"}

package/dist/testing/gate/index.js

@@ -0,0 +1,632 @@
+// src/testing/gate/round-trip.ts
+var RANK = {
+  roundTrip: { pass: 2, refused: 1, silent: 0 },
+  leftover: { refused: 2, pass: 2, silent: 0 }
+};
+var BEST = { roundTrip: "pass", leftover: "refused" };
+var UNKNOWN_WORD = "zzqx";
+function defaultValue(role, i) {
+  const types = role.expectedTypes ?? [];
+  if (types.includes("selector") && !types.includes("literal") && !types.includes("expression"))
+    return `#box${i}`;
+  return `v${i}x`;
+}
+function valuesFor(fixture, action, role, i) {
+  return fixture.values[`${action}.${role.role}`] ?? fixture.values[role.role] ?? [defaultValue(role, i)];
+}
+function commandsFor(schemas, fixture) {
+  const out = /* @__PURE__ */ new Set();
+  for (const schema of schemas) {
+    const required = schema.roles.filter((r) => r.required);
+    const variants = required.length === schema.roles.length ? [required] : [required, [...schema.roles]];
+    for (const roles of variants) {
+      const values = roles.map((r, i) => valuesFor(fixture, schema.action, r, i));
+      const write = (pick) => `[${schema.action}${roles.map((r, i) => ` ${r.role}:${pick[i]}`).join("")}]`;
+      const first = values.map((v) => v[0]);
+      out.add(write(first));
+      values.forEach((vs, i) => {
+        for (const v of vs.slice(1)) out.add(write(first.map((f, j) => j === i ? v : f)));
+      });
+    }
+  }
+  return [...out];
+}
+function roleEntries(node) {
+  return node.roles instanceof Map ? [...node.roles] : Object.entries(node.roles ?? {});
+}
+function roleValues(node) {
+  return Object.fromEntries(
+    roleEntries(node).map(([k, v]) => [
+      k,
+      String(v?.value ?? v?.raw ?? v)
+    ]).sort(([a], [b]) => a < b ? -1 : 1)
+  );
+}
+function summarize(node) {
+  const roles = roleEntries(node).map(([k, v]) => {
+    const t = v?.type;
+    const raw = v?.value ?? v?.raw ?? v;
+    const text = String(raw);
+    return `${k}${t ? `:${String(t)}` : ""}=${/[\s=]/.test(text) ? JSON.stringify(text) : text}`;
+  }).sort();
+  return [node.action, ...roles].join(" ");
+}
+function describeRead(want, ref, back) {
+  if (back.action !== ref.action) return `another command: ${back.action}`;
+  const got = roleValues(back);
+  const lost = Object.keys(want).filter((k) => !(k in got));
+  const added = Object.keys(got).filter((k) => !(k in want));
+  const changed = Object.keys(want).filter((k) => k in got && got[k] !== want[k]);
+  const parts = [
+    ...lost.map((k) => `${k} lost`),
+    ...added.map((k) => `${k} added (${got[k]})`),
+    ...changed.map((k) => `${k}: ${got[k]}`)
+  ];
+  return parts.join(", ");
+}
+var message = (e) => String(e?.message ?? e).slice(0, 160);
+function referenceCode(dsl, source) {
+  try {
+    const c = dsl.compile(source, "en");
+    return c.ok && typeof c.code === "string" ? c.code : null;
+  } catch {
+    return null;
+  }
+}
+function runRoundTrip(grammar, dsl, schemas, fixture) {
+  const languages = dsl.getSupportedLanguages();
+  const results = [];
+  for (const source of commandsFor(schemas, fixture)) {
+    const ref = dsl.parse(source, "en");
+    const want = roleValues(ref);
+    const sameNode = (back) => back.action === ref.action && JSON.stringify(roleValues(back)) === JSON.stringify(want);
+    const refCode = fixture.oracle === "compile" ? referenceCode(dsl, source) : null;
+    for (const lang of languages) {
+      const at = { grammar, source, lang };
+      const text = dsl.render?.(ref, lang) ?? null;
+      if (!text) {
+        results.push({ ...at, kind: "roundTrip", outcome: "refused", why: "no render" });
+        continue;
+      }
+      let back;
+      try {
+        back = dsl.parse(text, lang);
+      } catch (e) {
+        results.push({ ...at, kind: "roundTrip", outcome: "refused", text, why: message(e) });
+        continue;
+      }
+      let outcome = "silent";
+      let why;
+      if (sameNode(back)) outcome = "pass";
+      else if (refCode !== null) {
+        let code;
+        try {
+          const c = dsl.compile(text, lang);
+          code = c.ok ? c.code : void 0;
+        } catch {
+          code = void 0;
+        }
+        if (code === refCode) outcome = "pass";
+        else why = `compiles differently; reads ${describeRead(want, ref, back)}`;
+      } else why = `reads ${describeRead(want, ref, back)}`;
+      results.push({
+        ...at,
+        kind: "roundTrip",
+        outcome,
+        text,
+        ...why ? { why } : {},
+        read: summarize(back)
+      });
+      const extended = `${text} ${UNKNOWN_WORD}`;
+      try {
+        const read = dsl.parse(extended, lang);
+        results.push({
+          ...at,
+          kind: "leftover",
+          outcome: "silent",
+          text: extended,
+          why: `accepted as ${read.action}`
+        });
+      } catch {
+        results.push({ ...at, kind: "leftover", outcome: "refused", text: extended });
+      }
+    }
+  }
+  return results;
+}
+function listedOutcome(baseline, r) {
+  if (r.kind === "roundTrip")
+    return baseline.roundTrip[r.grammar]?.[r.source]?.[r.lang]?.outcome ?? "pass";
+  return baseline.leftover[r.grammar]?.[r.source]?.includes(r.lang) ? "silent" : "refused";
+}
+function compare(grammar, results, baseline) {
+  const drift = { worse: [], better: [], stale: [] };
+  const seen = /* @__PURE__ */ new Set();
+  for (const r of results) {
+    seen.add(`${r.kind}\0${r.source}\0${r.lang}`);
+    const listed = listedOutcome(baseline, r);
+    const rank = RANK[r.kind];
+    const entry = { kind: r.kind, source: r.source, lang: r.lang, listed, now: r.outcome };
+    if (rank[r.outcome] < rank[listed])
+      drift.worse.push({
+        ...entry,
+        ...r.text !== void 0 ? { text: r.text } : {},
+        ...r.why !== void 0 ? { why: r.why } : {}
+      });
+    else if (rank[r.outcome] > rank[listed]) drift.better.push(entry);
+  }
+  const listedCases = [
+    ...Object.entries(baseline.roundTrip[grammar] ?? {}).flatMap(
+      ([source, langs]) => Object.keys(langs).map((lang) => ({ kind: "roundTrip", source, lang }))
+    ),
+    ...Object.entries(baseline.leftover[grammar] ?? {}).flatMap(
+      ([source, langs]) => langs.map((lang) => ({ kind: "leftover", source, lang }))
+    )
+  ];
+  for (const c of listedCases)
+    if (!seen.has(`${c.kind}\0${c.source}\0${c.lang}`)) drift.stale.push(c);
+  return drift;
+}
+function listingOf(results) {
+  var _a, _b;
+  const roundTrip = {};
+  const leftover = {};
+  for (const r of results) {
+    if (r.kind === "roundTrip" && r.outcome !== "pass") {
+      (roundTrip[_a = r.source] ?? (roundTrip[_a] = {}))[r.lang] = {
+        outcome: r.outcome,
+        ...r.text ? { text: r.text } : {},
+        ...r.why ? { why: r.why } : {}
+      };
+    } else if (r.kind === "leftover" && r.outcome === "silent")
+      (leftover[_b = r.source] ?? (leftover[_b] = [])).push(r.lang);
+  }
+  return { roundTrip, leftover };
+}
+function countsOf(results) {
+  const counts = {
+    roundTrip: { cases: 0, pass: 0, refused: 0, silent: 0 },
+    leftover: { cases: 0, pass: 0, refused: 0, silent: 0 }
+  };
+  for (const r of results) {
+    counts[r.kind].cases++;
+    counts[r.kind][r.outcome]++;
+  }
+  return counts;
+}
+function snapshotOf(results) {
+  var _a;
+  const out = {};
+  for (const r of results)
+    if (r.kind === "roundTrip") (out[_a = r.source] ?? (out[_a] = {}))[r.lang] = r.read ?? "refused";
+  return out;
+}
+function snapshotChanges(grammar, results, snapshot) {
+  const now = snapshotOf(results);
+  const was = snapshot.parses[grammar] ?? {};
+  const changes = [];
+  for (const source of /* @__PURE__ */ new Set([...Object.keys(was), ...Object.keys(now)]))
+    for (const lang of /* @__PURE__ */ new Set([
+      ...Object.keys(was[source] ?? {}),
+      ...Object.keys(now[source] ?? {})
+    ])) {
+      const a = was[source]?.[lang];
+      const b = now[source]?.[lang];
+      if (a !== b)
+        changes.push({
+          source,
+          lang,
+          ...a !== void 0 ? { was: a } : {},
+          ...b !== void 0 ? { now: b } : {}
+        });
+    }
+  return changes;
+}
+
+// src/testing/gate/lint/rules/schema-rules.ts
+var RULE_ID = "schema-structure";
+var schemaStructureRule = (input) => {
+  const findings = [];
+  if (!input.validateSchema) return findings;
+  for (const schema of input.schemas) {
+    const result = input.validateSchema(schema);
+    for (const item of result.items) {
+      if (item.severity === "note") continue;
+      findings.push({
+        rule: `${RULE_ID}:${item.code}`,
+        severity: item.severity,
+        message: `[${schema.action}] ${item.message}`,
+        context: {
+          domain: input.name,
+          action: String(schema.action),
+          code: item.code
+        }
+      });
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/keyword-coverage.ts
+var RULE_ID2 = "keyword-coverage";
+var keywordCoverageRule = (input) => {
+  const findings = [];
+  for (const profile of input.profiles) {
+    for (const schema of input.schemas) {
+      const action = schema.action;
+      if (!profile.keywords[action]) {
+        findings.push({
+          rule: RULE_ID2,
+          severity: "error",
+          message: `profile [${profile.code}] has no keyword for action '${String(action)}'`,
+          context: {
+            domain: input.name,
+            lang: profile.code,
+            action: String(action)
+          }
+        });
+      }
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/marker-tokenization.ts
+var RULE_ID3 = "marker-tokenization";
+function tokenCount(tokenizer, word) {
+  const stream = tokenizer.tokenize(word);
+  return stream.tokens.length;
+}
+function collectLexicalItems(input) {
+  const items = [];
+  for (const schema of input.schemas) {
+    for (const role of schema.roles) {
+      if (!role.markerOverride) continue;
+      for (const [lang, word] of Object.entries(role.markerOverride)) {
+        items.push({
+          lang,
+          word,
+          source: `schema markerOverride ${String(schema.action)}.${role.role}`
+        });
+      }
+    }
+  }
+  for (const profile of input.profiles) {
+    for (const [action, entry] of Object.entries(profile.keywords)) {
+      if (entry.primary) {
+        items.push({
+          lang: profile.code,
+          word: entry.primary,
+          source: `profile keyword ${action}.primary`
+        });
+      }
+      for (const alt of entry.alternatives ?? []) {
+        items.push({
+          lang: profile.code,
+          word: alt,
+          source: `profile keyword ${action}.alternatives`
+        });
+      }
+    }
+  }
+  for (const profile of input.profiles) {
+    for (const [role, marker] of Object.entries(profile.roleMarkers ?? {})) {
+      if (marker.primary) {
+        items.push({
+          lang: profile.code,
+          word: marker.primary,
+          source: `profile roleMarker ${role}.primary`
+        });
+      }
+      for (const alt of marker.alternatives ?? []) {
+        items.push({
+          lang: profile.code,
+          word: alt,
+          source: `profile roleMarker ${role}.alternatives`
+        });
+      }
+    }
+  }
+  return items;
+}
+var markerTokenizationRule = (input) => {
+  const findings = [];
+  const items = collectLexicalItems(input);
+  for (const { lang, word, source } of items) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) {
+      continue;
+    }
+    if (word === "") continue;
+    if (/\s/.test(word)) continue;
+    const count = tokenCount(tokenizer, word);
+    if (count !== 1) {
+      findings.push({
+        rule: RULE_ID3,
+        severity: "error",
+        message: `"${word}" tokenizes as ${count} tokens in ${lang} (${source})`,
+        context: {
+          domain: input.name,
+          lang,
+          word,
+          source,
+          tokenCount: count
+        }
+      });
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/keyword-classification.ts
+var RULE_ID4 = "keyword-classification";
+var keywordClassificationRule = (input) => {
+  const findings = [];
+  for (const { lang, word, source } of collectLexicalItems(input)) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) continue;
+    if (word === "" || /\s/.test(word)) continue;
+    const kind = tokenizer.classifyToken(word);
+    if (kind !== "keyword" && kind !== "particle") {
+      findings.push({
+        rule: RULE_ID4,
+        severity: "error",
+        message: `"${word}" classifies as '${kind}' (not 'keyword') in ${lang} (${source})`,
+        context: {
+          domain: input.name,
+          lang,
+          word,
+          source,
+          kind
+        }
+      });
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/extractor-coverage.ts
+var RULE_ID5 = "extractor-coverage";
+var DIACRITIC_SAMPLES = {
+  es: "a\xF1adir",
+  // ñ
+  fr: "d\xE9j\xE0",
+  // é, à
+  pt: "cora\xE7\xE3o",
+  // ç, ã
+  de: "gr\xF6\xDFer",
+  // ö, ß
+  tr: "de\u011Fi\u015Ftir",
+  // ğ, ş
+  it: "per\xF2",
+  // ò
+  pl: "kocha\u0107",
+  // ć
+  cs: "p\u0159\xEDli\u0161"
+  // ř, í
+  // Not run for en, ja, ar, ko, zh, ru, hi, etc. — either pure ASCII
+  // (en) or non-Latin scripts where the Unicode extractor handles them.
+};
+var extractorCoverageRule = (input) => {
+  const findings = [];
+  for (const [lang, sample] of Object.entries(DIACRITIC_SAMPLES)) {
+    const tokenizer = input.tokenizers[lang];
+    if (!tokenizer) continue;
+    const stream = tokenizer.tokenize(sample);
+    if (stream.tokens.length !== 1) {
+      findings.push({
+        rule: RULE_ID5,
+        severity: "warning",
+        message: `[${lang}] diacritic probe "${sample}" tokenizes as ${stream.tokens.length} tokens \u2014 tokenizer likely missing LatinExtendedIdentifierExtractor`,
+        context: {
+          domain: input.name,
+          lang,
+          sample,
+          tokenCount: stream.tokens.length
+        }
+      });
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/position-ordering.ts
+var RULE_ID6 = "position-ordering";
+function findCollisions(values) {
+  const byPosition = /* @__PURE__ */ new Map();
+  for (const { role, position } of values) {
+    const existing = byPosition.get(position) ?? [];
+    existing.push(role);
+    byPosition.set(position, existing);
+  }
+  return Array.from(byPosition.entries()).filter(([, roles]) => roles.length > 1).map(([position, roles]) => ({ position, roles }));
+}
+var positionOrderingRule = (input) => {
+  const findings = [];
+  for (const schema of input.schemas) {
+    if (schema.roles.length < 2) continue;
+    const svoValues = schema.roles.filter((r) => typeof r.svoPosition === "number" && r.svoPosition !== 0).map((r) => ({ role: r.role, position: r.svoPosition }));
+    const sovValues = schema.roles.filter((r) => typeof r.sovPosition === "number" && r.sovPosition !== 0).map((r) => ({ role: r.role, position: r.sovPosition }));
+    for (const { position, roles } of findCollisions(svoValues)) {
+      findings.push({
+        rule: RULE_ID6,
+        severity: "warning",
+        message: `[${String(schema.action)}] roles ${roles.join(", ")} share svoPosition=${position} \u2014 order is undefined`,
+        context: {
+          domain: input.name,
+          action: String(schema.action),
+          wordOrder: "SVO",
+          position,
+          roles: roles.join(",")
+        }
+      });
+    }
+    for (const { position, roles } of findCollisions(sovValues)) {
+      findings.push({
+        rule: RULE_ID6,
+        severity: "warning",
+        message: `[${String(schema.action)}] roles ${roles.join(", ")} share sovPosition=${position} \u2014 order is undefined`,
+        context: {
+          domain: input.name,
+          action: String(schema.action),
+          wordOrder: "SOV",
+          position,
+          roles: roles.join(",")
+        }
+      });
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/renderer-coherence.ts
+var RULE_ID7 = "renderer-coherence";
+var rendererCoherenceRule = (input) => {
+  if (!input.renderer) return [];
+  const findings = [];
+  for (const [action, langMap] of Object.entries(input.renderer.commandKeywords)) {
+    for (const [lang, rendererWord] of Object.entries(langMap)) {
+      const profile = input.profiles.find((p) => p.code === lang);
+      if (!profile) continue;
+      const profileEntry = profile.keywords[action];
+      if (!profileEntry) {
+        findings.push({
+          rule: RULE_ID7,
+          severity: "warning",
+          message: `renderer has '${action}' keyword for ${lang} ("${rendererWord}") but profile has no entry for that action`,
+          context: { domain: input.name, lang, action, rendererWord }
+        });
+        continue;
+      }
+      const profilePrimary = profileEntry.primary;
+      const profileAlts = profileEntry.alternatives ?? [];
+      if (rendererWord !== profilePrimary && !profileAlts.includes(rendererWord)) {
+        findings.push({
+          rule: RULE_ID7,
+          severity: "warning",
+          message: `renderer '${action}' keyword for ${lang} is "${rendererWord}" but profile has primary="${profilePrimary}"${profileAlts.length ? ` (alternatives: ${profileAlts.join(", ")})` : ""}`,
+          context: {
+            domain: input.name,
+            lang,
+            action,
+            rendererWord,
+            profilePrimary
+          }
+        });
+      }
+    }
+  }
+  const markerByLang = /* @__PURE__ */ new Map();
+  for (const schema of input.schemas) {
+    for (const role of schema.roles) {
+      if (!role.markerOverride) continue;
+      for (const [lang, word] of Object.entries(role.markerOverride)) {
+        const set = markerByLang.get(lang) ?? /* @__PURE__ */ new Set();
+        set.add(word);
+        markerByLang.set(lang, set);
+      }
+    }
+  }
+  for (const [markerName, langMap] of Object.entries(input.renderer.markers)) {
+    for (const [lang, rendererWord] of Object.entries(langMap)) {
+      const schemaMarkers = markerByLang.get(lang);
+      if (!schemaMarkers) continue;
+      if (!schemaMarkers.has(rendererWord)) {
+        findings.push({
+          rule: RULE_ID7,
+          severity: "warning",
+          message: `renderer marker "${markerName}" in ${lang} is "${rendererWord}" but no schema role has that markerOverride value \u2014 likely drift`,
+          context: {
+            domain: input.name,
+            lang,
+            markerName,
+            rendererWord
+          }
+        });
+      }
+    }
+  }
+  return findings;
+};
+
+// src/testing/gate/lint/rules/index.ts
+var ALL_RULES = [
+  schemaStructureRule,
+  keywordCoverageRule,
+  keywordClassificationRule,
+  markerTokenizationRule,
+  extractorCoverageRule,
+  positionOrderingRule,
+  rendererCoherenceRule
+];
+function runRules(input) {
+  const findings = [];
+  for (const rule of ALL_RULES) {
+    findings.push(...rule(input));
+  }
+  return findings;
+}
+
+// src/testing/gate/lint/format.ts
+var SEVERITY_GLYPH = {
+  error: "\u2717",
+  warning: "\u26A0",
+  note: "\u2139"
+};
+function formatFinding(finding) {
+  const glyph = SEVERITY_GLYPH[finding.severity];
+  return `  ${glyph} [${finding.rule}] ${finding.message}`;
+}
+function formatResult(result) {
+  const header = `domain-toolkit: lint(${result.domain}) \u2014 ${result.errorCount} error(s), ${result.warningCount} warning(s)`;
+  if (result.findings.length === 0) return `${header}
+  (clean)`;
+  const body = result.findings.map(formatFinding).join("\n");
+  return `${header}
+${body}`;
+}
+
+// src/testing/gate/lint/index.ts
+function matchesWaiver(finding, waiver) {
+  if (finding.rule !== waiver.rule) return false;
+  if (!waiver.matches) return true;
+  const ctx = finding.context ?? {};
+  for (const [key, expected] of Object.entries(waiver.matches)) {
+    if (ctx[key] !== expected) return false;
+  }
+  return true;
+}
+function applyWaivers(findings, waivers) {
+  if (!waivers || waivers.length === 0) return findings;
+  return findings.filter((f) => !waivers.some((w) => matchesWaiver(f, w)));
+}
+function lintDomain(input) {
+  const raw = runRules(input);
+  const findings = applyWaivers(raw, input.waivers);
+  const errorCount = findings.filter((f) => f.severity === "error").length;
+  const warningCount = findings.filter((f) => f.severity === "warning").length;
+  return {
+    domain: input.name,
+    findings,
+    errorCount,
+    warningCount
+  };
+}
+export {
+  BEST,
+  RANK,
+  UNKNOWN_WORD,
+  commandsFor,
+  compare,
+  countsOf,
+  formatFinding,
+  formatResult,
+  lintDomain,
+  listingOf,
+  referenceCode,
+  runRoundTrip,
+  snapshotChanges,
+  snapshotOf,
+  summarize
+};
+//# sourceMappingURL=index.js.map