@lokascript/hyperscript-adapter 2.10.0 → 2.11.1

package/src/host-validate.ts

@@ -0,0 +1,63 @@
+/**
+ * Host-parser validity gate (review item F8).
+ *
+ * After the preprocessor rewrites an attribute to English, the plugin asks
+ * the HOST _hyperscript runtime — the same parser that will consume the
+ * rewrite moments later — whether the result actually parses. On rejection
+ * the plugin falls back to the author's original text, so any parse error
+ * the author then sees names code they wrote, not invisible generated
+ * English. This is the runtime analog of the offline R4 canonical-validity
+ * gate, and the F5 arc measured its failure class shipping in practice:
+ * until the whole-string-first reorder (#899), 256 corpus rows rendered
+ * English the engine rejects, with no warning anywhere.
+ *
+ * The engine has two failure channels — `parse().errors` collects grammar
+ * errors, and the tokenizer THROWS on an unknown character — folded here
+ * the same way test/whole-string-first.test.ts folds them.
+ *
+ * Zero-dependency module: shared by the full, slim, and lite plugin
+ * variants, which must not share heavier import chains (the slim/lite
+ * bundles exclude the full semantic package by construction).
+ */
+
+export interface HyperscriptParseHost {
+  parse?: (src: string) => { errors?: unknown[] } | null | undefined;
+}
+
+/**
+ * True when the host's parser accepts `src`. Also true when the host
+ * exposes no `parse()` — with nothing to validate against, the gate
+ * degrades to a no-op rather than suppressing translation on unusual
+ * builds (same graceful posture as the `addBeforeProcessHook` check).
+ */
+export function acceptedByHost(hs: HyperscriptParseHost, src: string): boolean {
+  if (typeof hs.parse !== 'function') return true;
+  try {
+    const result = hs.parse(src);
+    return !result?.errors || result.errors.length === 0;
+  } catch {
+    return false;
+  }
+}
+
+/** Languages already warned about a rejected translation this page load —
+ *  same warn-once-per-lang convention as the unchanged-translation warning
+ *  (and htmx-adapter's warnMissingLangOnce). */
+const warnedRejectedLang = new Set<string>();
+
+/** Reset the warn-once state. Mainly for tests. */
+export function resetHostValidationWarnings(): void {
+  warnedRejectedLang.clear();
+}
+
+export function warnRejectedOnce(lang: string, src: string, english: string): void {
+  if (warnedRejectedLang.has(lang)) return;
+  warnedRejectedLang.add(lang);
+  console.warn(
+    `[hyperscript-i18n] Translation for lang="${lang}" rendered hyperscript the host parser ` +
+      `rejects — falling back to the original text. ` +
+      `Source: "${src.length > 60 ? src.slice(0, 60) + '…' : src}" → ` +
+      `"${english.length > 60 ? english.slice(0, 60) + '…' : english}". ` +
+      'Further elements in this language stay quiet — enable { debug: true } for per-element detail.'
+  );
+}

package/src/hyperscript-renderer.ts

@@ -28,6 +28,27 @@ import type {
 import { SYNTAX } from './generated/syntax-table';
 export { SYNTAX };
 
+// ---------------------------------------------------------------------------
+// Statement-join rules. Mirrors the semantic package's renderer
+// (explicit/renderer.ts) — the slim path has to reach the same English, and a
+// chain word in the wrong seam is a hard parse error on the real engine, not a
+// style difference.
+// ---------------------------------------------------------------------------
+
+/**
+ * Block-header commands: their body follows the header DIRECTLY. `repeat 3
+ * times then add …` is rejected ("Expected 'end' but found 'then'"), as is
+ * `tell #panel then add …`.
+ */
+const BLOCK_HEADER_ACTIONS = new Set(['repeat', 'for', 'while', 'tell']);
+
+/**
+ * Commands whose captured body is an open-ended block that must be closed by an
+ * explicit `end` when a sibling follows: `js foo() then add …` otherwise bleeds
+ * the following hyperscript into the JavaScript body.
+ */
+const BLOCK_NEEDS_TRAILING_END = new Set(['js']);
+
 // ---------------------------------------------------------------------------
 // Main entry point
 // ---------------------------------------------------------------------------
@@ -53,10 +74,14 @@ export function renderToHyperscript(node: SemanticNode): string {
 function renderEventHandler(node: EventHandlerSemanticNode): string {
   const parts: string[] = ['on'];
 
-  // Event name
+  // Event name. Rendered BARE, never through renderValue: the parser
+  // produces the event as a string-typed literal, which renderValue would
+  // quote — and `on "click"` is a hard parse error on the real engine
+  // ("Expected event name"). Event names are identifiers in canonical
+  // hyperscript (`on click`, `on draggable:start`), never quoted strings.
   const event = node.roles.get('event');
   if (event) {
-    parts.push(renderValue(event));
+    parts.push(event.type === 'literal' ? String(event.value) : renderValue(event));
   }
 
   // Event source (from #element)
@@ -65,10 +90,13 @@ function renderEventHandler(node: EventHandlerSemanticNode): string {
     parts.push('from', renderValue(source));
   }
 
-  // Body commands
+  // Body commands. Space-joined, matching the semantic renderer: the chain word
+  // between sibling body commands is optional in canonical hyperscript, and
+  // joining with ` then ` unconditionally injected one after a block header
+  // (`on click tell #panel then add .open` — rejected by the real engine).
   if (node.body && node.body.length > 0) {
     const bodyParts = node.body.map(renderToHyperscript);
-    parts.push(bodyParts.join(' then '));
+    parts.push(bodyParts.join(' '));
   }
 
   return parts.join(' ');
@@ -76,7 +104,26 @@ function renderEventHandler(node: EventHandlerSemanticNode): string {
 
 function renderCompound(node: CompoundSemanticNode): string {
   const chainWord = node.chainType === 'async' ? 'async' : node.chainType;
-  return node.statements.map(renderToHyperscript).join(` ${chainWord} `);
+  const rendered = node.statements.map(renderToHyperscript);
+  let out = rendered[0] ?? '';
+  for (let i = 1; i < rendered.length; i++) {
+    const prev = node.statements[i - 1];
+    const cur = node.statements[i];
+    const afterBlockHeader = prev.kind === 'command' && BLOCK_HEADER_ACTIONS.has(prev.action);
+    // Consecutive top-level `bind` features are separate reactive features, not
+    // a then-chain: `bind $x to #a then bind $x to #b` is rejected ("Unexpected
+    // Token : then" between features).
+    const betweenBindFeatures =
+      prev.kind === 'command' &&
+      prev.action === 'bind' &&
+      cur.kind === 'command' &&
+      cur.action === 'bind';
+    if (prev.kind === 'command' && BLOCK_NEEDS_TRAILING_END.has(prev.action)) {
+      out += ' end';
+    }
+    out += (afterBlockHeader || betweenBindFeatures ? ' ' : ` ${chainWord} `) + rendered[i];
+  }
+  return out;
 }
 
 function renderCommand(node: SemanticNode): string {
@@ -92,6 +139,18 @@ function renderCommand(node: SemanticNode): string {
       // `add .active` not `add .active to me`; `remove .hidden` not `remove .hidden
       // from me`. Mirrors the semantic renderer's implicit-me suppression so the
       // full and slim (custom-renderer) paths agree.
+      //
+      // KNOWN GAP, deliberately unfixed here: semantic's renderer EXCEPTS a
+      // string-content patient (`add "<p>Line</p>" to me` keeps the
+      // destination — the bare form is rejected by the engine). Mirroring
+      // that exception is correct in isolation, but it may not ship before
+      // the generated repeat patterns capture quantity: measured on the
+      // parity corpus, the exception alone turns the es `repeat` row's slim
+      // output from engine-INVALID (host-validate gate → safe fallback to
+      // the author's text) into the engine-VALID `on click repeat add
+      // "<p>Line</p>" to me` — a bare `repeat` is FOREVER, so a warned
+      // no-op becomes a committed infinite loop. The parity slim test pins
+      // that row's output staying invalid until both fixes land together.
       if (
         (role === 'destination' || role === 'source') &&
         value.type === 'reference' &&

package/src/index.ts

@@ -22,6 +22,11 @@
  * _hyperscript(english);
  */
 
-export { hyperscriptI18n, preprocess, type PluginOptions } from './plugin';
+export {
+  hyperscriptI18n,
+  preprocess,
+  resetTranslationWarnings,
+  type PluginOptions,
+} from './plugin';
 export { preprocessToEnglish, type PreprocessorConfig } from './preprocessor';
 export { resolveLanguage } from './language-resolver';

package/src/language-resolver.ts

@@ -11,8 +11,16 @@
  * Resolution order:
  * 1. `data-lang` attribute on the element itself
  * 2. `data-hyperscript-lang` attribute on the element or closest ancestor
- * 3. `lang` attribute on `<html>` element
- * 4. null (assume English, no preprocessing needed)
+ * 3. `lang` attribute on the element or closest ancestor (the HTML-standard
+ *    cascade — a `<section lang="es">` localizes everything inside it, and a
+ *    nested `lang="en"` opts back out). This is how the paired htmx-adapter
+ *    (`langOf()`) and loka-js resolve language, so `hx-*` and `_` attributes
+ *    on the same element agree.
+ * 4. `lang` on `<html>` via `document.documentElement` — only reachable when
+ *    the element is DETACHED (step 3's ancestor walk covers `<html>` for
+ *    attached elements): a hook processing a not-yet-inserted fragment still
+ *    picks up the page default.
+ * 5. null (assume English, no preprocessing needed)
  */
 export function resolveLanguage(elt: Element): string | null {
   // 1. Explicit per-element
@@ -25,7 +33,11 @@ export function resolveLanguage(elt: Element): string | null {
     elt.closest?.('[data-hyperscript-lang]')?.getAttribute('data-hyperscript-lang');
   if (hsLang) return normalizeLangCode(hsLang);
 
-  // 3. Document-level lang
+  // 3. Standard lang cascade (nearest ancestor wins)
+  const closestLang = elt.closest?.('[lang]')?.getAttribute('lang');
+  if (closestLang) return normalizeLangCode(closestLang);
+
+  // 4. Document-level lang (detached elements only — see doc comment)
   const htmlLang = typeof document !== 'undefined' ? document.documentElement?.lang : null;
   if (htmlLang && htmlLang !== 'en') return normalizeLangCode(htmlLang);
 

package/src/plugin.ts

@@ -8,6 +8,12 @@
 import { resolveLanguage } from './language-resolver';
 import { preprocessToEnglish, type PreprocessorConfig } from './preprocessor';
 import { installAttributeTranslator, type HyperscriptHost } from './attribute-translator';
+import {
+  acceptedByHost,
+  warnRejectedOnce,
+  resetHostValidationWarnings,
+  type HyperscriptParseHost,
+} from './host-validate';
 
 export interface PluginOptions extends Partial<PreprocessorConfig> {
   /** Default language for all elements (overridable per-element). */
@@ -16,6 +22,38 @@ export interface PluginOptions extends Partial<PreprocessorConfig> {
   languageAttribute?: string;
   /** Enable debug logging to console. Default: false */
   debug?: boolean;
+  /**
+   * Validate rendered English on the HOST parser before committing the
+   * rewrite; on rejection, fall back to the original text (so parse errors
+   * name the author's code, not generated English). Default: true.
+   * No-op on host builds that expose no `parse()`.
+   */
+  validateWithHost?: boolean;
+}
+
+/** Languages already warned about an unchanged translation this page load.
+ *  Unchanged output is common and often legitimate (canonical-English
+ *  hyperscript under a non-en lang scope), so warning per element per
+ *  processNode was pure noise — mirror htmx-adapter's warn-once-per-lang
+ *  convention and leave per-element detail to `debug: true`. */
+const warnedUnchangedLang = new Set<string>();
+
+/** Reset the warn-once state (unchanged + host-rejected). Mainly for tests. */
+export function resetTranslationWarnings(): void {
+  warnedUnchangedLang.clear();
+  resetHostValidationWarnings();
+}
+
+function warnUnchangedOnce(lang: string, src: string): void {
+  if (warnedUnchangedLang.has(lang)) return;
+  warnedUnchangedLang.add(lang);
+  console.warn(
+    `[hyperscript-i18n] Translation unchanged for lang="${lang}": "${src.length > 60 ? src.slice(0, 60) + '…' : src}". ` +
+      'This is fine if the source is already canonical English; otherwise the input may not match ' +
+      'any known pattern, or the language may not be registered. Original text is passed to ' +
+      '_hyperscript as-is. Further elements in this language stay quiet — enable { debug: true } ' +
+      'for per-element detail.'
+  );
 }
 
 /**
@@ -35,7 +73,8 @@ export interface PluginOptions extends Partial<PreprocessorConfig> {
  */
 export function hyperscriptI18n(options: PluginOptions = {}) {
   return function plugin(hs: unknown): void {
-    installAttributeTranslator(hs as HyperscriptHost, (src, elt) => {
+    const host = hs as HyperscriptHost & HyperscriptParseHost;
+    installAttributeTranslator(host, (src, elt) => {
       // Resolve language
       const lang = resolveLanguageWithOptions(elt, options);
 
@@ -46,16 +85,27 @@ export function hyperscriptI18n(options: PluginOptions = {}) {
       const english = preprocessToEnglish(src, lang, options);
 
       if (english !== src) {
+        // Validity gate: the host parser is the consumer of this rewrite —
+        // if it rejects the English, committing it would only trade a
+        // translation gap for a parse error naming code the author never
+        // wrote. Fall back to the original text instead.
+        if (options.validateWithHost !== false && !acceptedByHost(host, english)) {
+          if (options.debug) {
+            console.log(
+              `[hyperscript-i18n] ${lang}: host rejected "${english}" — keeping "${src}"`
+            );
+          } else {
+            warnRejectedOnce(lang, src, english);
+          }
+          return src;
+        }
         if (options.debug) {
           console.log(`[hyperscript-i18n] ${lang}: "${src}" → "${english}"`);
         }
+      } else if (options.debug) {
+        console.log(`[hyperscript-i18n] ${lang}: unchanged "${src}"`);
       } else {
-        // Translation produced no change — likely a failure
-        console.warn(
-          `[hyperscript-i18n] Translation unchanged for lang="${lang}": "${src.length > 60 ? src.slice(0, 60) + '…' : src}". ` +
-            'The input may not match any known pattern, or the language may not be registered. ' +
-            'Original text will be passed to _hyperscript as-is.'
-        );
+        warnUnchangedOnce(lang, src);
       }
 
       return english;

package/src/preprocessor-core.ts

@@ -0,0 +1,217 @@
+/**
+ * Preprocessor core — the shared skeleton of the full and slim
+ * preprocessors.
+ *
+ * Everything that is NOT genuinely different between the two paths lives
+ * here exactly once: config defaults/threshold resolution, strategy
+ * ordering (semantic → optional i18n), event-prefix stripping, and the
+ * return-original fallback. The two real differences stay with each path,
+ * injected as hooks:
+ *
+ *   - HOW one statement is parsed and rendered to English
+ *     (`translateSingle`): the full path uses semantic's parseSemantic +
+ *     render + a translate() rescue for confident-but-nodeless parses;
+ *     the slim path uses parseWithConfidence + the custom
+ *     hyperscript-renderer (no English language data).
+ *   - WHERE the registry lookup comes from (`isLanguageRegistered`):
+ *     `@lokascript/semantic` vs `…/core` — under tsup's split dist these
+ *     are separate registry instances, so each path must bring its own.
+ *
+ * Behavior is pinned by the parity ratchet
+ * (test/preprocessor-parity.*.test.ts + the committed fixture): both paths
+ * are byte-identical to their pre-extraction outputs over the corpus.
+ *
+ * HISTORY: the skeleton used to carry a compound-splitting fallback
+ * (`splitStatements` on localized `then` keywords + newlines,
+ * `translateCompound` rejoining pieces with a hardcoded ` then `), run when
+ * the whole-string parse declined. Deleted 2026-08-07 after measurement:
+ * over all 3703 patterns.db corpus translations plus the parity corpus, on
+ * BOTH paths, the split arm produced ZERO final outputs (full: 115
+ * invocations, all returned null; slim: 32, all null) — post-whole-string-
+ * first (#899), an input whose pieces each parse would have parsed whole,
+ * so the arm's success condition is self-defeating. It was also defective
+ * twice over: the regex split was string-literal-blind (`say 'now then
+ * later'` split inside the literal) and the ` then ` rejoin is invalid
+ * immediately after block headers. Deleting it erased both defect classes
+ * wholesale; the parity fixture regenerated byte-identical.
+ */
+
+export interface PreprocessorConfig {
+  /**
+   * Minimum confidence threshold for semantic parsing (0-1). Default: 0.5.
+   * Can be a single number (applies to all languages) or a per-language map.
+   * Per-language thresholds are useful because SOV languages (ja, ko, tr) produce
+   * inherently lower confidence scores than SVO languages (es, fr, de).
+   *
+   * @example
+   * // Single threshold
+   * { confidenceThreshold: 0.5 }
+   *
+   * @example
+   * // Per-language thresholds
+   * { confidenceThreshold: { es: 0.7, ja: 0.1, ko: 0.05, '*': 0.5 } }
+   */
+  confidenceThreshold: number | Record<string, number>;
+  /** Strategy: 'semantic' (default), 'i18n', or 'auto' (semantic then i18n) */
+  strategy: 'semantic' | 'i18n' | 'auto';
+  /**
+   * @deprecated Never implemented — `preprocessToEnglish` always returns a
+   * string, and on translation failure that string is the original source
+   * (there is nothing else it could return). The option has had no effect in
+   * any released version and is ignored; it will be removed in a future major.
+   */
+  fallbackToOriginal?: boolean;
+  /** Optional i18n toEnglish function (loaded dynamically if available) */
+  i18nToEnglish?: (input: string, locale: string) => string;
+}
+
+/** The per-path seam — see module doc. */
+export interface PreprocessorHooks {
+  isLanguageRegistered(lang: string): boolean;
+  /**
+   * Parse ONE statement in `lang` and render it to English. Return null
+   * when it cannot be translated (below-threshold confidence, no node, …);
+   * the skeleton then tries the remaining strategies / falls back.
+   */
+  translateSingle(src: string, lang: string, threshold: number): string | null;
+}
+
+const DEFAULT_THRESHOLD = 0.5;
+
+const DEFAULT_CONFIG: PreprocessorConfig = {
+  confidenceThreshold: DEFAULT_THRESHOLD,
+  strategy: 'semantic',
+};
+
+/**
+ * Resolve the confidence threshold for a specific language.
+ * Supports both a single number and a per-language map with '*' as default.
+ */
+function resolveThreshold(threshold: number | Record<string, number>, lang: string): number {
+  if (typeof threshold === 'number') return threshold;
+  return threshold[lang] ?? threshold['*'] ?? DEFAULT_THRESHOLD;
+}
+
+/**
+ * Match _hyperscript event handler prefix: "on [every] <event>[filter][.modifiers] "
+ *
+ * Examples:
+ *   "on click toggle .active"            → prefix: "on click ", commands: "toggle .active"
+ *   "on every click toggle .active"      → prefix: "on every click ", commands: "toggle .active"
+ *   "on click.debounce(300) toggle .x"   → prefix: "on click.debounce(300) ", commands: "toggle .x"
+ *   "on keyup[key=='Enter'] set x to 1"  → prefix: "on keyup[key=='Enter'] ", commands: "set x to 1"
+ *   "on click from body toggle .active"  → prefix: "on click from body ", commands: "toggle .active"
+ */
+const EVENT_PREFIX_RE =
+  /^(on\s+(?:every\s+)?[\w-]+(?:\[.*?\])?(?:\.[\w-]+(?:\([^)]*\))?)*(?:\s+from\s+\S+)?(?:\s+queue\s+\w+)?\s+)/;
+
+function stripEventPrefix(src: string): { prefix: string; commands: string } | null {
+  const match = src.match(EVENT_PREFIX_RE);
+  if (!match) return null;
+  const prefix = match[1];
+  const commands = src.slice(prefix.length);
+  if (!commands) return null;
+  return { prefix, commands };
+}
+
+/**
+ * Build a `preprocessToEnglish` function from the per-path hooks.
+ *
+ * Handles _hyperscript feature prefixes (e.g. "on click", "on every keyup")
+ * by stripping them, translating only the command portion, then reassembling.
+ */
+export function createPreprocessToEnglish(
+  hooks: PreprocessorHooks
+): (src: string, lang: string, config?: Partial<PreprocessorConfig>) => string {
+  /**
+   * Try semantic translation: parse the WHOLE string in the source
+   * language, render it to English. Returns null if confidence is below
+   * threshold. The semantic parser handles `then`-sequences, newlines,
+   * loop/tell bodies and behavior blocks natively (top-level
+   * command-sequence support landed with the multiset-recall arc), so
+   * there is deliberately no pre-splitting here — see the module doc's
+   * HISTORY note for the measured deletion of the old split fallback.
+   */
+  function trySemanticTranslation(src: string, lang: string, threshold: number): string | null {
+    try {
+      // Check if the semantic parser can handle this
+      if (!hooks.isLanguageRegistered(lang)) return null;
+
+      return hooks.translateSingle(src, lang, threshold);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Try i18n grammar transformation to English.
+   * Returns null if the result is identical to input (no translation happened).
+   */
+  function tryI18nTranslation(
+    src: string,
+    lang: string,
+    toEnglish: (input: string, locale: string) => string
+  ): string | null {
+    try {
+      const result = toEnglish(src, lang);
+      return result !== src ? result : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Try all configured translation strategies on the input.
+   * Returns null if none succeed.
+   */
+  function tryTranslateWithStrategies(
+    src: string,
+    lang: string,
+    cfg: PreprocessorConfig
+  ): string | null {
+    // Strategy: semantic-only
+    if (cfg.strategy === 'semantic' || cfg.strategy === 'auto') {
+      const threshold = resolveThreshold(cfg.confidenceThreshold, lang);
+      const result = trySemanticTranslation(src, lang, threshold);
+      if (result !== null) return result;
+    }
+
+    // Strategy: i18n fallback (auto mode or i18n-only)
+    if ((cfg.strategy === 'auto' || cfg.strategy === 'i18n') && cfg.i18nToEnglish) {
+      const result = tryI18nTranslation(src, lang, cfg.i18nToEnglish);
+      if (result !== null) return result;
+    }
+
+    return null;
+  }
+
+  return function preprocessToEnglish(
+    src: string,
+    lang: string,
+    config: Partial<PreprocessorConfig> = {}
+  ): string {
+    // English→English is identity; skip semantic parsing which may mangle
+    // the input. (Historically full-path only — the slim path gained it in
+    // the shared-skeleton extraction, deliberately: it is a mangle guard.)
+    if (lang === 'en') return src;
+
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+
+    // Try translating the full string first
+    const fullResult = tryTranslateWithStrategies(src, lang, cfg);
+    if (fullResult !== null) return fullResult;
+
+    // If full translation failed, try stripping event/feature prefix.
+    // _hyperscript attributes often contain "on <event> <commands>" — the semantic
+    // parser only understands command syntax, not event declarations.
+    const stripped = stripEventPrefix(src);
+    if (stripped) {
+      const translated = tryTranslateWithStrategies(stripped.commands, lang, cfg);
+      if (translated !== null) return stripped.prefix + translated;
+    }
+
+    // Fallback: return original (unconditional — see fallbackToOriginal's
+    // deprecation note; the string contract leaves nothing else to return)
+    return src;
+  };
+}