@lokascript/hyperscript-adapter 2.9.4 → 2.11.0

package/src/preprocessor.ts

@@ -1,157 +1,30 @@
 /**
- * Preprocessor
+ * Preprocessor (full path)
  *
  * Translates non-English hyperscript to English using the semantic parser
  * (with optional i18n fallback), so the original _hyperscript can parse it.
+ *
+ * The strategy/split/strip skeleton lives in preprocessor-core.ts, shared
+ * with the slim path — only HOW one statement is parsed and rendered
+ * differs here: semantic's parseSemantic + render('en'), plus a
+ * translate() rescue when the parse is confident but yields no node.
  */
 
-import {
-  translate,
-  render,
-  parseSemantic,
-  isLanguageRegistered,
-  tryGetProfile,
-} from '@lokascript/semantic';
-
-export interface PreprocessorConfig {
-  /**
-   * Minimum confidence threshold for semantic parsing (0-1). Default: 0.5.
-   * Can be a single number (applies to all languages) or a per-language map.
-   * Per-language thresholds are useful because SOV languages (ja, ko, tr) produce
-   * inherently lower confidence scores than SVO languages (es, fr, de).
-   *
-   * @example
-   * // Single threshold
-   * { confidenceThreshold: 0.5 }
-   *
-   * @example
-   * // Per-language thresholds
-   * { confidenceThreshold: { es: 0.7, ja: 0.1, ko: 0.05, '*': 0.5 } }
-   */
-  confidenceThreshold: number | Record<string, number>;
-  /** Strategy: 'semantic' (default), 'i18n', or 'auto' (semantic then i18n) */
-  strategy: 'semantic' | 'i18n' | 'auto';
-  /** Whether to return original text on failure. Default: true */
-  fallbackToOriginal: boolean;
-  /** Optional i18n toEnglish function (loaded dynamically if available) */
-  i18nToEnglish?: (input: string, locale: string) => string;
-}
+import { translate, render, parseSemantic, isLanguageRegistered } from '@lokascript/semantic';
 
-const DEFAULT_THRESHOLD = 0.5;
+import { createPreprocessToEnglish, type PreprocessorConfig } from './preprocessor-core';
 
-const DEFAULT_CONFIG: PreprocessorConfig = {
-  confidenceThreshold: DEFAULT_THRESHOLD,
-  strategy: 'semantic',
-  fallbackToOriginal: true,
-};
-
-/**
- * Resolve the confidence threshold for a specific language.
- * Supports both a single number and a per-language map with '*' as default.
- */
-function resolveThreshold(threshold: number | Record<string, number>, lang: string): number {
-  if (typeof threshold === 'number') return threshold;
-  return threshold[lang] ?? threshold['*'] ?? DEFAULT_THRESHOLD;
-}
+export type { PreprocessorConfig };
 
 /**
  * Preprocess non-English hyperscript into English.
  *
  * Uses the semantic parser to parse the input in the source language,
  * then renders back to English. Falls through to i18n if configured.
- *
- * Handles _hyperscript feature prefixes (e.g. "on click", "on every keyup")
- * by stripping them, translating only the command portion, then reassembling.
- */
-export function preprocessToEnglish(
-  src: string,
-  lang: string,
-  config: Partial<PreprocessorConfig> = {}
-): string {
-  // English→English is identity; skip semantic parsing which may mangle the input
-  if (lang === 'en') return src;
-
-  const cfg = { ...DEFAULT_CONFIG, ...config };
-
-  // Try translating the full string first
-  const fullResult = tryTranslateWithStrategies(src, lang, cfg);
-  if (fullResult !== null) return fullResult;
-
-  // If full translation failed, try stripping event/feature prefix.
-  // _hyperscript attributes often contain "on <event> <commands>" — the semantic
-  // parser only understands command syntax, not event declarations.
-  const stripped = stripEventPrefix(src);
-  if (stripped) {
-    const translated = tryTranslateWithStrategies(stripped.commands, lang, cfg);
-    if (translated !== null) return stripped.prefix + translated;
-  }
-
-  // Fallback: return original
-  return cfg.fallbackToOriginal ? src : src;
-}
-
-/**
- * Try all configured translation strategies on the input.
- * Returns null if none succeed.
  */
-function tryTranslateWithStrategies(
-  src: string,
-  lang: string,
-  cfg: PreprocessorConfig
-): string | null {
-  // Strategy: semantic-only
-  if (cfg.strategy === 'semantic' || cfg.strategy === 'auto') {
-    const threshold = resolveThreshold(cfg.confidenceThreshold, lang);
-    const result = trySemanticTranslation(src, lang, threshold);
-    if (result !== null) return result;
-  }
-
-  // Strategy: i18n fallback (auto mode or i18n-only)
-  if ((cfg.strategy === 'auto' || cfg.strategy === 'i18n') && cfg.i18nToEnglish) {
-    const result = tryI18nTranslation(src, lang, cfg.i18nToEnglish);
-    if (result !== null) return result;
-  }
-
-  return null;
-}
-
-/**
- * Match _hyperscript event handler prefix: "on [every] <event>[filter][.modifiers] "
- *
- * Examples:
- *   "on click toggle .active"            → prefix: "on click ", commands: "toggle .active"
- *   "on every click toggle .active"      → prefix: "on every click ", commands: "toggle .active"
- *   "on click.debounce(300) toggle .x"   → prefix: "on click.debounce(300) ", commands: "toggle .x"
- *   "on keyup[key=='Enter'] set x to 1"  → prefix: "on keyup[key=='Enter'] ", commands: "set x to 1"
- *   "on click from body toggle .active"  → prefix: "on click from body ", commands: "toggle .active"
- */
-const EVENT_PREFIX_RE =
-  /^(on\s+(?:every\s+)?[\w-]+(?:\[.*?\])?(?:\.[\w-]+(?:\([^)]*\))?)*(?:\s+from\s+\S+)?(?:\s+queue\s+\w+)?\s+)/;
-
-function stripEventPrefix(src: string): { prefix: string; commands: string } | null {
-  const match = src.match(EVENT_PREFIX_RE);
-  if (!match) return null;
-  const prefix = match[1];
-  const commands = src.slice(prefix.length);
-  if (!commands) return null;
-  return { prefix, commands };
-}
-
-/**
- * Try semantic translation: parse in source language, render to English.
- * Returns null if confidence is below threshold.
- */
-function trySemanticTranslation(src: string, lang: string, threshold: number): string | null {
-  try {
-    // Handle compound statements (split on "then" boundaries and newlines)
-    const statements = splitStatements(src, lang);
-    if (statements.length > 1) {
-      return translateCompound(statements, lang, threshold);
-    }
-
-    // Check if the semantic parser can handle this
-    if (!isLanguageRegistered(lang)) return null;
-
+export const preprocessToEnglish = createPreprocessToEnglish({
+  isLanguageRegistered,
+  translateSingle(src, lang, threshold) {
     const result = parseSemantic(src, lang);
     if (result.confidence < threshold || !result.node) {
       // If we got SOME confidence but no node, fall back to translate()
@@ -163,79 +36,5 @@ function trySemanticTranslation(src: string, lang: string, threshold: number): s
 
     // Render the semantic node to English
     return render(result.node, 'en');
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Try i18n grammar transformation to English.
- * Returns null if the result is identical to input (no translation happened).
- */
-function tryI18nTranslation(
-  src: string,
-  lang: string,
-  toEnglish: (input: string, locale: string) => string
-): string | null {
-  try {
-    const result = toEnglish(src, lang);
-    return result !== src ? result : null;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Get all "then" keyword forms for a language (primary + alternatives).
- * Falls back to English "then" if profile is unavailable.
- */
-function getThenKeywords(lang: string): string[] {
-  const keywords = ['then']; // Always include English
-  const profile = tryGetProfile(lang);
-  if (profile?.keywords?.then) {
-    const kw = profile.keywords.then;
-    if (kw.primary && kw.primary !== 'then') keywords.push(kw.primary);
-    if (kw.alternatives) {
-      for (const alt of kw.alternatives) {
-        if (!keywords.includes(alt)) keywords.push(alt);
-      }
-    }
-  }
-  return keywords;
-}
-
-/**
- * Split a hyperscript source string into individual statements.
- * Splits on language-specific "then" keywords and newlines.
- */
-function splitStatements(src: string, lang: string): string[] {
-  const thenWords = getThenKeywords(lang);
-  // Escape regex special chars and build pattern
-  const escaped = thenWords.map(w => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
-  const pattern = new RegExp(`\\s+(?:${escaped.join('|')})\\s+`, 'i');
-
-  // Split on newlines first
-  const lines = src
-    .split('\n')
-    .map(l => l.trim())
-    .filter(l => l.length > 0);
-  const result: string[] = [];
-  for (const line of lines) {
-    const parts = line.split(pattern);
-    result.push(...parts);
-  }
-  return result;
-}
-
-/**
- * Translate compound statements individually and rejoin with " then ".
- */
-function translateCompound(statements: string[], lang: string, threshold: number): string | null {
-  const translated: string[] = [];
-  for (const stmt of statements) {
-    const result = trySemanticTranslation(stmt.trim(), lang, threshold);
-    if (result === null) return null; // If any part fails, fail the whole compound
-    translated.push(result);
-  }
-  return translated.join(' then ');
-}
+  },
+});

package/src/slim-plugin.ts

@@ -10,19 +10,34 @@ import { preprocessToEnglish } from './slim-preprocessor';
 import type { PreprocessorConfig } from './preprocessor';
 import type { PluginOptions } from './plugin';
 import { installAttributeTranslator, type HyperscriptHost } from './attribute-translator';
+import { acceptedByHost, warnRejectedOnce, type HyperscriptParseHost } from './host-validate';
 
 export type { PluginOptions };
 
 export function hyperscriptI18n(options: PluginOptions = {}) {
   return function plugin(hs: unknown): void {
-    installAttributeTranslator(hs as HyperscriptHost, (src, elt) => {
+    const host = hs as HyperscriptHost & HyperscriptParseHost;
+    installAttributeTranslator(host, (src, elt) => {
       const lang = resolveLanguageWithOptions(elt, options);
       if (!lang || lang === 'en') return src;
 
       const english = preprocessToEnglish(src, lang, options);
 
-      if (options.debug && english !== src) {
-        console.log(`[hyperscript-i18n] ${lang}: "${src}" → "${english}"`);
+      if (english !== src) {
+        // Host-parser validity gate — see plugin.ts / host-validate.ts.
+        if (options.validateWithHost !== false && !acceptedByHost(host, english)) {
+          if (options.debug) {
+            console.log(
+              `[hyperscript-i18n] ${lang}: host rejected "${english}" — keeping "${src}"`
+            );
+          } else {
+            warnRejectedOnce(lang, src, english);
+          }
+          return src;
+        }
+        if (options.debug) {
+          console.log(`[hyperscript-i18n] ${lang}: "${src}" → "${english}"`);
+        }
       }
 
       return english;

package/src/slim-preprocessor.ts

@@ -1,172 +1,34 @@
 /**
  * Slim Preprocessor
  *
- * Same logic as preprocessor.ts but imports from @lokascript/semantic/core
- * and uses a custom hyperscript renderer instead of the semantic package's
- * render(). This avoids importing English language data (tokenizer, patterns,
- * profile), saving ~35 KB per per-language bundle.
+ * Same skeleton as preprocessor.ts (shared via preprocessor-core.ts) but
+ * imports from @lokascript/semantic/core and renders through the custom
+ * hyperscript renderer instead of the semantic package's render(). This
+ * avoids importing English language data (tokenizer, patterns, profile),
+ * saving ~35 KB per per-language bundle.
  *
- * Used by per-language browser bundles for tree-shaking.
+ * Used by per-language browser bundles for tree-shaking. Must never
+ * import from '@lokascript/semantic' (the full package) at runtime.
  */
 
 // Import from /core — does NOT trigger all-language registration.
 // Languages are registered separately via side-effect imports in bundle entries.
-import {
-  parseWithConfidence,
-  isLanguageRegistered,
-  tryGetProfile,
-} from '@lokascript/semantic/core';
+import { parseWithConfidence, isLanguageRegistered } from '@lokascript/semantic/core';
 
 import { renderToHyperscript } from './hyperscript-renderer';
-import type { PreprocessorConfig } from './preprocessor';
-
-const DEFAULT_THRESHOLD = 0.5;
-
-const DEFAULT_CONFIG: PreprocessorConfig = {
-  confidenceThreshold: DEFAULT_THRESHOLD,
-  strategy: 'semantic',
-  fallbackToOriginal: true,
-};
+import { createPreprocessToEnglish, type PreprocessorConfig } from './preprocessor-core';
 
-function resolveThreshold(threshold: number | Record<string, number>, lang: string): number {
-  if (typeof threshold === 'number') return threshold;
-  return threshold[lang] ?? threshold['*'] ?? DEFAULT_THRESHOLD;
-}
+export type { PreprocessorConfig };
 
 /**
- * Preprocess non-English hyperscript into English.
- * Identical to preprocessToEnglish in preprocessor.ts but with slim imports.
- *
- * Handles _hyperscript feature prefixes (e.g. "on click", "on every keyup")
- * by stripping them, translating only the command portion, then reassembling.
+ * Preprocess non-English hyperscript into English (slim imports).
  */
-export function preprocessToEnglish(
-  src: string,
-  lang: string,
-  config: Partial<PreprocessorConfig> = {}
-): string {
-  const cfg = { ...DEFAULT_CONFIG, ...config };
-
-  // Try translating the full string first
-  const fullResult = tryTranslateWithStrategies(src, lang, cfg);
-  if (fullResult !== null) return fullResult;
-
-  // If full translation failed, try stripping event/feature prefix
-  const stripped = stripEventPrefix(src);
-  if (stripped) {
-    const translated = tryTranslateWithStrategies(stripped.commands, lang, cfg);
-    if (translated !== null) return stripped.prefix + translated;
-  }
-
-  return src;
-}
-
-function tryTranslateWithStrategies(
-  src: string,
-  lang: string,
-  cfg: PreprocessorConfig
-): string | null {
-  if (cfg.strategy === 'semantic' || cfg.strategy === 'auto') {
-    const threshold = resolveThreshold(cfg.confidenceThreshold, lang);
-    const result = trySemanticTranslation(src, lang, threshold);
-    if (result !== null) return result;
-  }
-
-  if ((cfg.strategy === 'auto' || cfg.strategy === 'i18n') && cfg.i18nToEnglish) {
-    const result = tryI18nTranslation(src, lang, cfg.i18nToEnglish);
-    if (result !== null) return result;
-  }
-
-  return null;
-}
-
-/** Match _hyperscript event handler prefix. */
-const EVENT_PREFIX_RE =
-  /^(on\s+(?:every\s+)?[\w-]+(?:\[.*?\])?(?:\.[\w-]+(?:\([^)]*\))?)*(?:\s+from\s+\S+)?(?:\s+queue\s+\w+)?\s+)/;
-
-function stripEventPrefix(src: string): { prefix: string; commands: string } | null {
-  const match = src.match(EVENT_PREFIX_RE);
-  if (!match) return null;
-  const prefix = match[1];
-  const commands = src.slice(prefix.length);
-  if (!commands) return null;
-  return { prefix, commands };
-}
-
-function trySemanticTranslation(src: string, lang: string, threshold: number): string | null {
-  try {
-    const statements = splitStatements(src, lang);
-    if (statements.length > 1) {
-      return translateCompound(statements, lang, threshold);
-    }
-
-    if (!isLanguageRegistered(lang)) return null;
-
+export const preprocessToEnglish = createPreprocessToEnglish({
+  isLanguageRegistered,
+  translateSingle(src, lang, threshold) {
     const result = parseWithConfidence(src, lang);
     if (result.confidence < threshold || !result.node) return null;
 
     return renderToHyperscript(result.node);
-  } catch {
-    return null;
-  }
-}
-
-function tryI18nTranslation(
-  src: string,
-  lang: string,
-  toEnglish: (input: string, locale: string) => string
-): string | null {
-  try {
-    const result = toEnglish(src, lang);
-    return result !== src ? result : null;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Get all "then" keyword forms for a language (primary + alternatives).
- * Falls back to English "then" if profile is unavailable.
- */
-function getThenKeywords(lang: string): string[] {
-  const keywords = ['then']; // Always include English
-  const profile = tryGetProfile(lang);
-  if (profile?.keywords?.then) {
-    const kw = profile.keywords.then;
-    if (kw.primary && kw.primary !== 'then') keywords.push(kw.primary);
-    if (kw.alternatives) {
-      for (const alt of kw.alternatives) {
-        if (!keywords.includes(alt)) keywords.push(alt);
-      }
-    }
-  }
-  return keywords;
-}
-
-function splitStatements(src: string, lang: string): string[] {
-  const thenWords = getThenKeywords(lang);
-  // Escape regex special chars and build pattern
-  const escaped = thenWords.map(w => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
-  const pattern = new RegExp(`\\s+(?:${escaped.join('|')})\\s+`, 'i');
-
-  const lines = src
-    .split('\n')
-    .map(l => l.trim())
-    .filter(l => l.length > 0);
-  const result: string[] = [];
-  for (const line of lines) {
-    const parts = line.split(pattern);
-    result.push(...parts);
-  }
-  return result;
-}
-
-function translateCompound(statements: string[], lang: string, threshold: number): string | null {
-  const translated: string[] = [];
-  for (const stmt of statements) {
-    const result = trySemanticTranslation(stmt.trim(), lang, threshold);
-    if (result === null) return null;
-    translated.push(result);
-  }
-  return translated.join(' then ');
-}
+  },
+});