@hyperfixi/patterns-reference 3.1.1 → 3.2.0

package/src/html-snippets.ts

@@ -40,6 +40,8 @@ export interface MinimalElement {
   tagName: string;
   attributes: ArrayLike<MinimalAttr>;
   textContent: string | null;
+  /** Read only on `<template>`, whose content `querySelectorAll` cannot reach. */
+  innerHTML?: string;
   getAttribute(name: string): string | null;
 }
 
@@ -53,7 +55,11 @@ export interface MinimalDocument {
 }
 
 export interface MarkupSnippets {
-  /** Every hyperscript source found, in document order. */
+  /**
+   * Every hyperscript source found, in document order — including the `_`
+   * attributes inside component template bodies, which run once the
+   * component renders.
+   */
   snippets: string[];
   /**
    * The markup uses at least one attribute upstream `_hyperscript` has no
@@ -61,6 +67,12 @@ export interface MarkupSnippets {
    * how its snippets parse.
    */
   hyperfixiOnly: boolean;
+  /**
+   * Custom-element names the markup defines as template components. A
+   * component's behavior is its RENDER, which no parse of its snippets can
+   * check, so a verifier should instantiate each one.
+   */
+  componentTags: string[];
 }
 
 /**
@@ -72,6 +84,13 @@ export interface MarkupSnippets {
  * `sse-swap` event names, `ws-connect` URLs) are not hyperscript and carry no
  * snippet to verify, but they do set `hyperfixiOnly`.
  *
+ * Template components — `<script type="text/hyperscript-template"
+ * component="x">` (upstream's form, which upstream's official `component`
+ * extension implements and @hyperfixi/components also accepts) and
+ * `<template component="x">` — are NOT hyperfixi-only. Their bodies are markup
+ * the DOM walk cannot reach (script text; template content), so each is
+ * extracted recursively and its `_` sources join `snippets`.
+ *
  * `doc` is any DOM `Document` — a jsdom window's, or the ambient one in a
  * browser/jsdom test environment. Markup that fails to parse yields no
  * snippets rather than throwing.
@@ -81,10 +100,11 @@ export function extractHyperscriptFromMarkup(doc: MinimalDocument, markup: strin
   try {
     container.innerHTML = markup;
   } catch {
-    return { snippets: [], hyperfixiOnly: false };
+    return { snippets: [], hyperfixiOnly: false, componentTags: [] };
   }
 
   const snippets: string[] = [];
+  const componentTags: string[] = [];
   let hyperfixiOnly = false;
 
   for (const el of Array.from(container.querySelectorAll('*'))) {
@@ -100,15 +120,24 @@ export function extractHyperscriptFromMarkup(doc: MinimalDocument, markup: strin
       }
     }
 
-    if (el.tagName === 'SCRIPT') {
-      const type = el.getAttribute('type');
-      if (type === 'text/hyperscript-template') {
-        hyperfixiOnly = true;
-      } else if (type === 'text/hyperscript' && el.textContent?.trim()) {
-        snippets.push(el.textContent);
-      }
+    const tag = el.getAttribute('component');
+    const isScriptTemplate =
+      el.tagName === 'SCRIPT' && el.getAttribute('type') === 'text/hyperscript-template';
+    if (tag && (isScriptTemplate || el.tagName === 'TEMPLATE')) {
+      componentTags.push(tag);
+      const body = isScriptTemplate ? (el.textContent ?? '') : (el.innerHTML ?? '');
+      const inner = extractHyperscriptFromMarkup(doc, body);
+      snippets.push(...inner.snippets);
+      componentTags.push(...inner.componentTags);
+      hyperfixiOnly ||= inner.hyperfixiOnly;
+    } else if (
+      el.tagName === 'SCRIPT' &&
+      el.getAttribute('type') === 'text/hyperscript' &&
+      el.textContent?.trim()
+    ) {
+      snippets.push(el.textContent);
     }
   }
 
-  return { snippets, hyperfixiOnly };
+  return { snippets, hyperfixiOnly, componentTags };
 }

package/src/index.ts

@@ -44,6 +44,7 @@ export type {
 
   // LLM types
   LLMExample,
+  ExampleOptions,
 
   // Language documentation types
   Command,
@@ -264,8 +265,8 @@ export function createPatternsReference(options?: ConnectionOptions): PatternsRe
     verifyTranslation: translation => translations.verifyTranslation(translation, options),
 
     // LLM
-    getLLMExamples: (prompt, language, limit) =>
-      llm.getLLMExamples(prompt, language, limit, options),
+    getLLMExamples: (prompt, language, limit, engine) =>
+      llm.getLLMExamples(prompt, language, limit, { ...options, engine }),
 
     // Stats
     getStats: () => patterns.getPatternStats(options),
@@ -283,6 +284,15 @@ export function createPatternsReference(options?: ConnectionOptions): PatternsRe
 export { extractHyperscriptFromMarkup } from './html-snippets';
 export type { MarkupSnippets } from './html-snippets';
 
+/**
+ * The corpus writer's own locator for the `_="…"` bodies it translates in a
+ * markup row. Exported so the testing-framework's en-reference-preservation
+ * gate checks exactly the bodies the writer renders — not the broader set
+ * `extractHyperscriptFromMarkup` finds for whole files.
+ */
+export { findHyperscriptAttributes, isMarkupRow } from './sync/markup-attributes';
+export type { AttributeSpan } from './sync/markup-attributes';
+
 /**
  * Version of the package.
  */

package/src/registry/patterns-provider.ts

@@ -122,7 +122,9 @@ export class DatabasePatternsProvider implements PatternsSource {
         command: p.primaryCommand,
         language: language || 'en',
         confidence: 1.0,
-        verified: true,
+        // The English source pattern: "verified" = some engine runs it
+        // (code_examples.engine, mechanically checked) — not a constant.
+        verified: p.engine !== null,
         title: p.title,
         category: p.category || undefined,
       }));

package/src/sync/db-stamp.ts

@@ -43,9 +43,9 @@ function walkTsFiles(dir: string, acc: string[]): void {
 }
 
 /**
- * The source files whose content determines `patterns.db`: the i18n transformer +
- * dictionaries + grammar profiles (translation text), the semantic parser + profiles
- * (stored confidence + the language set), and the seed / sync scripts.
+ * The source files whose content determines `patterns.db`: the semantic parser +
+ * renderer + profiles (translation text, stored confidence, the language set), the
+ * i18n sources, the seed / sync scripts, and the sync helpers the writer calls.
  */
 function dbInputFiles(dbPath: string): string[] {
   const root = repoRootFromDbPath(dbPath);
@@ -56,7 +56,10 @@ function dbInputFiles(dbPath: string): string[] {
   for (const f of [
     join(pr, 'scripts', 'init-db.ts'),
     join(pr, 'scripts', 'sync-translations.ts'),
-    join(pr, 'src', 'sync', 'span-mask.ts'),
+    // Decides which markup `_` bodies are translated (and which stay English).
+    join(pr, 'src', 'sync', 'markup-attributes.ts'),
+    // Writes every row's `verified_parses`.
+    join(pr, 'src', 'sync', 'verify-parses.ts'),
     // Committed engine-verification results are seeded into the engine
     // column by init-db.ts, so they are DB input too.
     join(pr, 'data', 'engine-verification.json'),

package/src/sync/markup-attributes.ts

@@ -88,6 +88,21 @@ export function isMarkupRow(code: string): boolean {
   return /^\s*<[a-zA-Z!]/.test(code);
 }
 
+/** `hx-live="…"` values are hyperscript too (the htmx-compat layer compiles them). */
+const HX_LIVE_ATTRIBUTE = /(^|[\s"'])hx-live\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
+
+/**
+ * The hyperscript a stored row carries: the row itself, or — for a markup row —
+ * each `_` value (including those inside component template bodies) and each
+ * `hx-live` value. Markup with neither carries none.
+ */
+export function hyperscriptBodies(code: string): string[] {
+  if (!isMarkupRow(code)) return [code];
+  const bodies = findHyperscriptAttributes(code).map(span => span.body);
+  for (const match of code.matchAll(HX_LIVE_ATTRIBUTE)) bodies.push(match[2] ?? match[3] ?? '');
+  return bodies.filter(body => body.trim().length > 0);
+}
+
 /**
  * Whether a parse carried the whole body: every non-whitespace character of the
  * source reappears in its own English re-render.

package/src/sync/verify-parses.ts

@@ -0,0 +1,42 @@
+/**
+ * `pattern_translations.verified_parses`, MEASURED.
+ *
+ * The flag answers one question: does `@lokascript/semantic`'s parser accept
+ * this stored row in its own language? It used to be written as
+ * `language === 'en' ? 1 : 0` — never measured — so every consumer asking for
+ * "successfully parsed" translations (`getVerifiedTranslations`, the semantic
+ * `PatternsProvider`, `getSupportedLanguages()`, testing-framework's
+ * `--verified-only`) silently got English only. Its other writer, the since
+ * retired `validate-all --fix`, set it from bracket/quote balance without
+ * parsing. This is now the only definition; sync-translations, `npm run verify`
+ * and `verifyTranslation()` all call it.
+ *
+ * "Parses" is not "faithful": a parse can be non-null while dropping commands.
+ * Fidelity is the multilingual gate's job (its R0–R5 ratchets, against the
+ * English parse) and the en-reference-preservation gate's (the English parse
+ * against the source); engine validity is `code_examples.engine`'s.
+ */
+
+import { canParse } from '@lokascript/semantic';
+import { hyperscriptBodies } from './markup-attributes';
+
+/**
+ * Whether every hyperscript body of a stored row parses in `language`.
+ *
+ * A row with no hyperscript at all (`sse-connect` wiring, a template with no
+ * `_`) has nothing that could parse and is NOT verified. A non-translatable row
+ * stores the same English markup in every language, and English is what the
+ * runtime reads from it, so its bodies parse as `en` whatever row it is.
+ */
+export function verifyParses(code: string, language: string, translatable = true): boolean {
+  const bodies = hyperscriptBodies(code);
+  if (bodies.length === 0) return false;
+  const parseLanguage = translatable ? language : 'en';
+  return bodies.every(body => {
+    try {
+      return canParse(body, parseLanguage);
+    } catch {
+      return false;
+    }
+  });
+}

package/src/types/better-sqlite3.d.ts

@@ -10,6 +10,7 @@ declare module 'better-sqlite3' {
       prepare<T = unknown>(sql: string): Statement<T>;
       exec(sql: string): this;
       pragma(pragma: string, options?: { simple?: boolean }): unknown;
+      transaction<A extends unknown[], R>(fn: (...args: A) => R): (...args: A) => R;
       close(): void;
     }
 
@@ -37,6 +38,7 @@ declare module 'better-sqlite3' {
     prepare<T = unknown>(sql: string): BetterSqlite3.Statement<T>;
     exec(sql: string): this;
     pragma(pragma: string, options?: { simple?: boolean }): unknown;
+    transaction<A extends unknown[], R>(fn: (...args: A) => R): (...args: A) => R;
     close(): void;
   }
 

package/src/types/index.ts

@@ -28,6 +28,12 @@ export interface Pattern {
   tags: string[];
   difficulty: 'beginner' | 'intermediate' | 'advanced';
   engine: EngineCompat | null;
+  /**
+   * Whether the corpus writer translates this row. `false` rows are copied
+   * verbatim into every language (markup whose attribute names are resolved by
+   * vocab modules, or markup with no hyperscript at all).
+   */
+  translatable: boolean;
   createdAt: Date;
 }
 
@@ -67,7 +73,8 @@ export type WordOrder = 'SVO' | 'SOV' | 'VSO' | 'V2';
  * - `grammar-transform-no-reference`: `best` only — the i18n row because semantic
  *   cannot parse the ENGLISH source (a parser-coverage gap, not a render loss).
  * - `keyword-substitute`: word-for-word fallback for a language with no grammar profile.
- * - `original`: the English row; `non-translatable-identity`: markup rows copied verbatim.
+ * - `original`: the English row; `non-translatable-identity`: a non-translatable row copied
+ *   verbatim (the markup rows, and intercept-cache-strategies).
  */
 export type TranslationMethod =
   | 'semantic-render'
@@ -185,8 +192,25 @@ export interface LLMExample {
   prompt: string;
   completion: string;
   qualityScore: number;
+  /** @deprecated 0 unless something calls trackExampleUsage() (see getMostUsedExamples). */
   usageCount: number;
   createdAt: Date;
+  /**
+   * The engine(s) verified to run this example's pattern
+   * (`code_examples.engine`). Never null in results: an example no engine
+   * runs is not served.
+   */
+  engine: EngineCompat | null;
+}
+
+/**
+ * Options for the LLM-example getters. `engine` narrows to examples whose
+ * pattern runs on that engine — 'hyperscript' = both + upstream-only,
+ * 'lokascript' = both + hyperfixi-only, 'both' = both. With or without it,
+ * an example whose pattern NO engine runs is never returned.
+ */
+export interface ExampleOptions extends ConnectionOptions {
+  engine?: EngineCompat;
 }
 
 // =============================================================================
@@ -233,10 +257,22 @@ export interface RoleAlignmentResult {
 // API Types
 // =============================================================================
 
+/** Honoured by searchPatterns and getAllPatterns; the page is taken after every filter. */
 export interface SearchOptions {
+  /**
+   * Patterns usable in this language: a translation there that parses
+   * (`verified_parses`), or no hyperscript at all to translate. searchPatterns
+   * also matches its query against that translation.
+   */
   language?: string;
+  /** The pattern's category (`Pattern.category`). */
   category?: string;
+  /** As inferred from the code (`Pattern.difficulty`). */
   difficulty?: 'beginner' | 'intermediate' | 'advanced';
+  /**
+   * Patterns that run on this engine ('hyperscript' / 'lokascript' include
+   * 'both'); `null` = the patterns no engine runs; omitted = no filter.
+   */
   engine?: EngineCompat | null;
   limit?: number;
   offset?: number;
@@ -443,7 +479,13 @@ export interface PatternsReference {
   verifyTranslation(translation: Translation): Promise<VerificationResult>;
 
   // LLM support
-  getLLMExamples(prompt: string, language?: string, limit?: number): Promise<LLMExample[]>;
+  /** Never returns an example no engine runs; `engine` narrows further (see ExampleOptions). */
+  getLLMExamples(
+    prompt: string,
+    language?: string,
+    limit?: number,
+    engine?: EngineCompat
+  ): Promise<LLMExample[]>;
 
   // Statistics
   getStats(): Promise<PatternStats>;

package/src/sync/span-mask.ts

@@ -1,166 +0,0 @@
-/**
- * Span-aware masking for hyperscript code prior to translation.
- *
- * The grammar transformer and keyword-substitute fallback both treat hyperscript
- * source as a flat token stream. They will happily reorder text inside HTML
- * elements, translate words inside string literals, and shred bracket
- * expressions like `[key is 'Escape']`. To avoid that, we replace
- * non-translatable spans with opaque placeholder tokens before translation,
- * then restore them after.
- *
- * Placeholders use the form `__HFXMSK_<idx>_<KIND>__`. Underscores on both
- * sides make the placeholder a single "word" to JS regex `\b` boundaries —
- * keyword substitution `\bword\b` cannot match a substring inside the
- * placeholder, regardless of case-insensitivity flags.
- */
-
-export type SpanKind =
-  | 'string-single'
-  | 'string-double'
-  | 'string-template'
-  | 'url'
-  | 'html-text'
-  | 'directive'
-  | 'bracket-expr'
-  | 'js-block';
-
-export interface MaskedSpan {
-  kind: SpanKind;
-  original: string;
-  placeholder: string;
-}
-
-export interface MaskResult {
-  masked: string;
-  spans: MaskedSpan[];
-}
-
-const KIND_TAG: Record<SpanKind, string> = {
-  'string-single': 'STR1',
-  'string-double': 'STR2',
-  'string-template': 'STRT',
-  url: 'URL',
-  'html-text': 'TEXT',
-  directive: 'DIR',
-  'bracket-expr': 'BRK',
-  'js-block': 'JS',
-};
-
-function makePlaceholder(kind: SpanKind, idx: number): string {
-  return `__HFXMSK_${idx}_${KIND_TAG[kind]}__`;
-}
-
-const PLACEHOLDER_RE = /__HFXMSK_(\d+)_(?:STR1|STR2|STRT|URL|TEXT|DIR|BRK|JS)__/g;
-
-/**
- * Detect non-translatable spans in hyperscript code and replace them with
- * opaque placeholders. The detection order is significant: earlier rules
- * consume characters that later rules would otherwise misinterpret.
- *
- *   1. Template literals  (consume `${...}` interpolation contents)
- *   2. Double-quoted strings (consume HTML attribute values)
- *   3. Single-quoted strings (escaping possessive `'s`)
- *   4. HTML inner text  (between `>` and `<` with non-whitespace)
- *   5. Bracket expressions  (event filters like `[key is 'Escape']`)
- *   6. URL-like tokens  (paths and protocol://host)
- *   7. Component template directives  (`#if`, `#end`, `#for`, ...)
- */
-export function maskSpans(input: string): MaskResult {
-  let working = input;
-  const spans: MaskedSpan[] = [];
-
-  const record = (kind: SpanKind, original: string): string => {
-    const placeholder = makePlaceholder(kind, spans.length);
-    spans.push({ kind, original, placeholder });
-    return placeholder;
-  };
-
-  // 0. Embedded JS blocks: `js(args) ... end` opens a raw-JavaScript escape
-  //    hatch inside hyperscript. Its body must be passed through verbatim;
-  //    the grammar transformer would otherwise tokenize JavaScript as if
-  //    it were hyperscript and silently drop unrecognized tokens. Mask this
-  //    first because the body can contain any other span kind.
-  working = working.replace(/\bjs\([^)]*\)[\s\S]*?\bend\b/g, m => record('js-block', m));
-
-  // 1. Template literals — match outermost backtick body with escape support.
-  //    The body may contain `${...}` interpolations; treat the whole literal
-  //    as one opaque span. Inner code stays English but is structurally safe.
-  working = working.replace(/`(?:\\.|[^`\\])*`/g, m => record('string-template', m));
-
-  // 2. Double-quoted strings. This also captures HTML attribute values so
-  //    `component="my-layout"` cannot have `my` translated.
-  working = working.replace(/"(?:\\.|[^"\\])*"/g, m => record('string-double', m));
-
-  // 3. Single-quoted strings, but NOT possessive `'s` (preceded by a word char).
-  //    The lookbehind ensures `me's value 'foo'` matches only `'foo'`.
-  working = working.replace(/(?<!\w)'(?:\\.|[^'\\])*'/g, m => record('string-single', m));
-
-  // 4. HTML inner text content. Match a non-whitespace text run between
-  //    a `>` and the next `<`, on the SAME line. Newline confinement is
-  //    critical: without it the regex would span multiple element boundaries
-  //    and consume nested tags as if they were text. The middle character
-  //    class `[^<\n\s]` ensures at least one printable non-`<` char exists
-  //    so pure indentation between elements is not masked.
-  //    The `(?<!\/)` lookbehind excludes a `>` that closes a SELF-CLOSING
-  //    selector literal (`<button/>`): hyperscript between two selector
-  //    literals (`… last <button/> in .modal focus first <button/> …`) is
-  //    real code, not element inner text — masking it hid the whole segment
-  //    from the transformer, which then emitted it untranslated (the
-  //    focus-trap `focus first` leak).
-  working = working.replace(
-    /(?<!\/)>([^<\n]*[^<\n\s][^<\n]*)</g,
-    (_m, text) => `>${record('html-text', text)}<`
-  );
-
-  // 5. Bracket expressions. Mask the entire `[...]` payload as one unit so
-  //    SOV/VSO reordering cannot move tokens out of the brackets. Non-nested
-  //    is fine for the current corpus; bracket-inside-bracket isn't used.
-  working = working.replace(/\[[^\]]*\]/g, m => record('bracket-expr', m));
-
-  // 6. URL-like tokens. Two forms:
-  //      - protocol://host  (http, https, ws, wss, file)
-  //      - /path  (whitespace-bounded, not preceded by a word char to avoid
-  //                matching arithmetic like `1/2`)
-  //    The path form matches a bare `/` too (precache root) when followed
-  //    by non-word boundary. Path chars: word, dash, dot, slash, glob, query.
-  working = working.replace(/(?:https?|wss?|file):\/\/[^\s,)]+|(?<![\w<])\/[\w\-./*?=&]*/g, m =>
-    record('url', m)
-  );
-
-  // 7. Component template directives. Mask the `#keyword` token only — the
-  //    expression after it (e.g. `^user.admin`) is left translatable, since
-  //    those are typically variable references that pass through unchanged.
-  working = working.replace(/#(?:if|else|elif|end|for|each)\b/g, m => record('directive', m));
-
-  return { masked: working, spans };
-}
-
-/**
- * Restore placeholders to their original spans. Iterates to a fixed point so
- * that an `original` field containing another placeholder (which shouldn't
- * happen with the current detection rules, but is cheap insurance) is fully
- * resolved.
- */
-export function unmaskSpans(translated: string, spans: MaskedSpan[]): string {
-  let result = translated;
-  let prev: string;
-  do {
-    prev = result;
-    result = result.replace(PLACEHOLDER_RE, (m, idx) => {
-      const span = spans[Number(idx)];
-      return span ? span.original : m;
-    });
-  } while (result !== prev);
-  return result;
-}
-
-/**
- * Convenience wrapper: mask, run a transform on the masked surface, then
- * unmask. The transform receives only the masked string and must preserve
- * placeholders (which it will, since they look like ordinary identifiers).
- */
-export function withMaskedSpans(input: string, transform: (masked: string) => string): string {
-  const { masked, spans } = maskSpans(input);
-  const transformed = transform(masked);
-  return unmaskSpans(transformed, spans);
-}