fimo 0.8.0 → 0.9.0-experimental.2

package/dist/scripts/render-parity/normalize-html.js

@@ -0,0 +1,223 @@
+import * as cheerio from 'cheerio';
+// `title` included: Next dev streams document metadata into the body end
+// until the shell settles — title text is never visible page content, and
+// whether it lands in head or body depends on layout asyncness (a wrap
+// migration legitimately makes the root layout async).
+const REMOVED_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'template',
+    'link',
+    'meta',
+    'svg',
+    'iframe',
+    'head',
+    'title',
+]);
+const BLOCK_TAGS = new Set([
+    'p',
+    'h1',
+    'h2',
+    'h3',
+    'h4',
+    'h5',
+    'h6',
+    'li',
+    'td',
+    'th',
+    'dt',
+    'dd',
+    'blockquote',
+    'figcaption',
+    'pre',
+    'button',
+    'label',
+    'summary',
+    'caption',
+    'option',
+]);
+const FALLBACK_BLOCK_TAGS = new Set(['div', 'section', 'article', 'main', 'aside', 'header', 'footer', 'nav', 'body']);
+const MARK_KIND = {
+    strong: 'strong',
+    b: 'strong',
+    em: 'em',
+    i: 'em',
+    u: 'u',
+    s: 's',
+    del: 's',
+    strike: 's',
+    code: 'code',
+    a: 'a',
+};
+function collapse(text) {
+    return text.replace(/ /g, ' ').replace(/\s+/g, ' ').trim();
+}
+/** Strip the capture origin so differing ports between runs compare equal. */
+export function normalizeHref(href, origin) {
+    if (href.startsWith(origin)) {
+        const stripped = href.slice(origin.length);
+        return stripped.startsWith('/') ? stripped : `/${stripped}`;
+    }
+    return href;
+}
+/**
+ * Media src normalization: origin strip + Next image-optimizer decode, so a
+ * pre-wrap `next/image` (`/_next/image?url=…&w=…`) and a post-wrap plain
+ * `<img src>` of the same asset compare equal.
+ */
+export function normalizeMediaSrc(src, origin) {
+    let value = normalizeHref(src, origin);
+    const optimizerMatch = value.match(/^\/_next\/image\?(.+)$/);
+    if (optimizerMatch) {
+        const params = new URLSearchParams(optimizerMatch[1]);
+        const url = params.get('url');
+        if (url) {
+            value = url;
+        }
+    }
+    return value;
+}
+function isExcluded(node) {
+    const tag = node.name?.toLowerCase() ?? '';
+    if (REMOVED_TAGS.has(tag)) {
+        return true;
+    }
+    // Hidden elements are not visible content — except Next streaming
+    // containers (div[hidden][id^="S:"]), whose content replaces Suspense
+    // holes; kept in document order so streamed content is captured
+    // symmetrically between baseline and check.
+    if (node.attribs && 'hidden' in node.attribs) {
+        const isStreamingContainer = tag === 'div' && (node.attribs.id ?? '').startsWith('S:');
+        if (!isStreamingContainer) {
+            return true;
+        }
+    }
+    return false;
+}
+function nearestBlock(node) {
+    let current = node.parent;
+    let fallback = null;
+    while (current) {
+        const tag = current.name?.toLowerCase();
+        if (tag && BLOCK_TAGS.has(tag)) {
+            return current;
+        }
+        if (!fallback && tag && FALLBACK_BLOCK_TAGS.has(tag)) {
+            fallback = current;
+        }
+        current = current.parent;
+    }
+    return fallback;
+}
+function collectText(node) {
+    if (node.type === 'text') {
+        return String(node.data ?? '');
+    }
+    if (node.type !== 'tag' || isExcluded(node)) {
+        return '';
+    }
+    return (node.children ?? []).map(collectText).join('');
+}
+export function extractRouteSignature(html, opts) {
+    const $ = cheerio.load(html);
+    const bodyNode = $('body').get(0);
+    const blocks = [];
+    const blockIndex = new Map();
+    const images = [];
+    const videos = [];
+    const links = [];
+    const visit = (node) => {
+        if (node.type === 'text') {
+            const text = collapse(String(node.data ?? ''));
+            if (!text) {
+                return;
+            }
+            const block = nearestBlock(node);
+            if (!block) {
+                return;
+            }
+            const existing = blockIndex.get(block);
+            if (existing === undefined) {
+                blockIndex.set(block, blocks.length);
+                blocks.push({ tag: block.name?.toLowerCase() ?? 'div', text, marks: [] });
+            }
+            else {
+                blocks[existing].text = collapse(`${blocks[existing].text} ${text}`);
+            }
+            return;
+        }
+        if (node.type !== 'tag' && node.type !== 'root') {
+            return;
+        }
+        if (node.type === 'tag' && isExcluded(node)) {
+            return;
+        }
+        const tag = node.name?.toLowerCase() ?? '';
+        const attribs = node.attribs ?? {};
+        // Inline-marks signature: the formatting that must survive wrapping and
+        // the TipTap seed round-trip. Attached to the nearest block.
+        const markKind = MARK_KIND[tag];
+        if (markKind) {
+            const text = collapse(collectText(node));
+            if (text) {
+                const block = nearestBlock(node);
+                if (block) {
+                    if (!blockIndex.has(block)) {
+                        blockIndex.set(block, blocks.length);
+                        blocks.push({ tag: block.name?.toLowerCase() ?? 'div', text: '', marks: [] });
+                    }
+                    const mark = { kind: markKind, text };
+                    if (markKind === 'a') {
+                        if (attribs.href) {
+                            mark.href = normalizeHref(attribs.href, opts.origin);
+                            blocks[blockIndex.get(block)].marks.push(mark);
+                        }
+                    }
+                    else {
+                        blocks[blockIndex.get(block)].marks.push(mark);
+                    }
+                }
+            }
+        }
+        if (tag === 'img' && attribs.src) {
+            images.push(normalizeMediaSrc(attribs.src, opts.origin));
+        }
+        if (tag === 'source' && attribs.src) {
+            const parentTag = node.parent?.name?.toLowerCase();
+            if (parentTag === 'picture') {
+                images.push(normalizeMediaSrc(attribs.src, opts.origin));
+            }
+            else if (parentTag === 'video') {
+                videos.push(normalizeMediaSrc(attribs.src, opts.origin));
+            }
+        }
+        if (tag === 'video') {
+            if (attribs.src) {
+                videos.push(normalizeMediaSrc(attribs.src, opts.origin));
+            }
+            if (attribs.poster) {
+                videos.push(normalizeMediaSrc(attribs.poster, opts.origin));
+            }
+        }
+        if (tag === 'a' && attribs.href) {
+            links.push({ href: normalizeHref(attribs.href, opts.origin), text: collapse(collectText(node)) });
+        }
+        for (const child of node.children ?? []) {
+            visit(child);
+        }
+    };
+    if (bodyNode) {
+        for (const child of bodyNode.children ?? []) {
+            visit(child);
+        }
+    }
+    // Blocks created empty by a mark preceding any direct text (e.g. a <p>
+    // starting with <strong>) get their text from the block's full content.
+    for (const [block, index] of blockIndex) {
+        if (!blocks[index].text) {
+            blocks[index].text = collapse(collectText(block));
+        }
+    }
+    return { status: opts.status, blocks: blocks.filter((block) => block.text.length > 0), images, videos, links };
+}

package/dist/scripts/render-parity/types.d.ts

@@ -0,0 +1,41 @@
+export declare const PARITY_BASELINE_VERSION = 1;
+export interface InlineMark {
+    kind: 'strong' | 'em' | 'u' | 's' | 'code' | 'a';
+    text: string;
+    href?: string;
+}
+export interface TextBlock {
+    tag: string;
+    text: string;
+    marks: InlineMark[];
+}
+export interface RouteSignature {
+    status: number;
+    blocks: TextBlock[];
+    images: string[];
+    videos: string[];
+    links: Array<{
+        href: string;
+        text: string;
+    }>;
+}
+export type RouteCapture = {
+    signature: RouteSignature;
+} | {
+    skipped: 'dynamic-params' | 'fetch-failed';
+    error?: string;
+};
+/**
+ * Normalized pre-wrap snapshot of every route's rendered content. Written to
+ * `.fimo/migration/parity-baseline.json`; `fimo migrate wrap --check`
+ * recaptures with the identical pipeline and compares.
+ */
+export interface ParityBaseline {
+    version: typeof PARITY_BASELINE_VERSION;
+    capturedAt: string;
+    engine: 'dev-server';
+    framework: string | null;
+    command: string;
+    unwrappedAtCapture?: number;
+    routes: Record<string, RouteCapture>;
+}

package/dist/scripts/render-parity/types.js

@@ -0,0 +1 @@
+export const PARITY_BASELINE_VERSION = 1;

package/dist/scripts/scan-wrappable.d.ts

@@ -0,0 +1,196 @@
+/**
+ * Inverse of extract-translations: finds content that is NOT yet wrapped with
+ * Fimo primitives — hardcoded JSX text, image/video sources, file links, and
+ * user-visible attribute strings. Drives the wrap migration's coverage gate:
+ * every item must end wrapped, ignored-with-a-reason, or unresolved.
+ */
+export type WrappableKind = 'text' | 'image' | 'video' | 'file-link' | 'attribute';
+export interface WrappableItem {
+    kind: WrappableKind;
+    file: string;
+    line: number;
+    preview: string;
+    /** AST span info for the --apply codemod (offsets into the file). */
+    span?: {
+        start: number;
+        end: number;
+        parentTag?: string;
+        openNameStart?: number;
+        openNameEnd?: number;
+        closeNameStart?: number;
+        closeNameEnd?: number;
+    };
+}
+/**
+ * In-code collection candidate: an array literal of same-shaped objects with
+ * user-visible string fields (services cards, FAQ items, testimonials, …).
+ * Flat labels are the wrong shape for these — they belong in a Fimo
+ * collection (schema + entries), migrated by the agent.
+ */
+export interface CollectionCandidate {
+    name: string;
+    file: string;
+    line: number;
+    count: number;
+    fields: string[];
+    sample: string;
+    /**
+     * Cross-collection references detected by value matching: this candidate's
+     * field values point at another candidate's identity values. Migrated as
+     * Fimo `reference` fields (`target`/`targets` + `many`), never as strings.
+     */
+    references?: CandidateReference[];
+    /** Per-field string values (capped) — internal input to reference
+     *  detection; stripped before the plan is written. */
+    fieldValues?: Record<string, string[]>;
+    /** First row with NATIVE value types — internal input to schema-draft
+     *  type inference; stripped before the plan is written. */
+    typedSample?: Record<string, unknown>;
+    /** All rows (capped) with NATIVE value types — internal input to the
+     *  rows-staging generator; stripped before the plan is written. */
+    typedRows?: Record<string, unknown>[];
+    manyFields?: string[];
+}
+export interface CandidateReference {
+    field: string;
+    /** Candidate name(s) the values point at: one = fixed target, several = polymorphic. */
+    targets: string[];
+    many: boolean;
+}
+interface ReferenceMatchable {
+    name: string;
+    references?: CandidateReference[];
+    fieldValues?: Record<string, string[]>;
+    manyFields?: string[];
+}
+/**
+ * Annotate candidates whose field values point at another candidate's
+ * identity values: one full-coverage target = fixed reference, several
+ * partial targets covering together = polymorphic. Mutates in place.
+ *
+ * Coverage is OCCURRENCE-weighted and dangling-adjusted: values matching no
+ * candidate's identity set are dangling references by definition (deleted
+ * targets, sentinels) and drop out of the denominator, weighted by how
+ * often they occur — 1,700 case studies pointing at 12 clients with 190
+ * distinct ghost values is still a reference field, because 89% of the ROWS
+ * resolve. Guards against coincidence: at least 2 distinct matched values
+ * and at least half the occurrences must resolve somewhere.
+ */
+export declare function annotateCandidateReferences(candidates: ReferenceMatchable[]): void;
+/**
+ * External live data source: content fetched at render time from a
+ * third-party API (GitHub stats, dev.to posts, a headless CMS). Invisible to
+ * the wrap scan by nature — the migration must ASK the user what to do with
+ * it (keep live vs ingest) instead of guessing.
+ */
+export interface ExternalSource {
+    host: string;
+    files: string[];
+    count: number;
+    /** 'cms' when detected via a headless-CMS SDK dependency: the site's
+     *  CONTENT lives there — migrating it is the point, not an option. */
+    kind?: 'api' | 'cms';
+}
+/** Identities of already-wrapped content, for the seeding gate. */
+export interface WrappedIds {
+    labels: string[];
+    media: string[];
+    richText: string[];
+    files: string[];
+}
+export interface WrapScanResult {
+    items: WrappableItem[];
+    wrappedCount: number;
+    filesScanned: number;
+    wrappedIds: WrappedIds;
+    collections: CollectionCandidate[];
+    externalSources: ExternalSource[];
+    dataFiles: DataCopyFile[];
+    /** `t()` calls bound to a FOREIGN i18n library (next-intl, i18next, …), not
+     *  Fimo — surfaced so the migration never claims them as editable coverage. */
+    foreignI18nCount: number;
+}
+/**
+ * A .json data file whose values are site copy (config-driven sites keep ALL
+ * their text there and templates render only expressions, so template/JSX
+ * scanning finds nothing). Each detected file must be triaged in the manifest
+ * (`wrap.dataFiles`) — otherwise a copy-in-config site would pass the
+ * coverage gate with zero editable content.
+ */
+export interface DataCopyFile {
+    file: string;
+    strings: number;
+    samples: string[];
+    collections: DataFileCollection[];
+    /** 'markdown' groups: prose documents (md/mdx) aggregated per directory —
+     *  content the wrap cannot label-ify (Tier 1b ingestion territory), but the
+     *  triage decision must still be explicit. */
+    kind?: 'markdown';
+    /** Number of documents behind a markdown group entry. */
+    count?: number;
+}
+export interface DataFileCollection {
+    path: string;
+    length: number;
+    copyFields: string[];
+    references?: CandidateReference[];
+    /** Internal input to reference detection; stripped before the plan is written. */
+    fieldValues?: Record<string, string[]>;
+    /** First row with NATIVE value types — internal input to schema-draft
+     *  type inference; stripped before the plan is written. */
+    typedSample?: Record<string, unknown>;
+    /** All rows (capped) with NATIVE value types — internal to rows staging. */
+    typedRows?: Record<string, unknown>[];
+    manyFields?: string[];
+    /** Fields forced to `string` regardless of inferred type — markdown/MDX
+     *  bodies stay verbatim so the user's existing renderer keeps working
+     *  (wrap mode never reshapes what the page already renders). */
+    verbatimFields?: string[];
+}
+export declare function scanWrappableFile(filePath: string, code: string, relPath: string, opts?: {
+    i18nFimoWired?: boolean;
+}): {
+    items: WrappableItem[];
+    wrapped: number;
+    wrappedIds: WrappedIds;
+    collections: CollectionCandidate[];
+    foreignI18n?: number;
+};
+/**
+ * Parse ONE markdown/MDX document into a typed row: YAML frontmatter fields
+ * (natively typed — dates, booleans, numbers, arrays) plus a `slug` (from
+ * frontmatter or the filename stem) and a `body` holding the raw markdown
+ * after the frontmatter, verbatim. The body stays a string so the user's
+ * existing markdown renderer keeps rendering it unchanged (wrap mode).
+ */
+export declare function parseMarkdownDoc(fileName: string, raw: string): Record<string, unknown> | null;
+/**
+ * Read every markdown/MDX document in a directory (non-recursively — one
+ * group per directory) into typed rows. Shared by the scanner (sampling for
+ * schema drafts + reference detection) and the rows stager (full dataset).
+ */
+export declare function readMarkdownGroupRows(dir: string): Array<Record<string, unknown>>;
+export declare function scanDataJson(relPath: string, raw: string): DataCopyFile | null;
+/**
+ * The label keys a `decision: "labels"` data file is expected to seed —
+ * one per prose string, keyed by JSON path under the recorded prefix. The
+ * seeding gate verifies each exists in the database.
+ */
+export declare function dataFileLabelKeys(raw: string, prefix: string, filePath?: string): string[];
+/** Find render-time fetches of third-party APIs (fetch with an http(s) literal or template prefix). */
+export declare function scanExternalSources(rootDir: string, dirs: string[]): ExternalSource[];
+/**
+ * Scan a project's source directories for unwrapped content. `dirs` defaults
+ * to the common Next/React layouts that exist under `rootDir`.
+ */
+export declare function scanWrappable(rootDir: string, opts?: {
+    dirs?: string[];
+    excludePrefixes?: string[];
+    dataDirs?: string[];
+    keepInternal?: boolean;
+    /** The app's i18n message loader is wired to Fimo (imports
+     *  `loadFimoMessages`), so its foreign `t()` calls ARE Fimo-backed and
+     *  editable — count them as covered instead of excluding them. */
+    i18nFimoWired?: boolean;
+}): WrapScanResult;
+export {};