@thallylabs/migrate 0.2.27 → 0.2.29

package/README.md

@@ -4,6 +4,56 @@ The shared migration engine used by the Thally CLI, MCP server, and Thally
 Cloud. It imports documentation repositories and public documentation URLs
 into one canonical bundle before any caller writes files.
 
+`runUrlMigration` adds an evidence-bearing, resumable URL path. It accepts a
+provider-neutral `captureProvider` (rendered HTML, read-only alternative states,
+ordered navigation and binary assets), or uses ordinary local fetching. Hosted
+callers must inject their own network-safe text/capture and binary boundaries.
+No browser, model, credentials, sandbox or publishing provider is bundled here.
+
+Server applications should import `@thallylabs/migrate/svg-policy` for the passive
+SVG validator, or `@thallylabs/migrate/url-migration` for the URL runner, renderer
+and their types. These independent entries avoid tracing repository cloning and
+filesystem discovery into application routes. The root entry remains compatible.
+
+The returned `checkpoint` is JSON-safe. `onRecord` lets a host persist a completed
+page before acknowledging it; `onCheckpoint` emits bounded deltas, including
+pending discoveries and checkpoint identity fields. Merge delta records by URL,
+then pass the reconstructed checkpoint back to resume. Converted page and asset
+hashes are checked before reuse, and `reconcile` verifies externally stored
+artifacts. Capture adapters should persist raw source/state evidence before
+returning an `artifactRef`. `maxCaptures` bounds a continuation; `signal` cancels
+capture and returns committed progress. Failed pages retry on resume by default.
+
+Page limits support 1–10,000 pages (default 1,000). This interface limit does not
+establish that an arbitrary site can be imported completely. Every inventory
+entry has an explicit outcome; pending/failed pages, missing anchors, technical
+payload differences, unavailable assets and unsupported interactive components
+make validation partial. Raster/PDF assets require matching MIME and signatures;
+SVG must pass the exported `isPassiveSvg` policy: static drawing elements with
+only in-document references, no scripts, external resources, styles or animations.
+Rejected SVG and executable downloads are reported for review.
+Responsive `picture`/`source`/`img` choices retain their descriptors, media
+conditions, sizing and literal alt text. Every candidate is localized; shared
+asset URLs are fetched once and identical payloads count once toward the byte
+budget, including resumed journals. Complete static tab panels are preserved as
+alternatives; empty/lazy panels still require a capture provider.
+Observed locale metadata and URL prefixes are projected onto the runtime layout:
+primary-language files are unprefixed and secondary-language files retain their
+locale prefix. A single-language localized site uses that language as primary.
+Missing primary counterparts and projected path collisions require review; the
+engine does not invent translations. Fresh HTML captures also observe `html lang`; conflicting or unsupported
+language evidence requires review. Capture adapters can supply `locale` when
+no language is declared in HTML or identified by the scoped URL path.
+The `url-evidence-v2` converter rejects earlier journals: replay retained raw
+captures into a new journal rather than relabelling previously hashed pages.
+
+`stopReason` distinguishes a resumable `capture-limit` from hard `page-limit`
+or `byte-limit` outcomes. Hosts must not automatically retry a hard limit as an
+unfinished capture slice. Discovery indexes drain independently of article caps.
+Source validation remains independent of generated-site build/browser checks.
+The CLI's programmatic `migrateDocs({ urlMigration: { ... } })` path retains
+those source validation results in its migration report.
+
 Repository migration includes dedicated Mintlify and Docusaurus adapters.
 Mintlify repositories are located inside monorepos automatically. Their
 `docs.json`/`mint.json` page and group trees are projected without flattening

package/dist/index.d.ts

@@ -1,352 +1,6 @@
-/**
- * Canonical migration contracts shared by repository, CLI, MCP, and hosted
- * import paths. Callers materialize this bundle only after discovery and
- * normalization finish, so a partial crawl can never leave a half-written site.
- */
-type MigrationPlatform = 'mintlify' | 'docusaurus' | 'fern' | 'gitbook' | 'nextra' | 'vitepress' | 'starlight' | 'thally' | 'generic' | 'unknown';
-interface MigrationNavigationGroup {
-    defaultOpen?: boolean;
-    group: string;
-    icon?: string;
-    hidden?: boolean;
-    pages: Array<string | MigrationNavigationGroup>;
-}
-interface MigrationNavigationTab {
-    tab: string;
-    /** Reader-facing label when `tab` includes a version for unique identity. */
-    displayLabel?: string;
-    version?: string;
-    description?: string;
-    icon?: string;
-    href?: string;
-    hidden?: boolean;
-    /** Root page/group nodes used when the source has no synthetic section heading. */
-    pages?: Array<string | MigrationNavigationGroup>;
-    groups?: Array<MigrationNavigationGroup>;
-    api?: {
-        source: string;
-        navigation?: boolean;
-    };
-}
-interface MigrationBannerConfig {
-    content: string | Record<string, string>;
-    dismissible?: boolean;
-    id?: string;
-    revision?: string;
-    type?: 'info' | 'warning' | 'critical';
-    color?: {
-        light?: string;
-        dark?: string;
-    };
-}
-interface MigrationNavbarConfig {
-    links?: Array<{
-        label: string;
-        href: string;
-        type?: 'github';
-        button?: {
-            background: string;
-            color?: string;
-        };
-    }>;
-    /** Explicit null suppresses the starter's sample call to action. */
-    primary?: {
-        label: string;
-        href: string;
-        type?: 'github';
-    } | null;
-    /** Local public assets used until an owner uploads a brand replacement. */
-    /** Explicit null requests a text-only brand when the source has no logo. */
-    logo?: {
-        light: string;
-        dark?: string;
-        showTitle?: boolean;
-        rightText?: string;
-    } | null;
-}
-interface MigrationFooterConfig {
-    socials?: Record<string, string>;
-    /** Literal footer attribution; `{year}` is resolved by the site at render time. */
-    copyright?: string;
-    links?: Array<{
-        heading: string;
-        items: Array<{
-            label: string;
-            href: string;
-        }>;
-    }>;
-}
-interface MigrationDocsConfig {
-    markdown?: {
-        enabled?: boolean;
-    };
-    /** Local site-authored CSS projected from a source docs site, served from public/. */
-    stylesheets?: Array<string>;
-    /** `false` hides the breadcrumb trail; Mintlify pages show only a group eyebrow. */
-    breadcrumbs?: boolean;
-    tabs: Array<MigrationNavigationTab>;
-    navigation?: {
-        /** How sibling documentation collections are presented to readers. */
-        display?: 'tabs' | 'dropdown';
-        versions?: Array<{
-            label: string;
-            prefix: string;
-            href: string;
-            default?: boolean;
-            hidden?: boolean;
-        }>;
-        shortcuts?: Array<{
-            label: string;
-            href: string;
-            icon?: string;
-        }>;
-    };
-    theme?: 'default' | 'maple' | 'sharp' | 'minimal';
-    appearance?: {
-        default?: 'system' | 'light' | 'dark';
-        showToggle?: boolean;
-    };
-    background?: {
-        image?: string;
-        imageDark?: string;
-        decoration?: 'none' | 'grid' | 'gradient';
-    };
-    /** Icon set for content `icon` names; Mintlify's `icons.library` carries through. */
-    contextual?: {
-        options?: Array<string>;
-    };
-    colors?: Partial<Record<'light' | 'dark', {
-        primary?: string;
-        accent?: string;
-    }>>;
-    icons?: {
-        library?: 'lucide' | 'fontawesome' | 'tabler';
-    };
-    banner?: MigrationBannerConfig;
-    navbar?: MigrationNavbarConfig;
-    favicon?: {
-        light: string;
-        dark?: string;
-    };
-    footer?: MigrationFooterConfig;
-    seo?: {
-        indexing?: 'navigable' | 'all';
-        titleSeparator?: string;
-        sitemap?: 'navigable';
-        metatags?: Record<string, string>;
-    };
-    fonts?: {
-        body?: {
-            family: string;
-            weight?: Array<string>;
-        };
-        heading?: {
-            family: string;
-            weight?: Array<string>;
-        };
-    };
-    feedback?: {
-        thumbsRating?: boolean;
-    };
-    customScripts?: Array<{
-        src: string;
-        strategy?: 'beforeInteractive' | 'afterInteractive' | 'lazyOnload';
-    }>;
-    ai?: {
-        chat?: boolean;
-        label?: string;
-        icon?: string;
-    };
-    admin?: {
-        enabled?: boolean;
-    };
-    analytics?: {
-        enabled?: boolean;
-    };
-    /** Manual API pages: default server(s) and auth for `api:` frontmatter pages. */
-    api?: {
-        mdx?: {
-            server?: string | Array<string>;
-            auth?: {
-                method?: 'bearer' | 'basic' | 'key';
-                name?: string;
-            };
-        };
-        /** Mintlify `api.playground.display`; `auth` is carried as `simple`. */
-        playground?: {
-            display: 'interactive' | 'simple' | 'none';
-        };
-        /** `false` hides the visible "OpenAPI specification" line; Mintlify does not show it. */
-        specLink?: boolean;
-    };
-    /** Third-party analytics in Mintlify's `integrations` shape; validated again by the renderer. */
-    integrations?: {
-        ga4?: {
-            measurementId: string;
-        };
-        gtm?: {
-            tagId: string;
-        };
-        posthog?: {
-            apiKey: string;
-            apiHost?: string;
-            sessionRecording?: boolean;
-        };
-        plausible?: {
-            domain: string;
-            server?: string;
-        };
-    };
-    redirects?: Array<{
-        source: string;
-        destination: string;
-        permanent?: boolean;
-    }>;
-    i18n?: {
-        defaultLocale: string;
-        locales: Array<{
-            code: string;
-            label: string;
-        }>;
-        /** Source-authored navigation labels and ordering for each translation. */
-        navigation?: Record<string, Array<MigrationNavigationTab>>;
-    };
-}
-interface MigrationPage {
-    /** Path below `src/content`, without the `.mdx` extension. */
-    id: string;
-    /** Locale-independent page id used by the shared navigation projection. */
-    navigationId: string;
-    locale?: string;
-    title: string;
-    /** Authored display H1 when the source uses a different metadata title. */
-    headingTitle?: string;
-    navTitle?: string;
-    /** Sidebar icon name or image URL from page frontmatter, with its optional style. */
-    icon?: string;
-    iconType?: 'regular' | 'solid' | 'outline' | 'brands';
-    description: string;
-    /** Keep SEO/search description without repeating it above Docusaurus body copy. */
-    descriptionPlacement?: 'body';
-    badge?: string;
-    keywords: Array<string>;
-    mode?: 'default' | 'wide' | 'custom' | 'center' | 'home';
-    hidden?: boolean;
-    noindex?: boolean;
-    /**
-     * Reader access groups (frontmatter `groups`): trimmed, non-empty names the
-     * runtime matches exactly. The page needs a signed-in reader in one of them.
-     */
-    groups?: Array<string>;
-    /** Frontmatter `public`: `true` opens the page to anyone, `false` needs any signed-in reader. */
-    public?: boolean;
-    /** Social/SEO overrides from `og:*` and `twitter:*` frontmatter, keyed by the migrated frontmatter field. */
-    meta?: Partial<Record<'ogTitle' | 'ogDescription' | 'ogImage' | 'twitterTitle' | 'twitterDescription' | 'twitterImage', string>>;
-    /** OpenAPI operation key rendered by Thally instead of ordinary MDX. */
-    openapi?: string;
-    /** Manual API page: `METHOD <url-or-path>`; rendered by Thally's playground. */
-    api?: string;
-    /** Page-level playground auth override: bearer | basic | key | none. */
-    authMethod?: string;
-    /** Page-level `playground` override: interactive | simple | none (`auth` becomes `simple`). */
-    playground?: 'interactive' | 'simple' | 'none';
-    body: string;
-    source: string;
-    /** Set when the page's frontmatter was invalid YAML; the page is kept with a best-effort salvage. */
-    frontmatterError?: string;
-}
-interface MigrationAsset {
-    /** Path below `public` (or below the project root when `projectRelative`), always normalized and traversal-free. */
-    path: string;
-    content: Uint8Array;
-    /** Write at the project root instead of `public/`, so the host never serves it statically. */
-    projectRelative?: boolean;
-}
-interface MigrationWarning {
-    code: 'collision' | 'invalid-page' | 'missing-page' | 'unsupported-config' | 'limit-reached' | 'fetch-failed' | 'skipped-file'
-    /** Access-restricted source content: migrated with access rules, withheld (quarantined), or a site-wide auth risk. */
-     | 'gated-page';
-    message: string;
-    source?: string;
-}
-/** A repository-configured remote OpenAPI spec, downloaded after the bundle is built. */
-interface RemoteApiSpec {
-    url: string;
-    tabLabel?: string;
-    parentTab?: string;
-    icon?: string;
-    hidden?: boolean;
-    /** Operation keys (`get /x`) only access-restricted pages document; marked `x-excluded` after download. */
-    withheldOperations?: Array<string>;
-    /** Operation keys published pages or docs.json also name; keeps a shared `$ref` path item public. */
-    keptOperations?: Array<string>;
-}
-interface MigrationBundle {
-    sourceUrl: string;
-    sourceKind: 'repository' | 'url';
-    platform: MigrationPlatform;
-    pages: Array<MigrationPage>;
-    assets: Array<MigrationAsset>;
-    /** Repository-configured remote specs awaiting a bounded network fetch by the host. */
-    remoteApiSpecs?: Array<RemoteApiSpec>;
-    /** Customer-owned component source and registry; paths are repository-relative. */
-    componentFiles?: Array<RenderedMigrationFile>;
-    /**
-     * Restricted content that cannot be published safely (pages whose access
-     * rules are malformed or contradictory, and assets used only by restricted
-     * pages), written outside every published path.
-     */
-    quarantinedFiles?: Array<RenderedMigrationFile>;
-    /** Access-restricted pages the file limit dropped: neither published nor saved in quarantinedFiles. */
-    droppedGatedPages?: number;
-    /** Mintlify `sourceRef` repositories imported under a mount path, with their page counts. */
-    sourceRefs?: Array<{
-        repo: string;
-        mountPath: string;
-        pages: number;
-    }>;
-    docsConfig: MigrationDocsConfig;
-    site?: {
-        name?: string;
-        description?: string;
-        /**
-         * Source theme accent color(s), each a `#rrggbb`/`#rgb` hex string.
-         * Follows Mintlify's own `colors` schema: `light` is the color used in
-         * dark mode and `dark` is the color used in light mode. Fern's colors
-         * are normal (its `light`/`dark` match the mode they paint), so
-         * extraction swaps them onto this shape for a single downstream contract.
-         */
-        colors?: {
-            primary?: string;
-            light?: string;
-            dark?: string;
-        };
-    };
-    warnings: Array<MigrationWarning>;
-    stats: {
-        discovered: number;
-        imported: number;
-        skipped: number;
-    };
-}
-interface MigrationFetchRequest {
-    accept: string;
-}
-interface MigrationFetchResponse {
-    finalUrl: URL;
-    body: string;
-    contentType: string;
-    headers?: Record<string, string | undefined>;
-}
-/**
- * Network boundary injected by each host. Thally Cloud supplies a DNS-pinned,
- * SSRF-safe implementation; the local CLI uses the user's normal network.
- */
-type MigrationFetcher = (url: URL, request: MigrationFetchRequest) => Promise<MigrationFetchResponse>;
-interface RenderedMigrationFile {
-    path: string;
-    content: string | Uint8Array;
-}
+import { M as MigrationPage, a as MigrationAsset, b as MigrationNavigationGroup, R as RenderedMigrationFile, c as MigrationWarning, d as MigrationFetcher, e as MigrationPlatform, f as MigrationBundle, g as MigrationDocsConfig } from './url-migration-CDjLZVaN.js';
+export { h as MigrationCapture, i as MigrationCaptureProvider, j as MigrationCaptureState, k as MigrationCapturedAsset, l as MigrationEvidence, m as MigrationFetchRequest, n as MigrationFetchResponse, o as MigrationJournalAsset, p as MigrationNavigationOccurrence, q as MigrationNavigationTab, U as UrlMigrationCheckpoint, r as UrlMigrationCheckpointDelta, s as UrlMigrationIssue, t as UrlMigrationRecord, u as UrlMigrationRun, v as UrlMigrationRunOptions, w as UrlMigrationValidation, x as classifyMigrationResponse, y as mergeMigrationConfig, z as renderMigrationFiles, A as runUrlMigration, B as validateUrlMigration } from './url-migration-CDjLZVaN.js';
+export { isPassiveSvg } from './svg-policy.js';
 
 /**
  * Mintlify `sourceRef` navigation nodes mount the docs of another repository
@@ -673,16 +327,6 @@ declare function projectMintlifyIntegrations(config: Record<string, unknown>): {
  */
 declare function validateIntegrations(integrations: unknown): Array<string>;
 
-/** Render canonical migration bundles into a portable Thally repository tree. */
-
-/** Merge imported tabs into an existing site without duplicating changelog tabs. */
-declare function mergeMigrationConfig(existing: MigrationDocsConfig, incoming: MigrationDocsConfig): MigrationDocsConfig;
-/** Materialize every canonical page, asset, and config as repository file changes. */
-declare function renderMigrationFiles(bundle: MigrationBundle, options?: {
-    existingConfig?: MigrationDocsConfig;
-    existingComponentRegistry?: string;
-}): Array<RenderedMigrationFile>;
-
 /**
  * Public URL adapter for the shared migration engine. Discovery prefers
  * machine-readable Markdown and sitemaps, remains scoped to the submitted docs
@@ -712,4 +356,4 @@ declare function migrateUrl(options: UrlMigrationOptions): Promise<MigrationBund
  */
 declare function hydrateRemoteApiSpecs(bundle: MigrationBundle, fetcher?: MigrationFetcher): Promise<MigrationBundle>;
 
-export { type GitHubRepositorySource, type MigrationAsset, type MigrationBundle, type MigrationDocsConfig, type MigrationFetchRequest, type MigrationFetchResponse, type MigrationFetcher, type MigrationNavigationGroup, type MigrationNavigationTab, type MigrationPage, type MigrationPlatform, type MigrationWarning, type RenderedMigrationFile, type RepositoryMigrationOptions, SOURCE_REF_MAX_BYTES, SOURCE_REF_MAX_FILES, SOURCE_REF_MAX_REPOSITORY_KB, type SourceRefImport, type SourceRefMapping, type UrlMigrationOptions, addMintlifyDirectoryRedirects, buildNavigationFromPages, cloneGitHubRepository, defaultMigrationFetcher, detectRepositoryDocsDir, detectRepositoryPlatform, gitmodulePaths, hydrateRemoteApiSpecs, importSourceRef, mergeMigrationConfig, migrateRepository, migrateUrl, normalizeMdx, parseGitHubRepositoryUrl, parseMarkdownPage, parseSourceRefFlags, projectFernNavigation, projectMintlifyIntegrations, projectMintlifyNavigation, pruneMissingNavigationPages, readFernConfig, readMintlifyConfig, renderMigrationFiles, sourceRefOversizeReason, validateIntegrations, validateMigrationUrl };
+export { type GitHubRepositorySource, MigrationAsset, MigrationBundle, MigrationDocsConfig, MigrationFetcher, MigrationNavigationGroup, MigrationPage, MigrationPlatform, MigrationWarning, RenderedMigrationFile, type RepositoryMigrationOptions, SOURCE_REF_MAX_BYTES, SOURCE_REF_MAX_FILES, SOURCE_REF_MAX_REPOSITORY_KB, type SourceRefImport, type SourceRefMapping, type UrlMigrationOptions, addMintlifyDirectoryRedirects, buildNavigationFromPages, cloneGitHubRepository, defaultMigrationFetcher, detectRepositoryDocsDir, detectRepositoryPlatform, gitmodulePaths, hydrateRemoteApiSpecs, importSourceRef, migrateRepository, migrateUrl, normalizeMdx, parseGitHubRepositoryUrl, parseMarkdownPage, parseSourceRefFlags, projectFernNavigation, projectMintlifyIntegrations, projectMintlifyNavigation, pruneMissingNavigationPages, readFernConfig, readMintlifyConfig, sourceRefOversizeReason, validateIntegrations, validateMigrationUrl };