@thallylabs/migrate 0.2.6 → 0.2.8

package/README.md

@@ -15,10 +15,113 @@ compatible page frontmatter, site identity, and OpenAPI configuration. For publi
 sites, the URL adapter reads the structured configuration embedded in the
 rendered application before falling back to DOM link inference.
 
+An `openapi` (or `asyncapi`) reference is resolved wherever Mintlify allows
+it — the top-level `api.openapi`, or on any individual
+tab/anchor/dropdown/product/version, e.g. `navigation.tabs[].openapi` — and
+binds to that exact tab, not only the first one found. Remote `http://` and
+`https://` specs are not downloaded during migration. The report names each
+remote URL and tells you to add its spec to `public/`, then set the API source
+in `docs.json` to that local path. Missing local specs also produce warnings
+instead of silently dropping that API tab's reference. AsyncAPI has no Thally
+renderer, so it always produces a clear warning naming the spec instead of
+disappearing.
+
 Docusaurus sidebars are parsed as bounded, data-only JSON5—never executed—and
 preserve explicit categories, autogenerated slices, document IDs/slugs,
 positions, generated indexes, admonitions, tabs, and `static/` assets where
-supported.
+supported. Every `@docusaurus/plugin-content-docs` instance declared in
+`docusaurus.config` (not just the default one) is imported as its own tab,
+routed under its `routeBasePath`; a `path` is resolved the way Docusaurus
+itself resolves it (relative to the detected project root), with a fallback
+to the repository root for monorepos that build by copying the project
+root's contents up into the repository root before generating the site
+(Playwright's own `nodejs/` is one real example) — a `path` still not found
+under either root is reported with a warning naming the plugin instance
+instead of being silently dropped. The same repository-root fallback applies
+to `@site/...` aliases, relative component imports, and relative asset
+`require()` calls in copied components; a `require()` that still can't be
+resolved is replaced with an empty string (a broken image, not a build
+failure) and warned about instead of left dangling. `@docusaurus/plugin-client-redirects`
+rules are imported whether declared inline in `docusaurus.config` or in a
+separate module it imports (a named export, e.g. `redirects.ts`); a `from`
+may be a single path or an array of paths. Its `createRedirects` function
+computes redirects dynamically and can't be evaluated statically, so its
+presence is reported with a warning instead of being silently ignored. A
+static `--ifm-color-primary` accent color in the classic theme's `customCss`
+file is imported the same way Mintlify's and Fern's theme colors are.
+Versioned docs under `versioned_docs/` and translated docs under
+`i18n/<locale>/docusaurus-plugin-content-docs/` are imported with their
+navigation, including locale-prefixed routes such as `/fr/...`. Each version
+and locale keeps its own route tree; sidebar entries without source documents
+are reported.
+Docusaurus heading ID comments such as `{/* #custom-id */}` become explicit
+HTML anchors so links to those headings keep working after migration. Links
+using Docusaurus' implicit heading slugs also get explicit aliases when the
+linked heading or option-table field can be matched uniquely. Repeated option
+fields use the nearest preceding row when all local references agree;
+otherwise ambiguous anchors remain warnings.
+Longer fenced code examples keep nested shorter fences inert. An archived
+admonition missing its closing delimiter is closed before the next top-level
+heading (or at the end of the page) so its remaining content stays importable.
+Docusaurus `mdx-code-block` wrappers are opened as live MDX when the resulting
+page compiles and its special fences contain no module declarations. Pages
+whose fragments still need Docusaurus theme imports or remark plugins remain
+fenced and are retained.
+
+Fern repositories are located by their `fern/docs.yml` + `fern/fern.config.json`
+pair. `docs.yml` is parsed as bounded YAML—never executed—and preserves tabs,
+nested sections (including `slug`/`skip-slug`), navbar links, redirects, the
+default Fern version's navigation, and theme accent colors. Only pages
+referenced from `docs.yml` are imported, matching Fern's own publishing model.
+Versioned `folder:` entries expand the Markdown files they contain, even when
+the folder or an individual page is outside `fern/` but still inside the
+checkout. An explicit empty tab `slug` keeps pages at the site root; a folder
+title changes its navigation label without changing its directory-based URL.
+Internal links are adjusted for the site's domain path, selected version, and
+source-file routes. Links to sections not imported into Thally stay on the
+published Fern site and produce a migration warning, so the generated site can
+build while the remaining content is moved separately.
+Every `api:` section's OpenAPI document is resolved from its own
+`generators.yml` (`api.specs[].openapi`, or the legacy `api:` string) and
+imported into its own tab — a docs.yml with several API sections (e.g. a REST
+API and a WebSocket API in separate tabs) gets a spec per tab, not just the
+first. The per-API `fern/apis/<name>/` layout is checked first, keyed by the
+node's `api-name` (the folder name) rather than `api` (its display title in
+the nav) — the two are commonly different. The project-root `generators.yml`
+is used only when the node has no `api-name` or the repo has no `fern/apis/`
+folder. `openapi:` is conventionally relative to `generators.yml`'s own
+directory rather than the Fern root, so it commonly resolves outside the API
+folder (or outside `fern/` entirely, e.g. `../../../openapi.yaml`); the spec
+is still found as long as it stays inside the repository checkout, and a path
+that escapes the checkout is reported with a warning naming it, instead of
+the API being silently dropped. A Fern Definition (a non-OpenAPI API with no
+document to attach), an AsyncAPI/OpenRPC-only `api:` section (Thally has no
+renderer for either), and an `api:` section for which no spec could be
+resolved at all, are each reported as a warning naming the API instead of the
+tab being silently dropped, as is `changelog`, which has no equivalent Thally
+field yet.
+
+A section/page/tab's explicit `slug` is used as literal Fern routing text
+(e.g. `slug: baml_client`, matching a `baml_client/` folder) even when
+Thally's own route still hyphenates it for consistency (`baml-client`); an
+in-body link written against that literal, unslugified form — which Fern's
+own file-path-based routing tolerates — gets a redirect to the route Thally
+actually serves, instead of 404ing after migration.
+
+KaTeX-style `$$...$$` math (block or inline) has no Thally renderer yet.
+Rather than crashing the MDX compiler and silently excluding the whole page,
+a block becomes a fenced ```` ```math ```` code block and an inline span
+becomes an inline code span — both preserve the original math text — with a
+warning.
+
+A `products:` docs.yml (Fern's multi-product/product-switcher layout, as on
+buildwithfern.com/learn) imports every product: each product's own
+`navigation:`/`tabs:` becomes a top-level Thally tab named by its
+`display-name`, routed under its `slug` (or a slugified display-name), with
+its pages read from that product's own directory. Local component imports
+from Fern MDX pages (`../../components/...`, resolved against the Fern
+project root) are migrated the same way Mintlify and Docusaurus components
+are — see "Mintlify compatibility" below.
 
 ## Mintlify compatibility
 
@@ -35,12 +138,90 @@ The importer creates a root redirect to the first available page in source
 navigation when no root introduction exists, while retaining a separate custom
 homepage and explicit redirects. Locale tags and recognized locale-prefixed
 sections are normalized without discarding original URL redirects.
+Mintlify's generated `param-*` links for settings headings and named fields
+become explicit anchors in migrated MDX.
 
 Simple interactive HTML blocks in MDX are extracted into client components.
+For a copied client component that receives a simple page-authored JSX callback,
+the importer moves the callback into a client wrapper and keeps the page. If
+the callback cannot be moved safely, it keeps the page and the control's child
+content as a passive fallback, with a warning identifying the control. A
+stateful local snippet component is kept as a client component instead of
+flattening away its hooks and state.
 The Mintlify `search-bar-entry` click trigger is mapped to Thally's search
-shortcut. Markdown mixed inside interactive JSX, computed imports, namespace
-imports, and external packages that require installation are reported for
-manual review. Unsupported source is preserved, not replaced with empty stubs.
-File paths cannot escape the documentation root; symlinks and oversized graphs
-are rejected. This is compatibility analysis, **not a code sandbox**: imported
-JavaScript executes when the developer builds or runs the resulting site.
+shortcut. Markdown mixed inside interactive JSX, computed imports, and
+namespace imports are reported for manual review; unsupported source is
+preserved, not replaced with empty stubs. An external package's JSX usage is
+dropped and replaced with a safe fallback (with a warning); if that package's
+binding is instead referenced outside JSX (an expression, a prop, or an
+inline declaration) the import can't be rewritten the same way, so the whole
+page is excluded — instead of shipping with an import `next build` can't
+resolve — and pruned from navigation, with a warning naming the page, the
+package, and the binding. A local/relative component import is fully owned by
+the migration instead: the whole file (and its own local dependency graph) is
+copied, keeping every binding's original name, so it makes no difference
+whether a binding is used as a bare JSX tag or only referenced elsewhere (an
+enum passed into a sibling component's prop, a member tag, a wrapping
+declaration, ...) — the page is excluded only when the copy itself genuinely
+fails (a missing file, a computed/dynamic dependency, a cycle, or a budget
+overrun), with a warning naming the page and the component. Packages the starter already installs (`next`,
+`react-dom`, `clsx`, `lucide-react`, `tailwind-merge`, and their subpaths) are
+kept as-is (except `react-dom/server`, whose string renderers throw under
+Next) and copied into any extracted client module. A source site's own
+`@/…` path aliases are treated like an unavailable package. A page that
+passes a page-authored function into a client component needs the client
+wrapper described above; unsupported callback shapes retain their content
+with a warning. An unconfirmed target is warned for review.
+File paths cannot escape the repository checkout; symlinks and oversized
+graphs are rejected. This is compatibility analysis, **not a code sandbox**:
+imported JavaScript executes when the developer builds or runs the resulting
+site.
+
+A copied Mintlify `.jsx` snippet that uses the implicit `MintlifyComponents`
+global (`const { Card } = MintlifyComponents`, Mintlify's own no-import
+convention for its built-ins) gets that binding provided automatically,
+reusing Thally's own built-in MDX component registry — the same way an
+implicit `React`/hook global is already provided.
+
+A copied component is prepended with `// @ts-nocheck`: the source
+platform's own build never ran `next build`'s full `tsc --noEmit` type
+check, so loose types, ambient globals (`window.ethereum`), and implicit
+`any` parameters that were fine there would otherwise fail the migrated
+build. A locally-owned wrapper component whose own body imports a known
+YouTube-embed package (e.g. a custom `LiteYouTube` wrapping
+`react-lite-youtube-embed`) gets the same working `<iframe>` fallback as a
+page that imports that package directly, instead of losing the embed to the
+generic unknown-component fallback.
+
+## Branding
+
+A source site's logo, favicon, and theme accent color(s) are extracted where
+each platform's config exposes them (Mintlify's `colors`, Fern's
+`colors.accent-primary`, Docusaurus's classic theme `customCss`). A valid
+accent color is wired into the scaffolded project's `brandPreset` the same
+way `create-thally-docs`'s own `--colors` flag does. Logo and favicon files
+are copied into `public/`, but Thally's site configuration has no static
+field for them — branding is managed from the admin dashboard at runtime —
+so each site gets one warning naming the copied files and pointing at the
+dashboard's branding settings instead of leaving the loss silent.
+
+## Cloning a GitHub repository
+
+`cloneGitHubRepository` retries a transient network-class failure with
+backoff, and each underlying git process (the clone, and any submodule
+fetch) is killed and treated as a retryable failure if it stalls — a
+generous 10-minute default, configurable via
+`THALLY_MIGRATE_CLONE_TIMEOUT_MS` (milliseconds) for an unusually large
+repository on a slow link. The clone neutralizes the `filter.lfs.*`
+smudge/clean/process filter driver for that one process only (via
+`GIT_CONFIG_COUNT`/`GIT_CONFIG_KEY_n`/`GIT_CONFIG_VALUE_n`, never the user's
+global git config), so a repository that uses Git LFS still clones even when
+the host has no `git-lfs` binary installed — including a host where
+`filter.lfs.*` is still registered globally from a previous install,
+pointing at a command that no longer exists; an asset that's still a Git
+LFS pointer file afterward (not its real content) is skipped with a warning
+instead of being copied as if it were the real file. Submodules are
+initialized one at a time, best-effort,
+after a successful clone — a submodule that can't be fetched (private,
+deleted, network trouble) is named in a warning instead of failing the
+whole migration.

package/dist/index.d.ts

@@ -3,7 +3,7 @@
  * import paths. Callers materialize this bundle only after discovery and
  * normalization finish, so a partial crawl can never leave a half-written site.
  */
-type MigrationPlatform = 'mintlify' | 'docusaurus' | 'gitbook' | 'nextra' | 'vitepress' | 'starlight' | 'thally' | 'generic' | 'unknown';
+type MigrationPlatform = 'mintlify' | 'docusaurus' | 'fern' | 'gitbook' | 'nextra' | 'vitepress' | 'starlight' | 'thally' | 'generic' | 'unknown';
 interface MigrationNavigationGroup {
     group: string;
     icon?: string;
@@ -154,6 +154,18 @@ interface MigrationBundle {
     site?: {
         name?: string;
         description?: string;
+        /**
+         * Source theme accent color(s), each a `#rrggbb`/`#rgb` hex string.
+         * Follows Mintlify's own `colors` schema: `light` is the color used in
+         * dark mode and `dark` is the color used in light mode. Fern's colors
+         * are normal (its `light`/`dark` match the mode they paint), so
+         * extraction swaps them onto this shape for a single downstream contract.
+         */
+        colors?: {
+            primary?: string;
+            light?: string;
+            dark?: string;
+        };
     };
     warnings: Array<MigrationWarning>;
     stats: {
@@ -182,9 +194,13 @@ interface RenderedMigrationFile {
 }
 
 /**
- * Repository adapter for the shared migration engine. Traversal is bounded,
- * symbolic links are ignored, and Git is always invoked with an argument array
- * so source-controlled branch names can never become shell commands.
+ * Repository adapter for the shared migration engine. Traversal is bounded;
+ * `scanFiles` follows a symbolic link only when its resolved real path stays
+ * inside the repository checkout (this covers a submodule mounted as a
+ * symlink, e.g. Oasis's `docs/core -> ../external/oasis-core/docs`) and
+ * guards against a cycle, but every other directory walk in this file still
+ * skips symlinks outright. Git is always invoked with an argument array so
+ * source-controlled branch names can never become shell commands.
  */
 
 interface GitHubRepositorySource {
@@ -205,13 +221,30 @@ interface RepositoryMigrationOptions {
     docusaurusSkipPlugins?: boolean;
     /** @internal Additional docs plugins default to autogenerated navigation. */
     docusaurusSkipSidebar?: boolean;
+    /** @internal A version's sidebar is parsed as static data. */
+    docusaurusSidebarPath?: string;
     /** @internal Static assets are shared across docs-plugin instances. */
     docusaurusSkipAssets?: boolean;
+    /** @internal Redirects are global config, read once, not per plugin instance. */
+    docusaurusSkipRedirects?: boolean;
 }
 /** Parse and validate a public GitHub repository URL. */
 declare function parseGitHubRepositoryUrl(rawUrl: string): GitHubRepositorySource;
-/** Clone a repository without a shell; callers own and remove `targetDir`. */
-declare function cloneGitHubRepository(source: GitHubRepositorySource, targetDir: string): Promise<void>;
+/** Every submodule path declared in a cloned repository's `.gitmodules`, in file order. */
+declare function gitmodulePaths(targetDir: string): Array<string>;
+/**
+ * Clone a repository without a shell; callers own and remove `targetDir`. A
+ * large repository on a flaky connection can drop mid-clone (`RPC failed`,
+ * `Recv failure`, a `curl 56`, …) — retry a network-class failure a couple
+ * of times with backoff instead of surfacing a hard failure on the first
+ * blip. A partial checkout from the failed attempt is removed first, or
+ * `git clone` refuses to reuse the (now non-empty) target directory. A
+ * stalled clone/submodule-init is killed by `runGit`'s own timeout, which
+ * surfaces as a retryable error. `warnings`, if given, collects a
+ * submodule-fetch-failure warning (this function otherwise returns exactly
+ * as before, so existing callers are unaffected).
+ */
+declare function cloneGitHubRepository(source: GitHubRepositorySource, targetDir: string, warnings?: Array<MigrationWarning>): Promise<void>;
 /** Detect a supported repository docs platform from unambiguous config files. */
 declare function detectRepositoryPlatform(repositoryDir: string, docsDir?: string): MigrationPlatform;
 /** Pick the first conventional content root containing Markdown. */
@@ -257,6 +290,65 @@ interface GeneratedNavigationOptions {
 }
 /** Build deterministic fallback navigation from imported default-locale pages. */
 declare function buildNavigationFromPages(pages: Array<MigrationPage>, options?: GeneratedNavigationOptions): MigrationDocsConfig;
+/**
+ * A source page can be excluded after navigation is projected (invalid MDX,
+ * a client-boundary function prop, a duplicate id). Its reference still sits
+ * in the projected tabs/groups at that point, across every tab, group, and
+ * locale, since the nav tree and the page list are built from the same
+ * source config independently. Drop those dangling references here so
+ * `thally check` never reports a nav entry with no backing MDX file, and
+ * drop any group left with no pages as a result. Shared by every platform's
+ * navigation projection (Mintlify, Fern, Docusaurus).
+ */
+declare function pruneMissingNavigationPages(config: MigrationDocsConfig, availableIds: ReadonlySet<string>): MigrationDocsConfig;
+
+/**
+ * Static Fern (`fern/docs.yml`) repository projection. Fern navigation and
+ * generator config are plain YAML data files, so this adapter only ever
+ * parses them with `yaml` and never imports or executes JavaScript.
+ */
+
+interface FernPageDescriptor {
+    /** Path to the referenced page file, relative to the Fern root. */
+    sourcePath: string;
+    navigationId: string;
+}
+/** One `api:` navigation node, resolved to the name/tab Thally needs to bind its spec. */
+interface FernApiSection {
+    /** The node's own value (a display name or, in multi-API repos, a `fern/apis/<name>` identifier). */
+    name: string;
+    /** True when `name` came from an explicit `api-name` field rather than falling back to the `api:` display title. */
+    nameExplicit: boolean;
+    /** Label of the tab that owns this node, if that tab also has other content and survived projection. */
+    tabLabel?: string;
+    /**
+     * Route segments (section slugs) leading to this node, used only to
+     * disambiguate two `api:` nodes that land on the same `tabLabel` — a
+     * "Production API Reference" and "Testnet API Reference" section both
+     * nested under one tab both want that tab's label; without this, the
+     * second one silently overwrites the first tab's binding instead of
+     * getting its own tab.
+     */
+    routeSegments: Array<string>;
+}
+interface FernNavigationResult {
+    docsConfig: MigrationDocsConfig;
+    descriptors: Array<FernPageDescriptor>;
+    warnings: Array<MigrationWarning>;
+    /** Every `api:` node found in navigation, in document order. A repo can declare several (e.g. a REST and a WebSocket API in separate tabs). */
+    apiSections: Array<FernApiSection>;
+}
+/** Read `fern/docs.yml` without executing anything. */
+declare function readFernConfig(fernRoot: string): {
+    config: Record<string, unknown>;
+    sourcePath: string;
+} | null;
+/** Project a Fern `docs.yml` navigation tree into Thally tabs/groups. */
+declare function projectFernNavigation(input: {
+    config: Record<string, unknown>;
+    fernRoot: string;
+    repositoryRoot?: string;
+}): FernNavigationResult;
 
 /** Markdown/MDX normalization that preserves every component Thally supports. */
 
@@ -266,7 +358,7 @@ interface MarkdownPageIdentity {
     locale?: string;
 }
 /** Normalize only syntax Thally cannot render; supported source JSX stays intact. */
-declare function normalizeMdx(body: string): string;
+declare function normalizeMdx(body: string, platform?: MigrationPlatform, unwrapMdxCodeBlocks?: boolean): string;
 /** Parse source Markdown or MDX into the canonical page representation. */
 declare function parseMarkdownPage(input: {
     id: string;
@@ -274,6 +366,8 @@ declare function parseMarkdownPage(input: {
     locale?: string;
     raw: string;
     source: string;
+    /** Gates platform-specific textual renames in `normalizeMdx`; omitted runs them all. */
+    platform?: MigrationPlatform;
     /** Resolve platform-specific routes from the already-parsed frontmatter. */
     resolveIdentity?: (frontmatter: Record<string, unknown>, fallback: MarkdownPageIdentity) => MarkdownPageIdentity;
 }): MigrationPage | null;
@@ -310,4 +404,4 @@ declare const defaultMigrationFetcher: MigrationFetcher;
 /** Crawl one public docs origin into a canonical, caller-independent bundle. */
 declare function migrateUrl(options: UrlMigrationOptions): Promise<MigrationBundle>;
 
-export { type GitHubRepositorySource, type MigrationAsset, type MigrationBundle, type MigrationDocsConfig, type MigrationFetchRequest, type MigrationFetchResponse, type MigrationFetcher, type MigrationNavigationGroup, type MigrationNavigationTab, type MigrationPage, type MigrationPlatform, type MigrationWarning, type RenderedMigrationFile, type RepositoryMigrationOptions, type UrlMigrationOptions, addMintlifyDirectoryRedirects, buildNavigationFromPages, cloneGitHubRepository, defaultMigrationFetcher, detectRepositoryDocsDir, detectRepositoryPlatform, mergeMigrationConfig, migrateRepository, migrateUrl, normalizeMdx, parseGitHubRepositoryUrl, parseMarkdownPage, projectMintlifyNavigation, readMintlifyConfig, renderMigrationFiles, validateMigrationUrl };
+export { type GitHubRepositorySource, type MigrationAsset, type MigrationBundle, type MigrationDocsConfig, type MigrationFetchRequest, type MigrationFetchResponse, type MigrationFetcher, type MigrationNavigationGroup, type MigrationNavigationTab, type MigrationPage, type MigrationPlatform, type MigrationWarning, type RenderedMigrationFile, type RepositoryMigrationOptions, type UrlMigrationOptions, addMintlifyDirectoryRedirects, buildNavigationFromPages, cloneGitHubRepository, defaultMigrationFetcher, detectRepositoryDocsDir, detectRepositoryPlatform, gitmodulePaths, mergeMigrationConfig, migrateRepository, migrateUrl, normalizeMdx, parseGitHubRepositoryUrl, parseMarkdownPage, projectFernNavigation, projectMintlifyNavigation, pruneMissingNavigationPages, readFernConfig, readMintlifyConfig, renderMigrationFiles, validateMigrationUrl };