@dogsbay/import 0.0.0-stage → 0.2.0-beta.163

package/README.md

@@ -1,3 +1,20 @@
-# Temporary Holding Version
+# @dogsbay/import
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A site made with GitBook, VitePress, Docusaurus or Starlight, as a Dogsbay project: the one
+import the playground and `dogsbay import` run. No Node.
+
+```ts
+import { importSite } from "@dogsbay/import";
+
+const { files, ledger, unread } = await importSite({
+  format: "starlight",
+  files,            // Map<path, string | Uint8Array>: what you have read so far
+  paths,            // every path of the site
+  more: async (want) => { /* read `want` into `files` */ },
+  choice: { languages: ["en", "fr"] },
+  siteName: "My docs",
+});
+```
+
+`siteChoices(format, files, paths, more)` lists what the user can choose. See
+docs/import.md and docs-dev/import-command.md in the dogsbay repo.

package/dist/index.d.ts

@@ -0,0 +1,164 @@
+/**
+ * `@dogsbay/import`: a site made with another tool, as a Dogsbay project (plans/import-command.md).
+ *
+ * The ONE import every host runs: the playground (files from a zip or GitHub) and `dogsbay import`
+ * (files from disk). It calls the format's browser entry (`@dogsbay/format-*\/migrate`) with the
+ * choices the user made, and in rounds asks the host for what the conversion read and was not
+ * given, then for the assets it copies. A host supplies only what is its own: reading files, the
+ * Docusaurus adapter object, Starlight's TypeScript-to-JavaScript converter.
+ *
+ * No Node, no network, nothing from the repo is run. Each format's conversion is loaded only when
+ * a site of that format is imported.
+ */
+import type { ImportLedgerEntry } from "@dogsbay/types";
+import type { DocusaurusAdapter } from "@dogsbay/format-docusaurus/adapter";
+/** The formats this import converts. */
+export declare const IMPORT_FORMATS: readonly ["gitbook", "vitepress", "docusaurus", "starlight"];
+export type ImportFormat = (typeof IMPORT_FORMATS)[number];
+export declare const isImportFormat: (f: string) => f is ImportFormat;
+export type SiteFiles = Map<string, string | Uint8Array>;
+/** Docusaurus products and versions ticked: per docs instance, its versions by name (none for an unversioned one). */
+export interface Selection {
+    instance: string;
+    versions?: string[];
+}
+/** What the user chose for the site. Each field applies to the formats it names. */
+export interface ImportChoice {
+    /** GitBook: one space of a multi-space repo; all when omitted. */
+    space?: string;
+    /** Docusaurus: a docs instance (its id); the config's only or default one when omitted. */
+    instance?: string;
+    /** Docusaurus: a version (versions.json); the current docs folder when omitted. */
+    version?: string;
+    /** Docusaurus: products and versions converted together (`migrateSelection`), in place of `instance` and `version`. */
+    select?: Selection[];
+    /** Docusaurus: an API viewer page split into API pages mounted at it (the default), or the whole API kept on its page. */
+    openapi?: "split" | "embed";
+    /** VitePress and Starlight: the languages kept (ids, the site's order; the first the default); every language when omitted. */
+    languages?: string[];
+    /** Starlight: the products kept (folders under `src/content/docs/`); the whole site when omitted or empty. */
+    products?: string[];
+    /** Starlight: a site adapter by name (`cloudflare`). Docusaurus takes the adapter object (`docusaurusAdapter`). */
+    adapter?: string;
+}
+export interface ImportSiteInput {
+    format: ImportFormat;
+    /** The site's files given so far, paths from the site's folder (a detection's `root`). */
+    files: SiteFiles;
+    /** Every path of the site, when the host lists more than it gave: what the conversion needs from the rest it asks for (`more`). */
+    paths?: Iterable<string>;
+    choice?: ImportChoice;
+    siteName?: string;
+    siteUrl?: string;
+    /** Where the site came from, for MIGRATION.md: the repo's URL, or its folder. */
+    sourceLabel?: string;
+    /** Docusaurus: how many products the site has; a selection from a site of several gets a `products:` block, so another can be added later. */
+    productCount?: number;
+    /** Docusaurus: the site adapter (`tigeraAdapter` …), loaded by the host. */
+    docusaurusAdapter?: DocusaurusAdapter;
+    /** Starlight with the Cloudflare adapter: `<TypeScriptExample>`'s JavaScript tab. */
+    typescriptToJs?: (ts: string) => Promise<string>;
+    /**
+     * Adds these site paths to `files` (those it can read): what the conversion read and was not
+     * given (`assets` false: it converts again), or the assets it copies (`assets` true). Without it
+     * the conversion runs once over what was given.
+     */
+    more?(paths: string[], assets: boolean): Promise<void>;
+    /** Runs again with the files asked for, at most, after the first (each round can find one more level of an include or `require` chain). Default 5, as the playground. */
+    rounds?: number;
+}
+export interface ImportSiteResult {
+    /** The Dogsbay project: `dogsbay.config.yml`, `content/**`, `MIGRATION.md` (and `data/**` when the site has data). */
+    files: SiteFiles;
+    ledger: ImportLedgerEntry[];
+    /** Site paths the conversion needed and was not given (the host could not read them, or found after the last round). */
+    unread: string[];
+}
+export declare const DEFAULT_ROUNDS = 5;
+/**
+ * The site as a Dogsbay project: the format's conversion with the user's choices, run again with
+ * the files it asks for while there are any (`rounds` at most), then the assets it copies placed
+ * where it says (`copyTo`) without converting again. Files the host could not give are left out of
+ * `paths`, so the conversion goes on without them, and are returned in `unread`.
+ */
+export declare function importSite(input: ImportSiteInput): Promise<ImportSiteResult>;
+/** A language the site has, to choose. */
+export interface LanguageOption {
+    id: string;
+    label: string;
+    isDefault: boolean;
+    pages: number;
+}
+/** A Starlight product (a folder of its docs), to choose. */
+export interface ProductOption {
+    id: string;
+    label: string;
+    pages: number;
+}
+/** A Docusaurus product (docs instance) and its versions, to choose. */
+export interface DocsProductOption {
+    name: string;
+    instance: string;
+    label: string;
+    versions: {
+        id: string;
+        name: string;
+        label?: string;
+        isDefault: boolean;
+        prerelease?: true;
+    }[];
+}
+/** What the user can choose for a site; an empty list is nothing to choose. */
+export interface SiteChoices {
+    /** GitBook: the spaces of a multi-space repo (two or more). */
+    spaces: string[];
+    /** VitePress, Starlight: the languages (two or more). */
+    languages: LanguageOption[];
+    /** Starlight: its products, when it has no languages. */
+    products: ProductOption[];
+    /** Docusaurus: its products and their versions. */
+    docs: DocsProductOption[];
+    /** Files the choices needed and were not given. */
+    unread: string[];
+}
+/**
+ * What the user can choose for the site (the playground's offer), from the paths and the config
+ * files: asked for in rounds (`more`), as `importSite` asks.
+ */
+export declare function siteChoices(format: ImportFormat, files: SiteFiles, paths: Iterable<string>, more?: (paths: string[]) => Promise<void>): Promise<SiteChoices>;
+/**
+ * The files a host gives up front, read as text: pages, config, data and scripts the conversions
+ * read. Anything else (images, fonts, PDFs …) is listed only, and given when asked for.
+ */
+export declare const TEXT_FILE: RegExp;
+/** Text a Dogsbay project holds (what the SSG and the playground read). */
+export declare const PROJECT_TEXT_EXTENSIONS: readonly string[];
+/** Images a Dogsbay project holds, as bytes. */
+export declare const PROJECT_IMAGE_EXTENSIONS: readonly string[];
+export declare const isProjectText: (path: string) => boolean;
+export declare const isProjectImage: (path: string) => boolean;
+/**
+ * The files of an import's result a project keeps, as every host keeps them: text as text, images
+ * as bytes; anything else (a PDF, a font) is left out. The playground's store holds only these, so
+ * a host that wrote more would give a different project.
+ */
+export declare function projectFiles(files: SiteFiles): SiteFiles;
+/**
+ * A site's build output and caches: never given up front (a host lists them; a conversion that
+ * asks for one still gets it). Generated bundles there can be hundreds of MB.
+ */
+export declare const BUILD_OUTPUT: RegExp;
+/** The TypeScript examples converted in one import, at most (characters): Cloudflare's whole site has about 2 MB of them. */
+export declare const EXAMPLES_BYTES: number;
+/**
+ * Starlight's `typescriptToJs`, each example converted once whatever the round (an import runs the
+ * conversion again each round), and at most `maxChars` of examples in all: past it an example keeps
+ * its TypeScript only (the conversion takes a failed one so, and says so). A repo's examples are its
+ * own text: a hostile one must not keep the host formatting for minutes.
+ */
+export declare function cappedTypescriptToJs(convert: (ts: string) => Promise<string>, maxChars?: number): (ts: string) => Promise<string>;
+/**
+ * The Starlight adapter whose site this is (paths from the site's folder, `read` its text): Cloudflare's,
+ * when an `astro.config.*` names developers.cloudflare.com as its `site`. As the playground offers it.
+ */
+export declare function starlightAdapterFor(paths: Iterable<string>, read: (path: string) => string | undefined): string | undefined;

package/dist/index.js

@@ -0,0 +1,204 @@
+import { gitbookSpaces } from "@dogsbay/detect";
+/** The formats this import converts. */
+export const IMPORT_FORMATS = ["gitbook", "vitepress", "docusaurus", "starlight"];
+export const isImportFormat = (f) => IMPORT_FORMATS.includes(f);
+export const DEFAULT_ROUNDS = 5;
+/**
+ * The site as a Dogsbay project: the format's conversion with the user's choices, run again with
+ * the files it asks for while there are any (`rounds` at most), then the assets it copies placed
+ * where it says (`copyTo`) without converting again. Files the host could not give are left out of
+ * `paths`, so the conversion goes on without them, and are returned in `unread`.
+ */
+export async function importSite(input) {
+    const run = await runner(input);
+    const all = input.paths ? [...input.paths] : undefined;
+    const gone = new Set();
+    const paths = () => all?.filter((p) => !gone.has(p));
+    const rounds = Math.max(0, input.rounds ?? DEFAULT_ROUNDS);
+    let result = await run(paths());
+    for (let round = 0; round < rounds && result.unfetched.read.length && input.more; round++) {
+        const wanted = result.unfetched.read;
+        await input.more(wanted, false);
+        const missing = wanted.filter((p) => !input.files.has(p) && !gone.has(p));
+        for (const p of missing)
+            gone.add(p);
+        // Nothing new given and nothing newly left out: another run would ask the same
+        if (wanted.every((p) => !input.files.has(p)) && !missing.length)
+            break;
+        result = await run(paths());
+    }
+    const { copyTo } = result.unfetched;
+    if (copyTo.length && input.more) {
+        await input.more([...new Set(copyTo.map((c) => c.from))], true);
+        for (const { from, to } of copyTo) {
+            const data = input.files.get(from);
+            if (data !== undefined)
+                result.files.set(to, data);
+        }
+    }
+    const unread = [...new Set([...gone, ...result.unfetched.read])].sort();
+    return { files: result.files, ledger: result.ledger, unread };
+}
+/** The format's conversion with the input's choices, as a function of the paths listed. */
+async function runner(input) {
+    const { files, choice = {}, siteName, siteUrl, sourceLabel } = input;
+    const common = { files, ...(siteName !== undefined ? { siteName } : {}), ...(siteUrl ? { siteUrl } : {}), ...(sourceLabel !== undefined ? { sourceLabel } : {}) };
+    const withPaths = (paths) => (paths ? { paths } : {});
+    switch (input.format) {
+        case "gitbook": {
+            const { migrate } = await import("@dogsbay/format-gitbook/migrate");
+            return (paths) => migrate({ ...common, ...withPaths(paths), ...(choice.space ? { choose: { space: choice.space } } : {}) });
+        }
+        case "vitepress": {
+            const { migrate } = await import("@dogsbay/format-vitepress/migrate");
+            return (paths) => migrate({ ...common, ...withPaths(paths), ...(choice.languages?.length ? { languages: choice.languages } : {}) });
+        }
+        case "docusaurus": {
+            const { migrate, migrateSelection } = await import("@dogsbay/format-docusaurus/migrate");
+            const shared = { ...(input.docusaurusAdapter ? { adapter: input.docusaurusAdapter } : {}), ...(choice.openapi ? { openapi: choice.openapi } : {}) };
+            const select = choice.select;
+            if (select?.length) {
+                // Several products in the site: a products: block even for one ticked, so another can be added later
+                const layout = input.productCount && input.productCount > 1 ? { layout: "products" } : {};
+                return (paths) => migrateSelection({ ...common, ...withPaths(paths), ...shared, select, ...layout });
+            }
+            const choose = choice.instance || choice.version ? { choose: { ...(choice.instance ? { instance: choice.instance } : {}), ...(choice.version ? { version: choice.version } : {}) } } : {};
+            return (paths) => migrate({ ...common, ...withPaths(paths), ...shared, ...choose });
+        }
+        case "starlight": {
+            const { migrate } = await import("@dogsbay/format-starlight/migrate");
+            // A site with languages is chosen by language, never by product too (the conversion refuses both)
+            const products = choice.products?.length ? { products: choice.products } : {};
+            const languages = !choice.products?.length && choice.languages?.length ? { languages: choice.languages } : {};
+            return (paths) => migrate({
+                ...common,
+                ...withPaths(paths),
+                ...products,
+                ...languages,
+                ...(choice.adapter ? { adapter: choice.adapter } : {}),
+                ...(input.typescriptToJs ? { typescriptToJs: input.typescriptToJs } : {}),
+            });
+        }
+    }
+}
+/**
+ * What the user can choose for the site (the playground's offer), from the paths and the config
+ * files: asked for in rounds (`more`), as `importSite` asks.
+ */
+export async function siteChoices(format, files, paths, more) {
+    const all = [...paths];
+    const out = { spaces: [], languages: [], products: [], docs: [], unread: [] };
+    /** `choose` again with the files it asks for (`read`), while there are new ones. */
+    const rounds = async (choose) => {
+        let got = choose();
+        const tried = new Set();
+        for (let round = 0; round < DEFAULT_ROUNDS && more && got.read.some((p) => !tried.has(p)); round++) {
+            const wanted = got.read.filter((p) => !tried.has(p));
+            for (const p of wanted)
+                tried.add(p);
+            await more(wanted);
+            got = choose();
+        }
+        out.unread = got.read.filter((p) => !files.has(p));
+        return got;
+    };
+    if (format === "gitbook") {
+        const spaces = gitbookSpaces(all);
+        if (spaces.length > 1)
+            out.spaces = spaces;
+    }
+    else if (format === "vitepress") {
+        const { importChoices } = await import("@dogsbay/format-vitepress/migrate");
+        const got = await rounds(() => importChoices(files, all));
+        if (got.languages.length > 1)
+            out.languages = got.languages.map(({ id, label, isDefault, pages }) => ({ id, label, isDefault, pages }));
+    }
+    else if (format === "starlight") {
+        const { importChoices } = await import("@dogsbay/format-starlight/migrate");
+        const got = await rounds(() => importChoices(files, all));
+        if (got.languages.length > 1)
+            out.languages = got.languages.map(({ id, label, isDefault, pages }) => ({ id, label, isDefault, pages }));
+        else
+            out.products = got.products.map(({ id, label, pages }) => ({ id, label, pages }));
+    }
+    else {
+        const { importChoices } = await import("@dogsbay/format-docusaurus/migrate");
+        out.docs = importChoices(files, all).products.map((p) => ({
+            name: p.name,
+            instance: p.instance,
+            label: p.label,
+            versions: p.versions.map((v) => ({ id: v.id, name: v.name, ...(v.label ? { label: v.label } : {}), isDefault: v.isDefault, ...(v.prerelease ? { prerelease: true } : {}) })),
+        }));
+    }
+    return out;
+}
+/**
+ * The files a host gives up front, read as text: pages, config, data and scripts the conversions
+ * read. Anything else (images, fonts, PDFs …) is listed only, and given when asked for.
+ */
+export const TEXT_FILE = /\.(md|mdx|markdown|ya?ml|json|jsonc|json5|[cm]?[jt]sx?|vue|astro|toml|txt|csv|tsv|html?|css|svg)$/i;
+/** Text a Dogsbay project holds (what the SSG and the playground read). */
+export const PROJECT_TEXT_EXTENSIONS = [".md", ".yml", ".yaml", ".json", ".css", ".txt", ".csv", ".tsv"];
+/** Images a Dogsbay project holds, as bytes. */
+export const PROJECT_IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg", ".ico"];
+const extOf = (path) => {
+    const name = path.slice(path.lastIndexOf("/") + 1).toLowerCase();
+    const dot = name.lastIndexOf(".");
+    return dot > 0 ? name.slice(dot) : "";
+};
+export const isProjectText = (path) => PROJECT_TEXT_EXTENSIONS.includes(extOf(path));
+export const isProjectImage = (path) => PROJECT_IMAGE_EXTENSIONS.includes(extOf(path));
+/**
+ * The files of an import's result a project keeps, as every host keeps them: text as text, images
+ * as bytes; anything else (a PDF, a font) is left out. The playground's store holds only these, so
+ * a host that wrote more would give a different project.
+ */
+export function projectFiles(files) {
+    const out = new Map();
+    for (const [path, value] of files) {
+        if (isProjectText(path))
+            out.set(path, typeof value === "string" ? value : new TextDecoder().decode(value));
+        else if (isProjectImage(path))
+            out.set(path, typeof value === "string" ? new TextEncoder().encode(value) : value);
+    }
+    return new Map([...out].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+}
+/**
+ * A site's build output and caches: never given up front (a host lists them; a conversion that
+ * asks for one still gets it). Generated bundles there can be hundreds of MB.
+ */
+export const BUILD_OUTPUT = /(^|\/)(\.vitepress\/(cache|dist)|\.docusaurus|\.astro|\.cache|build|dist|\.next|\.nuxt|\.output)\//;
+/** The TypeScript examples converted in one import, at most (characters): Cloudflare's whole site has about 2 MB of them. */
+export const EXAMPLES_BYTES = 8 * 1024 * 1024;
+/**
+ * Starlight's `typescriptToJs`, each example converted once whatever the round (an import runs the
+ * conversion again each round), and at most `maxChars` of examples in all: past it an example keeps
+ * its TypeScript only (the conversion takes a failed one so, and says so). A repo's examples are its
+ * own text: a hostile one must not keep the host formatting for minutes.
+ */
+export function cappedTypescriptToJs(convert, maxChars = EXAMPLES_BYTES) {
+    const converted = new Map();
+    let chars = 0;
+    return (ts) => {
+        let done = converted.get(ts);
+        if (!done) {
+            chars += ts.length;
+            done = chars > maxChars ? Promise.reject(new Error(`over the ${maxChars / 1024 / 1024} MB of TypeScript examples converted in one import`)) : convert(ts);
+            converted.set(ts, done);
+        }
+        return done;
+    };
+}
+/** A Starlight site's Astro config, at the site's folder. */
+const ASTRO_CONFIG = /^astro\.config\.(js|ts|mjs|mts)$/;
+/**
+ * The Starlight adapter whose site this is (paths from the site's folder, `read` its text): Cloudflare's,
+ * when an `astro.config.*` names developers.cloudflare.com as its `site`. As the playground offers it.
+ */
+export function starlightAdapterFor(paths, read) {
+    for (const p of paths) {
+        if (ASTRO_CONFIG.test(p) && /\bsite:\s*["'`]https?:\/\/developers\.cloudflare\.com\/?["'`]/.test(read(p) ?? ""))
+            return "cloudflare";
+    }
+    return undefined;
+}

package/package.json

@@ -1,6 +1,42 @@
 {
   "name": "@dogsbay/import",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0-beta.163",
+  "description": "A site made with another tool (GitBook, VitePress, Docusaurus, Starlight) as a Dogsbay project: the one import the playground and `dogsbay import` run, over a map of files, asking the host for what it needs in rounds. No Node.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "@dogsbay/detect": "0.2.0-beta.163",
+    "@dogsbay/format-gitbook": "0.2.0-beta.163",
+    "@dogsbay/format-starlight": "0.2.0-beta.163",
+    "@dogsbay/format-docusaurus": "0.2.0-beta.163",
+    "@dogsbay/format-vitepress": "0.2.0-beta.163",
+    "@dogsbay/types": "0.2.0-beta.163"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.1",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.10"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dogsbay/dogsbay.git",
+    "directory": "packages/import"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
 }