@dogsbay/format-gitbook 0.2.0-beta.131 → 0.2.0-beta.133

package/dist/corpus.d.ts

@@ -1,18 +1,2 @@
-export interface Corpus {
-    /** Every file, posix, relative to the corpus root; plain paths only (`isPlainPath`). */
-    paths: string[];
-    /** The file's text, or undefined when it is not in the corpus. */
-    text(path: string): string | undefined;
-    /**
-     * Does `path` still lie inside the folder `dir` once links resolve? A folder corpus answers
-     * (a symlink in the docs root must not bring in the repo's `.env` or `.git`); a map of files
-     * has no links and leaves it out.
-     */
-    contains?(path: string, dir: string): boolean;
-}
-export type CorpusFiles = Map<string, string | Uint8Array>;
-/**
- * A corpus over a map of files; bytes are read as UTF-8. A key that is not a plain path
- * (`../x`, `/etc/x`, `a/./b`, a backslash) is left out: the keys come from someone else's repo.
- */
-export declare function corpusFromMap(files: CorpusFiles): Corpus;
+/** The corpus an importer reads: `@dogsbay/corpus`, shared by every format's browser import. */
+export { corpusFromMap, type Corpus, type CorpusFiles, type Unfetched, unfetchedMessage } from "@dogsbay/corpus";

package/dist/corpus.js

@@ -1,26 +1,2 @@
-/**
- * A corpus as the importer sees it: its file paths (posix, relative to the corpus root) and a
- * way to read one as text. The CLI builds one from a folder (`loaders/folder.ts`, which also
- * keeps symlinks inside it); the browser builds one from a map of files (`corpusFromMap`).
- * Assets are never read through it: the project lists them as copies, and each host copies
- * the bytes its own way.
- */
-import { isPlainPath } from "@dogsbay/detect";
-/**
- * A corpus over a map of files; bytes are read as UTF-8. A key that is not a plain path
- * (`../x`, `/etc/x`, `a/./b`, a backslash) is left out: the keys come from someone else's repo.
- */
-export function corpusFromMap(files) {
-    const decoder = new TextDecoder();
-    const paths = [...files.keys()].filter(isPlainPath).sort();
-    const listed = new Set(paths);
-    return {
-        paths,
-        text(path) {
-            if (!listed.has(path))
-                return undefined;
-            const v = files.get(path);
-            return v === undefined ? undefined : typeof v === "string" ? v : decoder.decode(v);
-        },
-    };
-}
+/** The corpus an importer reads: `@dogsbay/corpus`, shared by every format's browser import. */
+export { corpusFromMap, unfetchedMessage } from "@dogsbay/corpus";

package/dist/loaders/folder.d.ts

@@ -1,19 +1,7 @@
-import type { Corpus } from "../corpus.js";
+import { readFolder, type FolderCorpus } from "@dogsbay/corpus/folder";
 import { type GitbookConfig } from "./config.js";
 import { type GitbookLoadOptions, type GitbookLoadResult } from "./load.js";
-export interface FolderCorpus extends Corpus {
-    /** The folder's absolute path. */
-    root: string;
-    /** A corpus path as an absolute path on disk. */
-    absolute(path: string): string;
-}
-/**
- * List `dir` (skipping `node_modules` and `.git`) and read files from it on demand. A
- * directory reached by two names is listed under the first one walked, so `first` names the
- * folders to walk before the rest (the docs roots): a `docs -> content-src` link must list the
- * pages under `docs/`, where `root: ./docs/` looks for them.
- */
-export declare function readFolder(dir: string, maxDepth?: number, first?: string[]): FolderCorpus;
+export { readFolder, type FolderCorpus };
 /** Read `.gitbook.yaml` from `dir` if present; else return defaults. */
 export declare function readGitbookConfig(dir: string): GitbookConfig;
 /**

package/dist/loaders/folder.js

@@ -1,126 +1,15 @@
 /**
- * Node only: a folder as a {@link Corpus}, and the CLI's on-disk detector. Everything else in
- * the loader reads the corpus, so the CLI and the browser import run the same code.
- *
- * The walk keeps to the folder once links resolve, and visits each directory once: a symlink
- * out of the repo read pages from elsewhere, and two links to `.` made the walk branch twice
- * per level until ELOOP (about 2^40 visits). Text is read only when asked for.
+ * Node only: a GitBook folder as a corpus (`@dogsbay/corpus/folder`, its docs roots walked
+ * first), and the CLI's on-disk detector. Everything else in the loader reads the corpus, so
+ * the CLI and the browser import run the same code.
  */
-import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
-import { join, sep } from "node:path";
-import { detectGitbook as detectGitbookPaths, gitbookSpaces, isPlainPath, joinPath, normalizePath } from "@dogsbay/detect";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { detectGitbook as detectGitbookPaths, gitbookSpaces, joinPath, normalizePath } from "@dogsbay/detect";
+import { readFolder } from "@dogsbay/corpus/folder";
 import { parseGitbookConfig } from "./config.js";
 import { loadGitbookCorpus } from "./load.js";
-/**
- * List `dir` (skipping `node_modules` and `.git`) and read files from it on demand. A
- * directory reached by two names is listed under the first one walked, so `first` names the
- * folders to walk before the rest (the docs roots): a `docs -> content-src` link must list the
- * pages under `docs/`, where `root: ./docs/` looks for them.
- */
-export function readFolder(dir, maxDepth = Infinity, first = []) {
-    const paths = [];
-    /** Each listed path's real path, for `contains`. */
-    const reals = new Map();
-    let realRoot;
-    try {
-        realRoot = realpathSync(dir);
-    }
-    catch {
-        realRoot = "";
-    }
-    if (realRoot) {
-        const seen = new Set([realRoot]);
-        const walk = (abs, rel, depth) => {
-            let entries;
-            try {
-                entries = readdirSync(abs).sort();
-            }
-            catch {
-                return;
-            }
-            for (const name of entries) {
-                if (name === "node_modules" || name === ".git")
-                    continue;
-                const full = join(abs, name);
-                const path = rel ? `${rel}/${name}` : name;
-                if (!isPlainPath(path))
-                    continue;
-                let real;
-                let st;
-                try {
-                    real = realpathSync(full);
-                    if (real !== realRoot && !real.startsWith(realRoot + sep))
-                        continue;
-                    st = statSync(real);
-                }
-                catch {
-                    continue;
-                }
-                if (st.isDirectory()) {
-                    if (seen.has(real) || depth + 1 > maxDepth)
-                        continue;
-                    seen.add(real);
-                    walk(full, path, depth + 1);
-                }
-                else if (st.isFile()) {
-                    paths.push(path);
-                    reals.set(path, real);
-                }
-            }
-        };
-        for (const d of first) {
-            if (!d || !isPlainPath(d))
-                continue;
-            let real;
-            try {
-                real = realpathSync(join(dir, d));
-                if (!real.startsWith(realRoot + sep) || seen.has(real) || !statSync(real).isDirectory())
-                    continue;
-            }
-            catch {
-                continue;
-            }
-            seen.add(real);
-            walk(join(dir, d), d, d.split("/").length);
-        }
-        walk(dir, "", 0);
-    }
-    const listed = new Set(paths);
-    const realDirs = new Map();
-    const realDir = (d) => {
-        if (!realDirs.has(d)) {
-            let r;
-            try {
-                r = realpathSync(join(dir, d));
-            }
-            catch {
-                r = undefined;
-            }
-            realDirs.set(d, r);
-        }
-        return realDirs.get(d);
-    };
-    return {
-        root: dir,
-        paths,
-        absolute: (path) => join(dir, path),
-        contains(path, d) {
-            const real = reals.get(path);
-            const root = realDir(d);
-            return !!real && !!root && (real === root || real.startsWith(root + sep));
-        },
-        text(path) {
-            if (!listed.has(path))
-                return undefined;
-            try {
-                return readFileSync(join(dir, path), "utf-8");
-            }
-            catch {
-                return undefined;
-            }
-        },
-    };
-}
+export { readFolder };
 /** Read `.gitbook.yaml` from `dir` if present; else return defaults. */
 export function readGitbookConfig(dir) {
     const p = join(dir, ".gitbook.yaml");

package/dist/migrate.d.ts

@@ -5,11 +5,17 @@
  * no console, clock or network. Find the corpus with `@dogsbay/detect` first.
  */
 import type { ImportLedgerEntry } from "@dogsbay/types";
-import { type CorpusFiles } from "./corpus.js";
-export type { CorpusFiles } from "./corpus.js";
+import { type CorpusFiles, type Unfetched } from "./corpus.js";
+export type { CorpusFiles, Unfetched } from "./corpus.js";
 export interface MigrateInput {
     /** The corpus: paths relative to its root (a detection's `root`), text or bytes. */
     files: CorpusFiles;
+    /**
+     * Every path of the project (relative to the same root as `files`), when the host lists more
+     * than it fetched (a GitHub tree). The import then sees the repo as it is, and `unfetched`
+     * says what it needed from those not given: fetch them and call `migrate` again.
+     */
+    paths?: Iterable<string>;
     /** One space of a multi-space repo (a detection's `choices.spaces`); all when omitted. */
     choose?: {
         space?: string;
@@ -28,5 +34,14 @@ export interface MigrateResult {
         from: string;
         to: string;
     }[];
+    /**
+     * The listed `paths` the conversion needed but `files` did not hold. `read`: it read them,
+     * so fetch them and convert again (each round can find one more level of an include or
+     * `require` chain: cap the rounds). While `read` is not empty the result is not a
+     * conversion; when the import could not go on, `files` is empty and the ledger says where
+     * it stopped. `copy`: assets the output copies; fetch them for the finished files, no
+     * rerun. Both empty: the result is complete.
+     */
+    unfetched: Unfetched;
 }
 export declare function migrate(input: MigrateInput): Promise<MigrateResult>;

package/dist/migrate.js

@@ -1,19 +1,38 @@
-import { corpusFromMap } from "./corpus.js";
+import { corpusFromMap, unfetchedMessage } from "./corpus.js";
 import { importGitbookCorpus } from "./import.js";
 import { gitbookProject } from "./project.js";
+/** The ledger line for a conversion that still needs files, or nothing. */
+function unfetchedLedger(read, stoppedOn) {
+    const message = unfetchedMessage(read, stoppedOn);
+    return message ? [{ severity: "warning", code: "unfetched", message }] : [];
+}
 export async function migrate(input) {
-    const corpus = corpusFromMap(input.files);
-    const imported = importGitbookCorpus(corpus, { space: input.choose?.space });
-    const project = gitbookProject(imported, corpus, {
-        siteName: input.siteName?.trim() || "Docs",
-        siteUrl: input.siteUrl,
-        sourceLabel: input.sourceLabel ?? "GitBook corpus",
-    });
+    const corpus = corpusFromMap(input.files, { paths: input.paths });
+    let project;
+    try {
+        const imported = importGitbookCorpus(corpus, { space: input.choose?.space });
+        project = gitbookProject(imported, corpus, {
+            siteName: input.siteName?.trim() || "Docs",
+            siteUrl: input.siteUrl,
+            sourceLabel: input.sourceLabel ?? "GitBook corpus",
+        });
+    }
+    catch (e) {
+        // A file it needed was not given: that, not the error, is the answer
+        const read = corpus.missing();
+        if (read.length)
+            return { files: new Map(), ledger: unfetchedLedger(read, e), redirects: [], unfetched: { read, copy: [] } };
+        throw e;
+    }
     const files = new Map(project.files);
+    const copy = [];
     for (const { from, to } of project.assets) {
         const bytes = input.files.get(from);
         if (bytes !== undefined)
             files.set(to, bytes);
+        else
+            copy.push(from);
     }
-    return { files, ledger: project.ledger, redirects: project.redirects };
+    const read = corpus.missing();
+    return { files, ledger: [...unfetchedLedger(read), ...project.ledger], redirects: project.redirects, unfetched: { read, copy: copy.sort() } };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogsbay/format-gitbook",
-  "version": "0.2.0-beta.131",
+  "version": "0.2.0-beta.133",
   "description": "GitBook format plugin for Dogsbay — imports GitBook Git Sync (and legacy gitbook-cli) corpora by translating the Git Sync markdown dialect to Dogsbay-MD. ./migrate converts a corpus given as a map of files in the browser (no Node), the same import the CLI runs.",
   "type": "module",
   "main": "dist/index.js",
@@ -25,9 +25,10 @@
   ],
   "dependencies": {
     "js-yaml": "^5.2.1",
-    "@dogsbay/detect": "0.2.0-beta.131",
-    "@dogsbay/format-dogsbay-md": "0.2.0-beta.131",
-    "@dogsbay/types": "0.2.0-beta.131"
+    "@dogsbay/corpus": "0.2.0-beta.133",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.133",
+    "@dogsbay/detect": "0.2.0-beta.133",
+    "@dogsbay/types": "0.2.0-beta.133"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",