@dogsbay/corpus 0.0.0-stage → 0.2.0-beta.133

package/README.md

@@ -1,3 +1,26 @@
-# Temporary Holding Version
+# @dogsbay/corpus
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A documentation repository as the Dogsbay importers read it: its file paths (posix, relative
+to the corpus root) and its text on demand. The same importer then runs in the CLI and in a
+browser.
+
+```ts
+import { corpusFromMap } from "@dogsbay/corpus";        // browser: a map of files
+import { readFolder } from "@dogsbay/corpus/folder";    // Node: a folder
+
+const corpus = corpusFromMap(new Map([["docs/index.md", "# Home\n"]]));
+corpus.paths;                  // ["docs/index.md"]
+corpus.text("docs/index.md");  // "# Home\n"
+```
+
+A corpus is someone else's repository:
+
+- only plain paths enter it: no `..`, `.`, empty segment, leading `/`, backslash or control
+  character (`isPlainPath` from `@dogsbay/detect`);
+- a folder corpus keeps to its folder once symlinks resolve, lists each real directory once,
+  skips `node_modules` and `.git`, and answers `contains(path, dir)`: does `path` still lie
+  in `dir` once links resolve? An importer uses it to keep pages and assets in the docs root.
+
+## License
+
+MIT

package/dist/folder.d.ts

@@ -0,0 +1,14 @@
+import type { Corpus } from "./index.js";
+export interface FolderCorpus extends Corpus {
+    /** The folder's absolute path. */
+    root: string;
+    /** A corpus path as an absolute path on disk. */
+    absolute(path: string): string;
+}
+/**
+ * List `dir` (skipping `node_modules` and `.git`) and read files from it on demand. A
+ * directory reached by two names is listed under the first one walked, so `first` names the
+ * folders to walk before the rest (the docs roots): a `docs -> content-src` link must list the
+ * pages under `docs/`, where `root: ./docs/` looks for them.
+ */
+export declare function readFolder(dir: string, maxDepth?: number, first?: string[]): FolderCorpus;

package/dist/folder.js

@@ -0,0 +1,121 @@
+/**
+ * Node only: a folder as a {@link Corpus}. Everything else in an importer reads the corpus, so
+ * the CLI and the browser import run the same code.
+ *
+ * The walk keeps to the folder once links resolve, and visits each directory once: a symlink
+ * out of the repo read pages from elsewhere, and two links to `.` made the walk branch twice
+ * per level until ELOOP (about 2^40 visits). Text is read only when asked for.
+ */
+import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { join, sep } from "node:path";
+import { isPlainPath } from "@dogsbay/detect";
+/**
+ * List `dir` (skipping `node_modules` and `.git`) and read files from it on demand. A
+ * directory reached by two names is listed under the first one walked, so `first` names the
+ * folders to walk before the rest (the docs roots): a `docs -> content-src` link must list the
+ * pages under `docs/`, where `root: ./docs/` looks for them.
+ */
+export function readFolder(dir, maxDepth = Infinity, first = []) {
+    const paths = [];
+    /** Each listed path's real path, for `contains`. */
+    const reals = new Map();
+    let realRoot;
+    try {
+        realRoot = realpathSync(dir);
+    }
+    catch {
+        realRoot = "";
+    }
+    if (realRoot) {
+        const seen = new Set([realRoot]);
+        const walk = (abs, rel, depth) => {
+            let entries;
+            try {
+                entries = readdirSync(abs).sort();
+            }
+            catch {
+                return;
+            }
+            for (const name of entries) {
+                if (name === "node_modules" || name === ".git")
+                    continue;
+                const full = join(abs, name);
+                const path = rel ? `${rel}/${name}` : name;
+                if (!isPlainPath(path))
+                    continue;
+                let real;
+                let st;
+                try {
+                    real = realpathSync(full);
+                    if (real !== realRoot && !real.startsWith(realRoot + sep))
+                        continue;
+                    st = statSync(real);
+                }
+                catch {
+                    continue;
+                }
+                if (st.isDirectory()) {
+                    if (seen.has(real) || depth + 1 > maxDepth)
+                        continue;
+                    seen.add(real);
+                    walk(full, path, depth + 1);
+                }
+                else if (st.isFile()) {
+                    paths.push(path);
+                    reals.set(path, real);
+                }
+            }
+        };
+        for (const d of first) {
+            if (!d || !isPlainPath(d))
+                continue;
+            let real;
+            try {
+                real = realpathSync(join(dir, d));
+                if (!real.startsWith(realRoot + sep) || seen.has(real) || !statSync(real).isDirectory())
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            seen.add(real);
+            walk(join(dir, d), d, d.split("/").length);
+        }
+        walk(dir, "", 0);
+    }
+    const listed = new Set(paths);
+    const realDirs = new Map();
+    const realDir = (d) => {
+        if (!realDirs.has(d)) {
+            let r;
+            try {
+                r = realpathSync(join(dir, d));
+            }
+            catch {
+                r = undefined;
+            }
+            realDirs.set(d, r);
+        }
+        return realDirs.get(d);
+    };
+    return {
+        root: dir,
+        paths,
+        absolute: (path) => join(dir, path),
+        contains(path, d) {
+            const real = reals.get(path);
+            const root = realDir(d);
+            return !!real && !!root && (real === root || real.startsWith(root + sep));
+        },
+        text(path) {
+            if (!listed.has(path))
+                return undefined;
+            try {
+                return readFileSync(join(dir, path), "utf-8");
+            }
+            catch {
+                return undefined;
+            }
+        },
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,48 @@
+export interface Corpus {
+    /** Every file, posix, relative to the corpus root; plain paths only (`isPlainPath`). */
+    paths: string[];
+    /** The file's text, or undefined when it is not in the corpus. */
+    text(path: string): string | undefined;
+    /**
+     * Does `path` still lie inside the folder `dir` once links resolve? A folder corpus answers
+     * (a symlink in the docs root must not bring in the repo's `.env` or `.git`); a map of files
+     * has no links and leaves it out.
+     */
+    contains?(path: string, dir: string): boolean;
+}
+export type CorpusFiles = Map<string, string | Uint8Array>;
+export interface MapCorpusOptions {
+    /**
+     * Every path of the repo (relative to the same root as `files`), when the host knows more
+     * files than it has fetched — the playground lists a GitHub tree and fetches as it goes.
+     * The corpus lists them all, so the import sees the repo as it is; reading one that `files`
+     * does not hold returns undefined and is recorded (`missing`).
+     */
+    paths?: Iterable<string>;
+}
+export interface MapCorpus extends Corpus {
+    /** Listed paths the import read but `files` did not hold, sorted: fetch them and run it again. */
+    missing(): string[];
+}
+/**
+ * A corpus over a map of files; bytes are read as UTF-8. A key that is not a plain path
+ * (`../x`, `/etc/x`, `a/./b`, a backslash) is left out: the keys come from someone else's repo.
+ */
+export declare function corpusFromMap(files: CorpusFiles, opts?: MapCorpusOptions): MapCorpus;
+/**
+ * What a browser conversion still needs, when the host listed more paths than it fetched:
+ * `read` the import read (fetch them and convert again: the result changes), `copy` the
+ * output copies as they are (fetch them for the finished files; nothing is converted again).
+ * Both empty: the result is complete. Each round can find one more level of an include or
+ * `require` chain, so a host caps the rounds; a result with `read` not empty is not a
+ * conversion (its `files` may be empty: the import could not go on without them).
+ */
+export interface Unfetched {
+    read: string[];
+    copy: string[];
+}
+/**
+ * The ledger message for a conversion that still needs files (undefined when none), with the
+ * error the import stopped on, when it stopped.
+ */
+export declare function unfetchedMessage(read: string[], stoppedOn?: unknown): string | undefined;

package/dist/index.js

@@ -0,0 +1,45 @@
+/**
+ * @dogsbay/corpus — a corpus as an importer sees it: its file paths (posix, relative to the
+ * corpus root) and a way to read one as text. The CLI builds one from a folder
+ * (`@dogsbay/corpus/folder`, which also keeps symlinks inside it); the browser builds one from
+ * a map of files (`corpusFromMap`). Assets are never read through it: a project lists them as
+ * copies, and each host copies the bytes its own way. No Node here.
+ *
+ * A corpus is someone else's repository: only plain paths enter it (`isPlainPath`).
+ */
+import { isPlainPath } from "@dogsbay/detect";
+/**
+ * A corpus over a map of files; bytes are read as UTF-8. A key that is not a plain path
+ * (`../x`, `/etc/x`, `a/./b`, a backslash) is left out: the keys come from someone else's repo.
+ */
+export function corpusFromMap(files, opts = {}) {
+    const decoder = new TextDecoder();
+    const paths = [...new Set([...files.keys(), ...(opts.paths ?? [])])].filter(isPlainPath).sort();
+    const listed = new Set(paths);
+    const missing = new Set();
+    return {
+        paths,
+        text(path) {
+            if (!listed.has(path))
+                return undefined;
+            const v = files.get(path);
+            if (v === undefined) {
+                missing.add(path);
+                return undefined;
+            }
+            return typeof v === "string" ? v : decoder.decode(v);
+        },
+        missing: () => [...missing].sort(),
+    };
+}
+/**
+ * The ledger message for a conversion that still needs files (undefined when none), with the
+ * error the import stopped on, when it stopped.
+ */
+export function unfetchedMessage(read, stoppedOn) {
+    if (!read.length)
+        return undefined;
+    const names = `${read.slice(0, 3).join(", ")}${read.length > 3 ? ", …" : ""}`;
+    const stopped = stoppedOn instanceof Error ? ` It stopped on: ${stoppedOn.message}` : "";
+    return `${read.length} file${read.length === 1 ? "" : "s"} the conversion reads ${read.length === 1 ? "was" : "were"} not given (${names}): fetch them and convert again.${stopped}`;
+}

package/package.json

@@ -1,6 +1,42 @@
 {
   "name": "@dogsbay/corpus",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0-beta.133",
+  "description": "A documentation repository as its importers read it, in the browser or in Node: plain paths plus text on demand, from a map of files or from a folder whose symlinks are kept inside it",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./folder": {
+      "types": "./dist/folder.d.ts",
+      "import": "./dist/folder.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "@dogsbay/detect": "0.2.0-beta.133"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.1",
+    "esbuild": "^0.25.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.10"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dogsbay/dogsbay.git",
+    "directory": "packages/corpus"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
 }