@microsoft/rayfin-docs 1.27.0

package/dist/site-cjs/discovery.js

@@ -0,0 +1,409 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultTrust = void 0;
+exports.hasExplicitDiscoverySource = hasExplicitDiscoverySource;
+exports.discoverRayfinDocsPackages = discoverRayfinDocsPackages;
+exports.validateRayfinDocsManifest = validateRayfinDocsManifest;
+exports.docKindToModule = docKindToModule;
+/**
+ * Per-package docs discovery - Phase 2 of `docs-package-architecture`.
+ *
+ * Walks a project's `node_modules` to find packages declaring the
+ * `rayfinDocs` field in their `package.json`. Returns a list of
+ * discovered package roots so {@link DocsService} can load each tree's
+ * markdown alongside any explicit `assetsRoot`.
+ *
+ * **Trust model.** Discovery is conservative by default: only
+ * `@microsoft/rayfin-*` scoped packages are auto-trusted. Anything else
+ * requires the caller to pass an explicit `trust` predicate. This
+ * matches the spec-review resolution that broad walking is too trusting
+ * (see `openspec/changes/docs-package-architecture/spec-review-findings.md`,
+ * finding B3).
+ *
+ * Discovery is intentionally explicit: callers choose either a fixed
+ * candidate list, workspace package roots, or all-installed node_modules
+ * walking via `scanInstalledPackages: true`.
+ */
+const fs_1 = require("fs");
+const module_1 = require("module");
+const path_1 = require("path");
+const url_1 = require("url");
+/** Default trust predicate. Auto-trusts the Microsoft Rayfin namespace. */
+const defaultTrust = (packageName) => packageName.startsWith('@microsoft/rayfin-');
+exports.defaultTrust = defaultTrust;
+/**
+ * Returns whether callers already selected a discovery source.
+ * Wrappers use this to decide whether to inject their own bounded defaults.
+ */
+function hasExplicitDiscoverySource(options) {
+    return countDiscoverySources(options) > 0;
+}
+function discoverRayfinDocsPackages(options) {
+    const trust = options.trust ?? exports.defaultTrust;
+    const fromDir = options.from.startsWith('file:') || options.from.includes('://')
+        ? (0, path_1.dirname)((0, url_1.fileURLToPath)(options.from))
+        : resolveExistingDir(options.from);
+    const requireFromBase = (0, module_1.createRequire)((0, path_1.join)(fromDir, 'noop.js'));
+    const nodeModulesChain = buildNodeModulesChain(fromDir);
+    const discovered = [];
+    const notInstalled = [];
+    const untrustedSkipped = [];
+    const invalidManifest = [];
+    const sourceCount = countDiscoverySources(options);
+    if (sourceCount === 0) {
+        throw new Error('discoverRayfinDocsPackages requires an explicit source: pass `candidates`, `packageRoots`, or `scanInstalledPackages: true`.');
+    }
+    if (sourceCount > 1) {
+        throw new Error('discoverRayfinDocsPackages requires exactly one explicit source: pass only one of `candidates`, `packageRoots`, or `scanInstalledPackages: true`.');
+    }
+    let packagesToRead;
+    if (options.packageRoots !== undefined) {
+        packagesToRead = resolvePackageRootJsons(options.packageRoots);
+    }
+    else if (options.candidates !== undefined) {
+        packagesToRead = resolveCandidatePackageJsons(options.candidates, requireFromBase, nodeModulesChain, notInstalled, untrustedSkipped, trust);
+    }
+    else if (options.scanInstalledPackages) {
+        packagesToRead = discoverInstalledPackageJsons(nodeModulesChain);
+    }
+    else {
+        throw new Error('Invariant violation: explicit discovery source count did not match selected source.');
+    }
+    for (const { packageName: candidate, pkgJsonPath } of packagesToRead) {
+        let pkgJson;
+        try {
+            pkgJson = JSON.parse((0, fs_1.readFileSync)(pkgJsonPath, 'utf8'));
+        }
+        catch (err) {
+            invalidManifest.push({
+                packageName: candidate,
+                reason: `unreadable package.json: ${err.message}`,
+            });
+            continue;
+        }
+        const parsedName = pkgJson?.name;
+        const packageName = typeof parsedName === 'string' ? parsedName : candidate;
+        const hasRayfinDocs = typeof pkgJson === 'object' &&
+            pkgJson !== null &&
+            'rayfinDocs' in pkgJson;
+        if (!hasRayfinDocs) {
+            continue;
+        }
+        const parsedVersion = pkgJson?.version;
+        if (typeof parsedVersion !== 'string' || parsedVersion.length === 0) {
+            invalidManifest.push({
+                packageName,
+                reason: 'package.json must declare a non-empty string version when rayfinDocs is present',
+            });
+            continue;
+        }
+        if (!trust(packageName)) {
+            untrustedSkipped.push(packageName);
+            continue;
+        }
+        const validation = validateRayfinDocsManifest(pkgJson, candidate);
+        if (!validation.ok) {
+            if (validation.kind === 'missing') {
+                // Package is installed but doesn't declare rayfinDocs. This is
+                // not an error - it's the common case for any package that
+                // hasn't migrated yet. Don't surface as invalid.
+                continue;
+            }
+            invalidManifest.push({
+                packageName: candidate,
+                reason: validation.reason,
+            });
+            continue;
+        }
+        const packageRoot = (0, path_1.dirname)(pkgJsonPath);
+        discovered.push({
+            packageName,
+            packageVersion: parsedVersion,
+            packageRoot,
+            manifest: validation.manifest,
+        });
+    }
+    return { discovered, notInstalled, untrustedSkipped, invalidManifest };
+}
+function countDiscoverySources(options) {
+    return [
+        options.packageRoots !== undefined,
+        options.candidates !== undefined,
+        options.scanInstalledPackages === true,
+    ].filter(Boolean).length;
+}
+function resolvePackageRootJsons(packageRoots) {
+    return packageRoots.map((packageRoot) => ({
+        packageName: packageRoot,
+        pkgJsonPath: (0, path_1.join)(resolveExistingDir(packageRoot), 'package.json'),
+    }));
+}
+function resolveCandidatePackageJsons(candidates, requireFromBase, nodeModulesChain, notInstalled, untrustedSkipped, trust) {
+    const resolved = [];
+    for (const candidate of candidates) {
+        if (!trust(candidate)) {
+            untrustedSkipped.push(candidate);
+            continue;
+        }
+        const pkgJsonPath = resolvePackageJson(requireFromBase, candidate, nodeModulesChain);
+        if (!pkgJsonPath) {
+            notInstalled.push(candidate);
+            continue;
+        }
+        resolved.push({ packageName: candidate, pkgJsonPath });
+    }
+    return resolved;
+}
+function discoverInstalledPackageJsons(nodeModulesChain) {
+    const discovered = [];
+    const seen = new Set();
+    for (const dir of nodeModulesChain) {
+        const nodeModules = (0, path_1.join)(dir, 'node_modules');
+        if (!(0, fs_1.existsSync)(nodeModules) || !(0, fs_1.statSync)(nodeModules).isDirectory()) {
+            continue;
+        }
+        for (const entry of (0, fs_1.readdirSync)(nodeModules, { withFileTypes: true })) {
+            if ((!entry.isDirectory() && !entry.isSymbolicLink()) ||
+                entry.name.startsWith('.')) {
+                continue;
+            }
+            if (entry.name.startsWith('@')) {
+                const scopeRoot = (0, path_1.join)(nodeModules, entry.name);
+                for (const scoped of (0, fs_1.readdirSync)(scopeRoot, { withFileTypes: true })) {
+                    if (!scoped.isDirectory() && !scoped.isSymbolicLink())
+                        continue;
+                    const packageName = `${entry.name}/${scoped.name}`;
+                    const pkgJsonPath = (0, path_1.join)(scopeRoot, scoped.name, 'package.json');
+                    if (!seen.has(packageName) && (0, fs_1.existsSync)(pkgJsonPath)) {
+                        seen.add(packageName);
+                        discovered.push({ packageName, pkgJsonPath });
+                    }
+                }
+            }
+            else {
+                const packageName = entry.name;
+                const pkgJsonPath = (0, path_1.join)(nodeModules, entry.name, 'package.json');
+                if (!seen.has(packageName) && (0, fs_1.existsSync)(pkgJsonPath)) {
+                    seen.add(packageName);
+                    discovered.push({ packageName, pkgJsonPath });
+                }
+            }
+        }
+    }
+    return discovered;
+}
+/**
+ * Validate the `rayfinDocs` field on a parsed `package.json`. Returns
+ * a discriminated result so callers can distinguish "no manifest"
+ * (silently skip) from "broken manifest" (surface as warning).
+ *
+ * Path validation rejects absolute paths and parent-directory
+ * traversal in `dir` - discovered packages must keep their docs
+ * inside their own root.
+ */
+function validateRayfinDocsManifest(pkgJson, packageName) {
+    if (!pkgJson || typeof pkgJson !== 'object') {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: 'package.json is not an object',
+        };
+    }
+    const rec = pkgJson;
+    const raw = rec['rayfinDocs'];
+    if (raw === undefined) {
+        return { ok: false, kind: 'missing' };
+    }
+    if (!raw || typeof raw !== 'object') {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs` field is not an object',
+        };
+    }
+    const m = raw;
+    if (m['version'] !== 1) {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: `unsupported rayfinDocs.version (got ${JSON.stringify(m['version'])}, expected 1); skipping ${packageName}`,
+        };
+    }
+    if (m['dir'] !== undefined && typeof m['dir'] !== 'string') {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs.dir` must be a non-empty string',
+        };
+    }
+    if (m['dir'] === '') {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs.dir` must be a non-empty string',
+        };
+    }
+    const dir = m['dir'] ?? 'assets/docs';
+    if ((0, path_1.isAbsolute)(dir)) {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs.dir` must be a relative path inside the package root',
+        };
+    }
+    // Reject `..` segments anywhere in the relative path. Segment-based
+    // check uniformly handles `..`, `./..`, `docs/..`, `a/../b`, and
+    // mixed separators.
+    const normalized = dir.replace(/\\/g, '/');
+    if (normalized.split('/').some((segment) => segment === '..')) {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs.dir` must not traverse outside the package root (no `..` segments)',
+        };
+    }
+    if (typeof m['module'] !== 'string' || m['module'].length === 0) {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: '`rayfinDocs.module` must be a non-empty string',
+        };
+    }
+    if (m['kind'] !== 'guide' &&
+        m['kind'] !== 'host' &&
+        m['kind'] !== 'api-reference') {
+        return {
+            ok: false,
+            kind: 'invalid',
+            reason: `\`rayfinDocs.kind\` must be one of "guide" | "host" | "api-reference" (got ${JSON.stringify(m['kind'])}). ` +
+                `If you need a new kind, request it in the rayfin-docs repo: ` +
+                `https://github.com/microsoft/project-rayfin/issues/new`,
+        };
+    }
+    return {
+        ok: true,
+        manifest: {
+            version: 1,
+            dir,
+            module: m['module'],
+            kind: m['kind'],
+        },
+    };
+}
+/**
+ * Map a `DocKind` to the legacy `DocModule` value used throughout the
+ * existing CLI/MCP surfaces. Phase 2 keeps the `DocModule` union for
+ * back-compat; `--module ts-sdk` continues to mean "all api-reference
+ * docs" for end users.
+ */
+function docKindToModule(kind) {
+    switch (kind) {
+        case 'guide':
+            return 'guide';
+        case 'host':
+            return 'host';
+        case 'api-reference':
+            return 'ts-sdk';
+    }
+}
+/**
+ * Compute the chain of `node_modules` ancestor directories from
+ * `fromDir` upward, capped at 16 levels. Used by the tier-3 fallback
+ * in {@link resolvePackageJson} to avoid re-walking the same ancestors
+ * once per candidate (`9 candidates × 16 walks = 144 stat calls` in
+ * the original implementation; this caches the chain to 16 stats).
+ */
+function buildNodeModulesChain(fromDir) {
+    const chain = [];
+    let dir = fromDir;
+    for (let i = 0; i < 16; i += 1) {
+        chain.push(dir);
+        const parent = (0, path_1.dirname)(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return chain;
+}
+/**
+ * Find a package's `package.json` path even when the package's
+ * `exports` field doesn't expose it as a subpath OR when the package
+ * is ESM-only (`exports."."` is `{ "import": ... }` without a CJS
+ * `default`/`require` branch).
+ *
+ * SDK packages with restricted `exports` typically don't list
+ * `./package.json` and so `require.resolve('<pkg>/package.json')`
+ * throws `ERR_PACKAGE_PATH_NOT_EXPORTED`. ESM-only packages additionally
+ * fail `require.resolve('<pkg>')` (CJS resolution can't see the
+ * `import:` branch). We finally fall back to walking the node_modules
+ * tree from the resolution base - this matches what every npm/pnpm
+ * tool does internally.
+ *
+ * Returns the absolute path to `package.json`, or `undefined` when the
+ * package isn't installed at all.
+ */
+function resolvePackageJson(req, packageName, nodeModulesChain) {
+    // Fast path: works when the package omits `exports` or explicitly
+    // includes `./package.json` in it.
+    try {
+        return req.resolve(`${packageName}/package.json`);
+    }
+    catch (err) {
+        const code = err?.code;
+        if (code === 'MODULE_NOT_FOUND') {
+            // Package isn't installed.
+            return undefined;
+        }
+        // ERR_PACKAGE_PATH_NOT_EXPORTED or similar: keep going.
+    }
+    // Second fall-back: resolve the main entry, then walk up directories
+    // searching for a package.json whose name matches.
+    try {
+        const entry = req.resolve(packageName);
+        let dir = (0, path_1.dirname)(entry);
+        for (let i = 0; i < 16; i += 1) {
+            const candidate = (0, path_1.join)(dir, 'package.json');
+            if ((0, fs_1.existsSync)(candidate)) {
+                try {
+                    const parsed = JSON.parse((0, fs_1.readFileSync)(candidate, 'utf8'));
+                    if (parsed.name === packageName) {
+                        return candidate;
+                    }
+                }
+                catch {
+                    // keep walking
+                }
+            }
+            const parent = (0, path_1.dirname)(dir);
+            if (parent === dir)
+                break;
+            dir = parent;
+        }
+    }
+    catch {
+        // ESM-only package or similar: fall through to the node_modules walk.
+    }
+    // Third fall-back: walk the precomputed `node_modules` ancestor
+    // chain looking for `<ancestor>/node_modules/<packageName>/package.json`.
+    // This is what npm/pnpm/yarn tooling does internally and works
+    // regardless of how the package's `exports` are configured.
+    for (const dir of nodeModulesChain) {
+        const candidate = (0, path_1.join)(dir, 'node_modules', packageName, 'package.json');
+        if ((0, fs_1.existsSync)(candidate)) {
+            return candidate;
+        }
+    }
+    return undefined;
+}
+/** Internal: ensure `from` exists for `createRequire` when given a dir. */
+function resolveExistingDir(from) {
+    const absolute = (0, path_1.resolve)(from);
+    if (!(0, fs_1.existsSync)(absolute)) {
+        throw new Error(`discoverRayfinDocsPackages: \`from\` path does not exist: ${absolute}`);
+    }
+    const st = (0, fs_1.statSync)(absolute);
+    if (!st.isDirectory()) {
+        throw new Error(`discoverRayfinDocsPackages: \`from\` must be a directory: ${absolute}`);
+    }
+    return absolute;
+}

package/dist/site-cjs/package.json

@@ -0,0 +1 @@
+{ "type": "commonjs" }\n

package/dist/site-cjs/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types.d.ts

@@ -0,0 +1,89 @@
+export type DocModule = 'guide' | 'host' | 'ts-sdk';
+export type DocKind = 'guide' | 'host' | 'api-reference';
+export type DocSearchScope = 'docs' | 'symbols' | 'all';
+/**
+ * `rayfinDocs` package.json field declaring where a package's docs live
+ * and what kind they are. Read by the discovery walker; the manifest
+ * is per-package and points at a single doc tree.
+ */
+export interface RayfinDocsManifest {
+    version: 1;
+    /** Path to the docs root, relative to package.json's directory. */
+    dir: string;
+    /** Stable, human-readable module identity. Conventionally the
+     *  unscoped package name (e.g. `rayfin-core`, `rayfin-guide`,
+     *  `rayfin-host`). */
+    module: string;
+    /** Doc kind. Maps to the legacy `DocModule` for back-compat:
+     *  - `guide` → `module: guide`
+     *  - `host` → `module: host`
+     *  - `api-reference` → `module: ts-sdk` */
+    kind: DocKind;
+}
+export interface DocSection {
+    heading: string;
+    level: number;
+    content: string;
+    symbols: string[];
+}
+export interface DocEntry {
+    id: string;
+    module: DocModule;
+    path: string;
+    title: string;
+    content: string;
+    symbols: string[];
+    sections: DocSection[];
+    /** When this entry came from a discovered package (Phase 2+), the
+     *  identifying package metadata. `undefined` for entries loaded from
+     *  the bundled `assetsRoot` (Phase 1 mode). */
+    source?: DocSource;
+}
+export interface DocSource {
+    /** Manifest module identity (e.g. `rayfin-core`). */
+    module: string;
+    /** Doc kind from the manifest. */
+    kind: DocKind;
+    /** Package name from the manifest's `package.json` (e.g.
+     *  `@microsoft/rayfin-core`). The package's filesystem path is
+     *  intentionally NOT exposed in this public type — it leaks the
+     *  user's home directory into MCP/CLI output. Internal callers that
+     *  need the absolute path use the `DiscoveredPackage.packageRoot`
+     *  field on the discovery report instead. */
+    packageName: string;
+    /** Version from the source package's `package.json`. */
+    packageVersion: string;
+}
+export interface DocListItem {
+    id: string;
+    module: DocModule;
+    path: string;
+    title: string;
+    source?: DocSource;
+}
+export interface DocSearchResult {
+    id: string;
+    module: DocModule;
+    path: string;
+    title: string;
+    heading?: string;
+    symbol?: string;
+    kind?: 'doc' | 'symbol';
+    snippet: string;
+    snippetStart: number;
+    snippetEnd: number;
+    score: number;
+    symbols: string[];
+    source?: DocSource;
+}
+export interface DocSectionRef {
+    symbol: string;
+    entryId: string;
+    module: DocModule;
+    path: string;
+    title: string;
+    heading: string;
+    content: string;
+    source?: DocSource;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@microsoft/rayfin-docs",
+  "version": "1.27.0",
+  "description": "Rayfin docs indexing and search library — used by the MCP server, CLI, and other doc-facing surfaces",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./_internal/build-index": {
+      "types": "./dist/_internal/build-index.d.ts",
+      "import": "./dist/_internal/build-index.js"
+    },
+    "./_internal/site-discovery": {
+      "types": "./dist/discovery.d.ts",
+      "import": "./dist/discovery.js",
+      "require": "./dist/site-cjs/discovery.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.json",
+    "!dist/cjs/**",
+    "!dist/**/__tests__/**",
+    "assets/catalog.json",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "minisearch": "~7.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.10",
+    "eslint": "^9.28.0",
+    "prettier": "^3.5.3",
+    "rimraf": "~6.0.1",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.3",
+    "@vitest/coverage-v8": "~3.2.4"
+  },
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com",
+    "access": "restricted"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/project-rayfin.git",
+    "directory": "packages/tools/docs-lib"
+  },
+  "license": "MIT",
+  "scripts": {
+    "build": "tsc && tsc -p tsconfig.site-cjs.json && node scripts/write-site-cjs-package-json.mjs",
+    "build:watch": "tsc --watch",
+    "clean": "rimraf dist .tsbuildinfo .tsbuildinfo.site-cjs",
+    "test": "vitest --run",
+    "test:watch": "vitest"
+  }
+}