docspack 0.0.1 → 0.1.1

package/src/http.ts

@@ -0,0 +1,190 @@
+import { DocspackError } from "./errors.js";
+
+export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
+
+export interface HttpResponse {
+  /** Final URL after redirects, when the runtime reports one. */
+  readonly url: string;
+  readonly contentType: string;
+  readonly text: string;
+  readonly bytes: number;
+}
+
+export interface GetOptions {
+  readonly maxBytes?: number;
+  readonly headers?: Readonly<Record<string, string>>;
+}
+
+export interface HttpClientOptions {
+  readonly fetchImpl?: FetchLike;
+  readonly timeoutMs?: number;
+  readonly maxBytes?: number;
+  readonly retries?: number;
+  readonly userAgent?: string;
+}
+
+const RETRY_DELAYS_MS = [250, 1000];
+const DEFAULT_ACCEPT = "text/markdown, text/plain, text/html;q=0.8, */*;q=0.1";
+
+/** Signals a transport-level failure worth retrying. */
+class RetryableHttpError extends Error {}
+
+const delay = (ms: number): Promise<void> => new Promise((done) => setTimeout(done, ms));
+
+/** Parses a URL and rejects anything that is not http(s), including `file:` and `data:`. */
+export function assertHttpUrl(url: string): URL {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new DocspackError(`"${url}" is not a valid URL`);
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new DocspackError(`Refusing to fetch "${url}": only http and https URLs are supported`);
+  }
+  return parsed;
+}
+
+/** Small HTTP client with byte caps, timeouts and retries. The fetch implementation is injectable. */
+export class HttpClient {
+  readonly #fetch: FetchLike;
+  readonly #timeoutMs: number;
+  readonly #maxBytes: number;
+  readonly #retries: number;
+  readonly #userAgent: string;
+
+  constructor(options: HttpClientOptions = {}) {
+    this.#fetch = options.fetchImpl ?? globalThis.fetch;
+    this.#timeoutMs = options.timeoutMs ?? 30_000;
+    this.#maxBytes = options.maxBytes ?? 2_000_000;
+    this.#retries = options.retries ?? 2;
+    this.#userAgent = options.userAgent ?? "docspack";
+  }
+
+  async get(url: string, options: GetOptions = {}): Promise<HttpResponse> {
+    const maxBytes = options.maxBytes ?? this.#maxBytes;
+    let lastError: unknown;
+
+    for (let attempt = 0; attempt <= this.#retries; attempt += 1) {
+      if (attempt > 0) await delay(RETRY_DELAYS_MS[attempt - 1] ?? 1000);
+      try {
+        return await this.#getOnce(url, maxBytes, options.headers);
+      } catch (error) {
+        if (!(error instanceof RetryableHttpError)) throw error;
+        lastError = error;
+      }
+    }
+
+    throw new DocspackError(
+      `Could not fetch ${url}: ${lastError instanceof Error ? lastError.message : "unknown error"}`,
+      {
+        hint: "Check your network connection and whether the documentation host is reachable.",
+        cause: lastError,
+      },
+    );
+  }
+
+  /** True when a HEAD request succeeds. Used to probe for optional documents. */
+  async exists(url: string, headers?: Readonly<Record<string, string>>): Promise<boolean> {
+    try {
+      const parsed = assertHttpUrl(url);
+      const response = await this.#fetch(parsed.toString(), {
+        method: "HEAD",
+        redirect: "follow",
+        signal: AbortSignal.timeout(this.#timeoutMs),
+        headers: { "user-agent": this.#userAgent, ...headers },
+      });
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  async #getOnce(
+    url: string,
+    maxBytes: number,
+    headers: Readonly<Record<string, string>> | undefined,
+  ): Promise<HttpResponse> {
+    const parsed = assertHttpUrl(url);
+
+    let response: Response;
+    try {
+      response = await this.#fetch(parsed.toString(), {
+        redirect: "follow",
+        signal: AbortSignal.timeout(this.#timeoutMs),
+        headers: { "user-agent": this.#userAgent, accept: DEFAULT_ACCEPT, ...headers },
+      });
+    } catch (error) {
+      throw new RetryableHttpError(error instanceof Error ? error.message : String(error));
+    }
+
+    if (!response.ok) {
+      const detail = `${response.status} ${response.statusText}`.trim();
+      if (response.status === 429 || response.status >= 500) {
+        throw new RetryableHttpError(detail);
+      }
+      throw new DocspackError(
+        `Could not fetch ${url}: ${detail}`,
+        response.status === 404
+          ? {
+              hint: "The document does not exist at that URL. Check the source or pick another one.",
+            }
+          : {},
+      );
+    }
+
+    // A redirect must not be able to move the request onto a non-http scheme.
+    if (response.url.length > 0) assertHttpUrl(response.url);
+
+    const declared = Number(response.headers.get("content-length") ?? Number.NaN);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+      throw new DocspackError(`${url} is ${declared} bytes, over the ${maxBytes} byte limit`, {
+        hint: "Raise the limit with --max-bytes, or use a smaller documentation source.",
+      });
+    }
+
+    const { text, bytes } = await this.#readCapped(response, url, maxBytes);
+
+    return {
+      url: response.url.length > 0 ? response.url : parsed.toString(),
+      contentType: response.headers.get("content-type") ?? "",
+      text: text.replace(/\r\n/g, "\n"),
+      bytes,
+    };
+  }
+
+  async #readCapped(
+    response: Response,
+    url: string,
+    maxBytes: number,
+  ): Promise<{ text: string; bytes: number }> {
+    const body = response.body;
+    if (body === null) {
+      const text = await response.text();
+      const bytes = Buffer.byteLength(text, "utf8");
+      if (bytes > maxBytes) throw this.#tooLarge(url, maxBytes);
+      return { text, bytes };
+    }
+
+    const reader = body.getReader();
+    const chunks: Uint8Array[] = [];
+    let bytes = 0;
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > maxBytes) {
+        await reader.cancel();
+        throw this.#tooLarge(url, maxBytes);
+      }
+      chunks.push(value);
+    }
+    return { text: Buffer.concat(chunks).toString("utf8"), bytes };
+  }
+
+  #tooLarge(url: string, maxBytes: number): DocspackError {
+    return new DocspackError(`${url} exceeds the ${maxBytes} byte limit`, {
+      hint: "Raise the limit with --max-bytes, or use a smaller documentation source.",
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,132 @@
+export { type BuildOptions, type BuildResult, buildPackage } from "./build.js";
+export {
+  type BuildConfig,
+  CONFIG_KEY,
+  type FeedbackChannel,
+  parseBuildConfig,
+  parseFeedbackChannel,
+  readBuildConfig,
+} from "./config.js";
+export {
+  defaultStorePath,
+  type IndexedChunk,
+  type IndexedPackage,
+  type SearchHit,
+  type SearchOptions,
+  Store,
+  silenceSqliteWarning,
+  toFtsQuery,
+} from "./db.js";
+export {
+  type DiscoveredPackage,
+  type Discovery,
+  discoverPackages,
+  projectPackageIds,
+  resolvePackageDir,
+} from "./discovery.js";
+export {
+  type DoctorOptions,
+  type DoctorReport,
+  type Finding,
+  runDoctor,
+  type Severity,
+} from "./doctor.js";
+export { DocspackError } from "./errors.js";
+export { type ExportSurface, readExportSurface } from "./exports.js";
+export {
+  type AddOptions,
+  type AddResult,
+  addFinding,
+  FEEDBACK_DIR,
+  FEEDBACK_FILE,
+  type FeedbackFinding,
+  feedbackPath,
+  fingerprintOf,
+  type RemoveOptions,
+  readFeedback,
+  removeFindings,
+} from "./feedback.js";
+export { type FetchLike, HttpClient } from "./http.js";
+export { type Detected, detectProject, type PackageManager } from "./init/detect.js";
+export {
+  type InitMode,
+  type InitOptions,
+  type InitPlan,
+  type InitTemplate,
+  type InputKind,
+  type PlannedFile,
+  planInit,
+  proposePackageName,
+} from "./init/plan.js";
+export { type InitResult, type RunInitOptions, runInit } from "./init/run.js";
+export {
+  type ApplyOptions,
+  applyPlan,
+  renderTree,
+  type WriteStatus,
+  type WrittenFile,
+} from "./init/write.js";
+export { type FindingKind, isFindingKind, KINDS } from "./kinds.js";
+export {
+  fetchableLinks,
+  type LlmsTxtDocument,
+  type LlmsTxtLink,
+  type LlmsTxtSection,
+  parseLlmsTxt,
+} from "./llms-txt.js";
+export { createMcpServer, type McpOptions, startMcpServer } from "./mcp.js";
+export { type PreviewOptions, type PreviewResult, previewPackage } from "./preview.js";
+export { type Choice, defaultStreams, Prompter, type PromptStreams } from "./prompt.js";
+export {
+  DEFAULT_LIMIT,
+  DEFAULT_MAX_TOKENS,
+  type QueryHit,
+  type QueryOptions,
+  type QueryResult,
+  queryDocs,
+  renderAnswer,
+  splitPackageId,
+} from "./search.js";
+export { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
+export {
+  CHUNKS_DIR,
+  type ChunkSpec,
+  chunkId,
+  estimateTokens,
+  isCommunityPackage,
+  isDocsPackage,
+  isVendorPackage,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  packageId,
+  parseManifest,
+  resolveChunkFile,
+  SCHEMA_URL,
+  serializeManifest,
+} from "./spec.js";
+export {
+  type BlockedFinding,
+  type BlockedReason,
+  issueUrl,
+  prepareSubmission,
+  type RoutedFinding,
+  renderBody,
+  type SubmitOptions,
+  type SubmitReport,
+} from "./submit.js";
+export {
+  type SyncedPackage,
+  type SyncOptions,
+  type SyncResult,
+  type SyncStatus,
+  syncProject,
+} from "./sync.js";
+export {
+  type DriftFinding,
+  type VerifiedPackage,
+  type VerifyOptions,
+  type VerifyReport,
+  type VerifyStatus,
+  verifyProject,
+} from "./verify.js";

package/src/init/detect.ts

@@ -0,0 +1,142 @@
+import type { Dirent } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+
+export type PackageManager = "pnpm" | "npm" | "yarn" | "bun";
+
+/** What `init` learned about the project it was run in. Everything is a proposal, never a decision. */
+export interface Detected {
+  readonly libraryName?: string;
+  readonly libraryVersion?: string;
+  readonly docsDir?: string;
+  readonly docsFiles?: number;
+  readonly openapi?: string;
+  readonly llmsTxt?: string;
+  readonly repository?: string;
+  readonly license?: string;
+  readonly packageManager: PackageManager;
+  readonly workspace: boolean;
+}
+
+const DOC_DIRS = ["docs", "documentation", "website/docs", "content/docs", "src/content/docs"];
+const OPENAPI_FILES = [
+  "openapi.json",
+  "api/openapi.json",
+  "spec/openapi.json",
+  "docs/openapi.json",
+];
+const LLMS_TXT_FILES = ["llms.txt", "public/llms.txt", "static/llms.txt"];
+const MARKDOWN = /\.(md|mdx|markdown)$/i;
+
+export async function detectProject(cwd: string): Promise<Detected> {
+  const manifest = await readJson(join(cwd, "package.json"));
+  const name = typeof manifest?.name === "string" ? manifest.name : undefined;
+  const version = typeof manifest?.version === "string" ? manifest.version : undefined;
+  const license = typeof manifest?.license === "string" ? manifest.license : undefined;
+
+  const docs = await findDocsDir(cwd);
+  const openapi = await findOpenApi(cwd);
+  const llmsTxt = await findFirstFile(cwd, LLMS_TXT_FILES);
+  const repository = await gitRemote(cwd);
+
+  return {
+    ...(name === undefined ? {} : { libraryName: name }),
+    ...(version === undefined ? {} : { libraryVersion: version }),
+    ...(docs === undefined ? {} : { docsDir: docs.dir, docsFiles: docs.files }),
+    ...(openapi === undefined ? {} : { openapi }),
+    ...(llmsTxt === undefined ? {} : { llmsTxt }),
+    ...(repository === undefined ? {} : { repository }),
+    ...(license === undefined ? {} : { license: await detectLicense(cwd, license) }),
+    packageManager: await detectPackageManager(cwd),
+    workspace:
+      manifest?.workspaces !== undefined || (await exists(join(cwd, "pnpm-workspace.yaml"))),
+  };
+}
+
+async function findDocsDir(cwd: string): Promise<{ dir: string; files: number } | undefined> {
+  for (const candidate of DOC_DIRS) {
+    const dir = join(cwd, candidate);
+    const files = await countMarkdown(dir);
+    if (files > 0) return { dir: candidate, files };
+  }
+  return undefined;
+}
+
+async function countMarkdown(dir: string): Promise<number> {
+  let found: Dirent[];
+  try {
+    found = await readdir(dir, { recursive: true, withFileTypes: true });
+  } catch {
+    return 0;
+  }
+  return found.filter((entry) => entry.isFile() && MARKDOWN.test(entry.name)).length;
+}
+
+/** Only counts as an OpenAPI document if it actually declares itself as one. */
+async function findOpenApi(cwd: string): Promise<string | undefined> {
+  for (const candidate of OPENAPI_FILES) {
+    const parsed = await readJson(join(cwd, candidate));
+    if (parsed === undefined) continue;
+    if (typeof parsed.openapi === "string" || typeof parsed.swagger === "string") return candidate;
+  }
+  return undefined;
+}
+
+async function findFirstFile(
+  cwd: string,
+  candidates: readonly string[],
+): Promise<string | undefined> {
+  for (const candidate of candidates) {
+    if (await exists(join(cwd, candidate))) return candidate;
+  }
+  return undefined;
+}
+
+async function detectPackageManager(cwd: string): Promise<PackageManager> {
+  if (await exists(join(cwd, "pnpm-lock.yaml"))) return "pnpm";
+  if ((await exists(join(cwd, "bun.lock"))) || (await exists(join(cwd, "bun.lockb")))) return "bun";
+  if (await exists(join(cwd, "yarn.lock"))) return "yarn";
+  return "npm";
+}
+
+async function detectLicense(cwd: string, declared: string): Promise<string> {
+  return declared.length > 0 || (await exists(join(cwd, "LICENSE"))) ? declared : declared;
+}
+
+/** Reads the origin URL straight out of .git/config rather than spawning git. */
+async function gitRemote(cwd: string): Promise<string | undefined> {
+  let config: string;
+  try {
+    config = await readFile(join(cwd, ".git", "config"), "utf8");
+  } catch {
+    return undefined;
+  }
+
+  const section = /\[remote "origin"\]([\s\S]*?)(?:\n\[|$)/.exec(config);
+  const url = section?.[1] === undefined ? undefined : /url\s*=\s*(\S+)/.exec(section[1])?.[1];
+  if (url === undefined) return undefined;
+
+  const ssh = /^git@([^:]+):(.+?)(?:\.git)?$/.exec(url);
+  if (ssh?.[1] !== undefined && ssh[2] !== undefined) return `https://${ssh[1]}/${ssh[2]}`;
+  return url.replace(/\.git$/, "");
+}
+
+async function readJson(path: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    return typeof parsed === "object" && parsed !== null
+      ? (parsed as Record<string, unknown>)
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/init/plan.ts

@@ -0,0 +1,215 @@
+import { posix } from "node:path";
+import { DocspackError } from "../errors.js";
+import { isCommunityPackage, isDocsPackage } from "../spec.js";
+import type { Detected } from "./detect.js";
+import { docSeeds, gitignore, packageJson, readme, workflow } from "./templates.js";
+
+export type InitMode = "standalone" | "in-repo";
+export type InitTemplate = "full" | "minimal";
+export type InputKind = "from" | "openapi" | "mirror" | "seeded";
+
+export interface InitOptions {
+  readonly name?: string;
+  readonly version?: string;
+  readonly out?: string;
+  readonly mode?: InitMode;
+  readonly from?: string;
+  readonly openapi?: string;
+  readonly mirror?: string;
+  readonly community?: boolean;
+  readonly workflow?: boolean;
+  readonly template?: InitTemplate;
+  /** Version of docspack recorded as the package's devDependency. */
+  readonly docspackVersion: string;
+}
+
+export interface PlannedFile {
+  /** POSIX-style path relative to the package directory. */
+  readonly path: string;
+  readonly contents: string;
+}
+
+export interface InitPlan {
+  readonly name: string;
+  readonly version: string;
+  /** Package directory, relative to where init was run. */
+  readonly dir: string;
+  readonly input: { readonly kind: InputKind; readonly value: string };
+  readonly files: readonly PlannedFile[];
+  readonly notes: readonly string[];
+}
+
+const SLUG = /[^a-z0-9-]+/g;
+
+/**
+ * Computes the whole scaffold from options and what was detected. Pure on purpose: `--dry-run`
+ * prints exactly what a real run would write, and tests can assert the file set without a disk.
+ */
+export function planInit(options: InitOptions, detected: Detected): InitPlan {
+  const community =
+    options.community === true || (options.name !== undefined && isCommunityPackage(options.name));
+  const name = options.name ?? proposePackageName(detected, community);
+  assertPackageName(name);
+
+  const notes: string[] = [];
+  const version = options.version ?? detected.libraryVersion ?? "0.1.0";
+  if (options.version === undefined && detected.libraryVersion === undefined) {
+    notes.push("No library version was detected, so the package starts at 0.1.0.");
+  } else if (options.version === undefined && detected.libraryVersion !== undefined) {
+    notes.push(
+      `Version ${version} matches ${detected.libraryName ?? "the surrounding package"}; keep them in step on every release.`,
+    );
+  }
+
+  const mode: InitMode = options.mode ?? "standalone";
+  const dir = options.out ?? (mode === "in-repo" ? "packages/docspack" : "docspack");
+  const input = resolveInput(options, detected, dir, notes);
+  const template: InitTemplate = options.template ?? "full";
+
+  const files: PlannedFile[] = [
+    {
+      path: "package.json",
+      contents: packageJson({
+        name,
+        version,
+        ...(detected.libraryName === undefined ? {} : { libraryName: detected.libraryName }),
+        ...(detected.repository === undefined ? {} : { repository: detected.repository }),
+        ...(detected.license === undefined ? {} : { license: detected.license }),
+        build: buildConfigFor(input),
+        docspackVersion: options.docspackVersion,
+      }),
+    },
+    { path: ".gitignore", contents: gitignore() },
+  ];
+
+  if (template === "full") {
+    files.push({
+      path: "README.md",
+      contents: readme(name, detected.libraryName),
+    });
+  }
+
+  if (input.kind === "seeded") {
+    files.push(...docSeeds(detected.libraryName ?? name));
+    notes.push(
+      "docs/ was seeded with three templates. Replace the TODO markers before publishing; `docspack doctor --strict` fails while they remain.",
+    );
+  }
+
+  if (options.workflow === true) {
+    files.push({
+      path: ".github/workflows/publish-docspack.yml",
+      contents: workflow(name, workflowArgs(input)),
+    });
+    notes.push("The release workflow needs an NPM_TOKEN secret with publish rights to the scope.");
+  }
+
+  if (!community) {
+    const scope = name.slice(0, name.indexOf("/"));
+    notes.push(`Publishing to ${scope} requires owning that npm scope.`);
+  }
+
+  return { name, version, dir, input, files, notes };
+}
+
+/** `@acme/sdk` becomes `@acme/docspack`; an unscoped `acme` becomes `@acme/docspack`. */
+export function proposePackageName(detected: Detected, community: boolean): string {
+  const library = detected.libraryName;
+  if (library === undefined) {
+    throw new DocspackError("Could not work out a package name", {
+      hint: "Run inside a project with a package.json, or pass --name @acme/docspack.",
+    });
+  }
+
+  const bare = library.startsWith("@") ? (library.split("/")[1] ?? library.slice(1)) : library;
+  const slug = bare
+    .toLowerCase()
+    .replace(SLUG, "-")
+    .replace(/^-+|-+$/g, "");
+  if (community) return `@docspack-community/${slug}`;
+
+  const scope = library.startsWith("@") ? library.slice(1, library.indexOf("/")) : slug;
+  return `@${scope}/docspack`;
+}
+
+function assertPackageName(name: string): void {
+  if (!isDocsPackage(name)) {
+    throw new DocspackError(`"${name}" is not a name docspack will discover`, {
+      hint: "Use @vendor/docspack for a package you own, or @docspack-community/<name> otherwise.",
+    });
+  }
+}
+
+function resolveInput(
+  options: InitOptions,
+  detected: Detected,
+  dir: string,
+  notes: string[],
+): { kind: InputKind; value: string } {
+  const chosen = [options.from, options.openapi, options.mirror].filter(
+    (value) => value !== undefined,
+  );
+  if (chosen.length > 1) {
+    throw new DocspackError("Choose one input: --from, --openapi or --mirror");
+  }
+
+  if (options.mirror !== undefined) {
+    if (!isCommunityPackage(options.name ?? "")) {
+      notes.push(
+        "A mirror republishes someone else's documentation; publish it under @docspack-community rather than the vendor's scope.",
+      );
+    }
+    return { kind: "mirror", value: options.mirror };
+  }
+  if (options.openapi !== undefined)
+    return { kind: "openapi", value: relativeToPackage(options.openapi, dir) };
+  if (options.from !== undefined)
+    return { kind: "from", value: relativeToPackage(options.from, dir) };
+
+  if (detected.docsDir !== undefined) {
+    notes.push(`Found ${detected.docsFiles} Markdown files in ${detected.docsDir}/.`);
+    return { kind: "from", value: relativeToPackage(detected.docsDir, dir) };
+  }
+  if (detected.openapi !== undefined) {
+    notes.push(`Found an OpenAPI document at ${detected.openapi}.`);
+    return { kind: "openapi", value: relativeToPackage(detected.openapi, dir) };
+  }
+
+  return { kind: "seeded", value: "./docs" };
+}
+
+/**
+ * Build inputs are recorded relative to the package directory, because that is where
+ * `docspack build` runs from.
+ */
+function relativeToPackage(target: string, dir: string): string {
+  if (/^https?:\/\//i.test(target)) return target;
+  const normalized = target.replace(/\\/g, "/").replace(/^\.\//, "");
+  const relative = posix.relative(dir.replace(/\\/g, "/"), normalized);
+  return relative.startsWith(".") ? relative : `./${relative}`;
+}
+
+function buildConfigFor(input: {
+  kind: InputKind;
+  value: string;
+}): Record<string, string | number> {
+  switch (input.kind) {
+    case "openapi":
+      return { openapi: input.value };
+    case "mirror":
+      return { source: input.value };
+    default:
+      return { from: input.value, maxChunkTokens: 800 };
+  }
+}
+
+function workflowArgs(input: { kind: InputKind; value: string }): string {
+  switch (input.kind) {
+    case "openapi":
+      return ` --openapi ${input.value}`;
+    case "mirror":
+      return ` ${input.value}`;
+    default:
+      return ` --from ${input.value}`;
+  }
+}