docspack 0.0.1 → 0.1.0

package/src/style.ts

@@ -0,0 +1,111 @@
+/**
+ * Style rules for documentation written to be read by a model.
+ *
+ * The goal is density, not brevity for its own sake. Stripping prose wholesale measurably hurts
+ * retrieval: the index matches on the words a user would type, and those live in sentences, not
+ * in headings. What is free to remove is filler — narration and marketing that carries neither a
+ * fact nor a word anyone would search for.
+ */
+
+/** Narration and marketing. None of these words appear in a question someone asks an agent. */
+const FILLER = [
+  /\bin this (guide|section|article|tutorial|post|chapter)\b/gi,
+  /\b(we|you)('ll| will) (now |then )?(learn|see|explore|cover|look at|dive into|walk through)\b/gi,
+  /\blet'?s (get started|begin|dive|take a look|explore)\b/gi,
+  /\bas you can see\b/gi,
+  /\bas (we )?(mentioned|discussed|noted) (above|earlier|before)\b/gi,
+  /\b(welcome to|congratulations|don'?t worry|no worries)\b/gi,
+  /\b(simply|just simply|easily) (call|use|add|run|pass|set)\b/gi,
+  /\b(blazing[- ]fast|lightning[- ]fast|seamlessly|effortlessly|cutting[- ]edge|world[- ]class|state[- ]of[- ]the[- ]art)\b/gi,
+  /\b(super|incredibly|extremely|amazingly) (fast|simple|easy|powerful)\b/gi,
+  /\bfirst things first\b/gi,
+];
+
+/** Boilerplate a documentation site adds around the content, never part of it. */
+const BOILERPLATE = [
+  /^\s*\[!\[[^\]]*\]\([^)]*\)\]\([^)]*\)\s*$/, // badge wrapped in a link
+  /^\s*\[!\[[^\]]*\]\([^)]*\)\s*$/, // bare badge
+  /^\s*(was this (page )?helpful\??|edit this page|on this page|table of contents)\s*$/i,
+  /^\s*(previous|next)\s*[:|]\s*\S/i,
+  /^\s*<!--[\s\S]*?-->\s*$/,
+];
+
+const SENTENCE_SPLIT = /(?<=[.!?])\s+/;
+/** A heading or list item starts a new unit, whatever the previous line ended with. */
+const BLOCK_START = /\n(?=\s*(?:[-*+]\s|\d+\.\s|#{1,6}\s|\|))/;
+const LONG_SENTENCE_WORDS = 40;
+
+export interface FillerHit {
+  readonly phrase: string;
+  readonly count: number;
+}
+
+/** Filler phrases found in a document, with how often each occurs. */
+export function findFiller(text: string): FillerHit[] {
+  const prose = withoutCode(text);
+  const found = new Map<string, number>();
+
+  for (const pattern of FILLER) {
+    for (const match of prose.matchAll(pattern)) {
+      const phrase = match[0].toLowerCase().replace(/\s+/g, " ");
+      found.set(phrase, (found.get(phrase) ?? 0) + 1);
+    }
+  }
+
+  return [...found]
+    .map(([phrase, count]) => ({ phrase, count }))
+    .sort((a, b) => b.count - a.count || a.phrase.localeCompare(b.phrase));
+}
+
+/**
+ * Sentences long enough that a reader — human or model — loses the thread. Headings, list items
+ * and paragraphs are separate units: a heading followed by a bulleted list is not one sentence,
+ * however little punctuation it contains.
+ */
+export function findLongSentences(text: string, maxWords = LONG_SENTENCE_WORDS): string[] {
+  return withoutCode(text)
+    .split(/\n{2,}/)
+    .flatMap((block) => block.split(BLOCK_START))
+    .flatMap((block) => block.split(SENTENCE_SPLIT))
+    .map((sentence) => sentence.replace(/\s+/g, " ").trim())
+    .filter((sentence) => sentence.split(" ").filter(Boolean).length > maxWords);
+}
+
+/**
+ * True when a chunk contains something concrete: a code block, an inline identifier, a list or a
+ * table. A chunk with none of those is usually narration about documentation rather than
+ * documentation.
+ */
+export function hasConcreteContent(text: string): boolean {
+  return (
+    /```|~~~/.test(text) ||
+    /`[^`\n]+`/.test(text) ||
+    /^\s*[-*+]\s|^\s*\d+\.\s/m.test(text) ||
+    /^\s*\|.*\|/m.test(text)
+  );
+}
+
+/** Normalized form used to spot two chunks that say the same thing. */
+export function contentFingerprint(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/<!--[\s\S]*?-->/g, "")
+    .replace(/[^a-z0-9]+/g, " ")
+    .trim();
+}
+
+/** Removes lines a documentation site wraps around its content. */
+export function stripBoilerplate(text: string): string {
+  return text
+    .split("\n")
+    .filter((line) => !BOILERPLATE.some((pattern) => pattern.test(line)))
+    .join("\n");
+}
+
+/** Text with fenced and inline code removed, so prose rules never fire on a code sample. */
+export function withoutCode(text: string): string {
+  return text
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/~~~[\s\S]*?~~~/g, " ")
+    .replace(/`[^`\n]*`/g, " ");
+}

package/src/submit.ts

@@ -0,0 +1,189 @@
+import type { FeedbackChannel } from "./config.js";
+import { readBuildConfig } from "./config.js";
+import { discoverPackages } from "./discovery.js";
+import { type FeedbackFinding, readFeedback } from "./feedback.js";
+import { KINDS } from "./kinds.js";
+import { packageId } from "./spec.js";
+
+/**
+ * A finding that a vendor has asked to receive, rendered and ready for a human to file.
+ *
+ * `url` is a prefilled GitHub issue link. docspack does not open it, does not follow it, and
+ * has no code that could: a human clicks it, reads what is there, and files it under their own
+ * name. That is the whole transmission mechanism.
+ */
+export interface RoutedFinding {
+  readonly fingerprint: string;
+  readonly chunkId: string;
+  readonly title: string;
+  readonly body: string;
+  readonly labels: readonly string[];
+  /** Undefined when the rendered report is too long to survive a URL. */
+  readonly url?: string;
+  /** True when the report contains a reproduction, which may quote the user's own code. */
+  readonly containsCode: boolean;
+}
+
+export type BlockedReason = "no-channel" | "not-accepted";
+
+export interface BlockedFinding {
+  readonly fingerprint: string;
+  readonly chunkId: string;
+  readonly kind: string;
+  readonly reason: BlockedReason;
+  /** What the vendor does accept, when they accept something narrower. */
+  readonly accepts?: readonly string[];
+}
+
+export interface SubmitReport {
+  readonly routed: readonly RoutedFinding[];
+  readonly blocked: readonly BlockedFinding[];
+  /** Channels involved, so the caller can show each vendor's policy once. */
+  readonly channels: readonly { readonly packageId: string; readonly channel: FeedbackChannel }[];
+}
+
+export interface SubmitOptions {
+  readonly cwd: string;
+  readonly docspackVersion: string;
+  /** Only findings recorded against this package. */
+  readonly packageFilter?: string;
+}
+
+/**
+ * A GitHub issue URL has to survive a browser address bar, and a reproduction can be long.
+ * Past this the report is printed for the human to paste instead of silently truncated.
+ */
+const MAX_URL = 6000;
+
+/**
+ * Prepares recorded findings for filing. Nothing is sent, and nothing here can send: this
+ * function returns text.
+ */
+export async function prepareSubmission(options: SubmitOptions): Promise<SubmitReport> {
+  const findings = await readFeedback(options.cwd);
+  const channels = await channelsFor(options.cwd);
+
+  const routed: RoutedFinding[] = [];
+  const blocked: BlockedFinding[] = [];
+  const used = new Map<string, FeedbackChannel>();
+
+  for (const finding of findings) {
+    const id = packageId(finding.package, finding.version);
+    if (options.packageFilter !== undefined && !finding.package.includes(options.packageFilter)) {
+      continue;
+    }
+
+    const channel = channels.get(id);
+    if (channel === undefined) {
+      blocked.push({ ...identity(finding), reason: "no-channel" });
+      continue;
+    }
+
+    // An omitted `accepts` means the vendor takes everything; declaring the channel was the
+    // opt-in. Narrowing it is how a vendor drowning in noise turns the volume down.
+    const accepts = channel.accepts ?? KINDS;
+    if (!accepts.includes(finding.kind)) {
+      blocked.push({ ...identity(finding), reason: "not-accepted", accepts });
+      continue;
+    }
+
+    used.set(id, channel);
+    routed.push(render(finding, channel, options.docspackVersion));
+  }
+
+  return {
+    routed,
+    blocked,
+    channels: [...used].map(([id, channel]) => ({ packageId: id, channel })),
+  };
+}
+
+function identity(finding: FeedbackFinding): {
+  fingerprint: string;
+  chunkId: string;
+  kind: string;
+} {
+  return { fingerprint: finding.fingerprint, chunkId: finding.chunkId, kind: finding.kind };
+}
+
+/** The declared channel of every docs package installed here, by package id. */
+async function channelsFor(cwd: string): Promise<Map<string, FeedbackChannel>> {
+  const { packages } = await discoverPackages(cwd);
+  const channels = new Map<string, FeedbackChannel>();
+
+  for (const pkg of packages) {
+    // A package whose config cannot be read simply has no channel; a broken vendor manifest
+    // must not stop a human from reviewing the rest of their findings.
+    const config = await readBuildConfig(pkg.dir).catch(() => ({ feedback: undefined }));
+    if (config.feedback !== undefined) channels.set(pkg.id, config.feedback);
+  }
+  return channels;
+}
+
+function render(
+  finding: FeedbackFinding,
+  channel: FeedbackChannel,
+  docspackVersion: string,
+): RoutedFinding {
+  const title = `docspack: ${finding.evidence}`.slice(0, 120);
+  const body = renderBody(finding, docspackVersion);
+  const labels = channel.labels ?? [];
+
+  const url = issueUrl(channel.github, title, body, labels);
+  return {
+    fingerprint: finding.fingerprint,
+    chunkId: finding.chunkId,
+    title,
+    body,
+    labels,
+    ...(url === undefined ? {} : { url }),
+    containsCode: finding.repro !== undefined,
+  };
+}
+
+/**
+ * The report a maintainer reads. Provenance is stated rather than hidden: maintainers have
+ * asked repeatedly to be able to tell machine-written reports apart, and concealing it is
+ * what makes them toxic.
+ */
+export function renderBody(finding: FeedbackFinding, docspackVersion: string): string {
+  const lines = [
+    `**Chunk**: \`${finding.chunkId}\``,
+    `**Kind**: ${finding.kind}${finding.kind === "drift" ? " — a name the docs use that the package does not declare" : ""}`,
+    `**Seen**: ${finding.seen} ${finding.seen === 1 ? "time" : "times"} in one project`,
+    "",
+    finding.evidence,
+  ];
+
+  if (finding.expected !== undefined || finding.actual !== undefined) {
+    lines.push("", `Expected: ${finding.expected ?? "—"}`, `Actual:   ${finding.actual ?? "—"}`);
+  }
+  if (finding.repro !== undefined) {
+    lines.push("", "Reproduction:", "", "```", finding.repro, "```");
+  }
+
+  lines.push(
+    "",
+    "---",
+    `Recorded with docspack ${docspackVersion}, read and filed by a human. docspack does not`,
+    "send reports; this was written locally and someone chose to bring it here.",
+  );
+  return lines.join("\n");
+}
+
+/**
+ * A prefilled issue URL, using GitHub's documented `title`, `body` and `labels` query
+ * parameters. Returns nothing when the result is too long to be usable as a link.
+ */
+export function issueUrl(
+  repo: string,
+  title: string,
+  body: string,
+  labels: readonly string[],
+): string | undefined {
+  const query = new URLSearchParams({ title, body });
+  if (labels.length > 0) query.set("labels", labels.join(","));
+
+  const url = `https://github.com/${repo}/issues/new?${query.toString()}`;
+  return url.length > MAX_URL ? undefined : url;
+}

package/src/sync.ts

@@ -0,0 +1,112 @@
+import { readFile } from "node:fs/promises";
+import type { IndexedChunk, Store } from "./db.js";
+import { type DiscoveredPackage, discoverPackages } from "./discovery.js";
+import { chunkId, estimateTokens, resolveChunkFile } from "./spec.js";
+
+export interface SyncOptions {
+  readonly cwd: string;
+  readonly store: Store;
+  /** Re-index packages that are already in the store. */
+  readonly force?: boolean;
+  readonly onProgress?: (message: string) => void;
+}
+
+export type SyncStatus = "indexed" | "cached";
+
+export interface SyncedPackage {
+  readonly id: string;
+  readonly name: string;
+  readonly version: string;
+  readonly chunks: number;
+  readonly tokens: number;
+  readonly status: SyncStatus;
+  readonly trusted: boolean;
+}
+
+export interface SyncResult {
+  readonly packages: readonly SyncedPackage[];
+  readonly problems: readonly string[];
+}
+
+/**
+ * Indexes every documentation package this project depends on. Packages already present in the
+ * global store are left alone, so the same `@stripe/docspack@2025.4.1` is only read once per
+ * machine rather than once per project.
+ */
+export async function syncProject(options: SyncOptions): Promise<SyncResult> {
+  const { packages, problems: discoveryProblems } = await discoverPackages(options.cwd);
+  const problems = [...discoveryProblems];
+  const synced: SyncedPackage[] = [];
+
+  for (const pkg of packages) {
+    if (options.force !== true && options.store.hasPackage(pkg.id)) {
+      synced.push({
+        id: pkg.id,
+        name: pkg.name,
+        version: pkg.version,
+        chunks: options.store.countChunks(pkg.id),
+        tokens: 0,
+        status: "cached",
+        trusted: pkg.trusted,
+      });
+      continue;
+    }
+
+    options.onProgress?.(`indexing ${pkg.id}`);
+    const { chunks, problems: chunkProblems } = await readChunks(pkg);
+    problems.push(...chunkProblems);
+
+    if (chunks.length === 0) {
+      problems.push(`${pkg.name}: no readable chunks, nothing indexed`);
+      continue;
+    }
+
+    options.store.indexPackage({ id: pkg.id, name: pkg.name, version: pkg.version }, chunks);
+    synced.push({
+      id: pkg.id,
+      name: pkg.name,
+      version: pkg.version,
+      chunks: chunks.length,
+      tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+      status: "indexed",
+      trusted: pkg.trusted,
+    });
+  }
+
+  return { packages: synced, problems };
+}
+
+async function readChunks(
+  pkg: DiscoveredPackage,
+): Promise<{ chunks: IndexedChunk[]; problems: string[] }> {
+  const chunks: IndexedChunk[] = [];
+  const problems: string[] = [];
+
+  for (const chunk of pkg.manifest.chunks) {
+    let content: string;
+    try {
+      content = await readFile(resolveChunkFile(pkg.llmsDir, chunk.file), "utf8");
+    } catch (error) {
+      problems.push(
+        `${pkg.name} chunk "${chunk.id}": ${error instanceof Error ? error.message : String(error)}`,
+      );
+      continue;
+    }
+
+    const text = content.trim();
+    if (text.length === 0) {
+      problems.push(`${pkg.name} chunk "${chunk.id}" is empty`);
+      continue;
+    }
+
+    chunks.push({
+      chunkId: chunkId(pkg.id, chunk.id),
+      filePath: chunk.file,
+      tokens: chunk.tokens > 0 ? chunk.tokens : estimateTokens(text),
+      content: text,
+      tags: [...chunk.tags, ...chunk.entities],
+    });
+  }
+
+  return { chunks, problems };
+}