@gmickel/gno 2.12.2 → 2.14.0

package/src/core/audit-collection-rules.ts

@@ -0,0 +1,331 @@
+/**
+ * Opt-in, report-only collection rules for the `links` audit category:
+ * `links.collection-policy`, `links.external-url` and `links.path-layout`.
+ * A rule is emitted only when an audited collection configures it.
+ */
+
+import type { Collection, CollectionAuditConfig } from "../config/types";
+import type { AuditLinkSnapshot } from "../store/sqlite/graph-link-resolver";
+import type { DocumentRow } from "../store/types";
+import type { AuditFindingDraft, AuditRuleContribution } from "./audit";
+
+import { buildLineOffsets, offsetToPosition } from "../ingestion/position";
+import { getExcludedRanges, rangeIntersectsExcluded } from "../ingestion/strip";
+import { isTraversableResolution } from "../store/sqlite/graph-link-resolver";
+import {
+  AUDIT_MAX_EVIDENCE_DETAIL_CHARS,
+  compareAuditCodeUnits,
+  compareAuditFindingDrafts,
+} from "./audit";
+
+export interface CollectionAuditRuleInput {
+  links: AuditLinkSnapshot;
+  /** Audited documents (the requested scope). */
+  documents: readonly DocumentRow[];
+  /** Configured collections. */
+  collections: readonly Collection[];
+  /** Collections the run audits; rules are emitted only for these. */
+  auditedCollections: readonly string[];
+  /** Indexed text by mirror hash, loaded only for `externalUrls: report`. */
+  contents: ReadonlyMap<string, string>;
+  truncated: boolean;
+  maxFindingsPerRule: number;
+}
+
+/** `scheme://` in prose or link syntax; the start must not continue a word. */
+const URL_PATTERN = /(?<![a-z0-9+.-])([a-z][a-z0-9+.-]*):\/\/[^\s<>"'`]+/gi;
+/** Local schemes are not external URLs. */
+const LOCAL_URL_SCHEMES = new Set(["gno", "file"]);
+const TRAILING_PUNCTUATION = /[.,;:!?*_~]$/;
+const MAX_URL_EVIDENCE_CHARS = 400;
+
+const clip = (value: string, max: number): string =>
+  value.length > max ? `${value.slice(0, max - 1)}…` : value;
+
+const lineLocation = (line: number, column: number): string =>
+  `L${line}:C${column}`;
+
+/** Trim sentence punctuation and closers that do not belong to the URL. */
+const trimUrl = (raw: string): string => {
+  let url = raw;
+  for (;;) {
+    const last = url.at(-1);
+    if (last === undefined) return url;
+    if (TRAILING_PUNCTUATION.test(last)) {
+      url = url.slice(0, -1);
+      continue;
+    }
+    const pairs: Record<string, string> = { ")": "(", "]": "[", "}": "{" };
+    const opener = pairs[last];
+    if (
+      opener !== undefined &&
+      url.split(last).length > url.split(opener).length
+    ) {
+      url = url.slice(0, -1);
+      continue;
+    }
+    return url;
+  }
+};
+
+export interface ExternalUrlMatch {
+  url: string;
+  line: number;
+  column: number;
+}
+
+/** External URLs in indexed text, outside frontmatter, code and comments. */
+export const findExternalUrls = (markdown: string): ExternalUrlMatch[] => {
+  const excluded = getExcludedRanges(markdown);
+  const lineOffsets = buildLineOffsets(markdown);
+  const matches: ExternalUrlMatch[] = [];
+  for (const match of markdown.matchAll(URL_PATTERN)) {
+    const scheme = match[1]?.toLowerCase() ?? "";
+    if (LOCAL_URL_SCHEMES.has(scheme)) continue;
+    const url = trimUrl(match[0]);
+    if (url.length <= scheme.length + 3) continue;
+    const start = match.index;
+    if (rangeIntersectsExcluded(start, start + url.length, excluded)) continue;
+    const position = offsetToPosition(start, lineOffsets);
+    matches.push({ url, line: position.line, column: position.col });
+  }
+  return matches;
+};
+
+const configuredFor = (
+  input: CollectionAuditRuleInput
+): Map<string, CollectionAuditConfig> => {
+  const audited = new Set(input.auditedCollections);
+  return new Map(
+    input.collections
+      .filter(({ name, audit }) => audit !== undefined && audited.has(name))
+      .map(({ name, audit }) => [name, audit as CollectionAuditConfig])
+  );
+};
+
+const documentPath = (document: DocumentRow): string =>
+  document.recordSourcePath ?? document.relPath;
+
+const contribution = (
+  ruleId: string,
+  findings: AuditFindingDraft[],
+  input: CollectionAuditRuleInput,
+  examinedCount: number,
+  messages: { partial: string; done: string }
+): AuditRuleContribution => {
+  const partial = input.truncated;
+  return {
+    ruleId,
+    category: "links",
+    status: partial ? "inconclusive" : findings.length > 0 ? "fail" : "pass",
+    message: partial ? messages.partial : messages.done,
+    findings: [...findings]
+      .sort(compareAuditFindingDrafts)
+      .slice(0, input.maxFindingsPerRule),
+    findingCount: findings.length,
+    examinedCount,
+    durationMs: 0,
+    skipReason: partial ? "snapshot_truncated" : null,
+  };
+};
+
+const collectionPolicy = (
+  input: CollectionAuditRuleInput,
+  configured: ReadonlyMap<string, CollectionAuditConfig>
+): AuditRuleContribution | null => {
+  const allowedBySource = new Map<string, Set<string>>();
+  for (const [name, audit] of configured) {
+    if (audit.linkCollections === undefined) continue;
+    allowedBySource.set(name, new Set([name, ...audit.linkCollections]));
+  }
+  if (allowedBySource.size === 0) return null;
+  const audited = new Set(
+    input.links.auditedDocumentIds ?? input.links.documents.map(({ id }) => id)
+  );
+  const documents = new Map(input.links.documents.map((doc) => [doc.id, doc]));
+  const findings: AuditFindingDraft[] = [];
+  let examined = 0;
+  for (const link of input.links.links) {
+    const allowed = allowedBySource.get(link.sourceCollection);
+    if (!allowed || !audited.has(link.sourceId)) continue;
+    examined += 1;
+    // Unresolved and ambiguous links belong to the broken-link rules.
+    if (!link.resolved || !isTraversableResolution(link.resolved)) continue;
+    if (link.resolved.matchCount > 1) continue;
+    const target = documents.get(link.resolved.targetId);
+    if (!target || allowed.has(target.collection)) continue;
+    findings.push({
+      subject: link.sourceUri,
+      location: lineLocation(link.startLine, link.startCol),
+      severity: "warning",
+      message: `Link into collection "${target.collection}", which audit.linkCollections does not list: ${target.uri}`,
+      evidence: [
+        {
+          kind: "collection-policy",
+          summary: `${target.collection}:${target.uri}`,
+          uri: link.sourceUri,
+          path: link.sourceRelPath,
+          detail: clip(
+            JSON.stringify({
+              sourceCollection: link.sourceCollection,
+              targetCollection: target.collection,
+              target: target.uri,
+              linkType: link.linkType,
+              allowed: [...allowed].sort(compareAuditCodeUnits),
+            }),
+            AUDIT_MAX_EVIDENCE_DETAIL_CHARS
+          ),
+        },
+      ],
+      guidance: [
+        `Remove or change the link, or add "${target.collection}" to audit.linkCollections`,
+      ],
+    });
+  }
+  return contribution("links.collection-policy", findings, input, examined, {
+    partial: "Collection link policy scan was truncated",
+    done: `${findings.length} links into collections outside audit.linkCollections`,
+  });
+};
+
+const externalUrls = (
+  input: CollectionAuditRuleInput,
+  configured: ReadonlyMap<string, CollectionAuditConfig>
+): AuditRuleContribution | null => {
+  const reporting = new Set(
+    [...configured]
+      .filter(([, audit]) => audit.externalUrls === "report")
+      .map(([name]) => name)
+  );
+  if (reporting.size === 0) return null;
+  const findings: AuditFindingDraft[] = [];
+  let examined = 0;
+  let unchecked = 0;
+  for (const document of input.documents) {
+    if (!reporting.has(document.collection)) continue;
+    examined += 1;
+    const text = document.mirrorHash
+      ? input.contents.get(document.mirrorHash)
+      : undefined;
+    if (text === undefined) {
+      unchecked += 1;
+      findings.push({
+        subject: document.uri,
+        location: null,
+        severity: "warning",
+        message: "Indexed text is unavailable; external URLs were not checked",
+        evidence: [
+          {
+            kind: "external-url-unchecked",
+            summary: "unchecked",
+            uri: document.uri,
+            path: documentPath(document),
+          },
+        ],
+        guidance: ["Run gno update so the document has indexed text"],
+      });
+      continue;
+    }
+    for (const match of findExternalUrls(text)) {
+      findings.push({
+        subject: document.uri,
+        location: lineLocation(match.line, match.column),
+        severity: "warning",
+        message: `External URL in document text: ${clip(match.url, MAX_URL_EVIDENCE_CHARS)}`,
+        evidence: [
+          {
+            kind: "external-url",
+            summary: clip(match.url, MAX_URL_EVIDENCE_CHARS),
+            uri: document.uri,
+            path: documentPath(document),
+            detail: JSON.stringify({ line: match.line, column: match.column }),
+          },
+        ],
+        guidance: [
+          "Remove the URL, or set audit.externalUrls: ignore for this collection",
+        ],
+      });
+    }
+  }
+  const urls = findings.length - unchecked;
+  return contribution("links.external-url", findings, input, examined, {
+    partial: "External URL scan was truncated",
+    done:
+      unchecked > 0
+        ? `${urls} external URLs; ${unchecked} documents unchecked`
+        : `${urls} external URLs`,
+  });
+};
+
+const pathLayout = (
+  input: CollectionAuditRuleInput,
+  configured: ReadonlyMap<string, CollectionAuditConfig>
+): AuditRuleContribution | null => {
+  const globsByCollection = new Map<
+    string,
+    { patterns: string[]; globs: Bun.Glob[] }
+  >();
+  for (const [name, audit] of configured) {
+    if (audit.paths === undefined) continue;
+    globsByCollection.set(name, {
+      patterns: audit.paths,
+      globs: audit.paths.map((pattern) => new Bun.Glob(pattern)),
+    });
+  }
+  if (globsByCollection.size === 0) return null;
+  const findings: AuditFindingDraft[] = [];
+  let examined = 0;
+  for (const document of input.documents) {
+    const expected = globsByCollection.get(document.collection);
+    if (!expected) continue;
+    examined += 1;
+    const path = documentPath(document);
+    if (expected.globs.some((glob) => glob.match(path))) continue;
+    findings.push({
+      subject: document.uri,
+      location: null,
+      severity: "warning",
+      message: `Document path matches no audit.paths glob: ${path}`,
+      evidence: [
+        {
+          kind: "path-layout",
+          summary: path,
+          uri: document.uri,
+          path,
+          detail: clip(
+            JSON.stringify({ paths: expected.patterns }),
+            AUDIT_MAX_EVIDENCE_DETAIL_CHARS
+          ),
+        },
+      ],
+      guidance: ["Move the document, or add a matching glob to audit.paths"],
+    });
+  }
+  return contribution("links.path-layout", findings, input, examined, {
+    partial: "Path layout scan was truncated",
+    done: `${findings.length} documents outside the audit.paths layout`,
+  });
+};
+
+/** True when any audited collection reports external URLs. */
+export const needsAuditContents = (
+  collections: readonly Collection[],
+  auditedCollections: readonly string[]
+): boolean => {
+  const audited = new Set(auditedCollections);
+  return collections.some(
+    ({ name, audit }) => audited.has(name) && audit?.externalUrls === "report"
+  );
+};
+
+export const evaluateCollectionAuditRules = (
+  input: CollectionAuditRuleInput
+): AuditRuleContribution[] => {
+  const configured = configuredFor(input);
+  if (configured.size === 0) return [];
+  return [
+    collectionPolicy(input, configured),
+    externalUrls(input, configured),
+    pathLayout(input, configured),
+  ].filter((rule): rule is AuditRuleContribution => rule !== null);
+};

package/src/core/audit-links.ts

@@ -17,8 +17,10 @@ import {
  * 1.1: workspace-wide resolution and reference/resolution/scope evidence.
  * 1.2: `links.outside-index` for workspace files that are not indexed.
  * 1.3: relative Markdown links to such files are `outside-index` too.
+ * 1.4: opt-in collection rules from `audit` config (collection-policy,
+ *      external-url, path-layout); unchanged without that config.
  */
-export const LINK_AUDIT_RULE_VERSION = "1.3" as const;
+export const LINK_AUDIT_RULE_VERSION = "1.4" as const;
 /** Default per-rule cap; runs pass the effective `maxFindings` instead. */
 export const LINK_AUDIT_MAX_FINDINGS_PER_RULE = 1000;
 

package/src/core/audit-workspace.ts

@@ -30,6 +30,10 @@ import {
   resolveAuditMaxFindings,
   runAudit,
 } from "./audit";
+import {
+  evaluateCollectionAuditRules,
+  needsAuditContents,
+} from "./audit-collection-rules";
 import { evaluateFreshnessAudit } from "./audit-freshness";
 import { evaluateLinkAudit } from "./audit-links";
 import { markOutsideIndexLinks } from "./audit-outside-index";
@@ -81,10 +85,23 @@ interface WorkspaceDocumentSnapshot {
 
 interface WorkspaceSnapshot {
   documents: WorkspaceDocumentSnapshot[];
+  /** Every selected (audited) document row. */
+  selected: DocumentRow[];
+  /** Indexed text for collections with `audit.externalUrls: report`. */
+  contents: Map<string, string>;
   links: AuditLinkSnapshot;
   truncated: boolean;
 }
 
+/** Collections an audit run covers: the collection filter, else all. */
+const auditedCollectionNames = (
+  options: WorkspaceAuditOptions,
+  filters: { collections: string[] }
+): string[] =>
+  filters.collections.length > 0
+    ? filters.collections
+    : options.collections.map(({ name }) => name);
+
 const normalizeValues = (values: readonly string[] | undefined): string[] =>
   [...new Set((values ?? []).map((value) => value.normalize("NFC").trim()))]
     .filter(Boolean)
@@ -391,6 +408,26 @@ const emptyLinkSnapshot = (): AuditLinkSnapshot => ({
   },
 });
 
+/** Indexed text for documents in collections that report external URLs. */
+const loadAuditContents = async (
+  options: WorkspaceAuditOptions,
+  documents: readonly DocumentRow[]
+): Promise<Map<string, string>> => {
+  const reporting = new Set(
+    options.collections
+      .filter(({ audit }) => audit?.externalUrls === "report")
+      .map(({ name }) => name)
+  );
+  const hashes = documents
+    .filter(
+      ({ collection, mirrorHash }) => reporting.has(collection) && mirrorHash
+    )
+    .map(({ mirrorHash }) => mirrorHash as string);
+  const loaded = await options.store.getContentBatch(hashes);
+  if (!loaded.ok) throw new Error(loaded.error.message);
+  return loaded.value;
+};
+
 const loadWorkspaceSnapshot = async (
   options: WorkspaceAuditOptions,
   filters: { collections: string[]; paths: string[]; tags: string[] }
@@ -437,8 +474,18 @@ const loadWorkspaceSnapshot = async (
     selected.truncated,
     filters.collections
   );
+  const contents =
+    options.categories.includes("links") &&
+    needsAuditContents(
+      options.collections,
+      auditedCollectionNames(options, filters)
+    )
+      ? await loadAuditContents(options, selected.documents)
+      : new Map<string, string>();
   return {
     documents: observed,
+    selected: selected.documents,
+    contents,
     links: options.categories.includes("links")
       ? await markOutsideIndexLinks(options.store.getRawDb(), links, {
           signal: options.signal,
@@ -514,13 +561,19 @@ const captureWorkspaceFingerprints = async (
     options.categories.includes("links") && selected.documents.length > 0
       ? captureAuditLinkSnapshot(options.store.getRawDb())
       : null;
+  // `audit` is covered only where present on an audited collection, the only
+  // place it changes the run, so other fingerprints stay unchanged.
+  const audited = new Set(auditedCollectionNames(options, filters));
   return {
     config: hashAuditCanonical({
-      collections: options.collections.map(({ name, path, pattern }) => ({
-        name,
-        path,
-        pattern,
-      })),
+      collections: options.collections.map(
+        ({ name, path, pattern, audit }) => ({
+          name,
+          path,
+          pattern,
+          ...(audit === undefined || !audited.has(name) ? {} : { audit }),
+        })
+      ),
       filters,
     }),
     source: hashAuditCanonical({ sourceStats, total: selected.total }),
@@ -665,6 +718,17 @@ export const runWorkspaceAudit = async (
               rootUris: options.orphanRoots ?? [],
               ignorePathPrefixes: options.orphanIgnorePrefixes ?? [],
               maxFindingsPerRule,
+            }),
+            ...evaluateCollectionAuditRules({
+              links: snapshot.links,
+              documents: snapshot.selected,
+              collections: options.collections,
+              auditedCollections: auditedCollectionNames(options, filters),
+              contents: snapshot.contents,
+              truncated:
+                snapshot.links.truncated.documents ||
+                snapshot.links.truncated.links,
+              maxFindingsPerRule,
             })
           );
           await options.onProgress?.({

package/src/core/collection-egress-policy-service.ts

@@ -410,7 +410,10 @@ export class CollectionEgressPolicyService {
         return {
           ok: false,
           code: "VALIDATION",
-          error: "Invalid collection egress scope",
+          error:
+            error.code === "EGRESS_SCOPE_LIMIT_EXCEEDED"
+              ? "Collection egress scope exceeds 128 collections; name the collections to check"
+              : "Invalid collection egress scope",
         };
       }
       if (!(error instanceof EgressDeniedError)) throw error;

package/src/core/collection-scope.ts

@@ -0,0 +1,105 @@
+/**
+ * Request-level collection allowlist shared by search, vsearch and query on
+ * every surface (CLI, MCP, REST, SDK).
+ *
+ * @module src/core/collection-scope
+ */
+
+/** Most collections one request may list; matches the audit scope bound. */
+export const MAX_REQUEST_COLLECTIONS = 256;
+
+export type CollectionListResult =
+  | { ok: true; value: string[] }
+  | { ok: false; message: string };
+
+/**
+ * Normalize a `collections` allowlist: trim, lowercase, de-duplicate, sort.
+ * Rejects non-arrays, non-string or blank members, an empty list, and lists
+ * longer than {@link MAX_REQUEST_COLLECTIONS} after normalization.
+ */
+export function normalizeCollectionList(input: unknown): CollectionListResult {
+  if (!Array.isArray(input)) {
+    return { ok: false, message: "collections must be an array of names" };
+  }
+  const names = new Set<string>();
+  for (const value of input) {
+    if (typeof value !== "string" || value.trim() === "") {
+      return {
+        ok: false,
+        message: "collections must contain only non-empty collection names",
+      };
+    }
+    names.add(value.trim().toLowerCase());
+  }
+  if (names.size === 0) {
+    return { ok: false, message: "collections must list at least one name" };
+  }
+  if (names.size > MAX_REQUEST_COLLECTIONS) {
+    return {
+      ok: false,
+      message: `collections must list at most ${MAX_REQUEST_COLLECTIONS} names`,
+    };
+  }
+  return {
+    ok: true,
+    value: [...names].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)),
+  };
+}
+
+/** Error text when a request names both `collection` and `collections`. */
+export const COLLECTION_AND_COLLECTIONS_MESSAGE =
+  "Pass either collection or collections, not both";
+
+/** Names in `requested` that are not configured, in request order. */
+export function unknownCollections(
+  requested: readonly string[],
+  configured: readonly { name: string }[]
+): string[] {
+  const known = new Set(configured.map(({ name }) => name.toLowerCase()));
+  return requested.filter((name) => !known.has(name));
+}
+
+/** The existing unknown-collection message, naming every unknown name. */
+export function unknownCollectionsMessage(names: readonly string[]): string {
+  return names.length === 1
+    ? `Collection not found: ${names[0]}`
+    : `Collections not found: ${names.join(", ")}`;
+}
+
+export type CollectionScopeResult =
+  | { ok: true; collections: string[] | undefined }
+  | { ok: false; kind: "validation" | "not_found"; message: string };
+
+/**
+ * Resolve a request's `collections` allowlist against the configured
+ * collections. Absent `collections` resolves to undefined (unchanged
+ * single-collection behaviour). Both inputs together, a malformed list, or
+ * any unknown name is refused.
+ */
+export function resolveCollectionScope(
+  input: { collection?: unknown; collections?: unknown },
+  configured: readonly { name: string }[]
+): CollectionScopeResult {
+  if (input.collections === undefined)
+    return { ok: true, collections: undefined };
+  if (input.collection !== undefined) {
+    return {
+      ok: false,
+      kind: "validation",
+      message: COLLECTION_AND_COLLECTIONS_MESSAGE,
+    };
+  }
+  const normalized = normalizeCollectionList(input.collections);
+  if (!normalized.ok) {
+    return { ok: false, kind: "validation", message: normalized.message };
+  }
+  const unknown = unknownCollections(normalized.value, configured);
+  if (unknown.length > 0) {
+    return {
+      ok: false,
+      kind: "not_found",
+      message: unknownCollectionsMessage(unknown),
+    };
+  }
+  return { ok: true, collections: normalized.value };
+}

package/src/core/egress-enforcement.ts

@@ -13,7 +13,10 @@ import type { EgressLineage, EgressLineageInput } from "./egress-provenance";
 
 import { resolveConfiguredEgressPolicy } from "../config/types";
 import { EgressAuditService } from "./egress-audit";
-import { evaluateEgressPolicy } from "./egress-policy";
+import {
+  collectionLimitExceededDecision,
+  evaluateEgressPolicy,
+} from "./egress-policy";
 import {
   EgressProvenanceError,
   resolveEgressLineage,
@@ -58,8 +61,9 @@ const collectionLineageInputs = (
         "Collection policy scope must be a readable dense bounded array"
       );
     }
+    // The configuration is not bounded here; the resolved scope is.
     const { length } = collections;
-    if (!Number.isSafeInteger(length) || length > 128) {
+    if (!Number.isSafeInteger(length)) {
       throw new EgressProvenanceError(
         "INVALID_EGRESS_LINEAGE",
         "Collection policy scope must be a readable dense bounded array"
@@ -100,11 +104,23 @@ const collectionLineageInputs = (
   }
 };
 
+/** A scope over the lineage bound is a policy denial, never an internal error. */
 const resolveCollectionEgressLineage = (
   collections: readonly Collection[],
   names?: readonly string[]
-): EgressLineage =>
-  resolveEgressLineage(collectionLineageInputs(collections), names);
+): EgressLineage => {
+  try {
+    return resolveEgressLineage(collectionLineageInputs(collections), names);
+  } catch (error) {
+    if (
+      error instanceof EgressProvenanceError &&
+      error.code === "EGRESS_SCOPE_LIMIT_EXCEEDED"
+    ) {
+      throw new EgressDeniedError(collectionLimitExceededDecision());
+    }
+    throw error;
+  }
+};
 
 export const collectionEgressStates = (
   collections: readonly Collection[],

package/src/core/egress-policy.ts

@@ -420,6 +420,10 @@ function evaluateSnapshot(snapshot: EgressEvaluationSnapshot): EgressDecision {
   return decision(true, "REMOTE_POLICY_AUTHENTICATED", audit);
 }
 
+/** The denial for a scope over the collection bound, naming no collection. */
+export const collectionLimitExceededDecision = (): EgressDecision =>
+  decision(false, "COLLECTION_LIMIT_EXCEEDED", limitExceededAudit());
+
 export function evaluateEgressPolicy(input: unknown): EgressDecision {
   try {
     const result = snapshotInput(input);