@vibe-agent-toolkit/resources 0.1.39-rc.9 → 0.1.39

package/src/external-link-validator.ts

@@ -1,6 +1,99 @@
+import { userInfo } from 'node:os';
+
+import type { IssueCode } from '@vibe-agent-toolkit/agent-schema';
+import {
+  resolveAuthenticatedUrl,
+  safePath,
+  type LinkAuthConfig,
+  type ResolveOutcome,
+} from '@vibe-agent-toolkit/utils';
 import markdownLinkCheck from 'markdown-link-check';
 
 import { ExternalLinkCache } from './external-link-cache.js';
+import { classifyAuthenticatedResponse } from './link-auth-classify.js';
+import { type LinkAuthDeps, wrapLinkAuthDepsWithMemo } from './link-auth-deps-memo.js';
+import { authTransport } from './link-auth-transport.js';
+
+/**
+ * Resolve the OS user for cache scoping. Falls back through several layers
+ * because `os.userInfo()` throws when the running user has no /etc/passwd
+ * entry (common in container environments) — that's recoverable, not fatal.
+ */
+function resolveOsUser(): string {
+  try {
+    const u = userInfo().username;
+    if (u !== '') return u;
+  } catch {
+    // userInfo() throws when the running user has no /etc/passwd entry —
+    // recover via env vars below.
+  }
+  // Use `||` (not `??`) so an empty-string USER/USERNAME also falls through.
+  // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+  const fromEnv = process.env['USER'] || process.env['USERNAME'];
+  if (fromEnv) return fromEnv;
+  // Last resort: two distinct OS users on the same host both end up here
+  // would share the auth cache (cross-user leak). Warn once so the
+  // collision is at least observable. Adopters whose container clears
+  // these env vars deliberately can set `USER=container-name` to scope.
+  warnDefaultUserFallbackOnce();
+  return 'default';
+}
+
+let defaultUserWarned = false;
+function warnDefaultUserFallbackOnce(): void {
+  if (defaultUserWarned) return;
+  defaultUserWarned = true;
+  console.warn(
+    "[vat] linkAuth: could not determine an OS user via os.userInfo() or USER/USERNAME env; " +
+      "scoping auth cache to 'default'. Two users on this host would share auth-cached results — " +
+      'set the USER env var explicitly to scope per-user.',
+  );
+}
+
+/**
+ * Make an OS username safe for use as a directory component. Replaces path
+ * separators (`/`, `\`), the parent-directory shorthand (`..`), and other
+ * characters that could escape the cacheDir. Windows can produce
+ * `DOMAIN\user` forms; tests inject pathological values like `../escaped`.
+ */
+function sanitizeOsUser(user: string): string {
+  // Replace any character outside `[A-Za-z0-9._-]` with `_`. Then collapse any
+  // remaining `..` so a name like `..foo` cannot recombine into a traversal.
+  const replaced = user.replaceAll(/[^A-Za-z0-9._-]/g, '_').replaceAll('..', '__');
+  return replaced.length > 0 ? replaced : 'default';
+}
+
+type VerifiedPlan = Extract<ResolveOutcome, { fetchUrl: string }>;
+
+/**
+ * Build a LinkValidationResult from a status code by re-classifying it under
+ * the current run's provider `check` block. Used by both the cache-hit and
+ * cache-miss paths so the two produce identical results for the same
+ * `(url, provider)` pair — the cache only persists `statusCode`, not the
+ * derived `code`, so the code must be re-derived on every read.
+ */
+function buildAuthResult(
+	originalUrl: string,
+	statusCode: number,
+	plan: VerifiedPlan,
+	cached: boolean,
+	cachedStatusMessage?: string,
+): LinkValidationResult {
+	const classified = classifyAuthenticatedResponse(statusCode, plan.check);
+	const result: LinkValidationResult = {
+		url: originalUrl,
+		status: classified?.outcome === 'alive' ? 'ok' : 'error',
+		statusCode,
+		cached,
+	};
+	if (classified?.code != null) {
+		result.code = classified.code;
+	}
+	if (result.status === 'error') {
+		result.error = cachedStatusMessage ?? `HTTP ${statusCode}`;
+	}
+	return result;
+}
 
 /**
  * Safely serialize an error to a string, preventing [object Object] issues.
@@ -51,6 +144,38 @@ export interface ExternalLinkValidatorOptions {
 	retries?: number;
 	/** User agent string for requests (default: generic) */
 	userAgent?: string;
+	/**
+	 * Optional linkAuth config (per issue #113). When set, URLs whose host is
+	 * claimed by a provider in this config bypass the anonymous markdown-link-check
+	 * path and use an authenticated direct fetch instead. URLs no provider claims
+	 * continue to use markdown-link-check.
+	 */
+	linkAuthConfig?: LinkAuthConfig;
+	/**
+	 * Override the `fetch` implementation used by the authenticated branch.
+	 * Defaults to `globalThis.fetch`. Tests inject a stub; advanced adopters
+	 * may inject a wrapper for corporate proxies, custom TLS, or telemetry.
+	 */
+	fetchImpl?: typeof fetch;
+	/**
+	 * Override the linkAuth engine's token-resolution dependencies (env map +
+	 * runCommand). Defaults to reading `process.env` and running real commands
+	 * via `safeExecSync`. Tests inject a deterministic env; advanced adopters
+	 * usually leave unset.
+	 */
+	linkAuthDeps?: LinkAuthDeps;
+	/**
+	 * Override the sleep used between 429 retries. Defaults to `setTimeout`.
+	 * Test-only — production benefits from real wall-clock delay so Retry-After
+	 * hints are honored.
+	 */
+	sleep?: (ms: number) => Promise<void>;
+	/**
+	 * OS user to scope the authenticated cache by (#113 §6.3). Defaults to
+	 * `os.userInfo().username` with env-var fallbacks. Tests inject a fixed
+	 * value; production callers omit.
+	 */
+	osUser?: string;
 }
 
 /**
@@ -67,6 +192,21 @@ export interface LinkValidationResult {
 	error?: string;
 	/** Whether result came from cache */
 	cached: boolean;
+	/**
+	 * Set when the authenticated branch classified the response (issue #113 §7).
+	 * Consumers use this directly instead of mapping `statusCode` to a code, so
+	 * `notFoundMeaning`-dependent routing (404 → `LINK_AUTH_DEAD` vs
+	 * `LINK_AUTH_DEAD_OR_UNAUTHORIZED`) reflects the matched provider's config.
+	 *
+	 * Invariant: `code` is set iff the URL hit the authenticated branch AND
+	 * the outcome maps to a `LINK_AUTH_*` code per §7 (`unverified`, or any
+	 * classified status: `unauthorized`, `forbidden`, `dead`,
+	 * `dead_or_unauthorized`). Unset on the anonymous markdown-link-check path,
+	 * on `unsupported` (no provider claimed the host), and on classifier-`null`
+	 * statuses (5xx, unclassified) where the consumer's status-code mapping
+	 * (`EXTERNAL_URL_*`) is the correct fallback.
+	 */
+	code?: IssueCode;
 }
 
 /**
@@ -90,7 +230,23 @@ export interface LinkValidationResult {
  */
 export class ExternalLinkValidator {
 	private readonly cache: ExternalLinkCache;
-	private readonly options: Required<ExternalLinkValidatorOptions>;
+	/**
+	 * Auth-branch cache — scoped to a per-OS-user subdirectory of `cacheDir`
+	 * (#113 §6.3) so two users on a shared machine never read each other's
+	 * authenticated results. Distinct from `cache` (the anonymous cache, which
+	 * stays shared across users for the markdown-link-check path).
+	 */
+	private readonly authCache: ExternalLinkCache;
+	private readonly options: {
+		cacheTtlHours: number;
+		timeout: number;
+		retries: number;
+		userAgent: string;
+	};
+	private readonly linkAuthConfig: LinkAuthConfig | undefined;
+	private readonly fetchImpl: typeof fetch;
+	private readonly linkAuthDeps: LinkAuthDeps;
+	private readonly sleep: ((ms: number) => Promise<void>) | undefined;
 
 	/**
 	 * Create a new external link validator
@@ -108,7 +264,22 @@ export class ExternalLinkValidator {
 				'Mozilla/5.0 (compatible; VAT-LinkChecker/1.0; +https://github.com/jdutton/vibe-agent-toolkit)',
 		};
 
+		this.linkAuthConfig = options.linkAuthConfig;
+		this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
+		// Wrap the configured runCommand (or the engine's default) with a memo
+		// keyed by stringified argv so each unique token-resolution command runs
+		// at most once per validator instance. Validating N links to the same
+		// host previously re-resolved the token N times — including subprocess
+		// spawns for `command` sources. Per #125 review: treat all token
+		// resolvers as expensive; the memo's lifetime equals one validate() run.
+		this.linkAuthDeps = wrapLinkAuthDepsWithMemo(options.linkAuthDeps);
+		this.sleep = options.sleep;
+
 		this.cache = new ExternalLinkCache(cacheDir, this.options.cacheTtlHours);
+
+		const osUser = sanitizeOsUser(options.osUser ?? resolveOsUser());
+		const authCacheDir = safePath.join(cacheDir, `auth-${osUser}`);
+		this.authCache = new ExternalLinkCache(authCacheDir, this.options.cacheTtlHours);
 	}
 
 	/**
@@ -118,6 +289,29 @@ export class ExternalLinkValidator {
 	 * @returns Validation result
 	 */
 	async validateLink(url: string): Promise<LinkValidationResult> {
+		// Authenticated branch (issue #113): if a provider in linkAuthConfig
+		// claims this URL's host, bypass markdown-link-check and do a direct
+		// authenticated fetch + per-§7 classify.
+		if (this.linkAuthConfig) {
+			const plan = resolveAuthenticatedUrl(url, this.linkAuthConfig, this.linkAuthDeps);
+			if ('fetchUrl' in plan) {
+				return this.validateAuthenticatedLink(url, plan);
+			}
+			if (plan.outcome === 'unverified') {
+				// Per §6.3: never cache unverified outcomes — the result flips the
+				// moment a token appears, so caching a "no token" result is wrong.
+				return {
+					url,
+					status: 'error',
+					statusCode: 0,
+					error: plan.reason,
+					cached: false,
+					code: 'LINK_AUTH_UNVERIFIED',
+				};
+			}
+			// 'unsupported' → fall through to anonymous markdown-link-check path
+		}
+
 		// Check cache first
 		const cached = await this.cache.get(url);
 		if (cached) {
@@ -165,6 +359,70 @@ export class ExternalLinkValidator {
 		return Promise.all(urls.map((url) => this.validateLink(url)));
 	}
 
+	/**
+	 * Issue an authenticated fetch for `originalUrl` using the engine's plan
+	 * (rewritten URL + auth headers + provider's check config), classify the
+	 * response per #113 §7, and write a status-cache entry. Cache is keyed by
+	 * the *rewritten* URL (§6.3) — the original `blob/` URL 404s, so caching
+	 * by original would poison results.
+	 */
+	private async validateAuthenticatedLink(
+		originalUrl: string,
+		plan: VerifiedPlan,
+	): Promise<LinkValidationResult> {
+		// Cache check, keyed by rewritten URL. IMPORTANT: re-classify the cached
+		// statusCode under the *current* run's provider `check` block, rather
+		// than treating cache-hit as a status-only short-circuit. The cache
+		// only persists `statusCode` — without re-classifying we'd drop the
+		// LINK_AUTH_* `code`, and the consumer would fall back to
+		// `EXTERNAL_URL_DEAD` (error) instead of `LINK_AUTH_DEAD_OR_UNAUTHORIZED`
+		// (warning). Cache-hit semantics must match cache-miss semantics for
+		// the same (url, provider) pair.
+		const cached = await this.authCache.get(plan.fetchUrl);
+		if (cached) {
+			return buildAuthResult(originalUrl, cached.statusCode, plan, true, cached.statusMessage);
+		}
+
+		// Fresh fetch via the auth-aware transport (handles cross-origin redirect
+		// Authorization stripping + 429/Retry-After per §5.2 §8). The conditional
+		// spread builds the object in one pass — AuthTransportOptions.sleep is
+		// readonly so post-construction assignment would be a TS error.
+		const transportOptions: Parameters<typeof authTransport>[3] = {
+			signal: AbortSignal.timeout(this.options.timeout),
+			...(this.sleep === undefined ? {} : { sleep: this.sleep }),
+		};
+
+		let response: Response;
+		try {
+			response = await authTransport(
+				plan.fetchUrl,
+				plan.headers,
+				this.fetchImpl,
+				transportOptions,
+			);
+		} catch (err) {
+			// Network-level failure (DNS/connect/TLS/timeout) — return error with
+			// no code so the consumer's existing statusCode→IssueCode mapping
+			// (EXTERNAL_URL_ERROR / EXTERNAL_URL_TIMEOUT) applies.
+			const message = safeSerializeError(err) ?? 'Authenticated fetch failed';
+			return {
+				url: originalUrl,
+				status: 'error',
+				statusCode: 0,
+				error: message,
+				cached: false,
+			};
+		}
+
+		const result = buildAuthResult(originalUrl, response.status, plan, false);
+		// Persist by rewritten URL so subsequent runs hit the cache. The cache
+		// only stores statusCode + statusMessage; the `code` is re-derived on
+		// read by re-running classifyAuthenticatedResponse against the current
+		// provider (see the authCache.get branch above).
+		await this.authCache.set(plan.fetchUrl, result.statusCode, result.error ?? 'OK');
+		return result;
+	}
+
 	/**
 	 * Check a link using markdown-link-check
 	 */
@@ -235,16 +493,21 @@ export class ExternalLinkValidator {
 	}
 
 	/**
-	 * Clear the validation cache
+	 * Clear both validation caches (anonymous + authenticated).
+	 *
+	 * Adopters call this after rotating a token or invalidating link data —
+	 * the operation must wipe both surfaces or stale auth results survive
+	 * (a real bug: a 401 cached under the old token would haunt the new one).
 	 */
 	async clearCache(): Promise<void> {
-		await this.cache.clear();
+		await Promise.all([this.cache.clear(), this.authCache.clear()]);
 	}
 
 	/**
-	 * Get cache statistics
+	 * Get combined cache statistics across both caches.
 	 */
 	async getCacheStats(): Promise<{ total: number; expired: number }> {
-		return this.cache.getStats();
+		const [anon, auth] = await Promise.all([this.cache.getStats(), this.authCache.getStats()]);
+		return { total: anon.total + auth.total, expired: anon.expired + auth.expired };
 	}
 }

package/src/index.ts

@@ -144,3 +144,26 @@ export {
   type SkillSourceDescriptor,
   type TestConfig,
 } from './schemas/project-config.js';
+
+// linkAuth content-fetch primitive (issue #113 slice 3). Ships standalone;
+// callers wire it into asset-reference / bundling consumers as they need.
+export {
+  fetchAuthenticated,
+  type ContentFetchResult,
+  type FetchAuthenticatedOptions,
+} from './link-auth-content-fetch.js';
+
+export { ContentCache, type ContentMetadata } from './content-cache.js';
+
+// Token-resolution memo wrapper. High-volume callers iterating many URLs
+// from the same provider wrap their deps once and reuse the result, so
+// expensive token sources (`gh auth token` etc.) run at most once.
+export {
+  wrapLinkAuthDepsWithMemo,
+  type LinkAuthDeps,
+} from './link-auth-deps-memo.js';
+
+// Bridge from adopter `resources.linkAuth` (Zod-validated) to the engine's
+// `LinkAuthConfig` (fully-expanded providers). Adopters carrying a parsed
+// config can hand it directly to `fetchAuthenticated` via this bridge.
+export { buildLinkAuthEngineConfig } from './link-auth-config-build.js';

package/src/link-auth-classify.ts

@@ -0,0 +1,61 @@
+/**
+ * Classify a response from an authenticated external-link fetch into one of
+ * the outcomes defined in design issue #113 §7.
+ *
+ * Pure function: takes the HTTP status from a completed fetch plus the
+ * matched provider's `check` block, returns the outcome name and the matching
+ * `LINK_AUTH_*` code from `CODE_REGISTRY`. Pre-fetch outcomes (`unsupported`,
+ * `unverified`) are produced upstream by `resolveAuthenticatedUrl` and never
+ * reach this classifier.
+ *
+ * Status codes that don't match any classified outcome return `null` — the
+ * caller falls through to the existing anonymous `EXTERNAL_URL_DEAD` path
+ * (per §7 "unsupported"), since the authenticated fetch produced no signal
+ * the per-provider table can speak to.
+ */
+
+import type { ProviderCheck } from '@vibe-agent-toolkit/utils';
+
+export type LinkAuthOutcome =
+  | { readonly outcome: 'alive'; readonly code: null }
+  | { readonly outcome: 'dead'; readonly code: 'LINK_AUTH_DEAD' }
+  | {
+      readonly outcome: 'dead_or_unauthorized';
+      readonly code: 'LINK_AUTH_DEAD_OR_UNAUTHORIZED';
+    }
+  | { readonly outcome: 'forbidden'; readonly code: 'LINK_AUTH_FORBIDDEN' }
+  | { readonly outcome: 'unauthorized'; readonly code: 'LINK_AUTH_UNAUTHORIZED' };
+
+export function classifyAuthenticatedResponse(
+  status: number,
+  check: ProviderCheck,
+): LinkAuthOutcome | null {
+  // aliveStatus wins first: a provider can in principle declare any code as
+  // alive (e.g. a future host where 204 means alive), so the membership test
+  // runs before the well-known-status branches.
+  if (check.aliveStatus.includes(status)) {
+    return { outcome: 'alive', code: null };
+  }
+
+  if (status === 401) {
+    return { outcome: 'unauthorized', code: 'LINK_AUTH_UNAUTHORIZED' };
+  }
+
+  if (status === 403) {
+    return { outcome: 'forbidden', code: 'LINK_AUTH_FORBIDDEN' };
+  }
+
+  if (status === 404 || status === 410) {
+    if (check.notFoundMeaning === 'dead') {
+      return { outcome: 'dead', code: 'LINK_AUTH_DEAD' };
+    }
+    return {
+      outcome: 'dead_or_unauthorized',
+      code: 'LINK_AUTH_DEAD_OR_UNAUTHORIZED',
+    };
+  }
+
+  // 2xx not in aliveStatus, 3xx redirects, 429 rate-limit, 5xx server errors
+  // — none of these are classified per §7. Caller decides what to do.
+  return null;
+}

package/src/link-auth-config-build.ts

@@ -0,0 +1,130 @@
+/**
+ * Bridge between the adopter-facing linkAuth config (Zod-validated, with
+ * macro references) and the engine-facing config (fully-expanded providers).
+ *
+ * Adopter config providers can be either:
+ *   - `{ use: <macro>, ...overrides }` — reference a shipped macro; deep-merge
+ *     `overrides` on top of the macro's defaults
+ *   - A full inline `{ match, rewrite, auth, token, check }` — used as-is
+ *
+ * This function walks each provider entry, runs macro expansion when needed,
+ * and re-validates each fully-expanded provider against `InlineProviderSchema`.
+ * Since the schemas are passthrough (per the repo Postel's Law rule for
+ * adopter input), unknown extra keys survive expansion silently — the engine
+ * ignores them at runtime, and a separate lint/warn pass is the right home
+ * for typo-catching DX. Post-expansion validation still catches the cases
+ * passthrough doesn't relax: missing required fields, and wrong types on
+ * declared fields (e.g. `notFoundMeaning: 'totally-invalid'`).
+ *
+ * Per design issue #113 §5 (macros are config, not a privileged code path)
+ * and §4 (engine vocabulary).
+ */
+
+import {
+  expandMacro,
+  type LinkAuthConfig,
+  type Provider,
+} from '@vibe-agent-toolkit/utils';
+
+import { InlineProviderSchema, type LinkAuthProjectConfig } from './schemas/link-auth.js';
+
+/**
+ * Compile-time drift defense: top-level field sets must match between the
+ * Zod schema and the engine's `Provider` interface. A strict structural
+ * check would also catch sub-type drift, but TS variance (readonly arrays in
+ * engine vs mutable in Zod inference) makes that noisy — the realistic
+ * drift mode is adding/renaming a top-level field on one side and forgetting
+ * the other, which this top-level key comparison catches at `tsc` time.
+ *
+ * The schema and engine type are kept in sync by review (per slice 1 design
+ * decision); this guard makes that review easier.
+ */
+// Use `.shape` (Zod's declared-key record) rather than `keyof z.infer<...>` —
+// passthrough() injects `string | number` into the inferred key set, which
+// defeats the drift comparison. `.shape` is unaffected: it's the set of
+// declared fields, exactly what we want to compare against the engine type.
+type _SchemaKeys = keyof typeof InlineProviderSchema.shape;
+type _EngineKeys = keyof Provider;
+type _KeysAgree = [_SchemaKeys] extends [_EngineKeys]
+  ? [_EngineKeys] extends [_SchemaKeys]
+    ? true
+    : { error: 'engine Provider has a field that InlineProviderSchema lacks'; missing: Exclude<_EngineKeys, _SchemaKeys> }
+  : { error: 'InlineProviderSchema has a field that engine Provider lacks'; missing: Exclude<_SchemaKeys, _EngineKeys> };
+// Type-level assert: the declaration must compile to `true` (i.e. both
+// directions of the key-set check pass). The exported function holds a
+// reference to the type so noUnusedLocals doesn't fire.
+export const _assertSchemaKeysAgreeWithEngine: _KeysAgree = true;
+
+export function buildLinkAuthEngineConfig(adopter: LinkAuthProjectConfig): LinkAuthConfig {
+  const providers: Provider[] = adopter.providers.map((entry, index) =>
+    expandProviderEntry(entry, index),
+  );
+  // Pass the cache block through to the engine config so the slice-3
+  // content-fetch primitive can read `ttlMinutes` from the same source of
+  // truth as the rest of the engine config. The engine itself doesn't consume
+  // this field — it stays stateless — but riding along on the config keeps
+  // callers from having to pass two objects to the primitive.
+  if (adopter.cache !== undefined) {
+    return { providers, cache: buildEngineCache(adopter.cache) };
+  }
+  return { providers };
+}
+
+function buildEngineCache(
+  adopterCache: NonNullable<LinkAuthProjectConfig['cache']>,
+): NonNullable<LinkAuthConfig['cache']> {
+  // Schema is passthrough so adopterCache may carry forward-compatible extras
+  // we don't know about — copy only the fields the engine type declares.
+  return adopterCache.ttlMinutes === undefined ? {} : { ttlMinutes: adopterCache.ttlMinutes };
+}
+
+function expandProviderEntry(entry: unknown, index: number): Provider {
+  // Discriminate via Object.hasOwn (not `'use' in entry`) so a prototype-
+  // injected `use` cannot reroute an inline entry into macro expansion. The
+  // Zod parser already produces plain JSON-shaped objects, but defending in
+  // depth costs nothing.
+  const isMacroRef =
+    typeof entry === 'object' &&
+    entry !== null &&
+    Object.hasOwn(entry, 'use');
+
+  if (!isMacroRef) {
+    return entry as Provider;
+  }
+
+  const { use, overrides } = splitMacroEntry(entry as Record<string, unknown>);
+  const expanded = expandMacro(use, overrides);
+
+  // Re-validate the fully-expanded shape. Unknown extra keys pass through
+  // (the engine ignores them); what we still catch are missing required
+  // fields and wrong types on declared fields — e.g. a macro override that
+  // sets `check.notFoundMeaning: 'bogus'` is rejected here even though the
+  // upstream macro schema's passthrough accepted it.
+  const parsed = InlineProviderSchema.safeParse(expanded);
+  if (!parsed.success) {
+    throw new Error(
+      `linkAuth providers[${index}] (use: ${JSON.stringify(use)}) ` +
+        `produced an invalid provider after macro expansion: ${parsed.error.message}`,
+    );
+  }
+  return parsed.data as Provider;
+}
+
+function splitMacroEntry(entry: Record<string, unknown>): {
+  use: string;
+  overrides: Record<string, unknown>;
+} {
+  // `use` value comes from a Zod-validated source so the type is string at
+  // this point — but defensively-check it anyway: `Object.hasOwn` upstream
+  // tells us the key exists, not what its value's type is.
+  const useValue = entry['use'];
+  if (typeof useValue !== 'string') {
+    throw new TypeError(`linkAuth provider \`use\` must be a string, got ${typeof useValue}`);
+  }
+  const overrides: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(entry)) {
+    if (key === 'use') continue;
+    overrides[key] = value;
+  }
+  return { use: useValue, overrides };
+}