@vibe-agent-toolkit/resources 0.1.39-rc.8 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-cache.d.ts +67 -0
- package/dist/content-cache.d.ts.map +1 -0
- package/dist/content-cache.js +160 -0
- package/dist/content-cache.js.map +1 -0
- package/dist/external-link-cache.d.ts +16 -2
- package/dist/external-link-cache.d.ts.map +1 -1
- package/dist/external-link-cache.js +43 -13
- package/dist/external-link-cache.js.map +1 -1
- package/dist/external-link-validator.d.ts +75 -2
- package/dist/external-link-validator.d.ts.map +1 -1
- package/dist/external-link-validator.js +184 -4
- package/dist/external-link-validator.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -1
- package/dist/link-auth-classify.d.ts +34 -0
- package/dist/link-auth-classify.d.ts.map +1 -0
- package/dist/link-auth-classify.js +42 -0
- package/dist/link-auth-classify.js.map +1 -0
- package/dist/link-auth-config-build.d.ts +47 -0
- package/dist/link-auth-config-build.d.ts.map +1 -0
- package/dist/link-auth-config-build.js +86 -0
- package/dist/link-auth-config-build.js.map +1 -0
- package/dist/link-auth-content-fetch.d.ts +75 -0
- package/dist/link-auth-content-fetch.d.ts.map +1 -0
- package/dist/link-auth-content-fetch.js +101 -0
- package/dist/link-auth-content-fetch.js.map +1 -0
- package/dist/link-auth-deps-memo.d.ts +40 -0
- package/dist/link-auth-deps-memo.d.ts.map +1 -0
- package/dist/link-auth-deps-memo.js +47 -0
- package/dist/link-auth-deps-memo.js.map +1 -0
- package/dist/link-auth-transport.d.ts +44 -0
- package/dist/link-auth-transport.d.ts.map +1 -0
- package/dist/link-auth-transport.js +151 -0
- package/dist/link-auth-transport.js.map +1 -0
- package/dist/link-parser.d.ts.map +1 -1
- package/dist/link-parser.js +8 -1
- package/dist/link-parser.js.map +1 -1
- package/dist/link-validator.d.ts +8 -8
- package/dist/link-validator.d.ts.map +1 -1
- package/dist/link-validator.js +51 -18
- package/dist/link-validator.js.map +1 -1
- package/dist/resource-registry.d.ts +2 -0
- package/dist/resource-registry.d.ts.map +1 -1
- package/dist/resource-registry.js +19 -4
- package/dist/resource-registry.js.map +1 -1
- package/dist/schemas/link-auth.d.ts +1049 -303
- package/dist/schemas/link-auth.d.ts.map +1 -1
- package/dist/schemas/link-auth.js +39 -20
- package/dist/schemas/link-auth.js.map +1 -1
- package/dist/schemas/project-config.d.ts +2942 -759
- package/dist/schemas/project-config.d.ts.map +1 -1
- package/dist/schemas/project-config.js +27 -1
- package/dist/schemas/project-config.js.map +1 -1
- package/dist/schemas/resource-metadata.d.ts +10 -9
- package/dist/schemas/resource-metadata.d.ts.map +1 -1
- package/dist/schemas/resource-metadata.js +2 -0
- package/dist/schemas/resource-metadata.js.map +1 -1
- package/package.json +4 -4
- package/src/content-cache.ts +190 -0
- package/src/external-link-cache.ts +50 -13
- package/src/external-link-validator.ts +268 -5
- package/src/index.ts +23 -0
- package/src/link-auth-classify.ts +61 -0
- package/src/link-auth-config-build.ts +130 -0
- package/src/link-auth-content-fetch.ts +146 -0
- package/src/link-auth-deps-memo.ts +56 -0
- package/src/link-auth-transport.ts +179 -0
- package/src/link-parser.ts +8 -1
- package/src/link-validator.ts +65 -17
- package/src/resource-registry.ts +25 -5
- package/src/schemas/link-auth.ts +47 -21
- package/src/schemas/project-config.ts +31 -1
- package/src/schemas/resource-metadata.ts +2 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public content-fetch primitive for the linkAuth feature (design issue #113 §6.2).
|
|
3
|
+
*
|
|
4
|
+
* Ships as a standalone primitive — no consumer wiring (asset-references,
|
|
5
|
+
* bundling) lands in this slice. The shape and security disciplines are
|
|
6
|
+
* designed so future callers can adopt it without reworking the contract.
|
|
7
|
+
*
|
|
8
|
+
* Responsibility split:
|
|
9
|
+
* - `resolveAuthenticatedUrl` (engine): pick provider, rewrite URL, resolve
|
|
10
|
+
* token, expand auth.headers AND fetch.headers against the same context.
|
|
11
|
+
* - `ContentCache`: persist `(bytes, metadata)` keyed by rewritten URL with
|
|
12
|
+
* 30-min default TTL; whitelists fields so tokens cannot be persisted.
|
|
13
|
+
* - `authTransport`: cross-origin auth strip, 429/Retry-After handling.
|
|
14
|
+
* - **This primitive**: wire them together, decide cache vs fetch, build
|
|
15
|
+
* metadata, return a typed result.
|
|
16
|
+
*
|
|
17
|
+
* Behavior matrix:
|
|
18
|
+
* - Engine returns `unsupported` → return as-is, no fetch, no cache touch.
|
|
19
|
+
* - Engine returns `unverified` → return as-is, no fetch, **no cache touch**
|
|
20
|
+
* even if a cache was supplied (§6.3: result flips when a token appears,
|
|
21
|
+
* so caching the no-token answer would poison future runs).
|
|
22
|
+
* - Engine returns success + cache present + not forceRefresh + cache hit →
|
|
23
|
+
* return `{ bytes, metadata, cached: true }`, no fetch.
|
|
24
|
+
* - Engine returns success otherwise → fetch via `authTransport` (using
|
|
25
|
+
* `fetch.headers` merged over `auth.headers` when present), write to
|
|
26
|
+
* cache if supplied, return `{ bytes, metadata, cached: false }`.
|
|
27
|
+
*
|
|
28
|
+
* **Throws** on network-level failures from the transport: DNS resolution
|
|
29
|
+
* failure, TLS handshake failure, connection refused, `AbortError` from the
|
|
30
|
+
* caller's `signal`, or any underlying `fetchImpl` rejection. The cache is
|
|
31
|
+
* only touched after the response body is fully read, so a transport failure
|
|
32
|
+
* cannot land a partial entry on disk. Consumers that need degradation
|
|
33
|
+
* semantics (validators, batch tools) should wrap calls in a try/catch.
|
|
34
|
+
*
|
|
35
|
+
* **Token never persisted.** Tokens are interpolated into headers in-memory;
|
|
36
|
+
* the metadata interface excludes header fields. The cache additionally
|
|
37
|
+
* whitelists the metadata fields it accepts. See cycle-6 token-persistence
|
|
38
|
+
* test for the on-disk assertion.
|
|
39
|
+
*
|
|
40
|
+
* **High-volume callers**: token resolution can be expensive (`gh auth token`
|
|
41
|
+
* spawns a subprocess). Callers iterating many URLs should wrap their `deps`
|
|
42
|
+
* once with `wrapLinkAuthDepsWithMemo` from `./link-auth-deps-memo.js` and
|
|
43
|
+
* pass the wrapped object to every invocation, so the same argv resolves at
|
|
44
|
+
* most once across the iteration.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import { resolveAuthenticatedUrl, type LinkAuthConfig } from '@vibe-agent-toolkit/utils';
|
|
48
|
+
|
|
49
|
+
import { type ContentCache, type ContentMetadata } from './content-cache.js';
|
|
50
|
+
import { type LinkAuthDeps } from './link-auth-deps-memo.js';
|
|
51
|
+
import { authTransport, type AuthTransportOptions } from './link-auth-transport.js';
|
|
52
|
+
|
|
53
|
+
export interface FetchAuthenticatedOptions {
|
|
54
|
+
/** Content cache to read from and write to. Omit to skip caching entirely. */
|
|
55
|
+
readonly cache?: ContentCache;
|
|
56
|
+
/** Bypass cache reads (still writes through on success). Default: false. */
|
|
57
|
+
readonly forceRefresh?: boolean;
|
|
58
|
+
/** Test/DI hook. Default: `globalThis.fetch`. */
|
|
59
|
+
readonly fetchImpl?: typeof fetch;
|
|
60
|
+
/** Token-resolution dependencies (env map, runCommand). Default: engine defaults. */
|
|
61
|
+
readonly deps?: LinkAuthDeps;
|
|
62
|
+
/** Propagated to the transport — caller's per-request timeout budget. */
|
|
63
|
+
readonly signal?: AbortSignal;
|
|
64
|
+
/** Per-call overrides for the transport's redirect / retry caps. */
|
|
65
|
+
readonly transportOptions?: AuthTransportOptions;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export type ContentFetchResult =
|
|
69
|
+
| {
|
|
70
|
+
readonly bytes: Uint8Array;
|
|
71
|
+
readonly metadata: ContentMetadata;
|
|
72
|
+
readonly cached: boolean;
|
|
73
|
+
}
|
|
74
|
+
| { readonly outcome: 'unsupported' }
|
|
75
|
+
| { readonly outcome: 'unverified'; readonly reason: string };
|
|
76
|
+
|
|
77
|
+
export async function fetchAuthenticated(
|
|
78
|
+
url: string,
|
|
79
|
+
config: LinkAuthConfig,
|
|
80
|
+
options: FetchAuthenticatedOptions = {},
|
|
81
|
+
): Promise<ContentFetchResult> {
|
|
82
|
+
const plan = resolveAuthenticatedUrl(url, config, options.deps);
|
|
83
|
+
// Discriminate via Object.hasOwn (not `'fetchUrl' in plan`) so a prototype-
|
|
84
|
+
// injected `fetchUrl` field cannot reroute a non-success outcome onto the
|
|
85
|
+
// cache/fetch path. The engine controls plan shape — defending in depth.
|
|
86
|
+
if (!Object.hasOwn(plan, 'fetchUrl')) {
|
|
87
|
+
// unsupported / unverified — both short-circuit without fetch or cache.
|
|
88
|
+
const failure = plan as Exclude<typeof plan, { fetchUrl: string }>;
|
|
89
|
+
return failure.outcome === 'unverified'
|
|
90
|
+
? { outcome: 'unverified', reason: failure.reason }
|
|
91
|
+
: { outcome: 'unsupported' };
|
|
92
|
+
}
|
|
93
|
+
const success = plan as Extract<typeof plan, { fetchUrl: string }>;
|
|
94
|
+
|
|
95
|
+
const cache = options.cache;
|
|
96
|
+
if (cache !== undefined && options.forceRefresh !== true) {
|
|
97
|
+
const hit = await cache.get(success.fetchUrl);
|
|
98
|
+
if (hit !== null) {
|
|
99
|
+
return { bytes: hit.bytes, metadata: hit.metadata, cached: true };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Merge headers: start from auth (covers Authorization + check-mode Accept),
|
|
104
|
+
// layer fetch.headers on top (overrides on conflict — e.g. swap the GitHub
|
|
105
|
+
// Accept from +json to .raw). Both header objects come from the engine, so
|
|
106
|
+
// both have the resolved token baked in (§6.2 dual-expand).
|
|
107
|
+
const requestHeaders: Record<string, string> = {
|
|
108
|
+
...success.headers,
|
|
109
|
+
...(success.fetchHeaders ?? {}),
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
const transportOptions: AuthTransportOptions = {
|
|
113
|
+
...(options.transportOptions ?? {}),
|
|
114
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
const response = await authTransport(
|
|
118
|
+
success.fetchUrl,
|
|
119
|
+
requestHeaders,
|
|
120
|
+
options.fetchImpl ?? globalThis.fetch,
|
|
121
|
+
transportOptions,
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
// Read full body as bytes. We use arrayBuffer (binary-clean) and wrap into a
|
|
125
|
+
// fresh Uint8Array so the caller doesn't hold a view onto fetch's internal
|
|
126
|
+
// buffer.
|
|
127
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
128
|
+
const bytes = new Uint8Array(arrayBuffer);
|
|
129
|
+
|
|
130
|
+
const metadata: ContentMetadata = {
|
|
131
|
+
status: response.status,
|
|
132
|
+
contentType: response.headers.get('content-type'),
|
|
133
|
+
etag: response.headers.get('etag'),
|
|
134
|
+
lastModified: response.headers.get('last-modified'),
|
|
135
|
+
fetchedAt: Date.now(),
|
|
136
|
+
rewrittenUrl: success.fetchUrl,
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
// Write-through. The ContentCache whitelists fields and fails soft on IO,
|
|
140
|
+
// so a write failure does not propagate.
|
|
141
|
+
if (cache !== undefined) {
|
|
142
|
+
await cache.set(success.fetchUrl, bytes, metadata);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return { bytes, metadata, cached: false };
|
|
146
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memoization wrapper for linkAuth `runCommand`-shaped token resolvers.
|
|
3
|
+
*
|
|
4
|
+
* Token resolution can be expensive — `gh auth token` spawns a subprocess on
|
|
5
|
+
* every call, env-source reads are cheap but consistent treatment is simpler
|
|
6
|
+
* to reason about — so the validator and the content-fetch primitive both
|
|
7
|
+
* wrap their `runCommand` once per session. Identical argv tuples return the
|
|
8
|
+
* cached result; distinct argv tuples are run independently.
|
|
9
|
+
*
|
|
10
|
+
* Lifetime of the memo is the lifetime of the returned `LinkAuthDeps` —
|
|
11
|
+
* one-per-session, never module-global. Callers that want a fresh resolve
|
|
12
|
+
* (e.g. token rotation between runs) construct a fresh validator / primitive
|
|
13
|
+
* call and get a fresh memo.
|
|
14
|
+
*
|
|
15
|
+
* Lifted out of `external-link-validator.ts` (where it shipped first in #125)
|
|
16
|
+
* so the slice-3 content-fetch primitive shares the same implementation —
|
|
17
|
+
* see CLAUDE.md "code duplication policy" (jscpd would flag a clone).
|
|
18
|
+
*
|
|
19
|
+
* Per design issue #113 review on PR #125 (memoize all token sources, not
|
|
20
|
+
* just `command` sources).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { defaultRunCommand, type resolveAuthenticatedUrl } from '@vibe-agent-toolkit/utils';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Re-exposed deps type — the third argument to `resolveAuthenticatedUrl`.
|
|
27
|
+
* Mirrors the engine's `Partial<TokenResolutionDeps>` without importing the
|
|
28
|
+
* private type name.
|
|
29
|
+
*/
|
|
30
|
+
export type LinkAuthDeps = Parameters<typeof resolveAuthenticatedUrl>[2];
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Wrap a `LinkAuthDeps` so its `runCommand` (used by the engine's
|
|
34
|
+
* `resolveToken`) caches results per unique argv. Without this, validating N
|
|
35
|
+
* URLs from the same host re-runs the token command N times.
|
|
36
|
+
*
|
|
37
|
+
* Caches by JSON-stringified argv so semantically-identical invocations
|
|
38
|
+
* share. The cache is per-call-site (one Map per wrapper); callers that share
|
|
39
|
+
* across multiple `resolveAuthenticatedUrl` invocations should call this
|
|
40
|
+
* once and reuse the wrapped object.
|
|
41
|
+
*/
|
|
42
|
+
export function wrapLinkAuthDepsWithMemo(deps: LinkAuthDeps): NonNullable<LinkAuthDeps> {
|
|
43
|
+
type RunCommandFn = NonNullable<NonNullable<LinkAuthDeps>['runCommand']>;
|
|
44
|
+
type RunCommandResult = ReturnType<RunCommandFn>;
|
|
45
|
+
const memo = new Map<string, RunCommandResult>();
|
|
46
|
+
const baseRunCommand: RunCommandFn = deps?.runCommand ?? defaultRunCommand;
|
|
47
|
+
const runCommand: RunCommandFn = (argv) => {
|
|
48
|
+
const key = JSON.stringify(argv);
|
|
49
|
+
const cached = memo.get(key);
|
|
50
|
+
if (cached !== undefined) return cached;
|
|
51
|
+
const fresh = baseRunCommand(argv);
|
|
52
|
+
memo.set(key, fresh);
|
|
53
|
+
return fresh;
|
|
54
|
+
};
|
|
55
|
+
return { ...(deps ?? {}), runCommand };
|
|
56
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auth-safe HTTP transport for the linkAuth feature (design issue #113 §5.2, §8).
|
|
3
|
+
*
|
|
4
|
+
* Lower-level than the public `fetchAuthenticated` primitive — this transport
|
|
5
|
+
* does not know about providers, token resolution, or caching. It takes a URL
|
|
6
|
+
* + already-built headers and adds two behaviors required by the design:
|
|
7
|
+
*
|
|
8
|
+
* §8 (cross-origin token leak defense) — `redirect: 'manual'`, then a
|
|
9
|
+
* bounded loop that follows Location headers. On any redirect to a
|
|
10
|
+
* different origin the `Authorization` header is stripped and stays
|
|
11
|
+
* stripped for subsequent hops (defeats token-laundering via a cross-origin
|
|
12
|
+
* bounce back to the original host).
|
|
13
|
+
*
|
|
14
|
+
* §5.2 (rate-limit handling) — on HTTP 429, parse `Retry-After` (seconds or
|
|
15
|
+
* HTTP-date), wait, retry. Bounded by `maxRetries` so a stuck host cannot
|
|
16
|
+
* hang validation; bounded by `maxRetryAfterMs` so a hostile or buggy host
|
|
17
|
+
* cannot pin a validation run for an hour with `Retry-After: 86400`.
|
|
18
|
+
*
|
|
19
|
+
* Pure-ish: `fetchImpl` and `sleep` are dependency-injected so tests do not
|
|
20
|
+
* touch the network or wall-clock. Production callers pass `globalThis.fetch`.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export interface AuthTransportOptions {
|
|
24
|
+
/** Maximum redirect hops to follow before returning the last 3xx response (default: 5). */
|
|
25
|
+
readonly maxRedirects?: number;
|
|
26
|
+
/** Maximum 429 retries before returning the final 429 response (default: 2). */
|
|
27
|
+
readonly maxRetries?: number;
|
|
28
|
+
/** Cap on Retry-After in ms; defends against hostile/buggy long values (default: 60_000). */
|
|
29
|
+
readonly maxRetryAfterMs?: number;
|
|
30
|
+
/** Abort signal propagated to every fetchImpl call (validator's per-request timeout budget). */
|
|
31
|
+
readonly signal?: AbortSignal;
|
|
32
|
+
/** Test-only sleep injection. Production callers omit. */
|
|
33
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const DEFAULT_MAX_REDIRECTS = 5;
|
|
37
|
+
const DEFAULT_MAX_RETRIES = 2;
|
|
38
|
+
const DEFAULT_MAX_RETRY_AFTER_MS = 60_000;
|
|
39
|
+
/**
|
|
40
|
+
* Minimum delay between 429 retries. A hostile host can send `Retry-After: 0`
|
|
41
|
+
* (or an HTTP-date in the past, parsing to 0); without a floor, we'd retry
|
|
42
|
+
* immediately, which is worse-than-useless for the host's rate-limiting and
|
|
43
|
+
* makes us a poor neighbor. Bounded by `maxRetries` regardless, but the floor
|
|
44
|
+
* gives the host's window a chance to slide.
|
|
45
|
+
*/
|
|
46
|
+
const MIN_RETRY_AFTER_MS = 250;
|
|
47
|
+
|
|
48
|
+
const defaultSleep = (ms: number): Promise<void> =>
|
|
49
|
+
new Promise((resolve) => {
|
|
50
|
+
setTimeout(resolve, ms);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Parse a `Retry-After` header value into milliseconds-to-wait.
|
|
55
|
+
*
|
|
56
|
+
* Returns `null` when the value is missing, empty, negative, or unparseable —
|
|
57
|
+
* callers treat `null` as "don't retry" (we can't pick a delay without a hint).
|
|
58
|
+
* HTTP-date values in the past return `0` (the server's hint is "you can retry
|
|
59
|
+
* now"). RFC 7231 §7.1.3.
|
|
60
|
+
*/
|
|
61
|
+
export function parseRetryAfter(value: string | null): number | null {
|
|
62
|
+
if (value === null) return null;
|
|
63
|
+
const trimmed = value.trim();
|
|
64
|
+
if (trimmed === '') return null;
|
|
65
|
+
|
|
66
|
+
// delta-seconds form: positive integer count of seconds.
|
|
67
|
+
if (/^\d+$/.test(trimmed)) {
|
|
68
|
+
return Number(trimmed) * 1000;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// HTTP-date form per RFC 7231 §7.1.1.1 — always contains weekday and month
|
|
72
|
+
// names. Guard with /[A-Za-z]/ because Date.parse() is permissive: '-5'
|
|
73
|
+
// parses as year-5 BC, '0' as 1 BC, etc. Requiring at least one letter
|
|
74
|
+
// restricts us to real HTTP-date strings.
|
|
75
|
+
if (!/[A-Za-z]/.test(trimmed)) return null;
|
|
76
|
+
const parsedMs = Date.parse(trimmed);
|
|
77
|
+
if (!Number.isFinite(parsedMs)) return null;
|
|
78
|
+
return Math.max(0, parsedMs - Date.now());
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export async function authTransport(
|
|
82
|
+
url: string,
|
|
83
|
+
headers: Record<string, string>,
|
|
84
|
+
fetchImpl: typeof fetch,
|
|
85
|
+
options: AuthTransportOptions = {},
|
|
86
|
+
): Promise<Response> {
|
|
87
|
+
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
88
|
+
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
89
|
+
const maxRetryAfterMs = options.maxRetryAfterMs ?? DEFAULT_MAX_RETRY_AFTER_MS;
|
|
90
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
91
|
+
|
|
92
|
+
let currentUrl = url;
|
|
93
|
+
let currentHeaders: Record<string, string> = { ...headers };
|
|
94
|
+
let redirects = 0;
|
|
95
|
+
let retries = 0;
|
|
96
|
+
|
|
97
|
+
// Bounded loop: every iteration either returns or strictly advances one of
|
|
98
|
+
// (retries, redirects), both capped, so termination is guaranteed within
|
|
99
|
+
// `maxRetries + maxRedirects + 1` fetchImpl calls.
|
|
100
|
+
while (redirects + retries < maxRedirects + maxRetries + 1) {
|
|
101
|
+
const init: RequestInit = {
|
|
102
|
+
headers: currentHeaders,
|
|
103
|
+
redirect: 'manual',
|
|
104
|
+
};
|
|
105
|
+
if (options.signal !== undefined) init.signal = options.signal;
|
|
106
|
+
|
|
107
|
+
const response = await fetchImpl(currentUrl, init);
|
|
108
|
+
|
|
109
|
+
const retryDelay =
|
|
110
|
+
retries < maxRetries ? computeRetryDelay(response, maxRetryAfterMs) : null;
|
|
111
|
+
if (retryDelay !== null) {
|
|
112
|
+
await sleep(retryDelay);
|
|
113
|
+
retries++;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (redirects < maxRedirects) {
|
|
118
|
+
const next = computeRedirect(response, currentUrl, currentHeaders);
|
|
119
|
+
if (next !== null) {
|
|
120
|
+
currentUrl = next.url;
|
|
121
|
+
currentHeaders = next.headers;
|
|
122
|
+
redirects++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return response;
|
|
128
|
+
}
|
|
129
|
+
// Unreachable: the loop returns or continues on every iteration, and the
|
|
130
|
+
// loop condition strictly bounds total iterations.
|
|
131
|
+
throw new Error('authTransport: unreachable iteration cap exceeded');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Decide the wait-time for a 429 response, or `null` if the response is not
|
|
136
|
+
* a 429 OR has no parseable Retry-After hint (no hint → caller returns the 429
|
|
137
|
+
* rather than retrying blindly).
|
|
138
|
+
*/
|
|
139
|
+
function computeRetryDelay(response: Response, maxRetryAfterMs: number): number | null {
|
|
140
|
+
if (response.status !== 429) return null;
|
|
141
|
+
const retryAfter = parseRetryAfter(response.headers.get('retry-after'));
|
|
142
|
+
if (retryAfter === null) return null;
|
|
143
|
+
// Two-sided clamp: never longer than the DoS cap, never shorter than the
|
|
144
|
+
// good-neighbor floor (defends against `Retry-After: 0` busy-loops).
|
|
145
|
+
return Math.min(Math.max(retryAfter, MIN_RETRY_AFTER_MS), maxRetryAfterMs);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Decide the next hop for a 3xx redirect, or `null` if the response is not a
|
|
150
|
+
* redirect / has no Location. Strips Authorization on cross-origin per §8.
|
|
151
|
+
*/
|
|
152
|
+
function computeRedirect(
|
|
153
|
+
response: Response,
|
|
154
|
+
currentUrl: string,
|
|
155
|
+
currentHeaders: Record<string, string>,
|
|
156
|
+
): { url: string; headers: Record<string, string> } | null {
|
|
157
|
+
if (response.status < 300 || response.status >= 400) return null;
|
|
158
|
+
const location = response.headers.get('location');
|
|
159
|
+
if (location === null) return null;
|
|
160
|
+
const nextUrl = new URL(location, currentUrl).toString();
|
|
161
|
+
const sameOrigin = new URL(nextUrl).origin === new URL(currentUrl).origin;
|
|
162
|
+
return {
|
|
163
|
+
url: nextUrl,
|
|
164
|
+
headers: sameOrigin ? currentHeaders : stripAuthorization(currentHeaders),
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Remove any header whose name is `Authorization` (case-insensitive). Returns
|
|
170
|
+
* a new object — does not mutate the input.
|
|
171
|
+
*/
|
|
172
|
+
function stripAuthorization(headers: Record<string, string>): Record<string, string> {
|
|
173
|
+
const result: Record<string, string> = {};
|
|
174
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
175
|
+
if (name.toLowerCase() === 'authorization') continue;
|
|
176
|
+
result[name] = value;
|
|
177
|
+
}
|
|
178
|
+
return result;
|
|
179
|
+
}
|
package/src/link-parser.ts
CHANGED
|
@@ -211,8 +211,15 @@ export function classifyLink(href: string): LinkType {
|
|
|
211
211
|
if (href.startsWith('#')) {
|
|
212
212
|
return 'anchor';
|
|
213
213
|
}
|
|
214
|
+
// Self-contained inline resources: a data: URI embeds its payload and a blob:
|
|
215
|
+
// URL references an in-memory object. Neither has a target to fetch or an
|
|
216
|
+
// anchor to resolve, so they are valid-but-nothing-to-validate (skipped), not
|
|
217
|
+
// "unknown". Common in HTML (inline SVG/PNG/GIF logos).
|
|
218
|
+
if (href.startsWith('data:') || href.startsWith('blob:')) {
|
|
219
|
+
return 'embedded';
|
|
220
|
+
}
|
|
214
221
|
// Any remaining href containing ':' is a protocol-like pattern we don't recognise
|
|
215
|
-
// (e.g., javascript:, tel:,
|
|
222
|
+
// (e.g., javascript:, tel:, ftp:) — classify as unknown rather than local file
|
|
216
223
|
if (href.includes(':')) {
|
|
217
224
|
return 'unknown';
|
|
218
225
|
}
|
package/src/link-validator.ts
CHANGED
|
@@ -59,6 +59,8 @@ export interface ValidateLinkOptions {
|
|
|
59
59
|
skipGitIgnoreCheck?: boolean;
|
|
60
60
|
/** Git tracker for efficient git-ignore checking (optional, improves performance) */
|
|
61
61
|
gitTracker?: GitTracker;
|
|
62
|
+
/** Strictly resolve HTML fragment anchors against element ids/names (default: false). HTML fragments are often runtime-defined by JS, so a static miss is not proof of breakage. */
|
|
63
|
+
checkHtmlAnchors?: boolean;
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
/**
|
|
@@ -93,7 +95,7 @@ export async function validateLink(
|
|
|
93
95
|
return await validateLocalFileLink(link, sourceFilePath, fragmentsByFile, options);
|
|
94
96
|
|
|
95
97
|
case 'anchor':
|
|
96
|
-
return await validateAnchorLink(link, sourceFilePath, fragmentsByFile, options
|
|
98
|
+
return await validateAnchorLink(link, sourceFilePath, fragmentsByFile, options);
|
|
97
99
|
|
|
98
100
|
case 'external':
|
|
99
101
|
// External URLs are not validated - don't report them
|
|
@@ -103,6 +105,11 @@ export async function validateLink(
|
|
|
103
105
|
// Email links are valid by default
|
|
104
106
|
return null;
|
|
105
107
|
|
|
108
|
+
case 'embedded':
|
|
109
|
+
// Self-contained inline resources (data:/blob:) have no target to
|
|
110
|
+
// validate — valid by default, don't report them.
|
|
111
|
+
return null;
|
|
112
|
+
|
|
106
113
|
case 'unknown':
|
|
107
114
|
return createRegistryIssue(
|
|
108
115
|
'LINK_UNKNOWN',
|
|
@@ -251,7 +258,12 @@ async function validateLocalFileLink(
|
|
|
251
258
|
if (gitIgnoreIssue) return gitIgnoreIssue;
|
|
252
259
|
|
|
253
260
|
if (resolved.anchor) {
|
|
254
|
-
const check = checkAnchor(
|
|
261
|
+
const check = checkAnchor(
|
|
262
|
+
resolved.anchor,
|
|
263
|
+
fileResult.resolvedPath,
|
|
264
|
+
fragmentsByFile,
|
|
265
|
+
options?.checkHtmlAnchors ?? false,
|
|
266
|
+
);
|
|
255
267
|
if (check === 'broken') {
|
|
256
268
|
return createRegistryIssue(
|
|
257
269
|
'LINK_BROKEN_ANCHOR',
|
|
@@ -279,13 +291,13 @@ async function validateAnchorLink(
|
|
|
279
291
|
link: ResourceLink,
|
|
280
292
|
sourceFilePath: string,
|
|
281
293
|
fragmentsByFile: FragmentIndex,
|
|
282
|
-
|
|
294
|
+
options?: ValidateLinkOptions,
|
|
283
295
|
): Promise<ValidationIssue | null> {
|
|
284
296
|
// Extract anchor (strip leading #)
|
|
285
297
|
const anchor = link.href.startsWith('#') ? link.href.slice(1) : link.href;
|
|
286
298
|
|
|
287
299
|
// Validate anchor exists in current file
|
|
288
|
-
const check = checkAnchor(anchor, sourceFilePath, fragmentsByFile);
|
|
300
|
+
const check = checkAnchor(anchor, sourceFilePath, fragmentsByFile, options?.checkHtmlAnchors ?? false);
|
|
289
301
|
|
|
290
302
|
switch (check) {
|
|
291
303
|
case 'skip':
|
|
@@ -295,7 +307,7 @@ async function validateAnchorLink(
|
|
|
295
307
|
return createRegistryIssue(
|
|
296
308
|
'LINK_BROKEN_ANCHOR',
|
|
297
309
|
`Anchor not found: ${link.href}`,
|
|
298
|
-
linkExtras(link, sourceFilePath, projectRoot, ''),
|
|
310
|
+
linkExtras(link, sourceFilePath, options?.projectRoot, ''),
|
|
299
311
|
);
|
|
300
312
|
default:
|
|
301
313
|
return assertNever(check);
|
|
@@ -383,37 +395,73 @@ export function fragmentIndex(entries: Iterable<readonly [string, Set<string>]>
|
|
|
383
395
|
return map;
|
|
384
396
|
}
|
|
385
397
|
|
|
398
|
+
/**
|
|
399
|
+
* Whether an HTML fragment is a structural, non-anchor hash: an SPA route
|
|
400
|
+
* (`#/route`) or a hash-encoded param string (`#id=1&mode=x`). Neither is
|
|
401
|
+
* ever a literal element id, regardless of `checkHtmlAnchors`.
|
|
402
|
+
*/
|
|
403
|
+
function isStructuralHtmlFragment(anchor: string): boolean {
|
|
404
|
+
return anchor.startsWith('/') || anchor.includes('=') || anchor.includes('&');
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Resolve an HTML-target anchor. Isolated from {@link checkAnchor} to keep
|
|
409
|
+
* cognitive complexity down.
|
|
410
|
+
*
|
|
411
|
+
* - Empty fragment / `top` (case-insensitive) — always `'valid'` (HTML
|
|
412
|
+
* top-navigation rule).
|
|
413
|
+
* - Structural non-anchor (`#/route`, `#k=v`, `#k=v&j=w`) — always `'skip'`;
|
|
414
|
+
* these are never element ids, so there is nothing to resolve.
|
|
415
|
+
* - `checkHtmlAnchors === false` (default) — `'skip'`: HTML fragments are
|
|
416
|
+
* frequently defined at runtime by JS (hash routers, hash query-params),
|
|
417
|
+
* so the static id/name set in the source HTML is not authoritative.
|
|
418
|
+
* - `checkHtmlAnchors === true` — resolve against the indexed ids/names
|
|
419
|
+
* (case-sensitive).
|
|
420
|
+
*/
|
|
421
|
+
function checkHtmlAnchor(
|
|
422
|
+
anchor: string,
|
|
423
|
+
entry: FragmentIndexEntry,
|
|
424
|
+
checkHtmlAnchors: boolean,
|
|
425
|
+
): AnchorCheck {
|
|
426
|
+
if (anchor === '' || anchor.toLowerCase() === 'top') {
|
|
427
|
+
return 'valid';
|
|
428
|
+
}
|
|
429
|
+
if (isStructuralHtmlFragment(anchor)) {
|
|
430
|
+
return 'skip';
|
|
431
|
+
}
|
|
432
|
+
if (!checkHtmlAnchors) {
|
|
433
|
+
return 'skip';
|
|
434
|
+
}
|
|
435
|
+
return entry.fragments.has(anchor) ? 'valid' : 'broken';
|
|
436
|
+
}
|
|
437
|
+
|
|
386
438
|
/**
|
|
387
439
|
* Check whether a fragment exists in the target file's anchor set.
|
|
388
440
|
*
|
|
389
441
|
* - `'skip'` — target file is not indexed; we cannot prove the anchor is
|
|
390
442
|
* broken, so callers must not emit an issue.
|
|
391
|
-
* -
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
* - For HTML targets the empty fragment (`#`) and `top` (ASCII
|
|
395
|
-
* case-insensitive) are always valid: per the HTML fragment-navigation
|
|
396
|
-
* algorithm both scroll to the top of the document regardless of whether a
|
|
397
|
-
* matching element exists, so `href="#"` / `href="#top"` are not broken.
|
|
443
|
+
* - HTML targets are delegated to {@link checkHtmlAnchor} (top-navigation
|
|
444
|
+
* rule, structural non-anchors, and the `checkHtmlAnchors` opt-in gate).
|
|
445
|
+
* - Markdown targets resolve against the indexed heading slugs, case-folded.
|
|
398
446
|
*
|
|
399
447
|
* @param anchor - Fragment without the leading `#`.
|
|
400
448
|
* @param targetFilePath - Absolute path of the file the fragment lives in.
|
|
401
449
|
* @param fragmentsByFile - Fragment index carrying each file's matching policy.
|
|
450
|
+
* @param checkHtmlAnchors - Strictly resolve HTML fragments against indexed
|
|
451
|
+
* element ids/names (default: false — see {@link checkHtmlAnchor}).
|
|
402
452
|
*/
|
|
403
453
|
export function checkAnchor(
|
|
404
454
|
anchor: string,
|
|
405
455
|
targetFilePath: string,
|
|
406
456
|
fragmentsByFile: FragmentIndex,
|
|
457
|
+
checkHtmlAnchors = false,
|
|
407
458
|
): AnchorCheck {
|
|
408
459
|
const entry = fragmentsByFile.get(targetFilePath);
|
|
409
460
|
if (!entry) {
|
|
410
461
|
return 'skip';
|
|
411
462
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
// HTML-format navigation rule, distinct from the case-folding policy above.
|
|
415
|
-
if (isHtmlPath(targetFilePath) && (anchor === '' || anchor.toLowerCase() === 'top')) {
|
|
416
|
-
return 'valid';
|
|
463
|
+
if (isHtmlPath(targetFilePath)) {
|
|
464
|
+
return checkHtmlAnchor(anchor, entry, checkHtmlAnchors);
|
|
417
465
|
}
|
|
418
466
|
const found = entry.caseSensitive
|
|
419
467
|
? entry.fragments.has(anchor)
|
package/src/resource-registry.ts
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
} from './frontmatter-link-validator.js';
|
|
24
24
|
import { validateFrontmatter } from './frontmatter-validator.js';
|
|
25
25
|
import { parseHtml } from './html-link-parser.js';
|
|
26
|
+
import { buildLinkAuthEngineConfig } from './link-auth-config-build.js';
|
|
26
27
|
import { parseMarkdown } from './link-parser.js';
|
|
27
28
|
import { fragmentIndexEntry, validateLink, type FragmentIndex, type ValidateLinkOptions } from './link-validator.js';
|
|
28
29
|
import type { ResourceCollectionInterface } from './resource-collection-interface.js';
|
|
@@ -93,6 +94,8 @@ export interface ValidateOptions {
|
|
|
93
94
|
validationMode?: 'strict' | 'permissive';
|
|
94
95
|
/** Check external URLs for validity (default: false) */
|
|
95
96
|
checkExternalUrls?: boolean;
|
|
97
|
+
/** Strictly validate HTML fragment anchors against element ids (default: false; HTML fragments are often runtime-defined by JS). */
|
|
98
|
+
checkHtmlAnchors?: boolean;
|
|
96
99
|
/** Disable cache for external URL checks (default: false) */
|
|
97
100
|
noCache?: boolean;
|
|
98
101
|
/**
|
|
@@ -527,7 +530,8 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
527
530
|
*/
|
|
528
531
|
private async validateAllLinks(
|
|
529
532
|
fragmentsByFile: FragmentIndex,
|
|
530
|
-
skipGitIgnoreCheck: boolean
|
|
533
|
+
skipGitIgnoreCheck: boolean,
|
|
534
|
+
checkHtmlAnchors: boolean
|
|
531
535
|
): Promise<ValidationIssue[]> {
|
|
532
536
|
const issues: ValidationIssue[] = [];
|
|
533
537
|
|
|
@@ -535,10 +539,11 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
535
539
|
for (const link of resource.links) {
|
|
536
540
|
// Only pass options if projectRoot is defined (exactOptionalPropertyTypes requirement)
|
|
537
541
|
const validateOptions = this.baseDir === undefined
|
|
538
|
-
? { skipGitIgnoreCheck }
|
|
542
|
+
? { skipGitIgnoreCheck, checkHtmlAnchors }
|
|
539
543
|
: {
|
|
540
544
|
projectRoot: this.baseDir,
|
|
541
545
|
skipGitIgnoreCheck,
|
|
546
|
+
checkHtmlAnchors,
|
|
542
547
|
...(this.gitTracker !== undefined && { gitTracker: this.gitTracker })
|
|
543
548
|
};
|
|
544
549
|
|
|
@@ -775,7 +780,8 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
775
780
|
// Validate each link in each resource
|
|
776
781
|
const linkIssues = await this.validateAllLinks(
|
|
777
782
|
fragmentsByFile,
|
|
778
|
-
options?.skipGitIgnoreCheck ?? false
|
|
783
|
+
options?.skipGitIgnoreCheck ?? false,
|
|
784
|
+
options?.checkHtmlAnchors ?? false
|
|
779
785
|
);
|
|
780
786
|
issues.push(...linkIssues);
|
|
781
787
|
|
|
@@ -838,10 +844,20 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
838
844
|
// Determine cache directory
|
|
839
845
|
const cacheDir = this.getCacheDirectory();
|
|
840
846
|
|
|
847
|
+
// Expand any `resources.linkAuth` providers from macro refs into the engine
|
|
848
|
+
// shape (#113 §5). When the adopter has no linkAuth config, the validator
|
|
849
|
+
// skips the authenticated branch and uses the existing markdown-link-check
|
|
850
|
+
// path for every URL — identical to pre-#113 behavior.
|
|
851
|
+
const adopterLinkAuth = this.config?.resources?.linkAuth;
|
|
852
|
+
const linkAuthConfig = adopterLinkAuth
|
|
853
|
+
? buildLinkAuthEngineConfig(adopterLinkAuth)
|
|
854
|
+
: undefined;
|
|
855
|
+
|
|
841
856
|
// Create validator
|
|
842
857
|
const validator = new ExternalLinkValidator(cacheDir, {
|
|
843
858
|
timeout: 15000,
|
|
844
859
|
cacheTtlHours: noCache ? 0 : 24,
|
|
860
|
+
...(linkAuthConfig !== undefined && { linkAuthConfig }),
|
|
845
861
|
});
|
|
846
862
|
|
|
847
863
|
// Collect all external URLs from all resources
|
|
@@ -911,7 +927,7 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
911
927
|
* @private
|
|
912
928
|
*/
|
|
913
929
|
private convertValidationResultsToIssues(
|
|
914
|
-
results: Array<{ url: string; status: 'ok' | 'error'; statusCode: number; error?: string }>,
|
|
930
|
+
results: Array<{ url: string; status: 'ok' | 'error'; statusCode: number; error?: string; code?: IssueCode }>,
|
|
915
931
|
urlsToValidate: Map<string, Array<{ resourcePath: string; line?: number }>>,
|
|
916
932
|
): ValidationIssue[] {
|
|
917
933
|
const issues: ValidationIssue[] = [];
|
|
@@ -926,7 +942,11 @@ export class ResourceRegistry implements ResourceCollectionInterface {
|
|
|
926
942
|
continue;
|
|
927
943
|
}
|
|
928
944
|
|
|
929
|
-
|
|
945
|
+
// Prefer a code surfaced by the validator's authenticated branch (#113 §7) —
|
|
946
|
+
// those LINK_AUTH_* codes encode per-provider notFoundMeaning routing that
|
|
947
|
+
// statusCode alone cannot express. Fall back to the statusCode mapping for
|
|
948
|
+
// the anonymous markdown-link-check path.
|
|
949
|
+
const issueCode = result.code ?? this.determineExternalUrlIssueCode(result.statusCode, result.error);
|
|
930
950
|
const errorMessage = result.error ?? `HTTP ${result.statusCode}`;
|
|
931
951
|
|
|
932
952
|
for (const location of locations) {
|