jamdesk 1.1.41 → 1.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/unit/vendored-sync.test.js +4 -0
- package/dist/__tests__/unit/vendored-sync.test.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/vendored/app/[[...slug]]/page.tsx +41 -28
- package/vendored/app/api/isr-health/route.ts +6 -4
- package/vendored/components/mdx/StepSlugContext.tsx +57 -0
- package/vendored/components/mdx/Steps.tsx +2 -2
- package/vendored/components/navigation/TableOfContents.tsx +77 -5
- package/vendored/lib/cache-tags.ts +25 -0
- package/vendored/lib/cache-utils.ts +19 -0
- package/vendored/lib/heading-extractor.ts +25 -6
- package/vendored/lib/indexnow.ts +1 -1
- package/vendored/lib/navigation-resolver.ts +1 -1
- package/vendored/lib/openapi-isr.ts +13 -8
- package/vendored/lib/r2-cleanup.ts +70 -0
- package/vendored/lib/r2-content.ts +0 -24
- package/vendored/lib/r2-manifest.ts +13 -3
- package/vendored/lib/revalidation-helpers.ts +41 -11
- package/vendored/lib/revalidation-trigger.ts +104 -28
- package/vendored/lib/scanner-blocklist.ts +256 -0
- package/vendored/lib/snippet-compiler-isr.ts +5 -2
- package/vendored/scripts/validate-links.cjs +17 -6
- package/vendored/workspace-package-lock.json +9 -9
- package/vendored/lib/cache-keys.ts +0 -117
|
@@ -15,19 +15,7 @@ export const R2_NOT_FOUND_ERROR_NAMES: ReadonlySet<string> = new Set([
|
|
|
15
15
|
]);
|
|
16
16
|
export const R2_NOT_FOUND_MESSAGE_PREFIX = 'Page not found';
|
|
17
17
|
|
|
18
|
-
export function evictOldest<K, V>(cache: Map<K, V>, maxSize: number): void {
|
|
19
|
-
if (cache.size <= maxSize) return;
|
|
20
|
-
const toDelete = cache.size - maxSize;
|
|
21
|
-
let count = 0;
|
|
22
|
-
for (const key of cache.keys()) {
|
|
23
|
-
if (count >= toDelete) break;
|
|
24
|
-
cache.delete(key);
|
|
25
|
-
count++;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
18
|
export function clearConfigCache(_projectSlug?: string): void {}
|
|
30
|
-
export function getConfigCacheSize(): number { return 0; }
|
|
31
19
|
|
|
32
20
|
export function isR2NotFound(_err: unknown): boolean { return false; }
|
|
33
21
|
|
|
@@ -54,18 +42,6 @@ export async function fetchOpenApiFile(_projectSlug: string, _specPath: string):
|
|
|
54
42
|
|
|
55
43
|
export async function listAllPaths(_projectSlug: string): Promise<string[]> { return []; }
|
|
56
44
|
|
|
57
|
-
export interface Manifest {
|
|
58
|
-
version: number;
|
|
59
|
-
timestamp: string;
|
|
60
|
-
projectSlug?: string;
|
|
61
|
-
files: Record<string, { hash: string; size?: number }>;
|
|
62
|
-
snippets?: Record<string, { hash: string }>;
|
|
63
|
-
openapi?: Record<string, { hash: string }>;
|
|
64
|
-
configHash?: string;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export async function fetchManifest(_projectSlug: string): Promise<Manifest | null> { return null; }
|
|
68
|
-
|
|
69
45
|
export interface AssetResult {
|
|
70
46
|
body: Uint8Array;
|
|
71
47
|
contentType: string;
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
S3Client,
|
|
11
11
|
GetObjectCommand,
|
|
12
12
|
} from '@aws-sdk/client-s3';
|
|
13
|
+
import { isR2NotFound } from './r2-content.js';
|
|
13
14
|
|
|
14
15
|
// R2 configuration from environment variables
|
|
15
16
|
const R2_CONFIG = {
|
|
@@ -52,12 +53,20 @@ export interface Manifest {
|
|
|
52
53
|
projectSlug?: string;
|
|
53
54
|
files: Record<string, { hash: string; size?: number }>;
|
|
54
55
|
snippets?: Record<string, { hash: string }>;
|
|
56
|
+
openapi?: Record<string, { hash: string }>;
|
|
55
57
|
assets?: Record<string, { hash: string }>;
|
|
58
|
+
staticFiles?: Record<string, { hash: string }>;
|
|
56
59
|
configHash?: string;
|
|
57
60
|
}
|
|
58
61
|
|
|
59
62
|
/**
|
|
60
|
-
* Fetch project manifest from R2
|
|
63
|
+
* Fetch project manifest from R2.
|
|
64
|
+
*
|
|
65
|
+
* Returns null only when the manifest object truly doesn't exist (NoSuchKey/404 —
|
|
66
|
+
* legitimate first build). Throws on transient R2 errors so the caller can
|
|
67
|
+
* distinguish a real first build versus a lost-state hiccup — silently treating
|
|
68
|
+
* a transient error as null causes the cleanup-removed-keys step to be skipped
|
|
69
|
+
* and forces a project-wide revalidation that wasn't actually warranted.
|
|
61
70
|
*/
|
|
62
71
|
export async function fetchManifest(
|
|
63
72
|
projectSlug: string
|
|
@@ -79,7 +88,8 @@ export async function fetchManifest(
|
|
|
79
88
|
|
|
80
89
|
const text = await response.Body.transformToString();
|
|
81
90
|
return JSON.parse(text);
|
|
82
|
-
} catch {
|
|
83
|
-
return null;
|
|
91
|
+
} catch (err) {
|
|
92
|
+
if (isR2NotFound(err)) return null;
|
|
93
|
+
throw err;
|
|
84
94
|
}
|
|
85
95
|
}
|
|
@@ -13,6 +13,7 @@ import { clearProjectCache as clearMcpProjectCache, clearCache as clearMcpCache
|
|
|
13
13
|
import { clearNavigationCache } from './navigation-resolver';
|
|
14
14
|
import { STATIC_REVALIDATION_PATHS } from './static-file-route';
|
|
15
15
|
import { clearRedirectCache } from './redirect-matcher';
|
|
16
|
+
import { mdxCacheTag, projectCacheTag } from './cache-tags';
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* Validate the revalidation secret using constant-time comparison.
|
|
@@ -119,15 +120,35 @@ export async function executeRevalidation(
|
|
|
119
120
|
): Promise<RevalidateResult> {
|
|
120
121
|
const revalidated: string[] = [];
|
|
121
122
|
|
|
122
|
-
//
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
123
|
+
// Whole-project clear: only when a project is named and the caller gave us
|
|
124
|
+
// nothing to scope on. A targeted request (tags or paths supplied) must NOT
|
|
125
|
+
// run the project-wide shims — clearConfigCache emits a revalidateTag that
|
|
126
|
+
// would silently widen the caller's surgical invalidation to bust unrelated
|
|
127
|
+
// cache namespaces, and the in-memory clears would discard parsed data the
|
|
128
|
+
// caller never asked to invalidate.
|
|
129
|
+
const isWholeProjectClear = Boolean(
|
|
130
|
+
request.project && request.tags.length === 0 && request.paths.length === 0
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
if (isWholeProjectClear) {
|
|
134
|
+
const project = request.project!;
|
|
135
|
+
clearConfigCache(project);
|
|
136
|
+
clearSnippetCache(project);
|
|
137
|
+
clearOpenApiCache(project);
|
|
138
|
+
clearMcpProjectCache(project);
|
|
139
|
+
await clearRedirectCache(project);
|
|
129
140
|
clearNavigationCache(); // Navigation cache keys include project name, clear all for simplicity
|
|
130
|
-
revalidated.push(`cache:${
|
|
141
|
+
revalidated.push(`cache:${project}`);
|
|
142
|
+
request.tags = [projectCacheTag(project)];
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Auto-add per-path mdx tags when paths are supplied with a project.
|
|
146
|
+
if (request.project && request.paths.length > 0) {
|
|
147
|
+
const pathTags = request.paths.map((p) => {
|
|
148
|
+
const normalized = p.replace(/^\//, '').replace(/\.mdx$/, '');
|
|
149
|
+
return mdxCacheTag(request.project!, normalized);
|
|
150
|
+
});
|
|
151
|
+
request.tags = [...request.tags, ...pathTags];
|
|
131
152
|
}
|
|
132
153
|
|
|
133
154
|
// Revalidate specific paths
|
|
@@ -143,10 +164,15 @@ export async function executeRevalidation(
|
|
|
143
164
|
}
|
|
144
165
|
}
|
|
145
166
|
|
|
146
|
-
// Revalidate by tag
|
|
167
|
+
// Revalidate by tag.
|
|
168
|
+
// `{ expire: 0 }` forces immediate purge — the next reader pays a fresh fetch
|
|
169
|
+
// rather than getting stale-while-revalidate content. Required for the
|
|
170
|
+
// publish→view loop: an author who just hit Publish must see their changes
|
|
171
|
+
// on first load, not on the second. The `'max'` profile would silently serve
|
|
172
|
+
// stale content to whoever loads the page first after a build.
|
|
147
173
|
if (request.tags.length > 0 && revalidateTag) {
|
|
148
174
|
for (const tag of request.tags) {
|
|
149
|
-
revalidateTag(tag);
|
|
175
|
+
revalidateTag(tag, { expire: 0 });
|
|
150
176
|
revalidated.push(`tag:${tag}`);
|
|
151
177
|
}
|
|
152
178
|
} else if (request.tags.length > 0) {
|
|
@@ -158,7 +184,11 @@ export async function executeRevalidation(
|
|
|
158
184
|
|
|
159
185
|
// Revalidate everything
|
|
160
186
|
if (request.all) {
|
|
161
|
-
|
|
187
|
+
// Bare {all: true} (no project scope) must also flush the in-memory parsed
|
|
188
|
+
// snippet/openapi caches — they store data in module-level Maps that
|
|
189
|
+
// revalidatePath alone won't invalidate. (clearConfigCache is per-project
|
|
190
|
+
// by design; admins running bare-all rely on the next reader hitting the
|
|
191
|
+
// unstable_cache TTL or use a follow-up project-scoped revalidate.)
|
|
162
192
|
clearSnippetCache();
|
|
163
193
|
clearOpenApiCache();
|
|
164
194
|
clearMcpCache();
|
|
@@ -147,61 +147,140 @@ async function purgeCloudflareCache(projectSlug: string): Promise<void> {
|
|
|
147
147
|
|
|
148
148
|
export interface Manifest {
|
|
149
149
|
files: Record<string, { hash: string }>;
|
|
150
|
+
snippets?: Record<string, { hash: string }>;
|
|
150
151
|
openapi?: Record<string, { hash: string }>;
|
|
152
|
+
assets?: Record<string, { hash: string }>;
|
|
153
|
+
staticFiles?: Record<string, { hash: string }>;
|
|
151
154
|
configHash?: string;
|
|
152
155
|
}
|
|
153
156
|
|
|
154
|
-
|
|
155
|
-
export function getChangedPaths(
|
|
157
|
+
function getMdxChangedPaths(
|
|
156
158
|
oldManifest: Manifest | null,
|
|
157
159
|
newManifest: Manifest
|
|
158
160
|
): string[] {
|
|
159
161
|
if (!oldManifest) {
|
|
160
|
-
// All files are new - return all paths
|
|
161
162
|
return Object.keys(newManifest.files).map(path => path.replace('.mdx', ''));
|
|
162
163
|
}
|
|
163
164
|
|
|
165
|
+
// Past the threshold the decision escalates to project-wide and the array
|
|
166
|
+
// is discarded, so cap collection at threshold+1 to avoid building huge
|
|
167
|
+
// path lists for bulk edits (e.g., 5k-page Prettier sweeps).
|
|
168
|
+
const cap = MANY_MDX_THRESHOLD + 1;
|
|
164
169
|
const changed: string[] = [];
|
|
165
170
|
|
|
166
|
-
// Check for new or modified files
|
|
167
171
|
for (const [path, info] of Object.entries(newManifest.files)) {
|
|
172
|
+
if (changed.length >= cap) return changed;
|
|
168
173
|
const oldInfo = oldManifest.files[path];
|
|
169
174
|
if (!oldInfo || oldInfo.hash !== info.hash) {
|
|
170
175
|
changed.push(path.replace('.mdx', ''));
|
|
171
176
|
}
|
|
172
177
|
}
|
|
173
178
|
|
|
174
|
-
// Check for deleted files
|
|
175
179
|
for (const path of Object.keys(oldManifest.files)) {
|
|
180
|
+
if (changed.length >= cap) return changed;
|
|
176
181
|
if (!(path in newManifest.files)) {
|
|
177
182
|
changed.push(path.replace('.mdx', ''));
|
|
178
183
|
}
|
|
179
184
|
}
|
|
180
185
|
|
|
181
|
-
// When config or OpenAPI specs changed (no MDX changes), revalidate all pages.
|
|
182
|
-
// Config changes (e.g., _hasCustomJs flag) affect every page's layout.
|
|
183
|
-
// OpenAPI changes can affect any endpoint page.
|
|
184
|
-
if (changed.length === 0 &&
|
|
185
|
-
(oldManifest.configHash !== newManifest.configHash ||
|
|
186
|
-
hasOpenapiChanges(oldManifest, newManifest))) {
|
|
187
|
-
return Object.keys(newManifest.files).map(p => p.replace('.mdx', ''));
|
|
188
|
-
}
|
|
189
|
-
|
|
190
186
|
return changed;
|
|
191
187
|
}
|
|
192
188
|
|
|
189
|
+
function hasConfigChanges(oldManifest: Manifest, newManifest: Manifest): boolean {
|
|
190
|
+
return oldManifest.configHash !== newManifest.configHash;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function hasHashMapChanges(
|
|
194
|
+
oldRecords: Record<string, { hash: string }> | undefined,
|
|
195
|
+
newRecords: Record<string, { hash: string }> | undefined
|
|
196
|
+
): boolean {
|
|
197
|
+
const oldEntries = oldRecords || {};
|
|
198
|
+
const newEntries = newRecords || {};
|
|
199
|
+
|
|
200
|
+
for (const [path, info] of Object.entries(newEntries)) {
|
|
201
|
+
if (!oldEntries[path] || oldEntries[path].hash !== info.hash) return true;
|
|
202
|
+
}
|
|
203
|
+
for (const path of Object.keys(oldEntries)) {
|
|
204
|
+
if (!(path in newEntries)) return true;
|
|
205
|
+
}
|
|
206
|
+
return false;
|
|
207
|
+
}
|
|
208
|
+
|
|
193
209
|
/** Check if any OpenAPI spec hashes changed between manifests. */
|
|
194
210
|
function hasOpenapiChanges(oldManifest: Manifest, newManifest: Manifest): boolean {
|
|
195
|
-
|
|
196
|
-
|
|
211
|
+
return hasHashMapChanges(oldManifest.openapi, newManifest.openapi);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function hasSnippetChanges(oldManifest: Manifest, newManifest: Manifest): boolean {
|
|
215
|
+
return hasHashMapChanges(oldManifest.snippets, newManifest.snippets);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function hasStaticFileChanges(oldManifest: Manifest, newManifest: Manifest): boolean {
|
|
219
|
+
return hasHashMapChanges(oldManifest.staticFiles, newManifest.staticFiles);
|
|
220
|
+
}
|
|
197
221
|
|
|
198
|
-
|
|
199
|
-
|
|
222
|
+
function hasAssetChanges(oldManifest: Manifest, newManifest: Manifest): boolean {
|
|
223
|
+
return hasHashMapChanges(oldManifest.assets, newManifest.assets);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export type RevalidationReason =
|
|
227
|
+
| 'first_build'
|
|
228
|
+
| 'mdx'
|
|
229
|
+
| 'many_mdx'
|
|
230
|
+
| 'config'
|
|
231
|
+
| 'openapi'
|
|
232
|
+
| 'snippets'
|
|
233
|
+
| 'static'
|
|
234
|
+
| 'assets';
|
|
235
|
+
|
|
236
|
+
export interface RevalidationDecision {
|
|
237
|
+
changedPaths: string[];
|
|
238
|
+
all: boolean;
|
|
239
|
+
reasons: RevalidationReason[];
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Above this number of MDX path changes we escalate to a project-wide
|
|
244
|
+
* invalidation rather than firing one revalidatePath per file. The current
|
|
245
|
+
* value is a heuristic — re-tune if production data shows a better tradeoff
|
|
246
|
+
* between per-call cost and Data Cache thrash for large bulk edits.
|
|
247
|
+
*/
|
|
248
|
+
const MANY_MDX_THRESHOLD = 50;
|
|
249
|
+
|
|
250
|
+
export function getRevalidationDecision(
|
|
251
|
+
oldManifest: Manifest | null,
|
|
252
|
+
newManifest: Manifest
|
|
253
|
+
): RevalidationDecision {
|
|
254
|
+
if (!oldManifest) {
|
|
255
|
+
return { changedPaths: [], all: true, reasons: ['first_build'] };
|
|
200
256
|
}
|
|
201
|
-
|
|
202
|
-
|
|
257
|
+
|
|
258
|
+
const changedPaths = getMdxChangedPaths(oldManifest, newManifest);
|
|
259
|
+
const reasons: RevalidationReason[] = [];
|
|
260
|
+
if (changedPaths.length > MANY_MDX_THRESHOLD) {
|
|
261
|
+
reasons.push('many_mdx');
|
|
262
|
+
} else if (changedPaths.length > 0) {
|
|
263
|
+
reasons.push('mdx');
|
|
203
264
|
}
|
|
204
|
-
|
|
265
|
+
if (hasConfigChanges(oldManifest, newManifest)) reasons.push('config');
|
|
266
|
+
if (hasOpenapiChanges(oldManifest, newManifest)) reasons.push('openapi');
|
|
267
|
+
if (hasSnippetChanges(oldManifest, newManifest)) reasons.push('snippets');
|
|
268
|
+
if (hasStaticFileChanges(oldManifest, newManifest)) reasons.push('static');
|
|
269
|
+
if (hasAssetChanges(oldManifest, newManifest)) reasons.push('assets');
|
|
270
|
+
|
|
271
|
+
// Operator-side rollback: if the targeted decision misbehaves in production
|
|
272
|
+
// (under-revalidates and serves stale content), set REVALIDATION_FORCE_ALL=true
|
|
273
|
+
// to escalate every non-empty change set to project-wide invalidation. The
|
|
274
|
+
// env-var check is per-build, so toggling on Cloud Run is a config update,
|
|
275
|
+
// not a redeploy.
|
|
276
|
+
const forceAll = process.env.REVALIDATION_FORCE_ALL === 'true' && reasons.length > 0;
|
|
277
|
+
const all = forceAll || reasons.some((reason) => reason !== 'mdx');
|
|
278
|
+
|
|
279
|
+
return {
|
|
280
|
+
changedPaths: all ? [] : changedPaths,
|
|
281
|
+
all,
|
|
282
|
+
reasons,
|
|
283
|
+
};
|
|
205
284
|
}
|
|
206
285
|
|
|
207
286
|
/** Trigger revalidation for changed files between manifests. */
|
|
@@ -210,19 +289,16 @@ export async function triggerRevalidationForChanges(
|
|
|
210
289
|
oldManifest: Manifest | null,
|
|
211
290
|
newManifest: Manifest
|
|
212
291
|
): Promise<void> {
|
|
213
|
-
const
|
|
292
|
+
const decision = getRevalidationDecision(oldManifest, newManifest);
|
|
214
293
|
|
|
215
|
-
if (
|
|
294
|
+
if (decision.reasons.length === 0) {
|
|
216
295
|
log('info', 'No changes detected, skipping revalidation', { projectSlug });
|
|
217
296
|
return;
|
|
218
297
|
}
|
|
219
298
|
|
|
220
|
-
// If more than 50 paths changed, just revalidate all
|
|
221
|
-
const shouldRevalidateAll = changedPaths.length > 50;
|
|
222
|
-
|
|
223
299
|
await triggerRevalidation({
|
|
224
300
|
projectSlug,
|
|
225
|
-
changedPaths:
|
|
226
|
-
all:
|
|
301
|
+
changedPaths: decision.changedPaths,
|
|
302
|
+
all: decision.all,
|
|
227
303
|
});
|
|
228
304
|
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// DO NOT EDIT — this file is auto-synced from shared/. Edit the source in shared/ and run ./scripts/sync-shared.sh
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Scanner probe blocklist.
|
|
5
|
+
*
|
|
6
|
+
* Returns a category tag for request paths that match well-known security-
|
|
7
|
+
* scanner patterns (.git, .env, wp-admin, phpmyadmin, etc.), or null for
|
|
8
|
+
* legitimate paths. Both Vercel proxies (build-service + proxy) use this
|
|
9
|
+
* as the first check in their middleware to short-circuit these requests
|
|
10
|
+
* with a 404 before any I/O — preventing the catch-all docs renderer / R2
|
|
11
|
+
* lookup from spending CPU on garbage traffic.
|
|
12
|
+
*
|
|
13
|
+
* The category is included in the proxy log line so we can rank patterns
|
|
14
|
+
* by volume (see scripts/analyze-scanner-blocks.sh) and promote the top
|
|
15
|
+
* ones to Vercel WAF rules at the network edge.
|
|
16
|
+
*
|
|
17
|
+
* Log-shape contract: each proxy emits a single-line JSON record formatted
|
|
18
|
+
* as `[middleware] scanner-block {"pathname":"...","category":"...","host":"...","clientIp":"...","clientIpSource":"..."}`.
|
|
19
|
+
* `clientIp` is sourced from `x-real-ip` (Vercel's edge sets it to the
|
|
20
|
+
* observed TCP peer — not spoofable from outside Vercel) with first-hop
|
|
21
|
+
* `x-forwarded-for` as a fallback for non-Vercel deploys / local dev.
|
|
22
|
+
* `clientIpSource` records WHICH header produced the value so the analyzer
|
|
23
|
+
* can filter to unspoofable IPs when promoting to a Vercel WAF block list —
|
|
24
|
+
* `x-real-ip` is trustworthy on Vercel; `x-forwarded-for` is attacker-
|
|
25
|
+
* controllable (Vercel preserves client-supplied XFF). Values:
|
|
26
|
+
* `x-real-ip` | `x-forwarded-for` | `none`. The analyzer's regex extractor
|
|
27
|
+
* (`"pathname":"[^"]+"`) assumes pathname/IP do NOT contain quote characters
|
|
28
|
+
* — true for IPs and real-world scanner probes, which use unescaped ASCII.
|
|
29
|
+
* If you extend the record to include fields that legitimately can contain
|
|
30
|
+
* quotes (user-agent, referer, query strings), update
|
|
31
|
+
* analyze-scanner-blocks.sh's regex to handle JSON-escaped quotes (`\"`)
|
|
32
|
+
* or it will silently truncate at the first one.
|
|
33
|
+
*
|
|
34
|
+
* Carve-out: /.well-known/* (with trailing slash) is always allowed for
|
|
35
|
+
* Let's Encrypt ACME, security.txt, app-site-association, etc.
|
|
36
|
+
*
|
|
37
|
+
* Case-insensitive: bots often randomize case to evade naive matchers.
|
|
38
|
+
*
|
|
39
|
+
* Defensive against // path traversal: even though Next.js normalizes
|
|
40
|
+
* pathname, we strip leading multiple slashes before matching.
|
|
41
|
+
*
|
|
42
|
+
* URL-encoding caveat: percent-encoded probes are NOT classified. Next.js's
|
|
43
|
+
* `request.nextUrl.pathname` preserves the percent-encoded form (per URL
|
|
44
|
+
* spec) — e.g. `/%2egit/config` arrives as `/%2egit/config`, not `/.git/config`,
|
|
45
|
+
* so the dotfile regex doesn't match. Same for double-encoding (`/%252egit/...`).
|
|
46
|
+
* In practice an encoded probe still 404s — the path doesn't exist — but it
|
|
47
|
+
* bypasses our fast-path 404 and pays full router fall-through. This is an
|
|
48
|
+
* intentional gap; the mitigation is Vercel WAF rules at the network edge,
|
|
49
|
+
* which decode before pattern-matching. Real-world scanners overwhelmingly
|
|
50
|
+
* use unencoded paths, so the helper's coverage is good in practice.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
export type ScannerCategory =
|
|
54
|
+
| 'dotfile' // /.git, /.env, /.aws, /.ssh, /.htaccess, /.idea, etc.
|
|
55
|
+
| 'wordpress' // /wp-admin, /wp-login.php, /wp-config.php, /xmlrpc.php
|
|
56
|
+
| 'phpmyadmin' // /phpmyadmin, /pma
|
|
57
|
+
| 'apache-status' // /server-status, /server-info
|
|
58
|
+
| 'php-config'; // /config.php, /configuration.php, /web.config, /adminer.php
|
|
59
|
+
|
|
60
|
+
// Exact-match → category (lowercase keys).
|
|
61
|
+
const SCANNER_EXACT: ReadonlyMap<string, ScannerCategory> = new Map([
|
|
62
|
+
['/wp-login.php', 'wordpress' as const],
|
|
63
|
+
['/wp-config.php', 'wordpress' as const],
|
|
64
|
+
['/wp-config.php.bak', 'wordpress' as const],
|
|
65
|
+
['/xmlrpc.php', 'wordpress' as const],
|
|
66
|
+
['/adminer.php', 'php-config' as const],
|
|
67
|
+
['/config.php', 'php-config' as const],
|
|
68
|
+
['/configuration.php', 'php-config' as const],
|
|
69
|
+
['/web.config', 'php-config' as const],
|
|
70
|
+
['/server-status', 'apache-status' as const],
|
|
71
|
+
['/server-info', 'apache-status' as const],
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
// Prefix-match → category (lowercase). Matches the exact prefix OR `${prefix}/...`.
|
|
75
|
+
const SCANNER_PREFIXES: ReadonlyArray<readonly [string, ScannerCategory]> = [
|
|
76
|
+
['/wp-admin', 'wordpress'],
|
|
77
|
+
['/wp-content', 'wordpress'],
|
|
78
|
+
['/wp-includes', 'wordpress'],
|
|
79
|
+
['/phpmyadmin', 'phpmyadmin'],
|
|
80
|
+
['/pma', 'phpmyadmin'],
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
export function classifyScannerProbe(pathname: string): ScannerCategory | null {
|
|
84
|
+
if (!pathname || pathname === '/') return null;
|
|
85
|
+
|
|
86
|
+
// Defensive: collapse leading multiple slashes so //.git/config → /.git/config.
|
|
87
|
+
// Next.js normalizes pathname, but this keeps the helper safe to call from
|
|
88
|
+
// anywhere.
|
|
89
|
+
const normalized = pathname.replace(/^\/+/, '/');
|
|
90
|
+
const lower = normalized.toLowerCase();
|
|
91
|
+
|
|
92
|
+
// Carve-out: legitimate /.well-known/* paths (ACME, security.txt, etc.).
|
|
93
|
+
// Note: requires trailing slash — bare /.well-known with no segment is
|
|
94
|
+
// still treated as a scanner probe (no real path is exactly /.well-known).
|
|
95
|
+
if (lower.startsWith('/.well-known/')) return null;
|
|
96
|
+
|
|
97
|
+
// Any root dotfile or dot-dir: /.git, /.env, /.aws/credentials, etc.
|
|
98
|
+
// Regex: starts with "/." followed by a non-slash char.
|
|
99
|
+
if (/^\/\.[^/]/.test(lower)) return 'dotfile';
|
|
100
|
+
|
|
101
|
+
const exactCategory = SCANNER_EXACT.get(lower);
|
|
102
|
+
if (exactCategory) return exactCategory;
|
|
103
|
+
|
|
104
|
+
for (const [prefix, category] of SCANNER_PREFIXES) {
|
|
105
|
+
if (lower === prefix || lower.startsWith(`${prefix}/`)) return category;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const LOG_PREFIX = '[middleware] scanner-block';
|
|
112
|
+
|
|
113
|
+
export type ClientIpSource = 'x-real-ip' | 'x-forwarded-for' | 'none';
|
|
114
|
+
|
|
115
|
+
export interface ScannerBlockLogPayload {
|
|
116
|
+
pathname: string;
|
|
117
|
+
category: ScannerCategory;
|
|
118
|
+
host: string;
|
|
119
|
+
clientIp: string;
|
|
120
|
+
clientIpSource: ClientIpSource;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Source tag lets the analyzer drop x-forwarded-for IPs (spoofable on Vercel)
|
|
124
|
+
// before WAF promotion — see file header for the full rationale.
|
|
125
|
+
export function clientIpFromHeaders(
|
|
126
|
+
xRealIp: string | null | undefined,
|
|
127
|
+
xForwardedFor: string | null | undefined,
|
|
128
|
+
): Pick<ScannerBlockLogPayload, 'clientIp' | 'clientIpSource'> {
|
|
129
|
+
const real = xRealIp?.trim();
|
|
130
|
+
if (real) return { clientIp: real, clientIpSource: 'x-real-ip' };
|
|
131
|
+
const xff = xForwardedFor?.split(',')[0]?.trim();
|
|
132
|
+
if (xff) return { clientIp: xff, clientIpSource: 'x-forwarded-for' };
|
|
133
|
+
return { clientIp: '', clientIpSource: 'none' };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Single-line JSON so `scripts/analyze-scanner-blocks.sh` can grep cleanly;
|
|
137
|
+
// `console.log(prefix, obj)` gets pretty-printed by Node and breaks the regex.
|
|
138
|
+
export function formatScannerBlockLog(payload: ScannerBlockLogPayload): string {
|
|
139
|
+
return `${LOG_PREFIX} ${JSON.stringify(payload)}`;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function parseScannerBlockLog(line: string): ScannerBlockLogPayload {
|
|
143
|
+
const expectedPrefix = `${LOG_PREFIX} `;
|
|
144
|
+
if (!line.startsWith(expectedPrefix)) {
|
|
145
|
+
throw new Error(`expected log line to start with "${expectedPrefix}"`);
|
|
146
|
+
}
|
|
147
|
+
return JSON.parse(line.slice(expectedPrefix.length)) as ScannerBlockLogPayload;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ── Persistence (Upstash Redis) ────────────────────────────────────────
|
|
151
|
+
//
|
|
152
|
+
// Edge middleware on each surface persists every blocked probe to a
|
|
153
|
+
// dedicated Upstash sorted set so the Mon/Thu digest can read the last
|
|
154
|
+
// 96 hours in one ZRANGEBYSCORE call. Score = unix ms timestamp; member
|
|
155
|
+
// = JSON-stringified ScannerBlockLogPayload. Retention is enforced
|
|
156
|
+
// prune-on-write (ZREMRANGEBYSCORE in the same pipeline as ZADD) so the
|
|
157
|
+
// set never exceeds 7 days even if the digest job stops.
|
|
158
|
+
|
|
159
|
+
export interface UpstashCreds {
|
|
160
|
+
/** Upstash REST API URL, e.g. https://xxxxxx.upstash.io */
|
|
161
|
+
url: string;
|
|
162
|
+
/** Upstash REST API bearer token. */
|
|
163
|
+
token: string;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Sorted-set key in Upstash. Single namespace shared across all three
|
|
167
|
+
* proxies. */
|
|
168
|
+
export const SCANNER_BLOCK_KEY = 'scanner-blocks';
|
|
169
|
+
|
|
170
|
+
/** Retention window. Prune-on-write trims anything older. */
|
|
171
|
+
export const SCANNER_BLOCK_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Best-effort write of a scanner-block event to Upstash, plus a prune of
|
|
175
|
+
* entries older than 7 days. Pipelined into a single round-trip. NEVER
|
|
176
|
+
* throws — middleware's primary job is the 404, persistence is a bonus.
|
|
177
|
+
*
|
|
178
|
+
* Use with `event.waitUntil()` from edge middleware so the 404 response
|
|
179
|
+
* is not blocked on this network call.
|
|
180
|
+
*/
|
|
181
|
+
export async function recordScannerBlock(
|
|
182
|
+
payload: ScannerBlockLogPayload,
|
|
183
|
+
creds: UpstashCreds,
|
|
184
|
+
now: number = Date.now(),
|
|
185
|
+
): Promise<void> {
|
|
186
|
+
try {
|
|
187
|
+
const member = JSON.stringify(payload);
|
|
188
|
+
const cutoff = now - SCANNER_BLOCK_RETENTION_MS;
|
|
189
|
+
const body = JSON.stringify([
|
|
190
|
+
['ZADD', SCANNER_BLOCK_KEY, String(now), member],
|
|
191
|
+
['ZREMRANGEBYSCORE', SCANNER_BLOCK_KEY, '-inf', String(cutoff)],
|
|
192
|
+
]);
|
|
193
|
+
await fetch(`${creds.url}/pipeline`, {
|
|
194
|
+
method: 'POST',
|
|
195
|
+
headers: {
|
|
196
|
+
Authorization: `Bearer ${creds.token}`,
|
|
197
|
+
'Content-Type': 'application/json',
|
|
198
|
+
},
|
|
199
|
+
body,
|
|
200
|
+
});
|
|
201
|
+
} catch {
|
|
202
|
+
// Swallow. Middleware must not fail because Upstash is flaky. The
|
|
203
|
+
// existing console.log emit still goes to Vercel logs, so
|
|
204
|
+
// analyze-scanner-blocks.sh keeps working as a fallback signal.
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Read all scanner-block events recorded since `sinceMs`. Throws on
|
|
210
|
+
* Upstash error — the digest job SHOULD fail loud if it can't reach
|
|
211
|
+
* storage; partial-data emails are misleading.
|
|
212
|
+
*/
|
|
213
|
+
export async function readScannerBlocks(
|
|
214
|
+
creds: UpstashCreds,
|
|
215
|
+
sinceMs: number,
|
|
216
|
+
): Promise<ScannerBlockLogPayload[]> {
|
|
217
|
+
const body = JSON.stringify([
|
|
218
|
+
'ZRANGEBYSCORE',
|
|
219
|
+
SCANNER_BLOCK_KEY,
|
|
220
|
+
String(sinceMs),
|
|
221
|
+
'+inf',
|
|
222
|
+
]);
|
|
223
|
+
const res = await fetch(creds.url, {
|
|
224
|
+
method: 'POST',
|
|
225
|
+
headers: {
|
|
226
|
+
Authorization: `Bearer ${creds.token}`,
|
|
227
|
+
'Content-Type': 'application/json',
|
|
228
|
+
},
|
|
229
|
+
body,
|
|
230
|
+
});
|
|
231
|
+
if (!res.ok) {
|
|
232
|
+
const errBody = await res.text();
|
|
233
|
+
throw new Error(`Upstash ZRANGEBYSCORE failed: ${res.status} ${errBody}`);
|
|
234
|
+
}
|
|
235
|
+
const data = (await res.json()) as { result?: unknown };
|
|
236
|
+
const members = Array.isArray(data.result) ? (data.result as string[]) : [];
|
|
237
|
+
const events: ScannerBlockLogPayload[] = [];
|
|
238
|
+
for (const m of members) {
|
|
239
|
+
try {
|
|
240
|
+
const parsed = JSON.parse(m) as Partial<ScannerBlockLogPayload>;
|
|
241
|
+
if (
|
|
242
|
+
parsed &&
|
|
243
|
+
typeof parsed.pathname === 'string' &&
|
|
244
|
+
typeof parsed.category === 'string' &&
|
|
245
|
+
typeof parsed.host === 'string' &&
|
|
246
|
+
typeof parsed.clientIp === 'string' &&
|
|
247
|
+
typeof parsed.clientIpSource === 'string'
|
|
248
|
+
) {
|
|
249
|
+
events.push(parsed as ScannerBlockLogPayload);
|
|
250
|
+
}
|
|
251
|
+
} catch {
|
|
252
|
+
// skip malformed entries
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return events;
|
|
256
|
+
}
|
|
@@ -77,9 +77,12 @@ export async function compileSnippetIsr(
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
|
-
* Clear snippet
|
|
80
|
+
* Clear the in-memory compiled-snippet Map for a project (or all projects).
|
|
81
81
|
*
|
|
82
|
-
*
|
|
82
|
+
* Does NOT touch the unstable_cache layer that wraps fetchSnippet — emitting a
|
|
83
|
+
* project: tag here would discard config/mdx/openapi/static cache entries
|
|
84
|
+
* unrelated to snippets. Callers needing that scope should dispatch the
|
|
85
|
+
* project tag themselves (executeRevalidation does this).
|
|
83
86
|
*/
|
|
84
87
|
export function clearSnippetCache(projectSlug?: string): void {
|
|
85
88
|
if (projectSlug) {
|
|
@@ -217,6 +217,16 @@ function generateSlug(text) {
|
|
|
217
217
|
.replace(/^-+|-+$/g, '');
|
|
218
218
|
}
|
|
219
219
|
|
|
220
|
+
/**
|
|
221
|
+
* Mirrors uniquifySlug in lib/heading-extractor.ts. First occurrence keeps the
|
|
222
|
+
* bare slug; later collisions get -2, -3, ... suffixes in source order.
|
|
223
|
+
*/
|
|
224
|
+
function uniquifySlug(seen, base) {
|
|
225
|
+
const count = seen.get(base) || 0;
|
|
226
|
+
seen.set(base, count + 1);
|
|
227
|
+
return count === 0 ? base : `${base}-${count + 1}`;
|
|
228
|
+
}
|
|
229
|
+
|
|
220
230
|
/**
|
|
221
231
|
* Extract heading slugs from MDX content.
|
|
222
232
|
* Includes markdown headings, <Update label="..."> anchors, and
|
|
@@ -229,6 +239,7 @@ function generateSlug(text) {
|
|
|
229
239
|
*/
|
|
230
240
|
function extractHeadingSlugs(content) {
|
|
231
241
|
const slugs = new Set();
|
|
242
|
+
const seen = new Map();
|
|
232
243
|
const lines = content.split('\n');
|
|
233
244
|
let inCodeBlock = false;
|
|
234
245
|
let fencePattern = '';
|
|
@@ -264,20 +275,20 @@ function extractHeadingSlugs(content) {
|
|
|
264
275
|
|
|
265
276
|
const headingMatch = line.match(/^#{1,6}\s+(.+)$/);
|
|
266
277
|
if (headingMatch) {
|
|
267
|
-
const
|
|
268
|
-
if (
|
|
278
|
+
const base = generateSlug(headingMatch[1].trim());
|
|
279
|
+
if (base) slugs.add(uniquifySlug(seen, base));
|
|
269
280
|
}
|
|
270
281
|
|
|
271
282
|
const updateMatch = line.match(/<Update\s+label=["']([^"']+)["']/);
|
|
272
283
|
if (updateMatch) {
|
|
273
|
-
const
|
|
274
|
-
if (
|
|
284
|
+
const base = generateSlug(updateMatch[1]);
|
|
285
|
+
if (base) slugs.add(uniquifySlug(seen, base));
|
|
275
286
|
}
|
|
276
287
|
|
|
277
288
|
if (inStepsBlock) {
|
|
278
289
|
for (const match of line.matchAll(STEP_TITLE_REGEX)) {
|
|
279
|
-
const
|
|
280
|
-
if (
|
|
290
|
+
const base = generateSlug(match[1] ?? match[2]);
|
|
291
|
+
if (base) slugs.add(uniquifySlug(seen, base));
|
|
281
292
|
}
|
|
282
293
|
}
|
|
283
294
|
|