jamdesk 1.1.156 → 1.1.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/vendored/app/not-found.tsx +6 -0
- package/vendored/components/mdx/Card.tsx +4 -0
- package/vendored/components/theme/ThemeToggle.tsx +24 -14
- package/vendored/lib/auth-resolver.ts +36 -0
- package/vendored/lib/build/r2-upload.ts +2 -0
- package/vendored/lib/openapi/operation-description.ts +110 -0
- package/vendored/lib/project-deploy-id.ts +197 -0
- package/vendored/lib/project-deploy-publisher.ts +95 -0
- package/vendored/lib/r2-cleanup.ts +159 -30
- package/vendored/lib/r2-feature-flags.ts +46 -0
- package/vendored/lib/r2-key-builder.ts +28 -0
- package/vendored/lib/r2-manifest.ts +2 -0
- package/vendored/lib/r2-public-fetch.ts +133 -0
- package/vendored/lib/r2-public-storage-guard.ts +113 -0
- package/vendored/lib/r2-upload-lock.ts +277 -0
- package/vendored/lib/render-doc-page.tsx +80 -1
- package/vendored/lib/revalidation-helpers.ts +7 -0
- package/vendored/lib/revalidation-trigger.ts +112 -0
- package/vendored/lib/static-artifacts.ts +8 -2
- package/vendored/lib/validate-page-frontmatter.ts +39 -3
- package/vendored/shared/status-reporter.ts +1 -0
- package/vendored/themes/base.css +22 -0
- package/vendored/themes/jam/variables.css +16 -5
- package/vendored/themes/nebula/variables.css +10 -0
- package/vendored/themes/pulsar/variables.css +5 -5
- package/vendored/workspace-package-lock.json +172 -256
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
import { getR2S3Client, getR2Config } from './r2.js';
|
|
15
15
|
import { isKnownLanguageCode } from './language-utils.js';
|
|
16
16
|
import { logger } from '../shared/logger.js';
|
|
17
|
+
import { isDualWriteEnabled } from './r2-feature-flags.js';
|
|
17
18
|
|
|
18
19
|
// AWS / R2 cap DeleteObjects to 1000 keys per request. Match ListObjectsV2's
|
|
19
20
|
// MaxKeys to that so each list page maps 1:1 to a delete page.
|
|
@@ -35,7 +36,23 @@ export interface CleanupResult {
|
|
|
35
36
|
* fails here won't reappear in the next build's diff — surface `failed` for
|
|
36
37
|
* manual follow-up rather than relying on a re-attempt.
|
|
37
38
|
*
|
|
39
|
+
* Dual-write coupling: mirrors `upload-to-r2.cjs`. Always deletes the
|
|
40
|
+
* versioned key `<slug>/<deployId>/<prefix><relKey>`. When
|
|
41
|
+
* `R2_DUAL_WRITE !== 'false'` (default true), ALSO deletes the legacy key
|
|
42
|
+
* `<slug>/<prefix><relKey>` so files dropped from the manifest don't leak
|
|
43
|
+
* at the legacy path during the dual-write phase.
|
|
44
|
+
*
|
|
45
|
+
* INTENTIONAL NO-OP: the versioned delete targets the CURRENT build's
|
|
46
|
+
* deployId, but a removed file was never uploaded under this build's prefix
|
|
47
|
+
* (the build only writes files present in the new manifest). S3 DeleteObject
|
|
48
|
+
* on a nonexistent key succeeds silently, so this arm is a harmless
|
|
49
|
+
* idempotent no-op kept for symmetry with the upload key scheme. Real
|
|
50
|
+
* deletions happen on the legacy arm; OLD deployIds' versioned trees are
|
|
51
|
+
* immutable by design and reclaimed wholesale by r2-deploy-gc, never
|
|
52
|
+
* per-file here.
|
|
53
|
+
*
|
|
38
54
|
* @param projectSlug Project namespace under the bucket
|
|
55
|
+
* @param deployId Current build/deploy id (versioned-path segment)
|
|
39
56
|
* @param oldEntries Hash map from the previous manifest (may be undefined)
|
|
40
57
|
* @param newEntries Hash map from the new manifest (may be undefined)
|
|
41
58
|
* @param keyPrefix Path under {slug}/ — empty for root files (custom.css),
|
|
@@ -43,6 +60,7 @@ export interface CleanupResult {
|
|
|
43
60
|
*/
|
|
44
61
|
export async function deleteRemovedR2Objects(
|
|
45
62
|
projectSlug: string,
|
|
63
|
+
deployId: string,
|
|
46
64
|
oldEntries: Record<string, HashEntry> | undefined,
|
|
47
65
|
newEntries: Record<string, HashEntry> | undefined,
|
|
48
66
|
keyPrefix: string,
|
|
@@ -54,27 +72,37 @@ export async function deleteRemovedR2Objects(
|
|
|
54
72
|
const removed = oldKeys.filter((k) => !newSet.has(k));
|
|
55
73
|
if (removed.length === 0) return { deleted: [], failed: [] };
|
|
56
74
|
|
|
75
|
+
// Single source of truth: isDualWriteEnabled() trims env-var whitespace.
|
|
76
|
+
// Bare `process.env.R2_DUAL_WRITE !== 'false'` would treat ' false ' as truthy.
|
|
77
|
+
const dualWrite = isDualWriteEnabled();
|
|
57
78
|
const client = getR2S3Client();
|
|
58
79
|
const { bucketName } = getR2Config();
|
|
59
80
|
const deleted: string[] = [];
|
|
60
81
|
const failed: string[] = [];
|
|
61
82
|
|
|
83
|
+
const deleteOne = async (fullKey: string) => {
|
|
84
|
+
try {
|
|
85
|
+
await client.send(
|
|
86
|
+
new DeleteObjectCommand({ Bucket: bucketName, Key: fullKey }),
|
|
87
|
+
);
|
|
88
|
+
deleted.push(fullKey);
|
|
89
|
+
} catch (err) {
|
|
90
|
+
failed.push(fullKey);
|
|
91
|
+
logger.warn('R2 delete failed (non-fatal)', {
|
|
92
|
+
projectSlug,
|
|
93
|
+
key: fullKey,
|
|
94
|
+
error: (err as Error).message,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
|
|
62
99
|
await Promise.all(
|
|
63
|
-
removed.
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
);
|
|
69
|
-
deleted.push(fullKey);
|
|
70
|
-
} catch (err) {
|
|
71
|
-
failed.push(fullKey);
|
|
72
|
-
logger.warn('R2 delete failed (non-fatal)', {
|
|
73
|
-
projectSlug,
|
|
74
|
-
key: fullKey,
|
|
75
|
-
error: (err as Error).message,
|
|
76
|
-
});
|
|
77
|
-
}
|
|
100
|
+
removed.flatMap((relKey) => {
|
|
101
|
+
const versionedKey = `${projectSlug}/${deployId}/${keyPrefix}${relKey}`;
|
|
102
|
+
const legacyKey = `${projectSlug}/${keyPrefix}${relKey}`;
|
|
103
|
+
return dualWrite
|
|
104
|
+
? [deleteOne(versionedKey), deleteOne(legacyKey)]
|
|
105
|
+
: [deleteOne(versionedKey)];
|
|
78
106
|
}),
|
|
79
107
|
);
|
|
80
108
|
|
|
@@ -118,7 +146,7 @@ export interface PrunableManifest {
|
|
|
118
146
|
commitTimestamp?: string;
|
|
119
147
|
}
|
|
120
148
|
|
|
121
|
-
export type PruneSkipReason = 'disabled' | 'empty-manifest' | 'mass-delete' | 'stale-build';
|
|
149
|
+
export type PruneSkipReason = 'disabled' | 'dual-write-off' | 'empty-manifest' | 'mass-delete' | 'stale-build';
|
|
122
150
|
|
|
123
151
|
/**
|
|
124
152
|
* Per-category context for a skipped prune, logged for instant triage — the
|
|
@@ -266,6 +294,16 @@ export async function pruneRemovedContent(
|
|
|
266
294
|
if (envFlag('JD_PRUNE_CONTENT_DISABLED')) {
|
|
267
295
|
return { deleted: [], failed: [], skipped: true, skipReason: 'disabled' };
|
|
268
296
|
}
|
|
297
|
+
// Prune deletes LEGACY keys only. With dual-write off, builds no longer
|
|
298
|
+
// WRITE legacy keys, so pruning must not touch them either (prune prefix
|
|
299
|
+
// must match upload prefix); the frozen legacy tree gets reclaimed
|
|
300
|
+
// wholesale in the final migration step, not per-build here.
|
|
301
|
+
// NOTE: while readers still fall back to legacy on public-fetch 404s,
|
|
302
|
+
// that frozen tree can serve deleted-content ghosts — remove the legacy
|
|
303
|
+
// fallback from r2-content.ts BEFORE ever setting R2_DUAL_WRITE=false.
|
|
304
|
+
if (!isDualWriteEnabled()) {
|
|
305
|
+
return { deleted: [], failed: [], skipped: true, skipReason: 'dual-write-off' };
|
|
306
|
+
}
|
|
269
307
|
if (Object.keys(newManifest.files ?? {}).length === 0) {
|
|
270
308
|
return {
|
|
271
309
|
deleted: [],
|
|
@@ -493,10 +531,41 @@ export async function deleteAllProjectR2Objects(
|
|
|
493
531
|
): Promise<number> {
|
|
494
532
|
const client = getR2S3Client();
|
|
495
533
|
const { bucketName } = getR2Config();
|
|
496
|
-
|
|
534
|
+
return deleteKeysUnderPrefix(client, bucketName, `${projectSlug}/`);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Whether a `{slug}/<segment>/` path segment is a versioned deploy prefix (vs.
|
|
539
|
+
* a legacy content directory like `content/`, `snippets/`, `assets/`,
|
|
540
|
+
* `openapi/`, or a locale code). A deploy ID is a 20-char alphanumeric
|
|
541
|
+
* Firestore auto-ID (server buildId) or a 24-char hex ID (CLI upload).
|
|
542
|
+
*
|
|
543
|
+
* MUST stay identical to `looksLikeDeployId` in `scripts/r2-deploy-gc.cjs` —
|
|
544
|
+
* both gate DESTRUCTIVE deletes, and a looser match would delete a customer's
|
|
545
|
+
* live legacy content. Pinned by `__tests__/r2-cleanup.test.ts`.
|
|
546
|
+
*/
|
|
547
|
+
export function isVersionedDeployPrefixSegment(segment: string): boolean {
|
|
548
|
+
return /^[A-Za-z0-9]{20}$/.test(segment) || /^[a-f0-9]{24}$/i.test(segment);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Shared primitive: page ListObjectsV2 + DeleteObjects to remove every key
|
|
553
|
+
* under `prefix`, returning the count of keys successfully deleted.
|
|
554
|
+
*
|
|
555
|
+
* R2 caps both list and delete at 1000 per request, so each list page maps 1:1
|
|
556
|
+
* to a delete page. Per-key failures inside an otherwise-successful batch are
|
|
557
|
+
* logged (greppable) but do NOT throw — callers (`deleteAllProjectR2Objects`
|
|
558
|
+
* on project delete, `purgeVersionedDeployPrefixes` on gating) treat this as
|
|
559
|
+
* best-effort. A `send()` rejection (list or delete) DOES propagate so the
|
|
560
|
+
* caller's own error handling fires.
|
|
561
|
+
*/
|
|
562
|
+
async function deleteKeysUnderPrefix(
|
|
563
|
+
client: ReturnType<typeof getR2S3Client>,
|
|
564
|
+
bucketName: string,
|
|
565
|
+
prefix: string,
|
|
566
|
+
): Promise<number> {
|
|
497
567
|
let deleted = 0;
|
|
498
568
|
let continuationToken: string | undefined;
|
|
499
|
-
|
|
500
569
|
do {
|
|
501
570
|
const listResp: {
|
|
502
571
|
Contents?: Array<{ Key?: string }>;
|
|
@@ -528,11 +597,8 @@ export async function deleteAllProjectR2Objects(
|
|
|
528
597
|
} = await client.send(
|
|
529
598
|
new DeleteObjectsCommand({
|
|
530
599
|
Bucket: bucketName,
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
// Quiet=false so we get per-key Deleted[] for the count.
|
|
534
|
-
Quiet: false,
|
|
535
|
-
},
|
|
600
|
+
// Quiet=false so we get per-key Deleted[] for the count + Errors[].
|
|
601
|
+
Delete: { Objects: keys.map((Key) => ({ Key })), Quiet: false },
|
|
536
602
|
}),
|
|
537
603
|
);
|
|
538
604
|
|
|
@@ -540,11 +606,8 @@ export async function deleteAllProjectR2Objects(
|
|
|
540
606
|
|
|
541
607
|
const errors = delResp.Errors || [];
|
|
542
608
|
if (errors.length > 0) {
|
|
543
|
-
// Per-key failures inside an otherwise-successful batch. Log so ops
|
|
544
|
-
// can grep for the slug; we don't throw because the caller treats
|
|
545
|
-
// this whole operation as best-effort.
|
|
546
609
|
logger.warn('R2 batch delete had per-key errors', {
|
|
547
|
-
|
|
610
|
+
prefix,
|
|
548
611
|
errorCount: errors.length,
|
|
549
612
|
firstError: {
|
|
550
613
|
key: errors[0]?.Key,
|
|
@@ -555,10 +618,76 @@ export async function deleteAllProjectR2Objects(
|
|
|
555
618
|
}
|
|
556
619
|
}
|
|
557
620
|
|
|
558
|
-
continuationToken = listResp.IsTruncated ?
|
|
559
|
-
listResp.NextContinuationToken :
|
|
560
|
-
undefined;
|
|
621
|
+
continuationToken = listResp.IsTruncated ? listResp.NextContinuationToken : undefined;
|
|
561
622
|
} while (continuationToken);
|
|
562
623
|
|
|
563
624
|
return deleted;
|
|
564
625
|
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* Delete every versioned deploy prefix `{slug}/<deployId>/*` for a project,
|
|
629
|
+
* leaving legacy keys (`{slug}/content/…`, `docs.json`, `manifest.json`, static
|
|
630
|
+
* artifacts) untouched.
|
|
631
|
+
*
|
|
632
|
+
* Used when a project becomes password-gated: the CF public custom domain
|
|
633
|
+
* serves the versioned keys, so a gated project must have NONE — this reclaims
|
|
634
|
+
* any snapshot written while the project was still public (which would
|
|
635
|
+
* otherwise stay publicly fetchable, and be protected forever by r2-deploy-gc
|
|
636
|
+
* once the deployId pointer freezes). Always-gated projects have no such
|
|
637
|
+
* prefixes, so this is a cheap no-op for them.
|
|
638
|
+
*
|
|
639
|
+
* Best-effort per prefix: a failed delete is logged (ERROR, greppable) and
|
|
640
|
+
* skipped, not thrown — the caller runs this inside a build and must not fail
|
|
641
|
+
* an otherwise-successful gated build on a cleanup blip; the next gated build
|
|
642
|
+
* retries idempotently. Caller must validate `projectSlug` against
|
|
643
|
+
* `^[A-Za-z0-9_-]{1,128}$`. Returns the deploy prefixes actually deleted.
|
|
644
|
+
*/
|
|
645
|
+
export async function purgeVersionedDeployPrefixes(
|
|
646
|
+
projectSlug: string,
|
|
647
|
+
): Promise<string[]> {
|
|
648
|
+
const client = getR2S3Client();
|
|
649
|
+
const { bucketName } = getR2Config();
|
|
650
|
+
const slugPrefix = `${projectSlug}/`;
|
|
651
|
+
|
|
652
|
+
// 1) Enumerate top-level `{slug}/<seg>/` common prefixes; keep only deploys.
|
|
653
|
+
const deployPrefixes: string[] = [];
|
|
654
|
+
let continuationToken: string | undefined;
|
|
655
|
+
do {
|
|
656
|
+
const listed: {
|
|
657
|
+
CommonPrefixes?: Array<{ Prefix?: string }>;
|
|
658
|
+
IsTruncated?: boolean;
|
|
659
|
+
NextContinuationToken?: string;
|
|
660
|
+
} = await client.send(
|
|
661
|
+
new ListObjectsV2Command({
|
|
662
|
+
Bucket: bucketName,
|
|
663
|
+
Prefix: slugPrefix,
|
|
664
|
+
Delimiter: '/',
|
|
665
|
+
ContinuationToken: continuationToken,
|
|
666
|
+
}),
|
|
667
|
+
);
|
|
668
|
+
for (const cp of listed.CommonPrefixes ?? []) {
|
|
669
|
+
const prefix = cp.Prefix;
|
|
670
|
+
if (typeof prefix !== 'string') continue;
|
|
671
|
+
// `{slug}/<seg>/` → `<seg>` (strip leading slug prefix + trailing slash).
|
|
672
|
+
const segment = prefix.slice(slugPrefix.length, -1);
|
|
673
|
+
if (isVersionedDeployPrefixSegment(segment)) deployPrefixes.push(prefix);
|
|
674
|
+
}
|
|
675
|
+
continuationToken = listed.IsTruncated ? listed.NextContinuationToken : undefined;
|
|
676
|
+
} while (continuationToken);
|
|
677
|
+
|
|
678
|
+
// 2) Delete all keys under each deploy prefix, batched.
|
|
679
|
+
const purged: string[] = [];
|
|
680
|
+
for (const prefix of deployPrefixes) {
|
|
681
|
+
try {
|
|
682
|
+
await deleteKeysUnderPrefix(client, bucketName, prefix);
|
|
683
|
+
purged.push(prefix);
|
|
684
|
+
} catch (err) {
|
|
685
|
+
logger.error('Failed to purge versioned deploy prefix (non-fatal)', {
|
|
686
|
+
projectSlug,
|
|
687
|
+
prefix,
|
|
688
|
+
error: err instanceof Error ? err.message : String(err),
|
|
689
|
+
});
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
return purged;
|
|
693
|
+
}
|
|
@@ -5,3 +5,49 @@
|
|
|
5
5
|
export function isR2ParallelCounterEnabled(): boolean {
|
|
6
6
|
return process.env.R2_PARALLEL_COUNTER?.trim() === 'true';
|
|
7
7
|
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* CF public-domain caching flags (cf-r2-public-domain-cache plan).
|
|
11
|
+
*
|
|
12
|
+
* - USE_R2_PUBLIC_FETCH: master switch. When 'true' AND R2_PUBLIC_DOMAIN
|
|
13
|
+
* is set, public (non-gated) projects read via the CF public domain.
|
|
14
|
+
* Otherwise reads use the S3 SDK (legacy behavior). Default: false.
|
|
15
|
+
*
|
|
16
|
+
* - R2_DUAL_WRITE: during transition, upload-to-r2.cjs writes to BOTH
|
|
17
|
+
* the versioned and legacy paths. Default: true. Flip to false once
|
|
18
|
+
* all projects have at least one versioned deploy AND legacy reads
|
|
19
|
+
* are gone.
|
|
20
|
+
*/
|
|
21
|
+
export function isPublicFetchEnabled(): boolean {
|
|
22
|
+
if (process.env.USE_R2_PUBLIC_FETCH?.trim() !== 'true') return false;
|
|
23
|
+
if (!process.env.R2_PUBLIC_DOMAIN?.trim()) return false;
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function isDualWriteEnabled(): boolean {
|
|
28
|
+
return process.env.R2_DUAL_WRITE?.trim() !== 'false';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Fail fast on the one flag combination that silently strands every site
|
|
33
|
+
* on its previous deploy: dual-write OFF (writes go to versioned keys only)
|
|
34
|
+
* while public fetch is OFF (reads resolve legacy keys only) — nothing
|
|
35
|
+
* would ever read what the build writes. Throwing turns a silent
|
|
36
|
+
* fleet-wide staleness incident into an immediate, attributable build
|
|
37
|
+
* failure.
|
|
38
|
+
*
|
|
39
|
+
* NOTE: this checks THIS process's env (Cloud Run / CLI uploader). The
|
|
40
|
+
* read flags must ALSO be live on the ISR app — mirror
|
|
41
|
+
* USE_R2_PUBLIC_FETCH + R2_PUBLIC_DOMAIN there FIRST, then disable
|
|
42
|
+
* dual-write here. (Mirrored cjs guard: scripts/upload-to-r2.cjs.)
|
|
43
|
+
*/
|
|
44
|
+
export function assertFlagSequencingValid(): void {
|
|
45
|
+
if (isDualWriteEnabled() || isPublicFetchEnabled()) return;
|
|
46
|
+
throw new Error(
|
|
47
|
+
'R2 flag sequencing error: R2_DUAL_WRITE=false while public fetch is disabled ' +
|
|
48
|
+
'(USE_R2_PUBLIC_FETCH/R2_PUBLIC_DOMAIN unset). New builds would write versioned ' +
|
|
49
|
+
'keys that no reader resolves, stranding every site on its previous deploy. ' +
|
|
50
|
+
'Enable USE_R2_PUBLIC_FETCH=true + R2_PUBLIC_DOMAIN on BOTH the ISR app and ' +
|
|
51
|
+
'this service before disabling dual-write.',
|
|
52
|
+
);
|
|
53
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* R2 key construction with deployId-versioned prefixing.
|
|
3
|
+
*
|
|
4
|
+
* Versioned shape: {slug}/{deployId}/{rest}
|
|
5
|
+
* Legacy shape: {slug}/{rest} (kept for migration & GC tooling)
|
|
6
|
+
*
|
|
7
|
+
* Versioned URLs auto-invalidate cache: each build writes a fresh prefix,
|
|
8
|
+
* old prefixes age out of CF cache as their URLs stop being requested.
|
|
9
|
+
* GC removes them from R2 after a 7-day retention window.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const DEPLOY_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
13
|
+
|
|
14
|
+
function stripLeadingSlash(s: string): string {
|
|
15
|
+
return s.startsWith('/') ? s.slice(1) : s;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function buildVersionedKey(slug: string, deployId: string, rest: string): string {
|
|
19
|
+
if (!deployId) throw new Error('deployId is required');
|
|
20
|
+
if (!DEPLOY_ID_PATTERN.test(deployId)) {
|
|
21
|
+
throw new Error(`Invalid deployId (must match ${DEPLOY_ID_PATTERN}): ${deployId}`);
|
|
22
|
+
}
|
|
23
|
+
return `${slug}/${deployId}/${stripLeadingSlash(rest)}`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function buildLegacyKey(slug: string, rest: string): string {
|
|
27
|
+
return `${slug}/${stripLeadingSlash(rest)}`;
|
|
28
|
+
}
|
|
@@ -50,6 +50,8 @@ function getR2Client(): S3Client {
|
|
|
50
50
|
export interface Manifest {
|
|
51
51
|
version: number;
|
|
52
52
|
timestamp: string;
|
|
53
|
+
/** Build ID that produced this manifest; doubles as R2 prefix when versioned uploads are active. */
|
|
54
|
+
currentDeployId?: string;
|
|
53
55
|
projectSlug?: string;
|
|
54
56
|
files: Record<string, { hash: string; size?: number }>;
|
|
55
57
|
snippets?: Record<string, { hash: string }>;
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* R2 Public Fetch Helper
|
|
3
|
+
*
|
|
4
|
+
* Reads R2 objects via the Cloudflare public custom domain (R2_PUBLIC_DOMAIN).
|
|
5
|
+
* Uses fetch() with an AbortController timeout (R2_PUBLIC_FETCH_TIMEOUT_MS,
|
|
6
|
+
* default 5s) that covers BOTH headers and body read. Emits [r2-timing] log lines including cf-cache-status so on-call
|
|
7
|
+
* can grep Vercel logs during rollout.
|
|
8
|
+
*
|
|
9
|
+
* Used by the public-fetch path in r2-content.ts (Task 5.3+) to swap
|
|
10
|
+
* public-project reads to the CF public custom domain.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { logger } from '../shared/logger';
|
|
14
|
+
|
|
15
|
+
const DEFAULT_TIMEOUT_MS = 5000;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Operator-tunable like the S3 path's R2_FETCH_TIMEOUT_MS (prod runs that
|
|
19
|
+
* at 15000ms) — separate var so tuning the S3 fallback doesn't silently
|
|
20
|
+
* stretch the public path's fail-fast budget. Bad/≤0 values floor to the
|
|
21
|
+
* default, mirroring resolveR2Timeout's semantics.
|
|
22
|
+
*/
|
|
23
|
+
function resolveTimeoutMs(): number {
|
|
24
|
+
const raw = Number(process.env.R2_PUBLIC_FETCH_TIMEOUT_MS);
|
|
25
|
+
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_TIMEOUT_MS;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export class R2PublicNotFoundError extends Error {
|
|
29
|
+
constructor(key: string) {
|
|
30
|
+
super(`R2 public 404: ${key}`);
|
|
31
|
+
this.name = 'R2PublicNotFoundError';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface PublicBytesResult {
|
|
36
|
+
body: Uint8Array;
|
|
37
|
+
/**
|
|
38
|
+
* Raw content-type header; null when the CDN omitted it. Callers own the
|
|
39
|
+
* fallback (r2-content.ts fetchAsset uses guessContentType for parity
|
|
40
|
+
* with the S3 path's `response.ContentType || guessContentType(...)`).
|
|
41
|
+
*/
|
|
42
|
+
contentType: string | null;
|
|
43
|
+
contentLength: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function publicDomain(): string {
|
|
47
|
+
const d = process.env.R2_PUBLIC_DOMAIN;
|
|
48
|
+
if (!d) {
|
|
49
|
+
throw new Error('R2_PUBLIC_DOMAIN env var is required for public fetch path');
|
|
50
|
+
}
|
|
51
|
+
return d.replace(/\/$/, '');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Misrouted-domain guard: R2 serves objects with their stored content-type
|
|
56
|
+
* (text/mdx, application/json, image/*…). A text/html 200 for a non-.html
|
|
57
|
+
* key means R2_PUBLIC_DOMAIN isn't actually fronting the bucket — e.g. a
|
|
58
|
+
* CNAME still pointed at Vercel returning an HTML shell with status 200.
|
|
59
|
+
* Without this check that HTML would be cached and rendered as page
|
|
60
|
+
* content; throwing routes the read to the S3 fallback instead.
|
|
61
|
+
*/
|
|
62
|
+
function assertNotMisroutedHtml(key: string, contentType: string | null): void {
|
|
63
|
+
if (!contentType) return;
|
|
64
|
+
if (!contentType.toLowerCase().startsWith('text/html')) return;
|
|
65
|
+
if (key.toLowerCase().endsWith('.html')) return;
|
|
66
|
+
throw new Error(
|
|
67
|
+
`R2 public fetch returned text/html for non-HTML key ${key} — R2_PUBLIC_DOMAIN likely misrouted`,
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function fetchPublicR2Body(
|
|
72
|
+
key: string,
|
|
73
|
+
read: 'text' | 'bytes',
|
|
74
|
+
): Promise<{ body: string | Uint8Array; contentType: string | null }> {
|
|
75
|
+
const url = `${publicDomain()}/${key.replace(/^\//, '')}`;
|
|
76
|
+
const controller = new AbortController();
|
|
77
|
+
const timer = setTimeout(() => controller.abort(), resolveTimeoutMs());
|
|
78
|
+
const start = performance.now();
|
|
79
|
+
const region = process.env.VERCEL_REGION || 'unknown';
|
|
80
|
+
try {
|
|
81
|
+
const res = await fetch(url, { signal: controller.signal });
|
|
82
|
+
const ms = Math.round(performance.now() - start);
|
|
83
|
+
logger.info('[r2-timing]', {
|
|
84
|
+
operation: `publicFetch(${key})`,
|
|
85
|
+
ms,
|
|
86
|
+
region,
|
|
87
|
+
cf: res.headers.get('cf-cache-status') || 'unknown',
|
|
88
|
+
status: res.status,
|
|
89
|
+
});
|
|
90
|
+
if (res.status === 404) throw new R2PublicNotFoundError(key);
|
|
91
|
+
if (!res.ok) throw new Error(`R2 public fetch ${res.status} for ${key}`);
|
|
92
|
+
const contentType = res.headers.get('content-type');
|
|
93
|
+
assertNotMisroutedHtml(key, contentType);
|
|
94
|
+
// Body is consumed while the abort timer is still armed — a CDN
|
|
95
|
+
// response whose headers arrive fast but whose body stalls gets cut
|
|
96
|
+
// at TIMEOUT_MS instead of wedging the render indefinitely.
|
|
97
|
+
const body =
|
|
98
|
+
read === 'text' ? await res.text() : new Uint8Array(await res.arrayBuffer());
|
|
99
|
+
return { body, contentType };
|
|
100
|
+
} catch (err) {
|
|
101
|
+
if (err instanceof R2PublicNotFoundError) throw err;
|
|
102
|
+
const ms = Math.round(performance.now() - start);
|
|
103
|
+
logger.warn('[r2-timing]', {
|
|
104
|
+
operation: `publicFetch(${key})`,
|
|
105
|
+
ms,
|
|
106
|
+
region,
|
|
107
|
+
error: (err as Error).message,
|
|
108
|
+
});
|
|
109
|
+
throw err;
|
|
110
|
+
} finally {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export async function fetchPublicR2Text(key: string): Promise<string> {
|
|
116
|
+
const { body } = await fetchPublicR2Body(key, 'text');
|
|
117
|
+
return body as string;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export async function fetchPublicR2Bytes(key: string): Promise<PublicBytesResult | null> {
|
|
121
|
+
try {
|
|
122
|
+
const { body, contentType } = await fetchPublicR2Body(key, 'bytes');
|
|
123
|
+
const buf = body as Uint8Array;
|
|
124
|
+
return {
|
|
125
|
+
body: buf,
|
|
126
|
+
contentType,
|
|
127
|
+
contentLength: buf.length,
|
|
128
|
+
};
|
|
129
|
+
} catch (err) {
|
|
130
|
+
if (err instanceof R2PublicNotFoundError) return null;
|
|
131
|
+
throw err;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { redis } from './redis.js';
|
|
2
|
+
import { logger } from '../shared/logger.js';
|
|
3
|
+
|
|
4
|
+
interface AuthSecret {
|
|
5
|
+
hash?: unknown;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
interface AuthPublic {
|
|
9
|
+
enabled?: unknown;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function publicDomainConfigured(): boolean {
|
|
13
|
+
return !!process.env.R2_PUBLIC_DOMAIN?.trim();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function overrideEnabled(): boolean {
|
|
17
|
+
return process.env.R2_ALLOW_GATED_PUBLIC_BUCKET?.trim() === 'true';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function isGated(secret: AuthSecret | null, pub: AuthPublic | null): boolean {
|
|
21
|
+
return typeof secret?.hash === 'string' && pub?.enabled === true;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* R2 write scheme for a build:
|
|
26
|
+
* - 'versioned+legacy': dual-write as normal (the versioned/public keys the
|
|
27
|
+
* CF custom domain serves are safe to write for this project).
|
|
28
|
+
* - 'legacy-only': password-gated project — write ONLY the legacy
|
|
29
|
+
* `{slug}/<file>` keys. The versioned public keys are never written, so
|
|
30
|
+
* gated content cannot enter the CF-served key space. The ISR reader keeps
|
|
31
|
+
* serving these projects from the S3 SDK path (isProjectGated=true), so the
|
|
32
|
+
* legacy keys are all it needs.
|
|
33
|
+
*/
|
|
34
|
+
export type R2WriteScheme = 'versioned+legacy' | 'legacy-only';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Storage-safety guard for the R2 public custom domain rollout: decide a
|
|
38
|
+
* build's R2 write scheme.
|
|
39
|
+
*
|
|
40
|
+
* `isProjectGated()` intentionally fails open because it only chooses a read
|
|
41
|
+
* mechanism after request auth has already happened. The write side is
|
|
42
|
+
* different: before writing objects into a bucket that is exposed by
|
|
43
|
+
* R2_PUBLIC_DOMAIN, it must fail closed unless we can prove the project is not
|
|
44
|
+
* password-gated or an explicit customer-acceptance override is set.
|
|
45
|
+
*
|
|
46
|
+
* A password-gated project resolves to `'legacy-only'` — its versioned/public
|
|
47
|
+
* keys are never written, so gated content stays out of the CF-served key
|
|
48
|
+
* space (the build still succeeds; the S3 read path serves it). Non-gated
|
|
49
|
+
* projects, the no-public-domain case, and the acceptance override all resolve
|
|
50
|
+
* to `'versioned+legacy'`. Throws when gated status cannot be determined (Redis
|
|
51
|
+
* unavailable / read fails after retries) — we must never write to a public
|
|
52
|
+
* bucket unverified, and a Redis outage during upload is a real failure worth
|
|
53
|
+
* surfacing (rare — the build has already done many Redis reads by this point).
|
|
54
|
+
*/
|
|
55
|
+
export async function resolveR2WriteScheme(
|
|
56
|
+
slug: string,
|
|
57
|
+
opts: { retryAttempts?: number; retryDelayMs?: number } = {},
|
|
58
|
+
): Promise<R2WriteScheme> {
|
|
59
|
+
const { retryAttempts = 3, retryDelayMs = 200 } = opts;
|
|
60
|
+
if (!publicDomainConfigured()) return 'versioned+legacy';
|
|
61
|
+
if (overrideEnabled()) {
|
|
62
|
+
// The override must never run silently: it disables the only write-side
|
|
63
|
+
// control keeping gated content out of a publicly fetchable bucket.
|
|
64
|
+
// WARN on every build so a forgotten env var stays visible in logs.
|
|
65
|
+
logger.warn(
|
|
66
|
+
'[r2-public-storage-guard] R2_ALLOW_GATED_PUBLIC_BUCKET override ACTIVE — gated-content write protection bypassed',
|
|
67
|
+
{ slug },
|
|
68
|
+
);
|
|
69
|
+
return 'versioned+legacy';
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (!redis) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Cannot verify gated status for ${slug}; refusing to write to a public R2 bucket`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
const client = redis;
|
|
78
|
+
|
|
79
|
+
// Retry a few times before failing closed: a transient Upstash blip during
|
|
80
|
+
// the upload phase otherwise hard-fails the whole build (we fail closed on
|
|
81
|
+
// any read error) even for non-gated projects. An outage that outlasts the
|
|
82
|
+
// retries still fails closed — correct, since we must never write to a
|
|
83
|
+
// public bucket unverified.
|
|
84
|
+
let secret: AuthSecret | null = null;
|
|
85
|
+
let pub: AuthPublic | null = null;
|
|
86
|
+
let lastErr: unknown;
|
|
87
|
+
let read = false;
|
|
88
|
+
for (let attempt = 1; attempt <= retryAttempts; attempt++) {
|
|
89
|
+
try {
|
|
90
|
+
const [secretResult, publicResult] = await Promise.all([
|
|
91
|
+
client.get(`projectAuthSecret:${slug}`),
|
|
92
|
+
client.get(`projectAuthPublic:${slug}`),
|
|
93
|
+
]);
|
|
94
|
+
secret = secretResult as AuthSecret | null;
|
|
95
|
+
pub = publicResult as AuthPublic | null;
|
|
96
|
+
read = true;
|
|
97
|
+
break;
|
|
98
|
+
} catch (err) {
|
|
99
|
+
lastErr = err;
|
|
100
|
+
if (attempt < retryAttempts && retryDelayMs > 0) {
|
|
101
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (!read) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`Cannot verify gated status for ${slug}; refusing to write to a public R2 bucket`,
|
|
108
|
+
{ cause: lastErr },
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return isGated(secret, pub) ? 'legacy-only' : 'versioned+legacy';
|
|
113
|
+
}
|