canopycms 0.0.68-int.100 → 0.0.68-int.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/api/branch.js +7 -0
- package/dist/api/client.d.ts +2 -1
- package/dist/api/client.js +2 -1
- package/dist/api/github-sync.d.ts +9 -0
- package/dist/api/github-sync.js +37 -0
- package/dist/assets/finalize.js +3 -0
- package/dist/assets/materialize.d.ts +45 -14
- package/dist/assets/materialize.js +145 -48
- package/dist/assets/store-local.d.ts +15 -5
- package/dist/assets/store-local.js +58 -10
- package/dist/assets/store-s3.d.ts +28 -6
- package/dist/assets/store-s3.js +78 -24
- package/dist/assets/types.d.ts +17 -3
- package/dist/auth/plugin.d.ts +1 -1
- package/dist/cli/asset-refs.d.ts +30 -8
- package/dist/cli/asset-refs.js +99 -73
- package/dist/cli/cli.d.ts +13 -0
- package/dist/cli/cli.js +2136 -1371
- package/dist/cli/configured-asset-store.d.ts +12 -0
- package/dist/cli/configured-asset-store.js +46 -0
- package/dist/cli/generate-ai-content.js +639 -178
- package/dist/cli/init.js +1 -0
- package/dist/client.d.ts +1 -1
- package/dist/client.js +1 -1
- package/dist/config/schemas/config.d.ts +3 -0
- package/dist/config/schemas/config.js +1 -0
- package/dist/config/types.d.ts +15 -0
- package/dist/content-reader.d.ts +9 -0
- package/dist/content-reader.js +8 -4
- package/dist/content-store.d.ts +4 -0
- package/dist/content-store.js +2 -0
- package/dist/editor/Editor.js +8 -2
- package/dist/editor/PreviewFrame.d.ts +24 -0
- package/dist/editor/PreviewFrame.js +140 -0
- package/dist/editor/context/ApiClientContext.d.ts +2 -1
- package/dist/editor/context/ApiClientContext.js +2 -1
- package/dist/editor/editor-utils.d.ts +2 -2
- package/dist/editor/editor-utils.js +2 -2
- package/dist/editor/fields/MarkdownField.js +6 -2
- package/dist/editor/fields/mdx-jsx-support.d.ts +5 -1
- package/dist/editor/fields/mdx-jsx-support.js +74 -70
- package/dist/editor/hooks/useBranchManager.d.ts +10 -0
- package/dist/editor/hooks/useBranchManager.js +125 -36
- package/dist/editor/preview-bridge.d.ts +7 -26
- package/dist/editor/preview-bridge.js +7 -140
- package/dist/github-service.d.ts +7 -0
- package/dist/github-service.js +14 -0
- package/dist/http/handler.js +7 -2
- package/dist/preview.d.ts +8 -0
- package/dist/preview.js +7 -0
- package/dist/server.d.ts +40 -0
- package/dist/server.js +26 -0
- package/dist/url-exclusivity-fixtures.d.ts +7 -0
- package/dist/url-exclusivity-fixtures.js +10 -0
- package/dist/utils/content-serialize.d.ts +5 -2
- package/dist/utils/content-serialize.js +98 -24
- package/dist/utils/yaml-source-splice.d.ts +58 -0
- package/dist/utils/yaml-source-splice.js +515 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worker/task-runner.js +60 -10
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -396,7 +396,7 @@ The editor loads entries on the client from `/api/canopycms/[branch]/entries`, s
|
|
|
396
396
|
_TODO_ show an example
|
|
397
397
|
|
|
398
398
|
7. **Split builds**
|
|
399
|
-
Keep your public build free of editor bundles by importing only from `canopycms` (server helpers + data loaders). Host the editor in a separate app or build target that imports from `canopycms/client` and mounts the API routes above.
|
|
399
|
+
Keep your public build free of editor bundles by importing only from `canopycms` (server helpers + data loaders) and `canopycms/preview` (the live-preview hook). Host the editor in a separate app or build target that imports from `canopycms/client` and mounts the API routes above.
|
|
400
400
|
|
|
401
401
|
_TODO_ show real examples of what to do
|
|
402
402
|
|
|
@@ -415,7 +415,7 @@ The `useCanopyPreview` hook provides live updates to your preview components as
|
|
|
415
415
|
```tsx
|
|
416
416
|
'use client'
|
|
417
417
|
|
|
418
|
-
import { useCanopyPreview } from 'canopycms/
|
|
418
|
+
import { useCanopyPreview } from 'canopycms/preview'
|
|
419
419
|
import type { PostContent } from './schemas'
|
|
420
420
|
|
|
421
421
|
export function PostView({ data }: { data: PostContent }) {
|
|
@@ -451,7 +451,7 @@ When using reference fields (foreign key relationships to other content), the ed
|
|
|
451
451
|
```tsx
|
|
452
452
|
'use client'
|
|
453
453
|
|
|
454
|
-
import { useCanopyPreview } from 'canopycms/
|
|
454
|
+
import { useCanopyPreview } from 'canopycms/preview'
|
|
455
455
|
|
|
456
456
|
export function PostView({ data }: { data: PostContent }) {
|
|
457
457
|
const { data: liveData, isLoading } = useCanopyPreview<PostContent>({
|
package/dist/api/branch.js
CHANGED
|
@@ -14,6 +14,7 @@ import { filePathExists } from '../utils/fs.js';
|
|
|
14
14
|
import { isNetworkRemoteUrl } from '../utils/git.js';
|
|
15
15
|
import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX, RESERVED_ROUTE_BRANCH_NAMES, isSettingsBranchName, } from '../paths/index.js';
|
|
16
16
|
import { GitManager } from '../git-manager.js';
|
|
17
|
+
import { syncDeleteRemoteBranch } from './github-sync.js';
|
|
17
18
|
import { branchNameSchema, branchParamSchema } from './validators.js';
|
|
18
19
|
const log = createDebugLogger({ prefix: 'BranchAPI' });
|
|
19
20
|
const createBranchBodySchema = z.object({
|
|
@@ -568,6 +569,12 @@ export const deleteBranchHandler = async (ctx, req, params) => {
|
|
|
568
569
|
console.warn(`CanopyCMS: failed to remove deleted branch's mirror head (branch ${sanitizedDeleted}, mirror ${mirrorPath}): ${getErrorMessage(err)}`);
|
|
569
570
|
}
|
|
570
571
|
}
|
|
572
|
+
// `branchContext` was read before the delete, so it still carries the PR number this gates on.
|
|
573
|
+
if (deletableHead) {
|
|
574
|
+
const warning = await syncDeleteRemoteBranch(ctx, branchContext);
|
|
575
|
+
if (warning)
|
|
576
|
+
cleanupWarning = cleanupWarning ? `${cleanupWarning}; ${warning}` : warning;
|
|
577
|
+
}
|
|
571
578
|
// Invalidate registry cache so next list() will regenerate without this branch
|
|
572
579
|
if (!ctx.services.registry) {
|
|
573
580
|
return {
|
package/dist/api/client.d.ts
CHANGED
|
@@ -32,7 +32,8 @@ export interface ApiClientOptions {
|
|
|
32
32
|
*/
|
|
33
33
|
trailingSlash?: boolean;
|
|
34
34
|
/**
|
|
35
|
-
* Called whenever a response
|
|
35
|
+
* Called whenever a response says the credential is no longer accepted: HTTP 401, or a
|
|
36
|
+
* handler's body with `status: 401` under the HTTP status `unauthenticatedStatus` chose. A
|
|
36
37
|
* notification, not a retry; the editor's auth gate uses it to show sign-in.
|
|
37
38
|
*/
|
|
38
39
|
onUnauthorized?: () => void;
|
package/dist/api/client.js
CHANGED
|
@@ -341,7 +341,8 @@ export class CanopyApiClient {
|
|
|
341
341
|
response = await this.fetchFn(url, init);
|
|
342
342
|
parsed = await response.json().catch(() => undefined);
|
|
343
343
|
}
|
|
344
|
-
|
|
344
|
+
// The body's status is the signal: `unauthenticatedStatus` may send it under HTTP 419.
|
|
345
|
+
if (response.status === 401 || (isApiResponseBody(parsed) && parsed.status === 401)) {
|
|
345
346
|
this.onUnauthorized?.();
|
|
346
347
|
return { ok: false, status: 401, error: errorFromBody(parsed) ?? 'Unauthorized' };
|
|
347
348
|
}
|
|
@@ -26,3 +26,12 @@ export declare function syncSubmitPr(ctx: ApiContext, context: BranchContext, su
|
|
|
26
26
|
* available, otherwise queues a task.
|
|
27
27
|
*/
|
|
28
28
|
export declare function syncConvertToDraft(ctx: ApiContext, context: BranchContext): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Used by delete, after the local delete succeeded. Deletes the branch on GitHub when the CMS
|
|
31
|
+
* put it there: a content branch reaches GitHub only through submit, which always opens a PR,
|
|
32
|
+
* so a recorded PR number is the proof. Without one, a same-named GitHub branch is someone
|
|
33
|
+
* else's (the create-time collision check is best-effort) and is never touched.
|
|
34
|
+
*
|
|
35
|
+
* Returns a client-facing warning when the delete could not be done or queued; never throws.
|
|
36
|
+
*/
|
|
37
|
+
export declare function syncDeleteRemoteBranch(ctx: ApiContext, context: BranchContext): Promise<string | undefined>;
|
package/dist/api/github-sync.js
CHANGED
|
@@ -4,6 +4,7 @@ import { clientOperatingStrategy } from '../operating-mode/index.js';
|
|
|
4
4
|
import { getErrorMessage } from '../utils/error.js';
|
|
5
5
|
import { sanitizeBranchName } from '../paths/branch-name.js';
|
|
6
6
|
import { buildPrSection, mergePrSection } from '../submission-attribution.js';
|
|
7
|
+
import { isRefAlreadyGoneError } from '../github-service.js';
|
|
7
8
|
/**
|
|
8
9
|
* Uses githubService directly if available, otherwise queues a task for the worker.
|
|
9
10
|
*
|
|
@@ -156,6 +157,42 @@ export async function syncConvertToDraft(ctx, context) {
|
|
|
156
157
|
});
|
|
157
158
|
}
|
|
158
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* Used by delete, after the local delete succeeded. Deletes the branch on GitHub when the CMS
|
|
162
|
+
* put it there: a content branch reaches GitHub only through submit, which always opens a PR,
|
|
163
|
+
* so a recorded PR number is the proof. Without one, a same-named GitHub branch is someone
|
|
164
|
+
* else's (the create-time collision check is best-effort) and is never touched.
|
|
165
|
+
*
|
|
166
|
+
* Returns a client-facing warning when the delete could not be done or queued; never throws.
|
|
167
|
+
*/
|
|
168
|
+
export async function syncDeleteRemoteBranch(ctx, context) {
|
|
169
|
+
if (!context.branch.pullRequestNumber)
|
|
170
|
+
return undefined;
|
|
171
|
+
const { githubService } = ctx.services;
|
|
172
|
+
const branch = context.branch.name;
|
|
173
|
+
if (githubService) {
|
|
174
|
+
try {
|
|
175
|
+
await githubService.deleteBranch(branch);
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
if (isRefAlreadyGoneError(err))
|
|
179
|
+
return undefined;
|
|
180
|
+
console.error(`CanopyCMS: Failed to delete GitHub branch for ${branch}:`, getErrorMessage(err));
|
|
181
|
+
return 'The branch could not be deleted on GitHub; delete it there by hand';
|
|
182
|
+
}
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
const mode = ctx.services.config.mode;
|
|
186
|
+
if (!clientOperatingStrategy(mode).supportsPullRequests())
|
|
187
|
+
return undefined;
|
|
188
|
+
const result = await enqueueGitHubTask(ctx, context, {
|
|
189
|
+
action: 'delete-remote-branch',
|
|
190
|
+
payload: { branch, pullRequestNumber: context.branch.pullRequestNumber },
|
|
191
|
+
});
|
|
192
|
+
return result.syncStatus === 'sync-failed'
|
|
193
|
+
? 'Deleting the branch on GitHub could not be queued; delete it there by hand'
|
|
194
|
+
: undefined;
|
|
195
|
+
}
|
|
159
196
|
async function enqueueGitHubTask(ctx, context, task) {
|
|
160
197
|
const taskDir = getTaskQueueDir(ctx.services.config);
|
|
161
198
|
try {
|
package/dist/assets/finalize.js
CHANGED
|
@@ -39,6 +39,9 @@ export async function finalizeAsset(store, input) {
|
|
|
39
39
|
const existing = await store.getMeta(meta.hash32);
|
|
40
40
|
if (existing)
|
|
41
41
|
return { ok: true, meta: existing };
|
|
42
|
+
// Either put may find its key taken (`already-exists`): by an earlier attempt that stopped
|
|
43
|
+
// before the meta commit, or by a concurrent identical upload. The object kept there is this
|
|
44
|
+
// content's, so finalize goes on to commit the meta either way.
|
|
42
45
|
await store.putOriginal({ hash32: meta.hash32, ext: meta.ext, data, contentType: meta.mime });
|
|
43
46
|
if (publicObject) {
|
|
44
47
|
await store.putPublicObject(publicObject);
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* before that build is released) and canopycms-cdk's lazy transform Lambda.
|
|
7
7
|
*/
|
|
8
8
|
import { type ParsedTransformPath } from './transform-directives.js';
|
|
9
|
-
import type { AssetStore } from './types.js';
|
|
9
|
+
import type { AssetStore, CreateOnlyResult } from './types.js';
|
|
10
10
|
/** Every transform output is stored under a content-addressed key, so it never changes. */
|
|
11
11
|
export declare const TRANSFORM_CACHE_CONTROL = "public, max-age=31536000, immutable";
|
|
12
12
|
/**
|
|
@@ -19,26 +19,32 @@ export declare const LAZY_TRANSFORM_TAG: {
|
|
|
19
19
|
readonly key: "canopy-transform";
|
|
20
20
|
readonly value: "lazy";
|
|
21
21
|
};
|
|
22
|
+
/**
|
|
23
|
+
* `data` is the computed output whether or not it was stored: `stored: 'already-exists'` means
|
|
24
|
+
* another writer stored the key first, and the store kept that object.
|
|
25
|
+
*/
|
|
22
26
|
export type StoreTransformResult = {
|
|
23
27
|
ok: true;
|
|
24
28
|
data: Uint8Array;
|
|
25
29
|
contentType: string;
|
|
30
|
+
stored: CreateOnlyResult;
|
|
26
31
|
} | {
|
|
27
32
|
ok: false;
|
|
28
33
|
status: 400 | 404 | 413 | 422;
|
|
29
34
|
error: string;
|
|
30
35
|
};
|
|
31
36
|
/**
|
|
32
|
-
* Compute the transform `parsed` names from the asset's original and write it under
|
|
33
|
-
*
|
|
34
|
-
* always match their key. A rejection is a fact
|
|
35
|
-
* retrying; a thrown error belongs to the store or
|
|
36
|
-
* was missing, so a caller answering a browser
|
|
37
|
+
* Compute the transform `parsed` names from the asset's original and write it under `key`: the
|
|
38
|
+
* canonical key, or that key beneath an output prefix. `parsed` must come from
|
|
39
|
+
* `canonicalizeTransformPath`, so the stored pixels always match their key. A rejection is a fact
|
|
40
|
+
* about the asset or the URL and is never worth retrying; a thrown error belongs to the store or
|
|
41
|
+
* the environment. A 404's `error` names what was missing, so a caller answering a browser
|
|
42
|
+
* replaces it.
|
|
37
43
|
*/
|
|
38
|
-
export declare function storeTransform(store: AssetStore, parsed: ParsedTransformPath,
|
|
44
|
+
export declare function storeTransform(store: AssetStore, parsed: ParsedTransformPath, key: string, options?: {
|
|
39
45
|
tags?: Readonly<Record<string, string>>;
|
|
40
46
|
}): Promise<StoreTransformResult>;
|
|
41
|
-
/**
|
|
47
|
+
/** One key a build references, and where it was referenced, for the failure report. */
|
|
42
48
|
export interface MaterializeTarget {
|
|
43
49
|
/** `assets/t/{directives}/{hash32}/{slug}.{ext}`. */
|
|
44
50
|
key: string;
|
|
@@ -56,17 +62,24 @@ export type MaterializeResult = {
|
|
|
56
62
|
routes: string[];
|
|
57
63
|
files: string[];
|
|
58
64
|
} & ({
|
|
59
|
-
status: 'existed' | 'created';
|
|
65
|
+
status: 'existed' | 'created' | 'copied';
|
|
60
66
|
} | {
|
|
61
67
|
status: 'failed';
|
|
62
68
|
failure: MaterializeFailureKind;
|
|
63
69
|
error: string;
|
|
64
70
|
});
|
|
71
|
+
/** The `schemaVersion` of every `MaterializeReport`; a consumer gating on the JSON checks it. */
|
|
72
|
+
export declare const MATERIALIZE_REPORT_SCHEMA_VERSION = 1;
|
|
65
73
|
export interface MaterializeReport {
|
|
74
|
+
schemaVersion: typeof MATERIALIZE_REPORT_SCHEMA_VERSION;
|
|
75
|
+
/** `MaterializeOptions.outputPrefix`, present only when one was given. */
|
|
76
|
+
outputPrefix?: string;
|
|
66
77
|
summary: {
|
|
67
78
|
total: number;
|
|
68
79
|
existed: number;
|
|
69
80
|
created: number;
|
|
81
|
+
/** Keys copied from the canonical prefix into an output prefix; 0 until one is configured. */
|
|
82
|
+
copied: number;
|
|
70
83
|
failed: number;
|
|
71
84
|
contentFailures: number;
|
|
72
85
|
storeFailures: number;
|
|
@@ -78,11 +91,18 @@ export interface MaterializeOptions {
|
|
|
78
91
|
store: AssetStore;
|
|
79
92
|
targets: readonly MaterializeTarget[];
|
|
80
93
|
/**
|
|
81
|
-
* `assets/{hash32}/{slug}.{ext}` keys (svg, pdf) a build references. These are
|
|
82
|
-
* finalize writes them at upload,
|
|
83
|
-
* unless the bucket itself is missing
|
|
94
|
+
* `assets/{hash32}/{slug}.{ext}` keys (svg, pdf) a build references. These are never produced
|
|
95
|
+
* here (finalize writes them at upload), only checked, or copied under an `outputPrefix`, so a
|
|
96
|
+
* missing one is a content failure, unless the bucket itself is missing: a store failure.
|
|
84
97
|
*/
|
|
85
98
|
statics?: readonly MaterializeTarget[];
|
|
99
|
+
/**
|
|
100
|
+
* Write every key `k` at `outputPrefix + k` instead, for a build whose writes must never land
|
|
101
|
+
* where production reads (a PR preview). A key production already stores is copied from there;
|
|
102
|
+
* the rest are transformed from the canonical originals. Production's own run, with no prefix,
|
|
103
|
+
* never reads beneath it. See `assertValidOutputPrefix` for what is accepted.
|
|
104
|
+
*/
|
|
105
|
+
outputPrefix?: string;
|
|
86
106
|
/** Store requests in flight at once. Default 8. */
|
|
87
107
|
concurrency?: number;
|
|
88
108
|
/**
|
|
@@ -106,6 +126,17 @@ export interface MaterializeOptions {
|
|
|
106
126
|
export declare class SharpUnavailableError extends Error {
|
|
107
127
|
constructor(cause: unknown);
|
|
108
128
|
}
|
|
129
|
+
/** `MaterializeOptions.outputPrefix` broke a rule of `assertValidOutputPrefix`; nothing was read or written. */
|
|
130
|
+
export declare class InvalidOutputPrefixError extends Error {
|
|
131
|
+
constructor(prefix: string, reason: string);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Throws `InvalidOutputPrefixError` unless `prefix` is relative, ends in `/`, and is a run of
|
|
135
|
+
* `[A-Za-z0-9._-]` segments, none empty, `.` or `..`, whose first segments are not a canopy
|
|
136
|
+
* prefix. Production trusts whatever is under its own prefixes, so a write beneath one is a write
|
|
137
|
+
* production serves. Compared by segment: `assets-x/` is accepted, `assets/x/` is not.
|
|
138
|
+
*/
|
|
139
|
+
export declare function assertValidOutputPrefix(prefix: string): void;
|
|
109
140
|
/**
|
|
110
141
|
* @internal Exported for tests. Throttling, 5xx and dropped connections are transient. Any other
|
|
111
142
|
* 4xx (AccessDenied, NoSuchBucket) is a configuration fault that a retry only delays.
|
|
@@ -115,8 +146,8 @@ export declare function isTransientStoreError(err: unknown): boolean;
|
|
|
115
146
|
* Make every referenced transform key exist in `store`, and check every static key does. A key
|
|
116
147
|
* already stored is left alone:
|
|
117
148
|
* keys are content-addressed, so an existing object is the right one. sharp is loaded only when
|
|
118
|
-
* something
|
|
119
|
-
* failing each key.
|
|
149
|
+
* something must be transformed, and a sharp that cannot load throws `SharpUnavailableError` rather
|
|
150
|
+
* than failing each key. An invalid `outputPrefix` throws `InvalidOutputPrefixError` first.
|
|
120
151
|
*/
|
|
121
152
|
export declare function materializeAssets(options: MaterializeOptions): Promise<MaterializeReport>;
|
|
122
153
|
export {};
|
|
@@ -20,13 +20,14 @@ export const TRANSFORM_CACHE_CONTROL = 'public, max-age=31536000, immutable';
|
|
|
20
20
|
*/
|
|
21
21
|
export const LAZY_TRANSFORM_TAG = { key: 'canopy-transform', value: 'lazy' };
|
|
22
22
|
/**
|
|
23
|
-
* Compute the transform `parsed` names from the asset's original and write it under
|
|
24
|
-
*
|
|
25
|
-
* always match their key. A rejection is a fact
|
|
26
|
-
* retrying; a thrown error belongs to the store or
|
|
27
|
-
* was missing, so a caller answering a browser
|
|
23
|
+
* Compute the transform `parsed` names from the asset's original and write it under `key`: the
|
|
24
|
+
* canonical key, or that key beneath an output prefix. `parsed` must come from
|
|
25
|
+
* `canonicalizeTransformPath`, so the stored pixels always match their key. A rejection is a fact
|
|
26
|
+
* about the asset or the URL and is never worth retrying; a thrown error belongs to the store or
|
|
27
|
+
* the environment. A 404's `error` names what was missing, so a caller answering a browser
|
|
28
|
+
* replaces it.
|
|
28
29
|
*/
|
|
29
|
-
export async function storeTransform(store, parsed,
|
|
30
|
+
export async function storeTransform(store, parsed, key, options = {}) {
|
|
30
31
|
const meta = await store.getMeta(parsed.hash32);
|
|
31
32
|
if (!meta) {
|
|
32
33
|
return { ok: false, status: 404, error: `Asset ${parsed.hash32} has no meta in the store` };
|
|
@@ -58,15 +59,17 @@ export async function storeTransform(store, parsed, canonicalKey, options = {})
|
|
|
58
59
|
if (!transformed.ok) {
|
|
59
60
|
return { ok: false, status: transformed.status, error: transformed.error };
|
|
60
61
|
}
|
|
61
|
-
await store.putPublicObject({
|
|
62
|
-
key
|
|
62
|
+
const stored = await store.putPublicObject({
|
|
63
|
+
key,
|
|
63
64
|
data: transformed.data,
|
|
64
65
|
contentType: transformed.contentType,
|
|
65
66
|
cacheControl: TRANSFORM_CACHE_CONTROL,
|
|
66
67
|
tags: options.tags,
|
|
67
68
|
});
|
|
68
|
-
return { ok: true, data: transformed.data, contentType: transformed.contentType };
|
|
69
|
+
return { ok: true, data: transformed.data, contentType: transformed.contentType, stored };
|
|
69
70
|
}
|
|
71
|
+
/** The `schemaVersion` of every `MaterializeReport`; a consumer gating on the JSON checks it. */
|
|
72
|
+
export const MATERIALIZE_REPORT_SCHEMA_VERSION = 1;
|
|
70
73
|
/** sharp cannot load in this process, so nothing missing can be produced here. */
|
|
71
74
|
export class SharpUnavailableError extends Error {
|
|
72
75
|
constructor(cause) {
|
|
@@ -75,6 +78,43 @@ export class SharpUnavailableError extends Error {
|
|
|
75
78
|
this.name = 'SharpUnavailableError';
|
|
76
79
|
}
|
|
77
80
|
}
|
|
81
|
+
/** `MaterializeOptions.outputPrefix` broke a rule of `assertValidOutputPrefix`; nothing was read or written. */
|
|
82
|
+
export class InvalidOutputPrefixError extends Error {
|
|
83
|
+
constructor(prefix, reason) {
|
|
84
|
+
super(`Invalid output prefix ${JSON.stringify(prefix)}: ${reason}`);
|
|
85
|
+
this.name = 'InvalidOutputPrefixError';
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const OUTPUT_PREFIX_SEGMENT_RE = /^[A-Za-z0-9._-]+$/;
|
|
89
|
+
const CANOPY_PREFIX_SEGMENTS = Object.values(ASSET_PREFIXES).map((prefix) => prefix.split('/'));
|
|
90
|
+
/**
|
|
91
|
+
* Throws `InvalidOutputPrefixError` unless `prefix` is relative, ends in `/`, and is a run of
|
|
92
|
+
* `[A-Za-z0-9._-]` segments, none empty, `.` or `..`, whose first segments are not a canopy
|
|
93
|
+
* prefix. Production trusts whatever is under its own prefixes, so a write beneath one is a write
|
|
94
|
+
* production serves. Compared by segment: `assets-x/` is accepted, `assets/x/` is not.
|
|
95
|
+
*/
|
|
96
|
+
export function assertValidOutputPrefix(prefix) {
|
|
97
|
+
const refuse = (reason) => {
|
|
98
|
+
throw new InvalidOutputPrefixError(prefix, reason);
|
|
99
|
+
};
|
|
100
|
+
if (prefix.startsWith('/'))
|
|
101
|
+
refuse('it must be relative, with no leading /');
|
|
102
|
+
if (!prefix.endsWith('/'))
|
|
103
|
+
refuse('it must end in /');
|
|
104
|
+
const segments = prefix.slice(0, -1).split('/');
|
|
105
|
+
for (const segment of segments) {
|
|
106
|
+
if (segment === '')
|
|
107
|
+
refuse('it has an empty segment');
|
|
108
|
+
if (segment === '.' || segment === '..')
|
|
109
|
+
refuse(`it has a ${segment} segment`);
|
|
110
|
+
if (!OUTPUT_PREFIX_SEGMENT_RE.test(segment)) {
|
|
111
|
+
refuse(`segment ${JSON.stringify(segment)} has a character outside [A-Za-z0-9._-]`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const canopy = CANOPY_PREFIX_SEGMENTS.find((prefixSegments) => prefixSegments.every((segment, i) => segments[i] === segment));
|
|
115
|
+
if (canopy)
|
|
116
|
+
refuse(`it begins with the canopy prefix ${canopy.join('/')}/`);
|
|
117
|
+
}
|
|
78
118
|
const DEFAULT_CONCURRENCY = 8;
|
|
79
119
|
const DEFAULT_TRANSFORM_CONCURRENCY = 2;
|
|
80
120
|
// eslint-disable-next-line security/detect-non-literal-regexp -- built from a constant
|
|
@@ -133,11 +173,14 @@ async function forEachBounded(items, limit, fn) {
|
|
|
133
173
|
* Make every referenced transform key exist in `store`, and check every static key does. A key
|
|
134
174
|
* already stored is left alone:
|
|
135
175
|
* keys are content-addressed, so an existing object is the right one. sharp is loaded only when
|
|
136
|
-
* something
|
|
137
|
-
* failing each key.
|
|
176
|
+
* something must be transformed, and a sharp that cannot load throws `SharpUnavailableError` rather
|
|
177
|
+
* than failing each key. An invalid `outputPrefix` throws `InvalidOutputPrefixError` first.
|
|
138
178
|
*/
|
|
139
179
|
export async function materializeAssets(options) {
|
|
140
|
-
const { store } = options;
|
|
180
|
+
const { store, outputPrefix } = options;
|
|
181
|
+
if (outputPrefix !== undefined)
|
|
182
|
+
assertValidOutputPrefix(outputPrefix);
|
|
183
|
+
const destOf = (key) => (outputPrefix ?? '') + key;
|
|
141
184
|
const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY);
|
|
142
185
|
const transformConcurrency = Math.max(1, options.transformConcurrency ?? DEFAULT_TRANSFORM_CONCURRENCY);
|
|
143
186
|
const listThreshold = options.listThreshold ?? DEFAULT_LIST_THRESHOLD;
|
|
@@ -200,37 +243,51 @@ export async function materializeAssets(options) {
|
|
|
200
243
|
group.push(target);
|
|
201
244
|
byDirectives.set(target.directives, group);
|
|
202
245
|
}
|
|
203
|
-
const
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
246
|
+
const listOnce = async (prefix) => {
|
|
247
|
+
if (!store.listPublicObjectKeys)
|
|
248
|
+
return undefined;
|
|
249
|
+
const listKeys = store.listPublicObjectKeys.bind(store);
|
|
250
|
+
try {
|
|
251
|
+
return new Set(await withRetry(async () => {
|
|
252
|
+
const keys = [];
|
|
253
|
+
for await (const key of listKeys(prefix))
|
|
254
|
+
keys.push(key);
|
|
255
|
+
return keys;
|
|
256
|
+
}));
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// A listing that keeps failing costs only speed: its keys are HEADed instead, and each HEAD
|
|
260
|
+
// reports its own failure.
|
|
261
|
+
return undefined;
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
const listingsByDirectives = new Map();
|
|
265
|
+
if (store.listPublicObjectKeys) {
|
|
266
|
+
for (const [directives, group] of byDirectives) {
|
|
267
|
+
if (group.length < listThreshold)
|
|
220
268
|
continue;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
}
|
|
269
|
+
const listPrefix = `${transformPrefix}${directives}/`;
|
|
270
|
+
const dest = await listOnce(destOf(listPrefix));
|
|
271
|
+
const needsCanonical = outputPrefix !== undefined && group.some((target) => !dest?.has(destOf(target.key)));
|
|
272
|
+
const canonical = needsCanonical ? await listOnce(listPrefix) : undefined;
|
|
273
|
+
listingsByDirectives.set(directives, { dest, canonical });
|
|
226
274
|
}
|
|
227
|
-
toHead.push(...group);
|
|
228
275
|
}
|
|
229
|
-
|
|
276
|
+
// Every presence check goes through here. Without an output prefix a key's destination is the
|
|
277
|
+
// key itself, so this is one listing lookup or HEAD and never `canonical`.
|
|
278
|
+
const resolvePresence = async (key, listings = {}) => {
|
|
279
|
+
const isStored = async (storeKey, listing) => listing ? listing.has(storeKey) : withRetry(() => store.hasPublicObject(storeKey));
|
|
280
|
+
const dest = destOf(key);
|
|
281
|
+
if (await isStored(dest, listings.dest))
|
|
282
|
+
return 'dest';
|
|
283
|
+
if (dest === key)
|
|
284
|
+
return 'absent';
|
|
285
|
+
return (await isStored(key, listings.canonical)) ? 'canonical' : 'absent';
|
|
286
|
+
};
|
|
287
|
+
const presence = new Map();
|
|
288
|
+
await forEachBounded(valid, concurrency, async (target) => {
|
|
230
289
|
try {
|
|
231
|
-
|
|
232
|
-
outcomes.set(target.key, { status: 'existed' });
|
|
233
|
-
}
|
|
290
|
+
presence.set(target.key, await resolvePresence(target.key, listingsByDirectives.get(target.directives)));
|
|
234
291
|
}
|
|
235
292
|
catch (err) {
|
|
236
293
|
fail(target.key, 'store', `Existence check failed: ${getErrorMessage(err)}`);
|
|
@@ -238,22 +295,51 @@ export async function materializeAssets(options) {
|
|
|
238
295
|
});
|
|
239
296
|
await forEachBounded(validStatics, concurrency, async (key) => {
|
|
240
297
|
try {
|
|
298
|
+
let found = await resolvePresence(key);
|
|
241
299
|
// A HEAD cannot tell a missing key from a missing bucket, and a static key has no
|
|
242
300
|
// transform's meta read to tell them apart afterwards; a GET names which.
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
301
|
+
if (found === 'absent' && (await withRetry(() => store.readPublicObject(key))) !== null) {
|
|
302
|
+
found = destOf(key) === key ? 'dest' : 'canonical';
|
|
303
|
+
}
|
|
304
|
+
presence.set(key, found);
|
|
305
|
+
}
|
|
306
|
+
catch (err) {
|
|
307
|
+
fail(key, 'store', `Existence check failed: ${getErrorMessage(err)}`);
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
const toCopy = [];
|
|
311
|
+
for (const [key, found] of presence) {
|
|
312
|
+
if (found === 'dest')
|
|
313
|
+
outcomes.set(key, { status: 'existed' });
|
|
314
|
+
else if (found === 'canonical')
|
|
315
|
+
toCopy.push(key);
|
|
316
|
+
}
|
|
317
|
+
await forEachBounded(toCopy, concurrency, async (key) => {
|
|
318
|
+
try {
|
|
319
|
+
let tries = 0;
|
|
320
|
+
const copied = await withRetry(() => {
|
|
321
|
+
tries++;
|
|
322
|
+
return store.copyPublicObject(key, destOf(key));
|
|
323
|
+
});
|
|
324
|
+
if (copied === 'source-missing') {
|
|
325
|
+
presence.set(key, 'absent');
|
|
247
326
|
}
|
|
248
327
|
else {
|
|
249
|
-
|
|
328
|
+
// After a failed try, `already-exists` may be that try's own copy, as for a transform.
|
|
329
|
+
const copiedElsewhere = copied === 'already-exists' && tries === 1;
|
|
330
|
+
outcomes.set(key, { status: copiedElsewhere ? 'existed' : 'copied' });
|
|
250
331
|
}
|
|
251
332
|
}
|
|
252
333
|
catch (err) {
|
|
253
|
-
fail(key, 'store', `
|
|
334
|
+
fail(key, 'store', `Copy failed: ${getErrorMessage(err)}`);
|
|
254
335
|
}
|
|
255
336
|
});
|
|
256
|
-
const
|
|
337
|
+
for (const key of validStatics) {
|
|
338
|
+
if (presence.get(key) === 'absent') {
|
|
339
|
+
fail(key, 'content', 'No stored object; an svg or pdf is written at upload only');
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
const missing = valid.filter((target) => presence.get(target.key) === 'absent');
|
|
257
343
|
if (missing.length > 0) {
|
|
258
344
|
try {
|
|
259
345
|
await loadSharp();
|
|
@@ -264,9 +350,17 @@ export async function materializeAssets(options) {
|
|
|
264
350
|
}
|
|
265
351
|
await forEachBounded(missing, transformConcurrency, async (target) => {
|
|
266
352
|
try {
|
|
267
|
-
|
|
353
|
+
let tries = 0;
|
|
354
|
+
const result = await withRetry(() => {
|
|
355
|
+
tries++;
|
|
356
|
+
return storeTransform(store, target.parsed, destOf(target.key));
|
|
357
|
+
});
|
|
268
358
|
if (result.ok) {
|
|
269
|
-
|
|
359
|
+
// After a failed attempt, `already-exists` may be that attempt's own write, whose response
|
|
360
|
+
// was lost, so it counts as `created`: a release waiting on its `created` keys then waits
|
|
361
|
+
// on a key it may not have written, never skips one it did.
|
|
362
|
+
const createdElsewhere = result.stored === 'already-exists' && tries === 1;
|
|
363
|
+
outcomes.set(target.key, { status: createdElsewhere ? 'existed' : 'created' });
|
|
270
364
|
}
|
|
271
365
|
else {
|
|
272
366
|
fail(target.key, 'content', `${result.status}: ${result.error}`);
|
|
@@ -292,10 +386,13 @@ export async function materializeAssets(options) {
|
|
|
292
386
|
});
|
|
293
387
|
const count = (predicate) => results.filter(predicate).length;
|
|
294
388
|
return {
|
|
389
|
+
schemaVersion: MATERIALIZE_REPORT_SCHEMA_VERSION,
|
|
390
|
+
...(outputPrefix !== undefined ? { outputPrefix } : {}),
|
|
295
391
|
summary: {
|
|
296
392
|
total: results.length,
|
|
297
393
|
existed: count((r) => r.status === 'existed'),
|
|
298
394
|
created: count((r) => r.status === 'created'),
|
|
395
|
+
copied: count((r) => r.status === 'copied'),
|
|
299
396
|
failed: count((r) => r.status === 'failed'),
|
|
300
397
|
contentFailures: count((r) => r.status === 'failed' && r.failure === 'content'),
|
|
301
398
|
storeFailures: count((r) => r.status === 'failed' && r.failure === 'store'),
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* bucket-prefix layout as S3AssetStore (see keys.ts) so content behaves
|
|
4
4
|
* identically across adapters.
|
|
5
5
|
*/
|
|
6
|
-
import type { AssetMeta, AssetStore, BeginUploadInput, PublicObject, StagedUploadTarget } from './types.js';
|
|
6
|
+
import type { AssetMeta, AssetStore, BeginUploadInput, CreateOnlyResult, PublicObject, StagedUploadTarget } from './types.js';
|
|
7
7
|
export interface LocalAssetStoreOptions {
|
|
8
8
|
/** Root directory. The five bucket prefixes are created as subdirectories under it. */
|
|
9
9
|
root: string;
|
|
@@ -26,11 +26,20 @@ export declare class LocalAssetStore implements AssetStore {
|
|
|
26
26
|
*/
|
|
27
27
|
private resolveKey;
|
|
28
28
|
private headersSidecarPath;
|
|
29
|
-
private writeHeadersSidecar;
|
|
30
29
|
private readHeadersSidecar;
|
|
31
30
|
private readFileOrNull;
|
|
32
31
|
private writeFile;
|
|
33
32
|
private deleteFileIfPresent;
|
|
33
|
+
/** Link `tempPath` in at `filePath`; `false` when something is already there. */
|
|
34
|
+
private linkIfAbsent;
|
|
35
|
+
/**
|
|
36
|
+
* Create-only write of a blob and its headers sidecar. Each is written whole to a temp file, then
|
|
37
|
+
* `link()`ed into place, which fails with EEXIST rather than replacing (see docs/concurrency.md).
|
|
38
|
+
* The sidecar is linked first, so a reader never finds a blob without its headers. A sidecar
|
|
39
|
+
* already there was linked by an earlier writer of this key and is kept, as the blob is: the
|
|
40
|
+
* first writer's headers win, as on S3 (a static's Content-Disposition names its uploaded file).
|
|
41
|
+
*/
|
|
42
|
+
private createExclusive;
|
|
34
43
|
beginUpload(_input: BeginUploadInput): Promise<StagedUploadTarget>;
|
|
35
44
|
/**
|
|
36
45
|
* Staging methods accept caller-influenced keys (finalize receives the key
|
|
@@ -47,7 +56,7 @@ export declare class LocalAssetStore implements AssetStore {
|
|
|
47
56
|
ext: string;
|
|
48
57
|
data: Uint8Array;
|
|
49
58
|
contentType: string;
|
|
50
|
-
}): Promise<
|
|
59
|
+
}): Promise<CreateOnlyResult>;
|
|
51
60
|
/** Ignores the `ext` hint: a directory read costs nothing here and needs no permission. */
|
|
52
61
|
readOriginal(hash32: string): Promise<{
|
|
53
62
|
data: Uint8Array;
|
|
@@ -62,7 +71,8 @@ export declare class LocalAssetStore implements AssetStore {
|
|
|
62
71
|
cacheControl?: string;
|
|
63
72
|
/** Ignored: tags exist for bucket lifecycle rules, and a local store has none. */
|
|
64
73
|
tags?: Readonly<Record<string, string>>;
|
|
65
|
-
}): Promise<
|
|
74
|
+
}): Promise<CreateOnlyResult>;
|
|
75
|
+
copyPublicObject(sourceKey: string, destKey: string): Promise<CreateOnlyResult | 'source-missing'>;
|
|
66
76
|
readPublicObject(key: string): Promise<PublicObject | null>;
|
|
67
77
|
hasPublicObject(key: string): Promise<boolean>;
|
|
68
78
|
/**
|
|
@@ -73,7 +83,7 @@ export declare class LocalAssetStore implements AssetStore {
|
|
|
73
83
|
* always clobbers the destination, which would let the second writer
|
|
74
84
|
* silently overwrite the first instead of losing cleanly.
|
|
75
85
|
*/
|
|
76
|
-
putMetaIfAbsent(hash32: string, meta: AssetMeta): Promise<
|
|
86
|
+
putMetaIfAbsent(hash32: string, meta: AssetMeta): Promise<CreateOnlyResult>;
|
|
77
87
|
getMeta(hash32: string): Promise<AssetMeta | null>;
|
|
78
88
|
listMeta(input?: {
|
|
79
89
|
cursor?: string;
|