canopycms 0.0.68-int.100 → 0.0.68-int.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +3 -3
  2. package/dist/api/branch.js +7 -0
  3. package/dist/api/github-sync.d.ts +9 -0
  4. package/dist/api/github-sync.js +37 -0
  5. package/dist/assets/finalize.js +3 -0
  6. package/dist/assets/materialize.d.ts +45 -14
  7. package/dist/assets/materialize.js +145 -48
  8. package/dist/assets/store-local.d.ts +15 -5
  9. package/dist/assets/store-local.js +58 -10
  10. package/dist/assets/store-s3.d.ts +28 -6
  11. package/dist/assets/store-s3.js +78 -24
  12. package/dist/assets/types.d.ts +17 -3
  13. package/dist/cli/asset-refs.d.ts +30 -8
  14. package/dist/cli/asset-refs.js +99 -73
  15. package/dist/cli/cli.d.ts +13 -0
  16. package/dist/cli/cli.js +2131 -1371
  17. package/dist/cli/configured-asset-store.d.ts +12 -0
  18. package/dist/cli/configured-asset-store.js +46 -0
  19. package/dist/cli/generate-ai-content.js +636 -178
  20. package/dist/client.d.ts +1 -1
  21. package/dist/client.js +1 -1
  22. package/dist/editor/Editor.js +8 -2
  23. package/dist/editor/PreviewFrame.d.ts +24 -0
  24. package/dist/editor/PreviewFrame.js +140 -0
  25. package/dist/editor/editor-utils.d.ts +2 -2
  26. package/dist/editor/editor-utils.js +2 -2
  27. package/dist/editor/hooks/useBranchManager.d.ts +10 -0
  28. package/dist/editor/hooks/useBranchManager.js +125 -36
  29. package/dist/editor/preview-bridge.d.ts +7 -26
  30. package/dist/editor/preview-bridge.js +7 -140
  31. package/dist/github-service.d.ts +7 -0
  32. package/dist/github-service.js +14 -0
  33. package/dist/preview.d.ts +8 -0
  34. package/dist/preview.js +7 -0
  35. package/dist/server.d.ts +40 -0
  36. package/dist/server.js +26 -0
  37. package/dist/utils/content-serialize.d.ts +5 -2
  38. package/dist/utils/content-serialize.js +98 -24
  39. package/dist/utils/yaml-source-splice.d.ts +58 -0
  40. package/dist/utils/yaml-source-splice.js +515 -0
  41. package/dist/version.d.ts +1 -1
  42. package/dist/version.js +1 -1
  43. package/dist/worker/task-runner.js +60 -10
  44. package/package.json +7 -2
package/README.md CHANGED
@@ -396,7 +396,7 @@ The editor loads entries on the client from `/api/canopycms/[branch]/entries`, s
396
396
  _TODO_ show an example
397
397
 
398
398
  7. **Split builds**
399
- Keep your public build free of editor bundles by importing only from `canopycms` (server helpers + data loaders). Host the editor in a separate app or build target that imports from `canopycms/client` and mounts the API routes above.
399
+ Keep your public build free of editor bundles by importing only from `canopycms` (server helpers + data loaders) and `canopycms/preview` (the live-preview hook). Host the editor in a separate app or build target that imports from `canopycms/client` and mounts the API routes above.
400
400
 
401
401
  _TODO_ show real examples of what to do
402
402
 
@@ -415,7 +415,7 @@ The `useCanopyPreview` hook provides live updates to your preview components as
415
415
  ```tsx
416
416
  'use client'
417
417
 
418
- import { useCanopyPreview } from 'canopycms/client'
418
+ import { useCanopyPreview } from 'canopycms/preview'
419
419
  import type { PostContent } from './schemas'
420
420
 
421
421
  export function PostView({ data }: { data: PostContent }) {
@@ -451,7 +451,7 @@ When using reference fields (foreign key relationships to other content), the ed
451
451
  ```tsx
452
452
  'use client'
453
453
 
454
- import { useCanopyPreview } from 'canopycms/client'
454
+ import { useCanopyPreview } from 'canopycms/preview'
455
455
 
456
456
  export function PostView({ data }: { data: PostContent }) {
457
457
  const { data: liveData, isLoading } = useCanopyPreview<PostContent>({
@@ -14,6 +14,7 @@ import { filePathExists } from '../utils/fs.js';
14
14
  import { isNetworkRemoteUrl } from '../utils/git.js';
15
15
  import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX, RESERVED_ROUTE_BRANCH_NAMES, isSettingsBranchName, } from '../paths/index.js';
16
16
  import { GitManager } from '../git-manager.js';
17
+ import { syncDeleteRemoteBranch } from './github-sync.js';
17
18
  import { branchNameSchema, branchParamSchema } from './validators.js';
18
19
  const log = createDebugLogger({ prefix: 'BranchAPI' });
19
20
  const createBranchBodySchema = z.object({
@@ -568,6 +569,12 @@ export const deleteBranchHandler = async (ctx, req, params) => {
568
569
  console.warn(`CanopyCMS: failed to remove deleted branch's mirror head (branch ${sanitizedDeleted}, mirror ${mirrorPath}): ${getErrorMessage(err)}`);
569
570
  }
570
571
  }
572
+ // `branchContext` was read before the delete, so it still carries the PR number this gates on.
573
+ if (deletableHead) {
574
+ const warning = await syncDeleteRemoteBranch(ctx, branchContext);
575
+ if (warning)
576
+ cleanupWarning = cleanupWarning ? `${cleanupWarning}; ${warning}` : warning;
577
+ }
571
578
  // Invalidate registry cache so next list() will regenerate without this branch
572
579
  if (!ctx.services.registry) {
573
580
  return {
@@ -26,3 +26,12 @@ export declare function syncSubmitPr(ctx: ApiContext, context: BranchContext, su
26
26
  * available, otherwise queues a task.
27
27
  */
28
28
  export declare function syncConvertToDraft(ctx: ApiContext, context: BranchContext): Promise<void>;
29
+ /**
30
+ * Used by delete, after the local delete succeeded. Deletes the branch on GitHub when the CMS
31
+ * put it there: a content branch reaches GitHub only through submit, which always opens a PR,
32
+ * so a recorded PR number is the proof. Without one, a same-named GitHub branch is someone
33
+ * else's (the create-time collision check is best-effort) and is never touched.
34
+ *
35
+ * Returns a client-facing warning when the delete could not be done or queued; never throws.
36
+ */
37
+ export declare function syncDeleteRemoteBranch(ctx: ApiContext, context: BranchContext): Promise<string | undefined>;
@@ -4,6 +4,7 @@ import { clientOperatingStrategy } from '../operating-mode/index.js';
4
4
  import { getErrorMessage } from '../utils/error.js';
5
5
  import { sanitizeBranchName } from '../paths/branch-name.js';
6
6
  import { buildPrSection, mergePrSection } from '../submission-attribution.js';
7
+ import { isRefAlreadyGoneError } from '../github-service.js';
7
8
  /**
8
9
  * Uses githubService directly if available, otherwise queues a task for the worker.
9
10
  *
@@ -156,6 +157,42 @@ export async function syncConvertToDraft(ctx, context) {
156
157
  });
157
158
  }
158
159
  }
160
+ /**
161
+ * Used by delete, after the local delete succeeded. Deletes the branch on GitHub when the CMS
162
+ * put it there: a content branch reaches GitHub only through submit, which always opens a PR,
163
+ * so a recorded PR number is the proof. Without one, a same-named GitHub branch is someone
164
+ * else's (the create-time collision check is best-effort) and is never touched.
165
+ *
166
+ * Returns a client-facing warning when the delete could not be done or queued; never throws.
167
+ */
168
+ export async function syncDeleteRemoteBranch(ctx, context) {
169
+ if (!context.branch.pullRequestNumber)
170
+ return undefined;
171
+ const { githubService } = ctx.services;
172
+ const branch = context.branch.name;
173
+ if (githubService) {
174
+ try {
175
+ await githubService.deleteBranch(branch);
176
+ }
177
+ catch (err) {
178
+ if (isRefAlreadyGoneError(err))
179
+ return undefined;
180
+ console.error(`CanopyCMS: Failed to delete GitHub branch for ${branch}:`, getErrorMessage(err));
181
+ return 'The branch could not be deleted on GitHub; delete it there by hand';
182
+ }
183
+ return undefined;
184
+ }
185
+ const mode = ctx.services.config.mode;
186
+ if (!clientOperatingStrategy(mode).supportsPullRequests())
187
+ return undefined;
188
+ const result = await enqueueGitHubTask(ctx, context, {
189
+ action: 'delete-remote-branch',
190
+ payload: { branch, pullRequestNumber: context.branch.pullRequestNumber },
191
+ });
192
+ return result.syncStatus === 'sync-failed'
193
+ ? 'Deleting the branch on GitHub could not be queued; delete it there by hand'
194
+ : undefined;
195
+ }
159
196
  async function enqueueGitHubTask(ctx, context, task) {
160
197
  const taskDir = getTaskQueueDir(ctx.services.config);
161
198
  try {
@@ -39,6 +39,9 @@ export async function finalizeAsset(store, input) {
39
39
  const existing = await store.getMeta(meta.hash32);
40
40
  if (existing)
41
41
  return { ok: true, meta: existing };
42
+ // Either put may find its key taken (`already-exists`): by an earlier attempt that stopped
43
+ // before the meta commit, or by a concurrent identical upload. The object kept there is this
44
+ // content's, so finalize goes on to commit the meta either way.
42
45
  await store.putOriginal({ hash32: meta.hash32, ext: meta.ext, data, contentType: meta.mime });
43
46
  if (publicObject) {
44
47
  await store.putPublicObject(publicObject);
@@ -6,7 +6,7 @@
6
6
  * before that build is released) and canopycms-cdk's lazy transform Lambda.
7
7
  */
8
8
  import { type ParsedTransformPath } from './transform-directives.js';
9
- import type { AssetStore } from './types.js';
9
+ import type { AssetStore, CreateOnlyResult } from './types.js';
10
10
  /** Every transform output is stored under a content-addressed key, so it never changes. */
11
11
  export declare const TRANSFORM_CACHE_CONTROL = "public, max-age=31536000, immutable";
12
12
  /**
@@ -19,26 +19,32 @@ export declare const LAZY_TRANSFORM_TAG: {
19
19
  readonly key: "canopy-transform";
20
20
  readonly value: "lazy";
21
21
  };
22
+ /**
23
+ * `data` is the computed output whether or not it was stored: `stored: 'already-exists'` means
24
+ * another writer stored the key first, and the store kept that object.
25
+ */
22
26
  export type StoreTransformResult = {
23
27
  ok: true;
24
28
  data: Uint8Array;
25
29
  contentType: string;
30
+ stored: CreateOnlyResult;
26
31
  } | {
27
32
  ok: false;
28
33
  status: 400 | 404 | 413 | 422;
29
34
  error: string;
30
35
  };
31
36
  /**
32
- * Compute the transform `parsed` names from the asset's original and write it under
33
- * `canonicalKey`. `parsed` must come from `canonicalizeTransformPath`, so the stored pixels
34
- * always match their key. A rejection is a fact about the asset or the URL and is never worth
35
- * retrying; a thrown error belongs to the store or the environment. A 404's `error` names what
36
- * was missing, so a caller answering a browser replaces it.
37
+ * Compute the transform `parsed` names from the asset's original and write it under `key`: the
38
+ * canonical key, or that key beneath an output prefix. `parsed` must come from
39
+ * `canonicalizeTransformPath`, so the stored pixels always match their key. A rejection is a fact
40
+ * about the asset or the URL and is never worth retrying; a thrown error belongs to the store or
41
+ * the environment. A 404's `error` names what was missing, so a caller answering a browser
42
+ * replaces it.
37
43
  */
38
- export declare function storeTransform(store: AssetStore, parsed: ParsedTransformPath, canonicalKey: string, options?: {
44
+ export declare function storeTransform(store: AssetStore, parsed: ParsedTransformPath, key: string, options?: {
39
45
  tags?: Readonly<Record<string, string>>;
40
46
  }): Promise<StoreTransformResult>;
41
- /** @internal One key a build references, and where it was referenced, for the failure report. */
47
+ /** One key a build references, and where it was referenced, for the failure report. */
42
48
  export interface MaterializeTarget {
43
49
  /** `assets/t/{directives}/{hash32}/{slug}.{ext}`. */
44
50
  key: string;
@@ -56,17 +62,24 @@ export type MaterializeResult = {
56
62
  routes: string[];
57
63
  files: string[];
58
64
  } & ({
59
- status: 'existed' | 'created';
65
+ status: 'existed' | 'created' | 'copied';
60
66
  } | {
61
67
  status: 'failed';
62
68
  failure: MaterializeFailureKind;
63
69
  error: string;
64
70
  });
71
+ /** The `schemaVersion` of every `MaterializeReport`; a consumer gating on the JSON checks it. */
72
+ export declare const MATERIALIZE_REPORT_SCHEMA_VERSION = 1;
65
73
  export interface MaterializeReport {
74
+ schemaVersion: typeof MATERIALIZE_REPORT_SCHEMA_VERSION;
75
+ /** `MaterializeOptions.outputPrefix`, present only when one was given. */
76
+ outputPrefix?: string;
66
77
  summary: {
67
78
  total: number;
68
79
  existed: number;
69
80
  created: number;
81
+ /** Keys copied from the canonical prefix into an output prefix; 0 until one is configured. */
82
+ copied: number;
70
83
  failed: number;
71
84
  contentFailures: number;
72
85
  storeFailures: number;
@@ -78,11 +91,18 @@ export interface MaterializeOptions {
78
91
  store: AssetStore;
79
92
  targets: readonly MaterializeTarget[];
80
93
  /**
81
- * `assets/{hash32}/{slug}.{ext}` keys (svg, pdf) a build references. These are only checked:
82
- * finalize writes them at upload, so a missing one is a content failure nothing here can fix,
83
- * unless the bucket itself is missing, which is a store failure.
94
+ * `assets/{hash32}/{slug}.{ext}` keys (svg, pdf) a build references. These are never produced
95
+ * here (finalize writes them at upload), only checked, or copied under an `outputPrefix`, so a
96
+ * missing one is a content failure, unless the bucket itself is missing: a store failure.
84
97
  */
85
98
  statics?: readonly MaterializeTarget[];
99
+ /**
100
+ * Write every key `k` at `outputPrefix + k` instead, for a build whose writes must never land
101
+ * where production reads (a PR preview). A key production already stores is copied from there;
102
+ * the rest are transformed from the canonical originals. Production's own run, with no prefix,
103
+ * never reads beneath it. See `assertValidOutputPrefix` for what is accepted.
104
+ */
105
+ outputPrefix?: string;
86
106
  /** Store requests in flight at once. Default 8. */
87
107
  concurrency?: number;
88
108
  /**
@@ -106,6 +126,17 @@ export interface MaterializeOptions {
106
126
  export declare class SharpUnavailableError extends Error {
107
127
  constructor(cause: unknown);
108
128
  }
129
+ /** `MaterializeOptions.outputPrefix` broke a rule of `assertValidOutputPrefix`; nothing was read or written. */
130
+ export declare class InvalidOutputPrefixError extends Error {
131
+ constructor(prefix: string, reason: string);
132
+ }
133
+ /**
134
+ * Throws `InvalidOutputPrefixError` unless `prefix` is relative, ends in `/`, and is a run of
135
+ * `[A-Za-z0-9._-]` segments, none empty, `.` or `..`, whose first segments are not a canopy
136
+ * prefix. Production trusts whatever is under its own prefixes, so a write beneath one is a write
137
+ * production serves. Compared by segment: `assets-x/` is accepted, `assets/x/` is not.
138
+ */
139
+ export declare function assertValidOutputPrefix(prefix: string): void;
109
140
  /**
110
141
  * @internal Exported for tests. Throttling, 5xx and dropped connections are transient. Any other
111
142
  * 4xx (AccessDenied, NoSuchBucket) is a configuration fault that a retry only delays.
@@ -115,8 +146,8 @@ export declare function isTransientStoreError(err: unknown): boolean;
115
146
  * Make every referenced transform key exist in `store`, and check every static key does. A key
116
147
  * already stored is left alone:
117
148
  * keys are content-addressed, so an existing object is the right one. sharp is loaded only when
118
- * something is missing, and a sharp that cannot load throws `SharpUnavailableError` rather than
119
- * failing each key.
149
+ * something must be transformed, and a sharp that cannot load throws `SharpUnavailableError` rather
150
+ * than failing each key. An invalid `outputPrefix` throws `InvalidOutputPrefixError` first.
120
151
  */
121
152
  export declare function materializeAssets(options: MaterializeOptions): Promise<MaterializeReport>;
122
153
  export {};
@@ -20,13 +20,14 @@ export const TRANSFORM_CACHE_CONTROL = 'public, max-age=31536000, immutable';
20
20
  */
21
21
  export const LAZY_TRANSFORM_TAG = { key: 'canopy-transform', value: 'lazy' };
22
22
  /**
23
- * Compute the transform `parsed` names from the asset's original and write it under
24
- * `canonicalKey`. `parsed` must come from `canonicalizeTransformPath`, so the stored pixels
25
- * always match their key. A rejection is a fact about the asset or the URL and is never worth
26
- * retrying; a thrown error belongs to the store or the environment. A 404's `error` names what
27
- * was missing, so a caller answering a browser replaces it.
23
+ * Compute the transform `parsed` names from the asset's original and write it under `key`: the
24
+ * canonical key, or that key beneath an output prefix. `parsed` must come from
25
+ * `canonicalizeTransformPath`, so the stored pixels always match their key. A rejection is a fact
26
+ * about the asset or the URL and is never worth retrying; a thrown error belongs to the store or
27
+ * the environment. A 404's `error` names what was missing, so a caller answering a browser
28
+ * replaces it.
28
29
  */
29
- export async function storeTransform(store, parsed, canonicalKey, options = {}) {
30
+ export async function storeTransform(store, parsed, key, options = {}) {
30
31
  const meta = await store.getMeta(parsed.hash32);
31
32
  if (!meta) {
32
33
  return { ok: false, status: 404, error: `Asset ${parsed.hash32} has no meta in the store` };
@@ -58,15 +59,17 @@ export async function storeTransform(store, parsed, canonicalKey, options = {})
58
59
  if (!transformed.ok) {
59
60
  return { ok: false, status: transformed.status, error: transformed.error };
60
61
  }
61
- await store.putPublicObject({
62
- key: canonicalKey,
62
+ const stored = await store.putPublicObject({
63
+ key,
63
64
  data: transformed.data,
64
65
  contentType: transformed.contentType,
65
66
  cacheControl: TRANSFORM_CACHE_CONTROL,
66
67
  tags: options.tags,
67
68
  });
68
- return { ok: true, data: transformed.data, contentType: transformed.contentType };
69
+ return { ok: true, data: transformed.data, contentType: transformed.contentType, stored };
69
70
  }
71
+ /** The `schemaVersion` of every `MaterializeReport`; a consumer gating on the JSON checks it. */
72
+ export const MATERIALIZE_REPORT_SCHEMA_VERSION = 1;
70
73
  /** sharp cannot load in this process, so nothing missing can be produced here. */
71
74
  export class SharpUnavailableError extends Error {
72
75
  constructor(cause) {
@@ -75,6 +78,43 @@ export class SharpUnavailableError extends Error {
75
78
  this.name = 'SharpUnavailableError';
76
79
  }
77
80
  }
81
+ /** `MaterializeOptions.outputPrefix` broke a rule of `assertValidOutputPrefix`; nothing was read or written. */
82
+ export class InvalidOutputPrefixError extends Error {
83
+ constructor(prefix, reason) {
84
+ super(`Invalid output prefix ${JSON.stringify(prefix)}: ${reason}`);
85
+ this.name = 'InvalidOutputPrefixError';
86
+ }
87
+ }
88
+ const OUTPUT_PREFIX_SEGMENT_RE = /^[A-Za-z0-9._-]+$/;
89
+ const CANOPY_PREFIX_SEGMENTS = Object.values(ASSET_PREFIXES).map((prefix) => prefix.split('/'));
90
+ /**
91
+ * Throws `InvalidOutputPrefixError` unless `prefix` is relative, ends in `/`, and is a run of
92
+ * `[A-Za-z0-9._-]` segments, none empty, `.` or `..`, whose first segments are not a canopy
93
+ * prefix. Production trusts whatever is under its own prefixes, so a write beneath one is a write
94
+ * production serves. Compared by segment: `assets-x/` is accepted, `assets/x/` is not.
95
+ */
96
+ export function assertValidOutputPrefix(prefix) {
97
+ const refuse = (reason) => {
98
+ throw new InvalidOutputPrefixError(prefix, reason);
99
+ };
100
+ if (prefix.startsWith('/'))
101
+ refuse('it must be relative, with no leading /');
102
+ if (!prefix.endsWith('/'))
103
+ refuse('it must end in /');
104
+ const segments = prefix.slice(0, -1).split('/');
105
+ for (const segment of segments) {
106
+ if (segment === '')
107
+ refuse('it has an empty segment');
108
+ if (segment === '.' || segment === '..')
109
+ refuse(`it has a ${segment} segment`);
110
+ if (!OUTPUT_PREFIX_SEGMENT_RE.test(segment)) {
111
+ refuse(`segment ${JSON.stringify(segment)} has a character outside [A-Za-z0-9._-]`);
112
+ }
113
+ }
114
+ const canopy = CANOPY_PREFIX_SEGMENTS.find((prefixSegments) => prefixSegments.every((segment, i) => segments[i] === segment));
115
+ if (canopy)
116
+ refuse(`it begins with the canopy prefix ${canopy.join('/')}/`);
117
+ }
78
118
  const DEFAULT_CONCURRENCY = 8;
79
119
  const DEFAULT_TRANSFORM_CONCURRENCY = 2;
80
120
  // eslint-disable-next-line security/detect-non-literal-regexp -- built from a constant
@@ -133,11 +173,14 @@ async function forEachBounded(items, limit, fn) {
133
173
  * Make every referenced transform key exist in `store`, and check every static key does. A key
134
174
  * already stored is left alone:
135
175
  * keys are content-addressed, so an existing object is the right one. sharp is loaded only when
136
- * something is missing, and a sharp that cannot load throws `SharpUnavailableError` rather than
137
- * failing each key.
176
+ * something must be transformed, and a sharp that cannot load throws `SharpUnavailableError` rather
177
+ * than failing each key. An invalid `outputPrefix` throws `InvalidOutputPrefixError` first.
138
178
  */
139
179
  export async function materializeAssets(options) {
140
- const { store } = options;
180
+ const { store, outputPrefix } = options;
181
+ if (outputPrefix !== undefined)
182
+ assertValidOutputPrefix(outputPrefix);
183
+ const destOf = (key) => (outputPrefix ?? '') + key;
141
184
  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY);
142
185
  const transformConcurrency = Math.max(1, options.transformConcurrency ?? DEFAULT_TRANSFORM_CONCURRENCY);
143
186
  const listThreshold = options.listThreshold ?? DEFAULT_LIST_THRESHOLD;
@@ -200,37 +243,51 @@ export async function materializeAssets(options) {
200
243
  group.push(target);
201
244
  byDirectives.set(target.directives, group);
202
245
  }
203
- const toHead = [];
204
- for (const [directives, group] of byDirectives) {
205
- if (store.listPublicObjectKeys && group.length >= listThreshold) {
206
- const listPrefix = `${transformPrefix}${directives}/`;
207
- const listKeys = store.listPublicObjectKeys.bind(store);
208
- try {
209
- const found = await withRetry(async () => {
210
- const keys = [];
211
- for await (const key of listKeys(listPrefix))
212
- keys.push(key);
213
- return keys;
214
- });
215
- const listed = new Set(found);
216
- for (const target of group) {
217
- if (listed.has(target.key))
218
- outcomes.set(target.key, { status: 'existed' });
219
- }
246
+ const listOnce = async (prefix) => {
247
+ if (!store.listPublicObjectKeys)
248
+ return undefined;
249
+ const listKeys = store.listPublicObjectKeys.bind(store);
250
+ try {
251
+ return new Set(await withRetry(async () => {
252
+ const keys = [];
253
+ for await (const key of listKeys(prefix))
254
+ keys.push(key);
255
+ return keys;
256
+ }));
257
+ }
258
+ catch {
259
+ // A listing that keeps failing costs only speed: its keys are HEADed instead, and each HEAD
260
+ // reports its own failure.
261
+ return undefined;
262
+ }
263
+ };
264
+ const listingsByDirectives = new Map();
265
+ if (store.listPublicObjectKeys) {
266
+ for (const [directives, group] of byDirectives) {
267
+ if (group.length < listThreshold)
220
268
  continue;
221
- }
222
- catch {
223
- // A listing that keeps failing costs only speed: HEAD the group instead, and let each
224
- // HEAD report its own failure.
225
- }
269
+ const listPrefix = `${transformPrefix}${directives}/`;
270
+ const dest = await listOnce(destOf(listPrefix));
271
+ const needsCanonical = outputPrefix !== undefined && group.some((target) => !dest?.has(destOf(target.key)));
272
+ const canonical = needsCanonical ? await listOnce(listPrefix) : undefined;
273
+ listingsByDirectives.set(directives, { dest, canonical });
226
274
  }
227
- toHead.push(...group);
228
275
  }
229
- await forEachBounded(toHead, concurrency, async (target) => {
276
+ // Every presence check goes through here. Without an output prefix a key's destination is the
277
+ // key itself, so this is one listing lookup or HEAD and never `canonical`.
278
+ const resolvePresence = async (key, listings = {}) => {
279
+ const isStored = async (storeKey, listing) => listing ? listing.has(storeKey) : withRetry(() => store.hasPublicObject(storeKey));
280
+ const dest = destOf(key);
281
+ if (await isStored(dest, listings.dest))
282
+ return 'dest';
283
+ if (dest === key)
284
+ return 'absent';
285
+ return (await isStored(key, listings.canonical)) ? 'canonical' : 'absent';
286
+ };
287
+ const presence = new Map();
288
+ await forEachBounded(valid, concurrency, async (target) => {
230
289
  try {
231
- if (await withRetry(() => store.hasPublicObject(target.key))) {
232
- outcomes.set(target.key, { status: 'existed' });
233
- }
290
+ presence.set(target.key, await resolvePresence(target.key, listingsByDirectives.get(target.directives)));
234
291
  }
235
292
  catch (err) {
236
293
  fail(target.key, 'store', `Existence check failed: ${getErrorMessage(err)}`);
@@ -238,22 +295,51 @@ export async function materializeAssets(options) {
238
295
  });
239
296
  await forEachBounded(validStatics, concurrency, async (key) => {
240
297
  try {
298
+ let found = await resolvePresence(key);
241
299
  // A HEAD cannot tell a missing key from a missing bucket, and a static key has no
242
300
  // transform's meta read to tell them apart afterwards; a GET names which.
243
- const exists = (await withRetry(() => store.hasPublicObject(key))) ||
244
- (await withRetry(() => store.readPublicObject(key))) !== null;
245
- if (exists) {
246
- outcomes.set(key, { status: 'existed' });
301
+ if (found === 'absent' && (await withRetry(() => store.readPublicObject(key))) !== null) {
302
+ found = destOf(key) === key ? 'dest' : 'canonical';
303
+ }
304
+ presence.set(key, found);
305
+ }
306
+ catch (err) {
307
+ fail(key, 'store', `Existence check failed: ${getErrorMessage(err)}`);
308
+ }
309
+ });
310
+ const toCopy = [];
311
+ for (const [key, found] of presence) {
312
+ if (found === 'dest')
313
+ outcomes.set(key, { status: 'existed' });
314
+ else if (found === 'canonical')
315
+ toCopy.push(key);
316
+ }
317
+ await forEachBounded(toCopy, concurrency, async (key) => {
318
+ try {
319
+ let tries = 0;
320
+ const copied = await withRetry(() => {
321
+ tries++;
322
+ return store.copyPublicObject(key, destOf(key));
323
+ });
324
+ if (copied === 'source-missing') {
325
+ presence.set(key, 'absent');
247
326
  }
248
327
  else {
249
- fail(key, 'content', 'No stored object; an svg or pdf is written at upload only');
328
+ // After a failed try, `already-exists` may be that try's own copy, as for a transform.
329
+ const copiedElsewhere = copied === 'already-exists' && tries === 1;
330
+ outcomes.set(key, { status: copiedElsewhere ? 'existed' : 'copied' });
250
331
  }
251
332
  }
252
333
  catch (err) {
253
- fail(key, 'store', `Existence check failed: ${getErrorMessage(err)}`);
334
+ fail(key, 'store', `Copy failed: ${getErrorMessage(err)}`);
254
335
  }
255
336
  });
256
- const missing = valid.filter((target) => !outcomes.has(target.key));
337
+ for (const key of validStatics) {
338
+ if (presence.get(key) === 'absent') {
339
+ fail(key, 'content', 'No stored object; an svg or pdf is written at upload only');
340
+ }
341
+ }
342
+ const missing = valid.filter((target) => presence.get(target.key) === 'absent');
257
343
  if (missing.length > 0) {
258
344
  try {
259
345
  await loadSharp();
@@ -264,9 +350,17 @@ export async function materializeAssets(options) {
264
350
  }
265
351
  await forEachBounded(missing, transformConcurrency, async (target) => {
266
352
  try {
267
- const result = await withRetry(() => storeTransform(store, target.parsed, target.key));
353
+ let tries = 0;
354
+ const result = await withRetry(() => {
355
+ tries++;
356
+ return storeTransform(store, target.parsed, destOf(target.key));
357
+ });
268
358
  if (result.ok) {
269
- outcomes.set(target.key, { status: 'created' });
359
+ // After a failed attempt, `already-exists` may be that attempt's own write, whose response
360
+ // was lost, so it counts as `created`: a release waiting on its `created` keys then waits
361
+ // on a key it may not have written, never skips one it did.
362
+ const createdElsewhere = result.stored === 'already-exists' && tries === 1;
363
+ outcomes.set(target.key, { status: createdElsewhere ? 'existed' : 'created' });
270
364
  }
271
365
  else {
272
366
  fail(target.key, 'content', `${result.status}: ${result.error}`);
@@ -292,10 +386,13 @@ export async function materializeAssets(options) {
292
386
  });
293
387
  const count = (predicate) => results.filter(predicate).length;
294
388
  return {
389
+ schemaVersion: MATERIALIZE_REPORT_SCHEMA_VERSION,
390
+ ...(outputPrefix !== undefined ? { outputPrefix } : {}),
295
391
  summary: {
296
392
  total: results.length,
297
393
  existed: count((r) => r.status === 'existed'),
298
394
  created: count((r) => r.status === 'created'),
395
+ copied: count((r) => r.status === 'copied'),
299
396
  failed: count((r) => r.status === 'failed'),
300
397
  contentFailures: count((r) => r.status === 'failed' && r.failure === 'content'),
301
398
  storeFailures: count((r) => r.status === 'failed' && r.failure === 'store'),
@@ -3,7 +3,7 @@
3
3
  * bucket-prefix layout as S3AssetStore (see keys.ts) so content behaves
4
4
  * identically across adapters.
5
5
  */
6
- import type { AssetMeta, AssetStore, BeginUploadInput, PublicObject, StagedUploadTarget } from './types.js';
6
+ import type { AssetMeta, AssetStore, BeginUploadInput, CreateOnlyResult, PublicObject, StagedUploadTarget } from './types.js';
7
7
  export interface LocalAssetStoreOptions {
8
8
  /** Root directory. The five bucket prefixes are created as subdirectories under it. */
9
9
  root: string;
@@ -26,11 +26,20 @@ export declare class LocalAssetStore implements AssetStore {
26
26
  */
27
27
  private resolveKey;
28
28
  private headersSidecarPath;
29
- private writeHeadersSidecar;
30
29
  private readHeadersSidecar;
31
30
  private readFileOrNull;
32
31
  private writeFile;
33
32
  private deleteFileIfPresent;
33
+ /** Link `tempPath` in at `filePath`; `false` when something is already there. */
34
+ private linkIfAbsent;
35
+ /**
36
+ * Create-only write of a blob and its headers sidecar. Each is written whole to a temp file, then
37
+ * `link()`ed into place, which fails with EEXIST rather than replacing (see docs/concurrency.md).
38
+ * The sidecar is linked first, so a reader never finds a blob without its headers. A sidecar
39
+ * already there was linked by an earlier writer of this key and is kept, as the blob is: the
40
+ * first writer's headers win, as on S3 (a static's Content-Disposition names its uploaded file).
41
+ */
42
+ private createExclusive;
34
43
  beginUpload(_input: BeginUploadInput): Promise<StagedUploadTarget>;
35
44
  /**
36
45
  * Staging methods accept caller-influenced keys (finalize receives the key
@@ -47,7 +56,7 @@ export declare class LocalAssetStore implements AssetStore {
47
56
  ext: string;
48
57
  data: Uint8Array;
49
58
  contentType: string;
50
- }): Promise<void>;
59
+ }): Promise<CreateOnlyResult>;
51
60
  /** Ignores the `ext` hint: a directory read costs nothing here and needs no permission. */
52
61
  readOriginal(hash32: string): Promise<{
53
62
  data: Uint8Array;
@@ -62,7 +71,8 @@ export declare class LocalAssetStore implements AssetStore {
62
71
  cacheControl?: string;
63
72
  /** Ignored: tags exist for bucket lifecycle rules, and a local store has none. */
64
73
  tags?: Readonly<Record<string, string>>;
65
- }): Promise<void>;
74
+ }): Promise<CreateOnlyResult>;
75
+ copyPublicObject(sourceKey: string, destKey: string): Promise<CreateOnlyResult | 'source-missing'>;
66
76
  readPublicObject(key: string): Promise<PublicObject | null>;
67
77
  hasPublicObject(key: string): Promise<boolean>;
68
78
  /**
@@ -73,7 +83,7 @@ export declare class LocalAssetStore implements AssetStore {
73
83
  * always clobbers the destination, which would let the second writer
74
84
  * silently overwrite the first instead of losing cleanly.
75
85
  */
76
- putMetaIfAbsent(hash32: string, meta: AssetMeta): Promise<'created' | 'already-exists'>;
86
+ putMetaIfAbsent(hash32: string, meta: AssetMeta): Promise<CreateOnlyResult>;
77
87
  getMeta(hash32: string): Promise<AssetMeta | null>;
78
88
  listMeta(input?: {
79
89
  cursor?: string;