@lightnet/cli 4.4.0 → 4.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,19 @@
1
1
  # @lightnet/cli
2
2
 
3
+ ## 4.4.1
4
+
5
+ ### Patch Changes
6
+
7
+ - [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Delete orphaned R2 objects from `check-files --fix --r2` in batches instead of spawning one rclone process per file.
8
+
9
+ - [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Show progress feedback while `check-files` and `check-links` perform long-running checks.
10
+
11
+ - [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Improve `check-files` performance by avoiding duplicate R2 listings and parallelizing independent filesystem work.
12
+
13
+ - [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Protect R2 objects referenced by media content links from `check-files --fix --r2` cleanup and warn that they should use upload content.
14
+
15
+ - [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Decode percent-encoded R2 public URLs when matching `check-files --r2` references against bucket objects.
16
+
3
17
  ## 4.4.0
4
18
 
5
19
  ### Minor Changes
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "description": "CLI for managing LightNet projects",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
- "version": "4.4.0",
6
+ "version": "4.4.1",
7
7
  "publishConfig": {
8
8
  "access": "public"
9
9
  },
@@ -3,7 +3,15 @@
3
3
  import { join, posix, resolve } from "node:path"
4
4
  import { cwd as processCwd, stdin, stdout } from "node:process"
5
5
 
6
- import { confirm, intro, isCancel, log, outro, text } from "@clack/prompts"
6
+ import {
7
+ confirm,
8
+ intro,
9
+ isCancel,
10
+ log,
11
+ outro,
12
+ spinner,
13
+ text,
14
+ } from "@clack/prompts"
7
15
 
8
16
  import { CliError } from "./support/cli-error.js"
9
17
  import { contentCollections } from "./support/content-collections.js"
@@ -48,6 +56,8 @@ const categoryImagesDir = "src/content/categories/images"
48
56
  * }} FileReference
49
57
  */
50
58
 
59
+ /** @typedef {MissingReference} WrongTypeReference */
60
+
51
61
  /**
52
62
  * @typedef {{
53
63
  * cwd?: string
@@ -79,14 +89,15 @@ export async function checkFiles(options, runtime = {}) {
79
89
  await assertLightNetSiteRoot(cwd)
80
90
 
81
91
  const collections = contentCollections(cwd)
82
- const mediaItems = await collections.getMediaItems()
83
- const categories = await collections.getCategories()
84
- log.message(
85
- `Checking ${mediaItems.length} media items and ${categories.length} categories.`,
86
- )
92
+ const [mediaItems, categories] = await Promise.all([
93
+ collections.getMediaItems(),
94
+ collections.getCategories(),
95
+ ])
87
96
 
88
97
  /** @type {MissingReference[]} */
89
98
  let missingContentFiles = []
99
+ /** @type {WrongTypeReference[]} */
100
+ let wrongTypeR2ContentFiles = []
90
101
  /** @type {string[]} */
91
102
  let orphanedContentFiles = []
92
103
  /** @type {string[]} */
@@ -104,18 +115,40 @@ export async function checkFiles(options, runtime = {}) {
104
115
  const categoryThumbnailStorage = files(cwd, { rootDir: categoryImagesDir })
105
116
 
106
117
  if (includeContentFiles) {
118
+ contentFileStorage = options.r2
119
+ ? createR2FileStorage({
120
+ cwd,
121
+ interactive,
122
+ promptConfirm,
123
+ promptText,
124
+ })
125
+ : contentFiles(cwd)
126
+ contentFileStorage = await contentFileStorage.init()
127
+ }
128
+
129
+ log.message(
130
+ `Checking ${mediaItems.length} media items and ${categories.length} categories.`,
131
+ )
132
+
133
+ if (includeContentFiles) {
134
+ if (!contentFileStorage) {
135
+ throw new CliError("Missing file storage for content validation.")
136
+ }
137
+ const initializedContentFileStorage = contentFileStorage
107
138
  if (options.r2) {
108
- contentFileStorage = createR2FileStorage({
109
- cwd,
110
- interactive,
111
- promptConfirm,
112
- promptText,
113
- })
114
- const r2Result = await validateContentFiles({
115
- mediaItems,
116
- storage: contentFileStorage,
139
+ const r2Result = await runSpinner({
140
+ error: "Content file check failed.",
141
+ start: "Listing R2 objects and checking content file references",
142
+ stop: (result) => formatContentCheckSummary(result),
143
+ task: () =>
144
+ validateContentFiles({
145
+ includeLinkReferences: true,
146
+ mediaItems,
147
+ storage: initializedContentFileStorage,
148
+ }),
117
149
  })
118
150
  missingContentFiles = r2Result.missingReferences
151
+ wrongTypeR2ContentFiles = r2Result.wrongTypeReferences
119
152
  orphanedContentFiles = r2Result.orphanedFiles
120
153
  if (r2Result.referenceCount === 0) {
121
154
  warnings.push(
@@ -123,10 +156,15 @@ export async function checkFiles(options, runtime = {}) {
123
156
  )
124
157
  }
125
158
  } else {
126
- contentFileStorage = contentFiles(cwd)
127
- const localContentResult = await validateContentFiles({
128
- mediaItems,
129
- storage: contentFileStorage,
159
+ const localContentResult = await runSpinner({
160
+ error: "Content file check failed.",
161
+ start: "Checking local content files",
162
+ stop: (result) => formatContentCheckSummary(result),
163
+ task: () =>
164
+ validateContentFiles({
165
+ mediaItems,
166
+ storage: initializedContentFileStorage,
167
+ }),
130
168
  })
131
169
  missingContentFiles = localContentResult.missingReferences
132
170
  orphanedContentFiles = localContentResult.orphanedFiles
@@ -139,12 +177,19 @@ export async function checkFiles(options, runtime = {}) {
139
177
  }
140
178
 
141
179
  if (includeThumbnails) {
142
- const thumbnailsResult = await validateThumbnails(
143
- mediaItems,
144
- categories,
145
- mediaThumbnailStorage,
146
- categoryThumbnailStorage,
147
- )
180
+ const thumbnailsResult = await runSpinner({
181
+ error: "Thumbnail check failed.",
182
+ start: "Checking thumbnails",
183
+ stop: (result) =>
184
+ `Checked thumbnails: ${result.orphanedMediaThumbnails.length} orphaned media, ${result.orphanedCategoryThumbnails.length} orphaned categories.`,
185
+ task: () =>
186
+ validateThumbnails(
187
+ mediaItems,
188
+ categories,
189
+ mediaThumbnailStorage,
190
+ categoryThumbnailStorage,
191
+ ),
192
+ })
148
193
  orphanedMediaThumbnails = thumbnailsResult.orphanedMediaThumbnails
149
194
  orphanedCategoryThumbnails = thumbnailsResult.orphanedCategoryThumbnails
150
195
  }
@@ -199,25 +244,52 @@ export async function checkFiles(options, runtime = {}) {
199
244
  }))
200
245
 
201
246
  if (shouldDelete) {
202
- const deletedContentTargets = contentFileStorage
203
- ? await contentFileStorage.delete(
204
- contentDeletions.map((deletion) => deletion.target),
205
- )
206
- : []
207
- const deletedLocalTargets = [
208
- ...(await mediaThumbnailStorage.delete(
209
- localDeletions
210
- .filter((deletion) => deletion.target.startsWith(mediaImagesDir))
211
- .map((deletion) => deletion.target),
212
- )),
213
- ...(await categoryThumbnailStorage.delete(
214
- localDeletions
215
- .filter((deletion) =>
216
- deletion.target.startsWith(categoryImagesDir),
217
- )
218
- .map((deletion) => deletion.target),
219
- )),
220
- ]
247
+ const { deletedContentTargets, deletedLocalTargets } = await runSpinner(
248
+ {
249
+ error: "File deletion failed.",
250
+ start: formatDeletionProgress("Deleting", deletions.length),
251
+ stop: (result) => {
252
+ const deletedCount =
253
+ result.deletedContentTargets.length +
254
+ result.deletedLocalTargets.length
255
+ return formatDeletionProgress("Deleted", deletedCount)
256
+ },
257
+ task: async () => {
258
+ const [
259
+ deletedContentTargets,
260
+ deletedMediaThumbnailTargets,
261
+ deletedCategoryThumbnailTargets,
262
+ ] = await Promise.all([
263
+ contentFileStorage
264
+ ? contentFileStorage.delete(
265
+ contentDeletions.map((deletion) => deletion.target),
266
+ )
267
+ : [],
268
+ mediaThumbnailStorage.delete(
269
+ localDeletions
270
+ .filter((deletion) =>
271
+ deletion.target.startsWith(mediaImagesDir),
272
+ )
273
+ .map((deletion) => deletion.target),
274
+ ),
275
+ categoryThumbnailStorage.delete(
276
+ localDeletions
277
+ .filter((deletion) =>
278
+ deletion.target.startsWith(categoryImagesDir),
279
+ )
280
+ .map((deletion) => deletion.target),
281
+ ),
282
+ ])
283
+ return {
284
+ deletedContentTargets,
285
+ deletedLocalTargets: [
286
+ ...deletedMediaThumbnailTargets,
287
+ ...deletedCategoryThumbnailTargets,
288
+ ],
289
+ }
290
+ },
291
+ },
292
+ )
221
293
  const deletedTargets = new Set([
222
294
  ...deletedContentTargets,
223
295
  ...deletedLocalTargets,
@@ -245,6 +317,7 @@ export async function checkFiles(options, runtime = {}) {
245
317
  const hasIssues =
246
318
  warnings.length > 0 ||
247
319
  missingContentFiles.length > 0 ||
320
+ wrongTypeR2ContentFiles.length > 0 ||
248
321
  orphanedContentFiles.length > 0 ||
249
322
  orphanedMediaThumbnails.length > 0 ||
250
323
  orphanedCategoryThumbnails.length > 0
@@ -262,6 +335,10 @@ export async function checkFiles(options, runtime = {}) {
262
335
  "Missing referenced content files",
263
336
  missingContentFiles,
264
337
  )
338
+ printWrongTypeReferenceSection(
339
+ "R2 objects referenced as links",
340
+ wrongTypeR2ContentFiles,
341
+ )
265
342
  printSection(
266
343
  options.r2 ? "Orphaned R2 objects" : "Orphaned local content files",
267
344
  orphanedContentFiles.map((filePath) =>
@@ -307,6 +384,64 @@ function printMissingReferenceSection(title, items) {
307
384
  }
308
385
  }
309
386
 
387
+ /**
388
+ * @param {string} title
389
+ * @param {WrongTypeReference[]} items
390
+ */
391
+ function printWrongTypeReferenceSection(title, items) {
392
+ if (items.length === 0) {
393
+ return
394
+ }
395
+ log.warn(`${title} (${items.length})`)
396
+ for (const item of items) {
397
+ log.message(
398
+ `• ${item.path} should use type "upload" (referenced by ${item.sources.toSorted().join(", ")})`,
399
+ )
400
+ }
401
+ }
402
+
403
+ /**
404
+ * @template T
405
+ * @param {{
406
+ * error: string
407
+ * start: string
408
+ * stop: (result: T) => string
409
+ * task: () => Promise<T>
410
+ * }} args
411
+ * @returns {Promise<T>}
412
+ */
413
+ async function runSpinner({ error, start, stop, task }) {
414
+ const activeSpinner = spinner()
415
+ activeSpinner.start(start)
416
+ try {
417
+ const result = await task()
418
+ activeSpinner.stop(stop(result))
419
+ return result
420
+ } catch (caughtError) {
421
+ activeSpinner.error(error)
422
+ throw caughtError
423
+ }
424
+ }
425
+
426
+ /**
427
+ * @param {{
428
+ * missingReferences: MissingReference[]
429
+ * orphanedFiles: string[]
430
+ * referenceCount: number
431
+ * }} result
432
+ */
433
+ function formatContentCheckSummary(result) {
434
+ return `Checked content files: ${result.referenceCount} references, ${result.missingReferences.length} missing, ${result.orphanedFiles.length} orphaned.`
435
+ }
436
+
437
+ /**
438
+ * @param {string} action
439
+ * @param {number} count
440
+ */
441
+ function formatDeletionProgress(action, count) {
442
+ return `${action} ${count} orphaned file${count === 1 ? "" : "s"}.`
443
+ }
444
+
310
445
  /**
311
446
  * @param {string|undefined} rawScope
312
447
  */
@@ -369,10 +504,15 @@ async function validateThumbnails(
369
504
  .filter((value) => value !== undefined),
370
505
  )
371
506
 
372
- const initializedMediaStorage = await mediaThumbnailStorage.init()
373
- const initializedCategoryStorage = await categoryThumbnailStorage.init()
374
- const mediaFiles = await initializedMediaStorage.list()
375
- const categoryFiles = await initializedCategoryStorage.list()
507
+ const [initializedMediaStorage, initializedCategoryStorage] =
508
+ await Promise.all([
509
+ mediaThumbnailStorage.init(),
510
+ categoryThumbnailStorage.init(),
511
+ ])
512
+ const [mediaFiles, categoryFiles] = await Promise.all([
513
+ initializedMediaStorage.list(),
514
+ initializedCategoryStorage.list(),
515
+ ])
376
516
 
377
517
  return {
378
518
  orphanedMediaThumbnails: mediaFiles.filter(
@@ -386,18 +526,28 @@ async function validateThumbnails(
386
526
 
387
527
  /**
388
528
  * @param {{
529
+ * includeLinkReferences?: boolean
389
530
  * mediaItems: MediaItem[]
390
531
  * storage: FileStorage
391
532
  * }} args
392
533
  */
393
- async function validateContentFiles({ mediaItems, storage }) {
534
+ async function validateContentFiles({
535
+ includeLinkReferences = false,
536
+ mediaItems,
537
+ storage,
538
+ }) {
394
539
  const initializedStorage = await storage.init()
395
- const references = collectContentReferences(mediaItems, initializedStorage)
540
+ const { references, wrongTypeReferences } = collectContentReferences(
541
+ mediaItems,
542
+ initializedStorage,
543
+ { includeLinkReferences },
544
+ )
396
545
  if (references.size === 0) {
397
546
  return {
398
547
  missingReferences: /** @type {MissingReference[]} */ ([]),
399
548
  orphanedFiles: /** @type {string[]} */ ([]),
400
549
  referenceCount: 0,
550
+ wrongTypeReferences: /** @type {WrongTypeReference[]} */ ([]),
401
551
  }
402
552
  }
403
553
 
@@ -416,41 +566,74 @@ async function validateContentFiles({ mediaItems, storage }) {
416
566
  .filter((filePath) => !references.has(filePath))
417
567
  .sort()
418
568
 
569
+ const existingWrongTypeReferences = [...wrongTypeReferences.entries()]
570
+ .filter(([path]) => fileSet.has(path))
571
+ .map(([, reference]) => ({
572
+ path: reference.displayPath,
573
+ sources: [...reference.sources].toSorted(),
574
+ }))
575
+ .sort((a, b) => a.path.localeCompare(b.path))
576
+
419
577
  return {
420
578
  missingReferences,
421
579
  orphanedFiles,
422
580
  referenceCount: references.size,
581
+ wrongTypeReferences: existingWrongTypeReferences,
423
582
  }
424
583
  }
425
584
 
426
585
  /**
427
586
  * @param {MediaItem[]} mediaItems
428
587
  * @param {FileStorage} storage
588
+ * @param {{includeLinkReferences:boolean}} options
429
589
  */
430
- function collectContentReferences(mediaItems, storage) {
590
+ function collectContentReferences(mediaItems, storage, options) {
431
591
  const references = new Map()
592
+ const wrongTypeReferences = new Map()
432
593
  for (const item of mediaItems) {
433
594
  const sourceFileName = posix.basename(toPosixPath(item.path))
434
595
  for (const contentItem of item.content) {
435
- if (contentItem.type !== "upload") {
596
+ const isLinkReference = contentItem.type === "link"
597
+ if (
598
+ contentItem.type !== "upload" &&
599
+ !(options.includeLinkReferences && isLinkReference)
600
+ ) {
436
601
  continue
437
602
  }
438
603
  const filePath = storage.toPath(contentItem.url)
439
604
  if (!filePath) {
440
605
  continue
441
606
  }
442
- const current = references.get(filePath)
443
- if (current) {
444
- current.sources.add(sourceFileName)
445
- } else {
446
- references.set(filePath, {
447
- displayPath: contentItem.url,
448
- sources: new Set([sourceFileName]),
449
- })
607
+ addFileReference(references, filePath, contentItem.url, sourceFileName)
608
+ if (isLinkReference) {
609
+ addFileReference(
610
+ wrongTypeReferences,
611
+ filePath,
612
+ contentItem.url,
613
+ sourceFileName,
614
+ )
450
615
  }
451
616
  }
452
617
  }
453
- return references
618
+ return { references, wrongTypeReferences }
619
+ }
620
+
621
+ /**
622
+ * @param {Map<string, FileReference>} references
623
+ * @param {string} filePath
624
+ * @param {string} displayPath
625
+ * @param {string} sourceFileName
626
+ */
627
+ function addFileReference(references, filePath, displayPath, sourceFileName) {
628
+ const current = references.get(filePath)
629
+ if (current) {
630
+ current.sources.add(sourceFileName)
631
+ return
632
+ }
633
+ references.set(filePath, {
634
+ displayPath,
635
+ sources: new Set([sourceFileName]),
636
+ })
454
637
  }
455
638
 
456
639
  /**
@@ -3,7 +3,7 @@
3
3
  import { posix, resolve } from "node:path"
4
4
  import { cwd as processCwd, stdin, stdout } from "node:process"
5
5
 
6
- import { intro, isCancel, log, outro, text } from "@clack/prompts"
6
+ import { intro, isCancel, log, outro, progress, text } from "@clack/prompts"
7
7
 
8
8
  import {
9
9
  cliConfigFileName,
@@ -96,12 +96,10 @@ export async function checkLinks(options, runtime = {}) {
96
96
  return true
97
97
  }
98
98
 
99
- const results = await mapConcurrent(
100
- references,
101
- defaultConcurrency,
102
- (reference) =>
103
- checkLink(reference.resolvedUrl, { fetch: runFetch, timeoutMs }),
104
- )
99
+ const results = await checkReferencesWithProgress(references, {
100
+ fetch: runFetch,
101
+ timeoutMs,
102
+ })
105
103
  const checkedLinks = references
106
104
  .map((reference, index) => ({ reference, result: results[index] }))
107
105
  .sort((a, b) =>
@@ -141,6 +139,46 @@ export async function checkLinks(options, runtime = {}) {
141
139
  return false
142
140
  }
143
141
 
142
+ /**
143
+ * @param {LinkReference[]} references
144
+ * @param {{fetch: typeof fetch, timeoutMs: number}} options
145
+ */
146
+ async function checkReferencesWithProgress(references, options) {
147
+ const linkProgress = progress({ max: references.length })
148
+ let completed = 0
149
+ linkProgress.start(`Checking links (0/${references.length})`)
150
+
151
+ try {
152
+ const results = await mapConcurrent(
153
+ references,
154
+ defaultConcurrency,
155
+ async (reference) => {
156
+ const result = await checkLink(reference.resolvedUrl, options)
157
+ completed += 1
158
+ linkProgress.advance(
159
+ 1,
160
+ `Checking links (${completed}/${references.length})`,
161
+ )
162
+ return result
163
+ },
164
+ )
165
+ const protectedCount = results.filter((result) =>
166
+ isProtectedLink(result),
167
+ ).length
168
+ const unavailableCount = results.filter(
169
+ (result) => !result.ok && !isProtectedLink(result),
170
+ ).length
171
+ const availableCount = results.length - protectedCount - unavailableCount
172
+ linkProgress.stop(
173
+ `Checked links: ${availableCount} available, ${protectedCount} protected, ${unavailableCount} unavailable.`,
174
+ )
175
+ return results
176
+ } catch (error) {
177
+ linkProgress.error("Link check failed.")
178
+ throw error
179
+ }
180
+ }
181
+
144
182
  /**
145
183
  * @param {{
146
184
  * cwd: string
@@ -58,23 +58,24 @@ export function files(cwd, options) {
58
58
  * @param {string[]} paths
59
59
  */
60
60
  async delete(paths) {
61
- /** @type {string[]} */
62
- const removed = []
63
- for (const path of paths) {
64
- try {
65
- await unlink(
66
- resolve(
67
- cwd,
68
- options.toAbsolutePath?.(path) ??
69
- toLocalFilePath(options.rootDir, path),
70
- ),
71
- )
72
- removed.push(path)
73
- } catch {
74
- // The caller prints per-file failures based on the returned paths.
75
- }
76
- }
77
- return removed
61
+ const removals = await Promise.all(
62
+ paths.map(async (path) => {
63
+ try {
64
+ await unlink(
65
+ resolve(
66
+ cwd,
67
+ options.toAbsolutePath?.(path) ??
68
+ toLocalFilePath(options.rootDir, path),
69
+ ),
70
+ )
71
+ return path
72
+ } catch {
73
+ // The caller prints per-file failures based on the returned paths.
74
+ return undefined
75
+ }
76
+ }),
77
+ )
78
+ return removals.filter((path) => path !== undefined)
78
79
  },
79
80
  /**
80
81
  * @param {string} url
@@ -161,13 +162,17 @@ async function walkFiles(dirPath) {
161
162
  /** @type {string[]} */
162
163
  const files = []
163
164
  const entries = await readdir(dirPath, { withFileTypes: true })
164
- for (const entry of entries) {
165
- const entryPath = join(dirPath, entry.name)
166
- if (entry.isDirectory()) {
167
- files.push(...(await walkFiles(entryPath)))
168
- } else if (entry.isFile()) {
169
- files.push(entryPath)
170
- }
165
+ const nestedFiles = await Promise.all(
166
+ entries.map(async (entry) => {
167
+ const entryPath = join(dirPath, entry.name)
168
+ if (entry.isDirectory()) {
169
+ return walkFiles(entryPath)
170
+ }
171
+ return entry.isFile() ? [entryPath] : []
172
+ }),
173
+ )
174
+ for (const item of nestedFiles) {
175
+ files.push(...item)
171
176
  }
172
177
  return files
173
178
  }
package/src/support/r2.js CHANGED
@@ -1,6 +1,9 @@
1
1
  // @ts-check
2
2
 
3
3
  import { execFile } from "node:child_process"
4
+ import { mkdtemp, rm, writeFile } from "node:fs/promises"
5
+ import { tmpdir } from "node:os"
6
+ import { join } from "node:path"
4
7
  import { env as processEnv } from "node:process"
5
8
  import { promisify } from "node:util"
6
9
 
@@ -15,6 +18,7 @@ import { CliError } from "./cli-error.js"
15
18
 
16
19
  const execFileAsync = promisify(execFile)
17
20
  const sessionSecretEnvName = "LIGHTNET_R2_SECRET_ACCESS_KEY"
21
+ const r2DeleteBatchSize = 1000
18
22
 
19
23
  /**
20
24
  * @typedef {{
@@ -117,18 +121,16 @@ export function createR2FileStorage({
117
121
  const storage = {
118
122
  async init() {
119
123
  await getConfig()
124
+ await getSecretAccessKey()
120
125
  return storage
121
126
  },
122
127
  async list() {
123
128
  return runWithRefresh(async (config) => {
124
- const secretAccessKey = await getSecretAccessKey()
125
- const sharedArgs = {
129
+ const bucketObjects = await listR2Objects({
126
130
  cwd,
127
131
  config,
128
- secretAccessKey,
129
- }
130
- await assertR2Access(sharedArgs)
131
- const bucketObjects = await listR2Objects(sharedArgs)
132
+ secretAccessKey: await getSecretAccessKey(),
133
+ })
132
134
  return bucketObjects.map((item) => item.key).sort()
133
135
  })
134
136
  },
@@ -138,22 +140,12 @@ export function createR2FileStorage({
138
140
  async delete(paths) {
139
141
  const r2Config = await getConfig()
140
142
  const secretAccessKey = await getSecretAccessKey()
141
- /** @type {string[]} */
142
- const removed = []
143
- for (const path of paths) {
144
- try {
145
- await runConfiguredRclone({
146
- args: ["deletefile", makeR2Path(r2Config, path)],
147
- cwd,
148
- config: r2Config,
149
- secretAccessKey,
150
- })
151
- removed.push(path)
152
- } catch {
153
- // The caller prints per-file failures based on the returned paths.
154
- }
155
- }
156
- return removed
143
+ return deleteR2Objects({
144
+ config: r2Config,
145
+ cwd,
146
+ paths,
147
+ secretAccessKey,
148
+ })
157
149
  },
158
150
  /**
159
151
  * @param {string} url
@@ -167,32 +159,134 @@ export function createR2FileStorage({
167
159
  return undefined
168
160
  }
169
161
  const path = url.slice(normalizedBase.length).replace(/^\/+/, "")
170
- return path || undefined
162
+ return decodeR2ObjectPath(path) || undefined
171
163
  },
172
164
  }
173
165
  return storage
174
166
  }
175
167
 
168
+ /**
169
+ * @param {string} path
170
+ */
171
+ function decodeR2ObjectPath(path) {
172
+ try {
173
+ return decodeURIComponent(path)
174
+ } catch {
175
+ return path
176
+ }
177
+ }
178
+
176
179
  /**
177
180
  * @param {{
181
+ * config: R2Config
178
182
  * cwd: string
183
+ * paths: string[]
184
+ * secretAccessKey: string
185
+ * }} args
186
+ */
187
+ async function deleteR2Objects({ config, cwd, paths, secretAccessKey }) {
188
+ /** @type {string[]} */
189
+ const removed = []
190
+ const batchablePaths = paths.filter((path) => !path.includes("\n"))
191
+ const singleDeletePaths = paths.filter((path) => path.includes("\n"))
192
+
193
+ for (
194
+ let index = 0;
195
+ index < batchablePaths.length;
196
+ index += r2DeleteBatchSize
197
+ ) {
198
+ const batch = batchablePaths.slice(index, index + r2DeleteBatchSize)
199
+ try {
200
+ await deleteR2ObjectBatch({ config, cwd, paths: batch, secretAccessKey })
201
+ removed.push(...batch)
202
+ } catch {
203
+ removed.push(
204
+ ...(await deleteR2ObjectsIndividually({
205
+ config,
206
+ cwd,
207
+ paths: batch,
208
+ secretAccessKey,
209
+ })),
210
+ )
211
+ }
212
+ }
213
+
214
+ removed.push(
215
+ ...(await deleteR2ObjectsIndividually({
216
+ config,
217
+ cwd,
218
+ paths: singleDeletePaths,
219
+ secretAccessKey,
220
+ })),
221
+ )
222
+
223
+ return removed
224
+ }
225
+
226
+ /**
227
+ * @param {{
179
228
  * config: R2Config
229
+ * cwd: string
230
+ * paths: string[]
180
231
  * secretAccessKey: string
181
232
  * }} args
182
233
  */
183
- async function assertR2Access(args) {
234
+ async function deleteR2ObjectBatch({ config, cwd, paths, secretAccessKey }) {
235
+ if (paths.length === 0) {
236
+ return
237
+ }
238
+
239
+ const tempDir = await mkdtemp(join(tmpdir(), "lightnet-r2-delete-"))
240
+ const fileListPath = join(tempDir, "files.txt")
184
241
  try {
242
+ await writeFile(fileListPath, `${paths.join("\n")}\n`, "utf8")
185
243
  await runConfiguredRclone({
186
- args: ["lsjson", makeR2Path(args.config), "--files-only"],
187
- ...args,
244
+ args: [
245
+ "delete",
246
+ makeR2Path(config),
247
+ "--files-from-raw",
248
+ fileListPath,
249
+ "--fast-list",
250
+ ],
251
+ cwd,
252
+ config,
253
+ secretAccessKey,
188
254
  })
189
- } catch (error) {
190
- throw new CliError(
191
- error instanceof Error
192
- ? `Unable to access R2 bucket "${args.config.bucket}": ${error.message}`
193
- : `Unable to access R2 bucket "${args.config.bucket}".`,
194
- )
255
+ } finally {
256
+ await rm(tempDir, { force: true, recursive: true })
257
+ }
258
+ }
259
+
260
+ /**
261
+ * @param {{
262
+ * config: R2Config
263
+ * cwd: string
264
+ * paths: string[]
265
+ * secretAccessKey: string
266
+ * }} args
267
+ */
268
+ async function deleteR2ObjectsIndividually({
269
+ config,
270
+ cwd,
271
+ paths,
272
+ secretAccessKey,
273
+ }) {
274
+ /** @type {string[]} */
275
+ const removed = []
276
+ for (const path of paths) {
277
+ try {
278
+ await runConfiguredRclone({
279
+ args: ["deletefile", makeR2Path(config, path)],
280
+ cwd,
281
+ config,
282
+ secretAccessKey,
283
+ })
284
+ removed.push(path)
285
+ } catch {
286
+ // The caller prints per-file failures based on the returned paths.
287
+ }
195
288
  }
289
+ return removed
196
290
  }
197
291
 
198
292
  /**
@@ -203,18 +297,28 @@ async function assertR2Access(args) {
203
297
  * }} args
204
298
  */
205
299
  async function listR2Objects(args) {
206
- const { stdout: listJson } = await runConfiguredRclone({
207
- args: [
208
- "lsjson",
209
- makeR2Path(args.config),
210
- "--recursive",
211
- "--files-only",
212
- "--fast-list",
213
- "--no-mimetype",
214
- "--no-modtime",
215
- ],
216
- ...args,
217
- })
300
+ let listJson
301
+ try {
302
+ const result = await runConfiguredRclone({
303
+ args: [
304
+ "lsjson",
305
+ makeR2Path(args.config),
306
+ "--recursive",
307
+ "--files-only",
308
+ "--fast-list",
309
+ "--no-mimetype",
310
+ "--no-modtime",
311
+ ],
312
+ ...args,
313
+ })
314
+ listJson = result.stdout
315
+ } catch (error) {
316
+ throw new CliError(
317
+ error instanceof Error
318
+ ? `Unable to access R2 bucket "${args.config.bucket}": ${error.message}`
319
+ : `Unable to access R2 bucket "${args.config.bucket}".`,
320
+ )
321
+ }
218
322
 
219
323
  let parsed
220
324
  try {