@lightnet/cli 4.4.0 → 4.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/package.json +3 -3
- package/src/check-files.js +324 -75
- package/src/check-links.js +66 -18
- package/src/check-translations.js +16 -8
- package/src/index.js +26 -15
- package/src/support/filesystem.js +29 -24
- package/src/support/prompt-cancel.js +13 -0
- package/src/support/r2.js +150 -45
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# @lightnet/cli
|
|
2
2
|
|
|
3
|
+
## 4.4.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#433](https://github.com/LightNetDev/LightNet/pull/433) [`dbde36e`](https://github.com/LightNetDev/LightNet/commit/dbde36e7a25bf6470783e9594260b666e09c2783) - Make `check-files` summaries clearer by explaining what was compared and what needs attention.
|
|
8
|
+
|
|
9
|
+
- [#433](https://github.com/LightNetDev/LightNet/pull/433) [`dbde36e`](https://github.com/LightNetDev/LightNet/commit/dbde36e7a25bf6470783e9594260b666e09c2783) - Handle Ctrl+C cancellation consistently for interactive `check-*` command prompts.
|
|
10
|
+
|
|
11
|
+
- [#433](https://github.com/LightNetDev/LightNet/pull/433) [`dbde36e`](https://github.com/LightNetDev/LightNet/commit/dbde36e7a25bf6470783e9594260b666e09c2783) - Replace `check-files --fix --no-confirm` with `check-files --fix-without-confirm` for non-interactive cleanup.
|
|
12
|
+
|
|
13
|
+
- [#433](https://github.com/LightNetDev/LightNet/pull/433) [`dbde36e`](https://github.com/LightNetDev/LightNet/commit/dbde36e7a25bf6470783e9594260b666e09c2783) - Treat warning-only `check-files` runs as successful when no local content file references are found.
|
|
14
|
+
|
|
15
|
+
- [#433](https://github.com/LightNetDev/LightNet/pull/433) [`dbde36e`](https://github.com/LightNetDev/LightNet/commit/dbde36e7a25bf6470783e9594260b666e09c2783) - Improve `check-files` and `check-links` output for long referenced paths and URLs.
|
|
16
|
+
|
|
17
|
+
## 4.4.1
|
|
18
|
+
|
|
19
|
+
### Patch Changes
|
|
20
|
+
|
|
21
|
+
- [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Delete orphaned R2 objects from `check-files --fix --r2` in batches instead of spawning one rclone process per file.
|
|
22
|
+
|
|
23
|
+
- [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Show progress feedback while `check-files` and `check-links` perform long-running checks.
|
|
24
|
+
|
|
25
|
+
- [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Improve `check-files` performance by avoiding duplicate R2 listings and parallelizing independent filesystem work.
|
|
26
|
+
|
|
27
|
+
- [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Protect R2 objects referenced by media content links from `check-files --fix --r2` cleanup and warn that they should use upload content.
|
|
28
|
+
|
|
29
|
+
- [#431](https://github.com/LightNetDev/LightNet/pull/431) [`aa5968f`](https://github.com/LightNetDev/LightNet/commit/aa5968f26112f93121f03e7aafdfb04018186032) - Decode percent-encoded R2 public URLs when matching `check-files --r2` references against bucket objects.
|
|
30
|
+
|
|
3
31
|
## 4.4.0
|
|
4
32
|
|
|
5
33
|
### Minor Changes
|
package/package.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"description": "CLI for managing LightNet projects",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
|
-
"version": "4.4.
|
|
6
|
+
"version": "4.4.2",
|
|
7
7
|
"publishConfig": {
|
|
8
8
|
"access": "public"
|
|
9
9
|
},
|
|
@@ -25,8 +25,8 @@
|
|
|
25
25
|
"CHANGELOG.md"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"
|
|
29
|
-
"
|
|
28
|
+
"@clack/prompts": "^1.7.0",
|
|
29
|
+
"commander": "^15.0.0"
|
|
30
30
|
},
|
|
31
31
|
"engines": {
|
|
32
32
|
"node": ">=22"
|
package/src/check-files.js
CHANGED
|
@@ -3,7 +3,15 @@
|
|
|
3
3
|
import { join, posix, resolve } from "node:path"
|
|
4
4
|
import { cwd as processCwd, stdin, stdout } from "node:process"
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
confirm,
|
|
8
|
+
intro,
|
|
9
|
+
isCancel,
|
|
10
|
+
log,
|
|
11
|
+
outro,
|
|
12
|
+
spinner,
|
|
13
|
+
text,
|
|
14
|
+
} from "@clack/prompts"
|
|
7
15
|
|
|
8
16
|
import { CliError } from "./support/cli-error.js"
|
|
9
17
|
import { contentCollections } from "./support/content-collections.js"
|
|
@@ -13,6 +21,7 @@ import {
|
|
|
13
21
|
pathExists,
|
|
14
22
|
toLocalContentDisplayPath,
|
|
15
23
|
} from "./support/filesystem.js"
|
|
24
|
+
import { cancelPrompt } from "./support/prompt-cancel.js"
|
|
16
25
|
import { createR2FileStorage } from "./support/r2.js"
|
|
17
26
|
|
|
18
27
|
const supportedScopes = new Set(["content-files", "thumbnails"])
|
|
@@ -25,7 +34,7 @@ const categoryImagesDir = "src/content/categories/images"
|
|
|
25
34
|
* @typedef {{
|
|
26
35
|
* scope?: string
|
|
27
36
|
* fix?: boolean
|
|
28
|
-
*
|
|
37
|
+
* fixWithoutConfirm?: boolean
|
|
29
38
|
* r2?: boolean
|
|
30
39
|
* }} CheckFilesOptions
|
|
31
40
|
*/
|
|
@@ -48,6 +57,8 @@ const categoryImagesDir = "src/content/categories/images"
|
|
|
48
57
|
* }} FileReference
|
|
49
58
|
*/
|
|
50
59
|
|
|
60
|
+
/** @typedef {MissingReference} WrongTypeReference */
|
|
61
|
+
|
|
51
62
|
/**
|
|
52
63
|
* @typedef {{
|
|
53
64
|
* cwd?: string
|
|
@@ -79,14 +90,15 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
79
90
|
await assertLightNetSiteRoot(cwd)
|
|
80
91
|
|
|
81
92
|
const collections = contentCollections(cwd)
|
|
82
|
-
const mediaItems = await
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
)
|
|
93
|
+
const [mediaItems, categories] = await Promise.all([
|
|
94
|
+
collections.getMediaItems(),
|
|
95
|
+
collections.getCategories(),
|
|
96
|
+
])
|
|
87
97
|
|
|
88
98
|
/** @type {MissingReference[]} */
|
|
89
99
|
let missingContentFiles = []
|
|
100
|
+
/** @type {WrongTypeReference[]} */
|
|
101
|
+
let wrongTypeR2ContentFiles = []
|
|
90
102
|
/** @type {string[]} */
|
|
91
103
|
let orphanedContentFiles = []
|
|
92
104
|
/** @type {string[]} */
|
|
@@ -104,18 +116,40 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
104
116
|
const categoryThumbnailStorage = files(cwd, { rootDir: categoryImagesDir })
|
|
105
117
|
|
|
106
118
|
if (includeContentFiles) {
|
|
119
|
+
contentFileStorage = options.r2
|
|
120
|
+
? createR2FileStorage({
|
|
121
|
+
cwd,
|
|
122
|
+
interactive,
|
|
123
|
+
promptConfirm,
|
|
124
|
+
promptText,
|
|
125
|
+
})
|
|
126
|
+
: contentFiles(cwd)
|
|
127
|
+
contentFileStorage = await contentFileStorage.init()
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
log.message(
|
|
131
|
+
`Checking ${mediaItems.length} media items and ${categories.length} categories.`,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if (includeContentFiles) {
|
|
135
|
+
if (!contentFileStorage) {
|
|
136
|
+
throw new CliError("Missing file storage for content validation.")
|
|
137
|
+
}
|
|
138
|
+
const initializedContentFileStorage = contentFileStorage
|
|
107
139
|
if (options.r2) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
140
|
+
const r2Result = await runSpinner({
|
|
141
|
+
error: "Content file check failed.",
|
|
142
|
+
start: "Listing R2 objects and checking content file references",
|
|
143
|
+
stop: (result) => formatContentCheckSummary(result),
|
|
144
|
+
task: () =>
|
|
145
|
+
validateContentFiles({
|
|
146
|
+
includeLinkReferences: true,
|
|
147
|
+
mediaItems,
|
|
148
|
+
storage: initializedContentFileStorage,
|
|
149
|
+
}),
|
|
117
150
|
})
|
|
118
151
|
missingContentFiles = r2Result.missingReferences
|
|
152
|
+
wrongTypeR2ContentFiles = r2Result.wrongTypeReferences
|
|
119
153
|
orphanedContentFiles = r2Result.orphanedFiles
|
|
120
154
|
if (r2Result.referenceCount === 0) {
|
|
121
155
|
warnings.push(
|
|
@@ -123,33 +157,44 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
123
157
|
)
|
|
124
158
|
}
|
|
125
159
|
} else {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
160
|
+
const localContentResult = await runSpinner({
|
|
161
|
+
error: "Content file check failed.",
|
|
162
|
+
start: "Checking local content files",
|
|
163
|
+
stop: (result) => formatContentCheckSummary(result),
|
|
164
|
+
task: () =>
|
|
165
|
+
validateContentFiles({
|
|
166
|
+
mediaItems,
|
|
167
|
+
storage: initializedContentFileStorage,
|
|
168
|
+
}),
|
|
130
169
|
})
|
|
131
170
|
missingContentFiles = localContentResult.missingReferences
|
|
132
171
|
orphanedContentFiles = localContentResult.orphanedFiles
|
|
133
172
|
if (localContentResult.referenceCount === 0) {
|
|
134
173
|
warnings.push(
|
|
135
|
-
'No local "/files" content file references found.
|
|
174
|
+
'No local "/files" content file references found. If your content files are stored in R2, consider re-running with "--r2".',
|
|
136
175
|
)
|
|
137
176
|
}
|
|
138
177
|
}
|
|
139
178
|
}
|
|
140
179
|
|
|
141
180
|
if (includeThumbnails) {
|
|
142
|
-
const thumbnailsResult = await
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
181
|
+
const thumbnailsResult = await runSpinner({
|
|
182
|
+
error: "Thumbnail check failed.",
|
|
183
|
+
start: "Checking thumbnails",
|
|
184
|
+
stop: (result) => formatThumbnailCheckSummary(result),
|
|
185
|
+
task: () =>
|
|
186
|
+
validateThumbnails(
|
|
187
|
+
mediaItems,
|
|
188
|
+
categories,
|
|
189
|
+
mediaThumbnailStorage,
|
|
190
|
+
categoryThumbnailStorage,
|
|
191
|
+
),
|
|
192
|
+
})
|
|
148
193
|
orphanedMediaThumbnails = thumbnailsResult.orphanedMediaThumbnails
|
|
149
194
|
orphanedCategoryThumbnails = thumbnailsResult.orphanedCategoryThumbnails
|
|
150
195
|
}
|
|
151
196
|
|
|
152
|
-
if (options.fix) {
|
|
197
|
+
if (options.fix || options.fixWithoutConfirm) {
|
|
153
198
|
/** @type {{displayPath:string, target:string}[]} */
|
|
154
199
|
const deletions = []
|
|
155
200
|
/** @type {{displayPath:string, target:string}[]} */
|
|
@@ -185,10 +230,10 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
185
230
|
}
|
|
186
231
|
|
|
187
232
|
if (deletions.length > 0) {
|
|
188
|
-
const shouldSkipConfirmation = options.
|
|
233
|
+
const shouldSkipConfirmation = options.fixWithoutConfirm === true
|
|
189
234
|
if (!shouldSkipConfirmation && !interactive) {
|
|
190
235
|
throw new CliError(
|
|
191
|
-
'Deletion requires confirmation. Re-run with "--fix
|
|
236
|
+
'Deletion requires confirmation. Re-run with "--fix-without-confirm" in non-interactive environments.',
|
|
192
237
|
)
|
|
193
238
|
}
|
|
194
239
|
const shouldDelete =
|
|
@@ -199,25 +244,52 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
199
244
|
}))
|
|
200
245
|
|
|
201
246
|
if (shouldDelete) {
|
|
202
|
-
const deletedContentTargets =
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
247
|
+
const { deletedContentTargets, deletedLocalTargets } = await runSpinner(
|
|
248
|
+
{
|
|
249
|
+
error: "File deletion failed.",
|
|
250
|
+
start: formatDeletionProgress("Deleting", deletions.length),
|
|
251
|
+
stop: (result) => {
|
|
252
|
+
const deletedCount =
|
|
253
|
+
result.deletedContentTargets.length +
|
|
254
|
+
result.deletedLocalTargets.length
|
|
255
|
+
return formatDeletionProgress("Deleted", deletedCount)
|
|
256
|
+
},
|
|
257
|
+
task: async () => {
|
|
258
|
+
const [
|
|
259
|
+
deletedContentTargets,
|
|
260
|
+
deletedMediaThumbnailTargets,
|
|
261
|
+
deletedCategoryThumbnailTargets,
|
|
262
|
+
] = await Promise.all([
|
|
263
|
+
contentFileStorage
|
|
264
|
+
? contentFileStorage.delete(
|
|
265
|
+
contentDeletions.map((deletion) => deletion.target),
|
|
266
|
+
)
|
|
267
|
+
: [],
|
|
268
|
+
mediaThumbnailStorage.delete(
|
|
269
|
+
localDeletions
|
|
270
|
+
.filter((deletion) =>
|
|
271
|
+
deletion.target.startsWith(mediaImagesDir),
|
|
272
|
+
)
|
|
273
|
+
.map((deletion) => deletion.target),
|
|
274
|
+
),
|
|
275
|
+
categoryThumbnailStorage.delete(
|
|
276
|
+
localDeletions
|
|
277
|
+
.filter((deletion) =>
|
|
278
|
+
deletion.target.startsWith(categoryImagesDir),
|
|
279
|
+
)
|
|
280
|
+
.map((deletion) => deletion.target),
|
|
281
|
+
),
|
|
282
|
+
])
|
|
283
|
+
return {
|
|
284
|
+
deletedContentTargets,
|
|
285
|
+
deletedLocalTargets: [
|
|
286
|
+
...deletedMediaThumbnailTargets,
|
|
287
|
+
...deletedCategoryThumbnailTargets,
|
|
288
|
+
],
|
|
289
|
+
}
|
|
290
|
+
},
|
|
291
|
+
},
|
|
292
|
+
)
|
|
221
293
|
const deletedTargets = new Set([
|
|
222
294
|
...deletedContentTargets,
|
|
223
295
|
...deletedLocalTargets,
|
|
@@ -243,25 +315,29 @@ export async function checkFiles(options, runtime = {}) {
|
|
|
243
315
|
}
|
|
244
316
|
|
|
245
317
|
const hasIssues =
|
|
246
|
-
warnings.length > 0 ||
|
|
247
318
|
missingContentFiles.length > 0 ||
|
|
319
|
+
wrongTypeR2ContentFiles.length > 0 ||
|
|
248
320
|
orphanedContentFiles.length > 0 ||
|
|
249
321
|
orphanedMediaThumbnails.length > 0 ||
|
|
250
322
|
orphanedCategoryThumbnails.length > 0
|
|
251
323
|
|
|
324
|
+
for (const warning of warnings) {
|
|
325
|
+
log.warn(warning)
|
|
326
|
+
}
|
|
327
|
+
|
|
252
328
|
if (!hasIssues && removedItems.length === 0) {
|
|
253
329
|
outro("No issues found. 🎉")
|
|
254
330
|
return true
|
|
255
331
|
}
|
|
256
332
|
|
|
257
|
-
for (const warning of warnings) {
|
|
258
|
-
log.warn(warning)
|
|
259
|
-
}
|
|
260
|
-
|
|
261
333
|
printMissingReferenceSection(
|
|
262
334
|
"Missing referenced content files",
|
|
263
335
|
missingContentFiles,
|
|
264
336
|
)
|
|
337
|
+
printWrongTypeReferenceSection(
|
|
338
|
+
"R2 objects referenced as links",
|
|
339
|
+
wrongTypeR2ContentFiles,
|
|
340
|
+
)
|
|
265
341
|
printSection(
|
|
266
342
|
options.r2 ? "Orphaned R2 objects" : "Orphaned local content files",
|
|
267
343
|
orphanedContentFiles.map((filePath) =>
|
|
@@ -300,13 +376,128 @@ function printMissingReferenceSection(title, items) {
|
|
|
300
376
|
return
|
|
301
377
|
}
|
|
302
378
|
log.error(`${title} (${items.length})`)
|
|
379
|
+
for (const item of items) {
|
|
380
|
+
log.message(`• ${item.path}\n${formatReferenceSources(item.sources)}`)
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* @param {string} title
|
|
386
|
+
* @param {WrongTypeReference[]} items
|
|
387
|
+
*/
|
|
388
|
+
function printWrongTypeReferenceSection(title, items) {
|
|
389
|
+
if (items.length === 0) {
|
|
390
|
+
return
|
|
391
|
+
}
|
|
392
|
+
log.warn(`${title} (${items.length})`)
|
|
303
393
|
for (const item of items) {
|
|
304
394
|
log.message(
|
|
305
|
-
`• ${item.path}
|
|
395
|
+
`• ${item.path}\n Expected type: "upload"\n${formatReferenceSources(item.sources)}`,
|
|
306
396
|
)
|
|
307
397
|
}
|
|
308
398
|
}
|
|
309
399
|
|
|
400
|
+
/**
|
|
401
|
+
* @param {string[]} sources
|
|
402
|
+
*/
|
|
403
|
+
function formatReferenceSources(sources) {
|
|
404
|
+
const sortedSources = sources.toSorted()
|
|
405
|
+
if (sortedSources.length === 1) {
|
|
406
|
+
return ` Referenced by: ${sortedSources[0]}`
|
|
407
|
+
}
|
|
408
|
+
return [
|
|
409
|
+
" Referenced by:",
|
|
410
|
+
...sortedSources.map((source) => ` - ${source}`),
|
|
411
|
+
].join("\n")
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* @template T
|
|
416
|
+
* @param {{
|
|
417
|
+
* error: string
|
|
418
|
+
* start: string
|
|
419
|
+
* stop: (result: T) => string
|
|
420
|
+
* task: () => Promise<T>
|
|
421
|
+
* }} args
|
|
422
|
+
* @returns {Promise<T>}
|
|
423
|
+
*/
|
|
424
|
+
async function runSpinner({ error, start, stop, task }) {
|
|
425
|
+
const activeSpinner = spinner()
|
|
426
|
+
activeSpinner.start(start)
|
|
427
|
+
try {
|
|
428
|
+
const result = await task()
|
|
429
|
+
activeSpinner.stop(stop(result))
|
|
430
|
+
return result
|
|
431
|
+
} catch (caughtError) {
|
|
432
|
+
activeSpinner.error(error)
|
|
433
|
+
throw caughtError
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* @param {{
|
|
439
|
+
* fileCount: number
|
|
440
|
+
* missingReferences: MissingReference[]
|
|
441
|
+
* orphanedFiles: string[]
|
|
442
|
+
* referenceCount: number
|
|
443
|
+
* }} result
|
|
444
|
+
*/
|
|
445
|
+
function formatContentCheckSummary(result) {
|
|
446
|
+
const issues = [
|
|
447
|
+
formatFindingCount(result.missingReferences.length, "missing reference"),
|
|
448
|
+
formatFindingCount(result.orphanedFiles.length, "orphaned file"),
|
|
449
|
+
]
|
|
450
|
+
|
|
451
|
+
if (result.referenceCount === 0 && result.fileCount === 0) {
|
|
452
|
+
return `Content files: none found.`
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
return `Content files: ${formatCount(result.referenceCount, "reference")}, ${formatCount(result.fileCount, "file")}; ${issues.join(", ")}.`
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* @param {{
|
|
460
|
+
* fileCount: number
|
|
461
|
+
* orphanedCategoryThumbnails: string[]
|
|
462
|
+
* orphanedMediaThumbnails: string[]
|
|
463
|
+
* referenceCount: number
|
|
464
|
+
* }} result
|
|
465
|
+
*/
|
|
466
|
+
function formatThumbnailCheckSummary(result) {
|
|
467
|
+
const orphanedThumbnails =
|
|
468
|
+
result.orphanedMediaThumbnails.length +
|
|
469
|
+
result.orphanedCategoryThumbnails.length
|
|
470
|
+
|
|
471
|
+
return `Thumbnails: ${formatCount(result.referenceCount, "reference")}, ${formatCount(result.fileCount, "file")}; orphaned: ${orphanedThumbnails} (${result.orphanedMediaThumbnails.length} media, ${result.orphanedCategoryThumbnails.length} categories).`
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* @param {number} count
|
|
476
|
+
* @param {string} singular
|
|
477
|
+
*/
|
|
478
|
+
function formatFindingCount(count, singular) {
|
|
479
|
+
if (count === 0) {
|
|
480
|
+
return `no ${singular}s`
|
|
481
|
+
}
|
|
482
|
+
return formatCount(count, singular)
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* @param {number} count
|
|
487
|
+
* @param {string} singular
|
|
488
|
+
*/
|
|
489
|
+
function formatCount(count, singular) {
|
|
490
|
+
return `${count} ${singular}${count === 1 ? "" : "s"}`
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* @param {string} action
|
|
495
|
+
* @param {number} count
|
|
496
|
+
*/
|
|
497
|
+
function formatDeletionProgress(action, count) {
|
|
498
|
+
return `${action} ${count} orphaned file${count === 1 ? "" : "s"}.`
|
|
499
|
+
}
|
|
500
|
+
|
|
310
501
|
/**
|
|
311
502
|
* @param {string|undefined} rawScope
|
|
312
503
|
*/
|
|
@@ -369,39 +560,57 @@ async function validateThumbnails(
|
|
|
369
560
|
.filter((value) => value !== undefined),
|
|
370
561
|
)
|
|
371
562
|
|
|
372
|
-
const initializedMediaStorage =
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
563
|
+
const [initializedMediaStorage, initializedCategoryStorage] =
|
|
564
|
+
await Promise.all([
|
|
565
|
+
mediaThumbnailStorage.init(),
|
|
566
|
+
categoryThumbnailStorage.init(),
|
|
567
|
+
])
|
|
568
|
+
const [mediaFiles, categoryFiles] = await Promise.all([
|
|
569
|
+
initializedMediaStorage.list(),
|
|
570
|
+
initializedCategoryStorage.list(),
|
|
571
|
+
])
|
|
376
572
|
|
|
377
573
|
return {
|
|
574
|
+
fileCount: mediaFiles.length + categoryFiles.length,
|
|
378
575
|
orphanedMediaThumbnails: mediaFiles.filter(
|
|
379
576
|
(file) => !mediaReferences.has(file),
|
|
380
577
|
),
|
|
381
578
|
orphanedCategoryThumbnails: categoryFiles.filter(
|
|
382
579
|
(file) => !categoryReferences.has(file),
|
|
383
580
|
),
|
|
581
|
+
referenceCount: mediaReferences.size + categoryReferences.size,
|
|
384
582
|
}
|
|
385
583
|
}
|
|
386
584
|
|
|
387
585
|
/**
|
|
388
586
|
* @param {{
|
|
587
|
+
* includeLinkReferences?: boolean
|
|
389
588
|
* mediaItems: MediaItem[]
|
|
390
589
|
* storage: FileStorage
|
|
391
590
|
* }} args
|
|
392
591
|
*/
|
|
393
|
-
async function validateContentFiles({
|
|
592
|
+
async function validateContentFiles({
|
|
593
|
+
includeLinkReferences = false,
|
|
594
|
+
mediaItems,
|
|
595
|
+
storage,
|
|
596
|
+
}) {
|
|
394
597
|
const initializedStorage = await storage.init()
|
|
395
|
-
const references = collectContentReferences(
|
|
598
|
+
const { references, wrongTypeReferences } = collectContentReferences(
|
|
599
|
+
mediaItems,
|
|
600
|
+
initializedStorage,
|
|
601
|
+
{ includeLinkReferences },
|
|
602
|
+
)
|
|
603
|
+
const files = await initializedStorage.list()
|
|
396
604
|
if (references.size === 0) {
|
|
397
605
|
return {
|
|
606
|
+
fileCount: files.length,
|
|
398
607
|
missingReferences: /** @type {MissingReference[]} */ ([]),
|
|
399
608
|
orphanedFiles: /** @type {string[]} */ ([]),
|
|
400
609
|
referenceCount: 0,
|
|
610
|
+
wrongTypeReferences: /** @type {WrongTypeReference[]} */ ([]),
|
|
401
611
|
}
|
|
402
612
|
}
|
|
403
613
|
|
|
404
|
-
const files = await initializedStorage.list()
|
|
405
614
|
const fileSet = new Set(files)
|
|
406
615
|
|
|
407
616
|
const missingReferences = [...references.entries()]
|
|
@@ -416,41 +625,75 @@ async function validateContentFiles({ mediaItems, storage }) {
|
|
|
416
625
|
.filter((filePath) => !references.has(filePath))
|
|
417
626
|
.sort()
|
|
418
627
|
|
|
628
|
+
const existingWrongTypeReferences = [...wrongTypeReferences.entries()]
|
|
629
|
+
.filter(([path]) => fileSet.has(path))
|
|
630
|
+
.map(([, reference]) => ({
|
|
631
|
+
path: reference.displayPath,
|
|
632
|
+
sources: [...reference.sources].toSorted(),
|
|
633
|
+
}))
|
|
634
|
+
.sort((a, b) => a.path.localeCompare(b.path))
|
|
635
|
+
|
|
419
636
|
return {
|
|
637
|
+
fileCount: files.length,
|
|
420
638
|
missingReferences,
|
|
421
639
|
orphanedFiles,
|
|
422
640
|
referenceCount: references.size,
|
|
641
|
+
wrongTypeReferences: existingWrongTypeReferences,
|
|
423
642
|
}
|
|
424
643
|
}
|
|
425
644
|
|
|
426
645
|
/**
|
|
427
646
|
* @param {MediaItem[]} mediaItems
|
|
428
647
|
* @param {FileStorage} storage
|
|
648
|
+
* @param {{includeLinkReferences:boolean}} options
|
|
429
649
|
*/
|
|
430
|
-
function collectContentReferences(mediaItems, storage) {
|
|
650
|
+
function collectContentReferences(mediaItems, storage, options) {
|
|
431
651
|
const references = new Map()
|
|
652
|
+
const wrongTypeReferences = new Map()
|
|
432
653
|
for (const item of mediaItems) {
|
|
433
654
|
const sourceFileName = posix.basename(toPosixPath(item.path))
|
|
434
655
|
for (const contentItem of item.content) {
|
|
435
|
-
|
|
656
|
+
const isLinkReference = contentItem.type === "link"
|
|
657
|
+
if (
|
|
658
|
+
contentItem.type !== "upload" &&
|
|
659
|
+
!(options.includeLinkReferences && isLinkReference)
|
|
660
|
+
) {
|
|
436
661
|
continue
|
|
437
662
|
}
|
|
438
663
|
const filePath = storage.toPath(contentItem.url)
|
|
439
664
|
if (!filePath) {
|
|
440
665
|
continue
|
|
441
666
|
}
|
|
442
|
-
|
|
443
|
-
if (
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
667
|
+
addFileReference(references, filePath, contentItem.url, sourceFileName)
|
|
668
|
+
if (isLinkReference) {
|
|
669
|
+
addFileReference(
|
|
670
|
+
wrongTypeReferences,
|
|
671
|
+
filePath,
|
|
672
|
+
contentItem.url,
|
|
673
|
+
sourceFileName,
|
|
674
|
+
)
|
|
450
675
|
}
|
|
451
676
|
}
|
|
452
677
|
}
|
|
453
|
-
return references
|
|
678
|
+
return { references, wrongTypeReferences }
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* @param {Map<string, FileReference>} references
|
|
683
|
+
* @param {string} filePath
|
|
684
|
+
* @param {string} displayPath
|
|
685
|
+
* @param {string} sourceFileName
|
|
686
|
+
*/
|
|
687
|
+
function addFileReference(references, filePath, displayPath, sourceFileName) {
|
|
688
|
+
const current = references.get(filePath)
|
|
689
|
+
if (current) {
|
|
690
|
+
current.sources.add(sourceFileName)
|
|
691
|
+
return
|
|
692
|
+
}
|
|
693
|
+
references.set(filePath, {
|
|
694
|
+
displayPath,
|
|
695
|
+
sources: new Set([sourceFileName]),
|
|
696
|
+
})
|
|
454
697
|
}
|
|
455
698
|
|
|
456
699
|
/**
|
|
@@ -512,7 +755,10 @@ function toPosixPath(filePath) {
|
|
|
512
755
|
*/
|
|
513
756
|
async function defaultPromptText(message) {
|
|
514
757
|
const answer = await text({ message })
|
|
515
|
-
|
|
758
|
+
if (isCancel(answer)) {
|
|
759
|
+
cancelPrompt()
|
|
760
|
+
}
|
|
761
|
+
return String(answer)
|
|
516
762
|
}
|
|
517
763
|
|
|
518
764
|
/**
|
|
@@ -523,5 +769,8 @@ async function defaultPromptConfirm(message) {
|
|
|
523
769
|
message,
|
|
524
770
|
initialValue: false,
|
|
525
771
|
})
|
|
526
|
-
|
|
772
|
+
if (isCancel(answer)) {
|
|
773
|
+
cancelPrompt()
|
|
774
|
+
}
|
|
775
|
+
return answer
|
|
527
776
|
}
|
package/src/check-links.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { posix, resolve } from "node:path"
|
|
4
4
|
import { cwd as processCwd, stdin, stdout } from "node:process"
|
|
5
5
|
|
|
6
|
-
import { intro, isCancel, log, outro, text } from "@clack/prompts"
|
|
6
|
+
import { intro, isCancel, log, outro, progress, text } from "@clack/prompts"
|
|
7
7
|
|
|
8
8
|
import {
|
|
9
9
|
cliConfigFileName,
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
import { CliError } from "./support/cli-error.js"
|
|
14
14
|
import { contentCollections } from "./support/content-collections.js"
|
|
15
15
|
import { pathExists } from "./support/filesystem.js"
|
|
16
|
+
import { cancelPrompt } from "./support/prompt-cancel.js"
|
|
16
17
|
|
|
17
18
|
const mediaDir = "src/content/media"
|
|
18
19
|
const defaultConcurrency = 24
|
|
@@ -96,12 +97,10 @@ export async function checkLinks(options, runtime = {}) {
|
|
|
96
97
|
return true
|
|
97
98
|
}
|
|
98
99
|
|
|
99
|
-
const results = await
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
checkLink(reference.resolvedUrl, { fetch: runFetch, timeoutMs }),
|
|
104
|
-
)
|
|
100
|
+
const results = await checkReferencesWithProgress(references, {
|
|
101
|
+
fetch: runFetch,
|
|
102
|
+
timeoutMs,
|
|
103
|
+
})
|
|
105
104
|
const checkedLinks = references
|
|
106
105
|
.map((reference, index) => ({ reference, result: results[index] }))
|
|
107
106
|
.sort((a, b) =>
|
|
@@ -128,11 +127,7 @@ export async function checkLinks(options, runtime = {}) {
|
|
|
128
127
|
log.error(`Unavailable links (${unavailableLinks.length})`)
|
|
129
128
|
for (const { reference, result } of unavailableLinks) {
|
|
130
129
|
log.message(
|
|
131
|
-
|
|
132
|
-
...reference.sources,
|
|
133
|
-
]
|
|
134
|
-
.toSorted()
|
|
135
|
-
.join(", ")})`,
|
|
130
|
+
`• ${reference.displayUrl}\n Result: ${formatFailure(result)}\n${formatReferenceSources(reference.sources)}`,
|
|
136
131
|
)
|
|
137
132
|
}
|
|
138
133
|
|
|
@@ -141,6 +136,46 @@ export async function checkLinks(options, runtime = {}) {
|
|
|
141
136
|
return false
|
|
142
137
|
}
|
|
143
138
|
|
|
139
|
+
/**
|
|
140
|
+
* @param {LinkReference[]} references
|
|
141
|
+
* @param {{fetch: typeof fetch, timeoutMs: number}} options
|
|
142
|
+
*/
|
|
143
|
+
async function checkReferencesWithProgress(references, options) {
|
|
144
|
+
const linkProgress = progress({ max: references.length })
|
|
145
|
+
let completed = 0
|
|
146
|
+
linkProgress.start(`Checking links (0/${references.length})`)
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
const results = await mapConcurrent(
|
|
150
|
+
references,
|
|
151
|
+
defaultConcurrency,
|
|
152
|
+
async (reference) => {
|
|
153
|
+
const result = await checkLink(reference.resolvedUrl, options)
|
|
154
|
+
completed += 1
|
|
155
|
+
linkProgress.advance(
|
|
156
|
+
1,
|
|
157
|
+
`Checking links (${completed}/${references.length})`,
|
|
158
|
+
)
|
|
159
|
+
return result
|
|
160
|
+
},
|
|
161
|
+
)
|
|
162
|
+
const protectedCount = results.filter((result) =>
|
|
163
|
+
isProtectedLink(result),
|
|
164
|
+
).length
|
|
165
|
+
const unavailableCount = results.filter(
|
|
166
|
+
(result) => !result.ok && !isProtectedLink(result),
|
|
167
|
+
).length
|
|
168
|
+
const availableCount = results.length - protectedCount - unavailableCount
|
|
169
|
+
linkProgress.stop(
|
|
170
|
+
`Checked links: ${availableCount} available, ${protectedCount} protected, ${unavailableCount} unavailable.`,
|
|
171
|
+
)
|
|
172
|
+
return results
|
|
173
|
+
} catch (error) {
|
|
174
|
+
linkProgress.error("Link check failed.")
|
|
175
|
+
throw error
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
144
179
|
/**
|
|
145
180
|
* @param {{
|
|
146
181
|
* cwd: string
|
|
@@ -294,15 +329,25 @@ function printProtectedLinks(links) {
|
|
|
294
329
|
log.warn(`Protected or forbidden links (${links.length})`)
|
|
295
330
|
for (const { reference, result } of links) {
|
|
296
331
|
log.message(
|
|
297
|
-
|
|
298
|
-
...reference.sources,
|
|
299
|
-
]
|
|
300
|
-
.toSorted()
|
|
301
|
-
.join(", ")})`,
|
|
332
|
+
`• ${reference.displayUrl}\n Result: ${formatFailure(result)}\n${formatReferenceSources(reference.sources)}`,
|
|
302
333
|
)
|
|
303
334
|
}
|
|
304
335
|
}
|
|
305
336
|
|
|
337
|
+
/**
|
|
338
|
+
* @param {Set<string>} sources
|
|
339
|
+
*/
|
|
340
|
+
function formatReferenceSources(sources) {
|
|
341
|
+
const sortedSources = [...sources].toSorted()
|
|
342
|
+
if (sortedSources.length === 1) {
|
|
343
|
+
return ` Referenced by: ${sortedSources[0]}`
|
|
344
|
+
}
|
|
345
|
+
return [
|
|
346
|
+
" Referenced by:",
|
|
347
|
+
...sortedSources.map((source) => ` - ${source}`),
|
|
348
|
+
].join("\n")
|
|
349
|
+
}
|
|
350
|
+
|
|
306
351
|
/**
|
|
307
352
|
* @param {string} url
|
|
308
353
|
* @param {{
|
|
@@ -445,7 +490,10 @@ function toPosixPath(filePath) {
|
|
|
445
490
|
*/
|
|
446
491
|
async function defaultPromptText(message) {
|
|
447
492
|
const answer = await text({ message })
|
|
448
|
-
|
|
493
|
+
if (isCancel(answer)) {
|
|
494
|
+
cancelPrompt()
|
|
495
|
+
}
|
|
496
|
+
return String(answer)
|
|
449
497
|
}
|
|
450
498
|
|
|
451
499
|
/**
|
|
@@ -5,7 +5,9 @@ import { readFile } from "node:fs/promises"
|
|
|
5
5
|
import { resolve } from "node:path"
|
|
6
6
|
import { cwd } from "node:process"
|
|
7
7
|
|
|
8
|
-
import { confirm, intro, log, outro, taskLog } from "@clack/prompts"
|
|
8
|
+
import { confirm, intro, isCancel, log, outro, taskLog } from "@clack/prompts"
|
|
9
|
+
|
|
10
|
+
import { cancelPrompt } from "./support/prompt-cancel.js"
|
|
9
11
|
|
|
10
12
|
/**
|
|
11
13
|
* @typedef {{
|
|
@@ -100,13 +102,7 @@ export async function checkTranslations(options = {}) {
|
|
|
100
102
|
* @returns
|
|
101
103
|
*/
|
|
102
104
|
async function runBuild(build) {
|
|
103
|
-
const shouldRunBuild =
|
|
104
|
-
build ??
|
|
105
|
-
(await confirm({
|
|
106
|
-
message:
|
|
107
|
-
"Run pnpm build now? Command requires an up-to-date dist/ directory.",
|
|
108
|
-
initialValue: false,
|
|
109
|
-
}))
|
|
105
|
+
const shouldRunBuild = build ?? (await promptRunBuild())
|
|
110
106
|
if (!shouldRunBuild) {
|
|
111
107
|
return true
|
|
112
108
|
}
|
|
@@ -158,6 +154,18 @@ async function runBuild(build) {
|
|
|
158
154
|
}
|
|
159
155
|
}
|
|
160
156
|
|
|
157
|
+
async function promptRunBuild() {
|
|
158
|
+
const answer = await confirm({
|
|
159
|
+
message:
|
|
160
|
+
"Run pnpm build now? Command requires an up-to-date dist/ directory.",
|
|
161
|
+
initialValue: false,
|
|
162
|
+
})
|
|
163
|
+
if (isCancel(answer)) {
|
|
164
|
+
cancelPrompt()
|
|
165
|
+
}
|
|
166
|
+
return answer
|
|
167
|
+
}
|
|
168
|
+
|
|
161
169
|
/**
|
|
162
170
|
*
|
|
163
171
|
* @param {{title:string, action:string}} source
|
package/src/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { checkFiles } from "./check-files.js"
|
|
|
8
8
|
import { checkLinks } from "./check-links.js"
|
|
9
9
|
import { checkTranslations } from "./check-translations.js"
|
|
10
10
|
import { CliError } from "./support/cli-error.js"
|
|
11
|
+
import { PromptCancelled } from "./support/prompt-cancel.js"
|
|
11
12
|
const { version } = pkg
|
|
12
13
|
|
|
13
14
|
const program = new Command()
|
|
@@ -24,8 +25,12 @@ program
|
|
|
24
25
|
.option("--build", "run pnpm build before checking translations")
|
|
25
26
|
.option("--no-build", "skip the build prompt and use the latest dist/ output")
|
|
26
27
|
.action(async (options) => {
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
try {
|
|
29
|
+
const checkSuccessful = await checkTranslations(options)
|
|
30
|
+
commandExitCode = checkSuccessful ? 0 : 1
|
|
31
|
+
} catch (error) {
|
|
32
|
+
handleCommandError(error)
|
|
33
|
+
}
|
|
29
34
|
})
|
|
30
35
|
|
|
31
36
|
program
|
|
@@ -34,7 +39,7 @@ program
|
|
|
34
39
|
"check for missing and orphaned content files and thumbnails in a LightNet site",
|
|
35
40
|
)
|
|
36
41
|
.option("--fix", "remove orphaned files")
|
|
37
|
-
.option("--
|
|
42
|
+
.option("--fix-without-confirm", "remove orphaned files without confirmation")
|
|
38
43
|
.option(
|
|
39
44
|
"--r2",
|
|
40
45
|
"validate remote content files in Cloudflare R2 instead of public/files",
|
|
@@ -48,12 +53,7 @@ program
|
|
|
48
53
|
const checkSuccessful = await checkFiles(options)
|
|
49
54
|
commandExitCode = checkSuccessful ? 0 : 1
|
|
50
55
|
} catch (error) {
|
|
51
|
-
|
|
52
|
-
console.error(error.message)
|
|
53
|
-
commandExitCode = 1
|
|
54
|
-
return
|
|
55
|
-
}
|
|
56
|
-
throw error
|
|
56
|
+
handleCommandError(error)
|
|
57
57
|
}
|
|
58
58
|
})
|
|
59
59
|
|
|
@@ -66,14 +66,25 @@ program
|
|
|
66
66
|
const checkSuccessful = await checkLinks(options)
|
|
67
67
|
commandExitCode = checkSuccessful ? 0 : 1
|
|
68
68
|
} catch (error) {
|
|
69
|
-
|
|
70
|
-
console.error(error.message)
|
|
71
|
-
commandExitCode = 1
|
|
72
|
-
return
|
|
73
|
-
}
|
|
74
|
-
throw error
|
|
69
|
+
handleCommandError(error)
|
|
75
70
|
}
|
|
76
71
|
})
|
|
77
72
|
|
|
78
73
|
await program.parseAsync()
|
|
79
74
|
process.exitCode = commandExitCode
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* @param {unknown} error
|
|
78
|
+
*/
|
|
79
|
+
function handleCommandError(error) {
|
|
80
|
+
if (error instanceof PromptCancelled) {
|
|
81
|
+
commandExitCode = 0
|
|
82
|
+
return
|
|
83
|
+
}
|
|
84
|
+
if (error instanceof CliError) {
|
|
85
|
+
console.error(error.message)
|
|
86
|
+
commandExitCode = 1
|
|
87
|
+
return
|
|
88
|
+
}
|
|
89
|
+
throw error
|
|
90
|
+
}
|
|
@@ -58,23 +58,24 @@ export function files(cwd, options) {
|
|
|
58
58
|
* @param {string[]} paths
|
|
59
59
|
*/
|
|
60
60
|
async delete(paths) {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
61
|
+
const removals = await Promise.all(
|
|
62
|
+
paths.map(async (path) => {
|
|
63
|
+
try {
|
|
64
|
+
await unlink(
|
|
65
|
+
resolve(
|
|
66
|
+
cwd,
|
|
67
|
+
options.toAbsolutePath?.(path) ??
|
|
68
|
+
toLocalFilePath(options.rootDir, path),
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
return path
|
|
72
|
+
} catch {
|
|
73
|
+
// The caller prints per-file failures based on the returned paths.
|
|
74
|
+
return undefined
|
|
75
|
+
}
|
|
76
|
+
}),
|
|
77
|
+
)
|
|
78
|
+
return removals.filter((path) => path !== undefined)
|
|
78
79
|
},
|
|
79
80
|
/**
|
|
80
81
|
* @param {string} url
|
|
@@ -161,13 +162,17 @@ async function walkFiles(dirPath) {
|
|
|
161
162
|
/** @type {string[]} */
|
|
162
163
|
const files = []
|
|
163
164
|
const entries = await readdir(dirPath, { withFileTypes: true })
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
165
|
+
const nestedFiles = await Promise.all(
|
|
166
|
+
entries.map(async (entry) => {
|
|
167
|
+
const entryPath = join(dirPath, entry.name)
|
|
168
|
+
if (entry.isDirectory()) {
|
|
169
|
+
return walkFiles(entryPath)
|
|
170
|
+
}
|
|
171
|
+
return entry.isFile() ? [entryPath] : []
|
|
172
|
+
}),
|
|
173
|
+
)
|
|
174
|
+
for (const item of nestedFiles) {
|
|
175
|
+
files.push(...item)
|
|
171
176
|
}
|
|
172
177
|
return files
|
|
173
178
|
}
|
package/src/support/r2.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import { execFile } from "node:child_process"
|
|
4
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises"
|
|
5
|
+
import { tmpdir } from "node:os"
|
|
6
|
+
import { join } from "node:path"
|
|
4
7
|
import { env as processEnv } from "node:process"
|
|
5
8
|
import { promisify } from "node:util"
|
|
6
9
|
|
|
@@ -12,9 +15,11 @@ import {
|
|
|
12
15
|
writeCliConfig,
|
|
13
16
|
} from "./cli-config.js"
|
|
14
17
|
import { CliError } from "./cli-error.js"
|
|
18
|
+
import { cancelPrompt } from "./prompt-cancel.js"
|
|
15
19
|
|
|
16
20
|
const execFileAsync = promisify(execFile)
|
|
17
21
|
const sessionSecretEnvName = "LIGHTNET_R2_SECRET_ACCESS_KEY"
|
|
22
|
+
const r2DeleteBatchSize = 1000
|
|
18
23
|
|
|
19
24
|
/**
|
|
20
25
|
* @typedef {{
|
|
@@ -117,18 +122,16 @@ export function createR2FileStorage({
|
|
|
117
122
|
const storage = {
|
|
118
123
|
async init() {
|
|
119
124
|
await getConfig()
|
|
125
|
+
await getSecretAccessKey()
|
|
120
126
|
return storage
|
|
121
127
|
},
|
|
122
128
|
async list() {
|
|
123
129
|
return runWithRefresh(async (config) => {
|
|
124
|
-
const
|
|
125
|
-
const sharedArgs = {
|
|
130
|
+
const bucketObjects = await listR2Objects({
|
|
126
131
|
cwd,
|
|
127
132
|
config,
|
|
128
|
-
secretAccessKey,
|
|
129
|
-
}
|
|
130
|
-
await assertR2Access(sharedArgs)
|
|
131
|
-
const bucketObjects = await listR2Objects(sharedArgs)
|
|
133
|
+
secretAccessKey: await getSecretAccessKey(),
|
|
134
|
+
})
|
|
132
135
|
return bucketObjects.map((item) => item.key).sort()
|
|
133
136
|
})
|
|
134
137
|
},
|
|
@@ -138,22 +141,12 @@ export function createR2FileStorage({
|
|
|
138
141
|
async delete(paths) {
|
|
139
142
|
const r2Config = await getConfig()
|
|
140
143
|
const secretAccessKey = await getSecretAccessKey()
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
cwd,
|
|
148
|
-
config: r2Config,
|
|
149
|
-
secretAccessKey,
|
|
150
|
-
})
|
|
151
|
-
removed.push(path)
|
|
152
|
-
} catch {
|
|
153
|
-
// The caller prints per-file failures based on the returned paths.
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
return removed
|
|
144
|
+
return deleteR2Objects({
|
|
145
|
+
config: r2Config,
|
|
146
|
+
cwd,
|
|
147
|
+
paths,
|
|
148
|
+
secretAccessKey,
|
|
149
|
+
})
|
|
157
150
|
},
|
|
158
151
|
/**
|
|
159
152
|
* @param {string} url
|
|
@@ -167,32 +160,134 @@ export function createR2FileStorage({
|
|
|
167
160
|
return undefined
|
|
168
161
|
}
|
|
169
162
|
const path = url.slice(normalizedBase.length).replace(/^\/+/, "")
|
|
170
|
-
return path || undefined
|
|
163
|
+
return decodeR2ObjectPath(path) || undefined
|
|
171
164
|
},
|
|
172
165
|
}
|
|
173
166
|
return storage
|
|
174
167
|
}
|
|
175
168
|
|
|
169
|
+
/**
|
|
170
|
+
* @param {string} path
|
|
171
|
+
*/
|
|
172
|
+
function decodeR2ObjectPath(path) {
|
|
173
|
+
try {
|
|
174
|
+
return decodeURIComponent(path)
|
|
175
|
+
} catch {
|
|
176
|
+
return path
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
176
180
|
/**
|
|
177
181
|
* @param {{
|
|
182
|
+
* config: R2Config
|
|
178
183
|
* cwd: string
|
|
184
|
+
* paths: string[]
|
|
185
|
+
* secretAccessKey: string
|
|
186
|
+
* }} args
|
|
187
|
+
*/
|
|
188
|
+
async function deleteR2Objects({ config, cwd, paths, secretAccessKey }) {
|
|
189
|
+
/** @type {string[]} */
|
|
190
|
+
const removed = []
|
|
191
|
+
const batchablePaths = paths.filter((path) => !path.includes("\n"))
|
|
192
|
+
const singleDeletePaths = paths.filter((path) => path.includes("\n"))
|
|
193
|
+
|
|
194
|
+
for (
|
|
195
|
+
let index = 0;
|
|
196
|
+
index < batchablePaths.length;
|
|
197
|
+
index += r2DeleteBatchSize
|
|
198
|
+
) {
|
|
199
|
+
const batch = batchablePaths.slice(index, index + r2DeleteBatchSize)
|
|
200
|
+
try {
|
|
201
|
+
await deleteR2ObjectBatch({ config, cwd, paths: batch, secretAccessKey })
|
|
202
|
+
removed.push(...batch)
|
|
203
|
+
} catch {
|
|
204
|
+
removed.push(
|
|
205
|
+
...(await deleteR2ObjectsIndividually({
|
|
206
|
+
config,
|
|
207
|
+
cwd,
|
|
208
|
+
paths: batch,
|
|
209
|
+
secretAccessKey,
|
|
210
|
+
})),
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
removed.push(
|
|
216
|
+
...(await deleteR2ObjectsIndividually({
|
|
217
|
+
config,
|
|
218
|
+
cwd,
|
|
219
|
+
paths: singleDeletePaths,
|
|
220
|
+
secretAccessKey,
|
|
221
|
+
})),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
return removed
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* @param {{
|
|
179
229
|
* config: R2Config
|
|
230
|
+
* cwd: string
|
|
231
|
+
* paths: string[]
|
|
180
232
|
* secretAccessKey: string
|
|
181
233
|
* }} args
|
|
182
234
|
*/
|
|
183
|
-
async function
|
|
235
|
+
async function deleteR2ObjectBatch({ config, cwd, paths, secretAccessKey }) {
|
|
236
|
+
if (paths.length === 0) {
|
|
237
|
+
return
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const tempDir = await mkdtemp(join(tmpdir(), "lightnet-r2-delete-"))
|
|
241
|
+
const fileListPath = join(tempDir, "files.txt")
|
|
184
242
|
try {
|
|
243
|
+
await writeFile(fileListPath, `${paths.join("\n")}\n`, "utf8")
|
|
185
244
|
await runConfiguredRclone({
|
|
186
|
-
args: [
|
|
187
|
-
|
|
245
|
+
args: [
|
|
246
|
+
"delete",
|
|
247
|
+
makeR2Path(config),
|
|
248
|
+
"--files-from-raw",
|
|
249
|
+
fileListPath,
|
|
250
|
+
"--fast-list",
|
|
251
|
+
],
|
|
252
|
+
cwd,
|
|
253
|
+
config,
|
|
254
|
+
secretAccessKey,
|
|
188
255
|
})
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
256
|
+
} finally {
|
|
257
|
+
await rm(tempDir, { force: true, recursive: true })
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* @param {{
|
|
263
|
+
* config: R2Config
|
|
264
|
+
* cwd: string
|
|
265
|
+
* paths: string[]
|
|
266
|
+
* secretAccessKey: string
|
|
267
|
+
* }} args
|
|
268
|
+
*/
|
|
269
|
+
async function deleteR2ObjectsIndividually({
|
|
270
|
+
config,
|
|
271
|
+
cwd,
|
|
272
|
+
paths,
|
|
273
|
+
secretAccessKey,
|
|
274
|
+
}) {
|
|
275
|
+
/** @type {string[]} */
|
|
276
|
+
const removed = []
|
|
277
|
+
for (const path of paths) {
|
|
278
|
+
try {
|
|
279
|
+
await runConfiguredRclone({
|
|
280
|
+
args: ["deletefile", makeR2Path(config, path)],
|
|
281
|
+
cwd,
|
|
282
|
+
config,
|
|
283
|
+
secretAccessKey,
|
|
284
|
+
})
|
|
285
|
+
removed.push(path)
|
|
286
|
+
} catch {
|
|
287
|
+
// The caller prints per-file failures based on the returned paths.
|
|
288
|
+
}
|
|
195
289
|
}
|
|
290
|
+
return removed
|
|
196
291
|
}
|
|
197
292
|
|
|
198
293
|
/**
|
|
@@ -203,18 +298,28 @@ async function assertR2Access(args) {
|
|
|
203
298
|
* }} args
|
|
204
299
|
*/
|
|
205
300
|
async function listR2Objects(args) {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
301
|
+
let listJson
|
|
302
|
+
try {
|
|
303
|
+
const result = await runConfiguredRclone({
|
|
304
|
+
args: [
|
|
305
|
+
"lsjson",
|
|
306
|
+
makeR2Path(args.config),
|
|
307
|
+
"--recursive",
|
|
308
|
+
"--files-only",
|
|
309
|
+
"--fast-list",
|
|
310
|
+
"--no-mimetype",
|
|
311
|
+
"--no-modtime",
|
|
312
|
+
],
|
|
313
|
+
...args,
|
|
314
|
+
})
|
|
315
|
+
listJson = result.stdout
|
|
316
|
+
} catch (error) {
|
|
317
|
+
throw new CliError(
|
|
318
|
+
error instanceof Error
|
|
319
|
+
? `Unable to access R2 bucket "${args.config.bucket}": ${error.message}`
|
|
320
|
+
: `Unable to access R2 bucket "${args.config.bucket}".`,
|
|
321
|
+
)
|
|
322
|
+
}
|
|
218
323
|
|
|
219
324
|
let parsed
|
|
220
325
|
try {
|
|
@@ -308,7 +413,7 @@ async function promptRequiredSecret() {
|
|
|
308
413
|
message: "R2 secretAccessKey:",
|
|
309
414
|
})
|
|
310
415
|
if (isCancel(value)) {
|
|
311
|
-
|
|
416
|
+
cancelPrompt()
|
|
312
417
|
}
|
|
313
418
|
if (value.trim()) {
|
|
314
419
|
processEnv[sessionSecretEnvName] = value
|