@stacksjs/defaults 0.74.32 → 0.74.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/skills/stacks-analytics/SKILL.md +104 -26
- package/ai/skills/stacks-auto-imports/SKILL.md +1 -1
- package/ai/skills/stacks-dashboard/SKILL.md +60 -0
- package/ai/skills/stacks-orm/SKILL.md +1 -1
- package/ai/skills/stacks-storage/SKILL.md +58 -4
- package/ai/skills/stacks-technical-diagrams/SKILL.md +1 -1
- package/app/Actions/Dashboard/Content/FileDuplicateAction.ts +25 -0
- package/app/Actions/Dashboard/Content/FileFavoriteAction.ts +25 -0
- package/app/Actions/Dashboard/Content/FileRenameAction.ts +25 -0
- package/app/Actions/Dashboard/Content/FileReprocessAction.ts +31 -0
- package/app/Actions/Dashboard/Content/FileTagsAction.ts +27 -0
- package/app/Actions/Dashboard/Content/FileVisibilityAction.ts +25 -0
- package/app/Actions/Dashboard/Content/file-manager.test.ts +227 -9
- package/app/Actions/Dashboard/Content/file-manager.ts +544 -11
- package/app/Actions/Dashboard/Content/file-metadata-store.ts +432 -0
- package/app/Actions/Dashboard/Content/file-metadata.test.ts +357 -0
- package/app/Actions/Dashboard/Content/file-metadata.ts +550 -0
- package/app/Actions/Dashboard/Content/file-pipeline.test.ts +344 -0
- package/app/Jobs/OptimizeStorageImageJob.ts +74 -0
- package/app/Jobs/TagStorageMediaJob.ts +122 -0
- package/app/Jobs/TranscodeStorageVideoJob.ts +88 -0
- package/app/Models/StorageItem.ts +123 -0
- package/app/Models/StorageItemTask.ts +134 -0
- package/ide/vscode/package.json +1 -1
- package/package.json +2 -2
- package/routes/dashboard-api.ts +13 -0
- package/vcs/github/workflows/buddy-bot.yml +109 -0
- package/vcs/github/workflows/ci.yml +3 -3
- package/vcs/github/workflows/release.yml +1 -1
- package/resources/assets/fonts/Monaco.ttf +0 -0
- package/vcs/github/renovate.json +0 -5
|
@@ -1,14 +1,35 @@
|
|
|
1
1
|
import type { ResponseStatus } from '@stacksjs/bun-router'
|
|
2
2
|
import { statfs } from 'node:fs/promises'
|
|
3
3
|
import { posix } from 'node:path'
|
|
4
|
-
import type {
|
|
4
|
+
import type { JobName } from '@stacksjs/queue'
|
|
5
|
+
import type { StorageAdapter, StorageManager, UploadedFileLike, Visibility } from '@stacksjs/storage'
|
|
5
6
|
import { Storage } from '@stacksjs/storage'
|
|
7
|
+
import type { StorageItemTask, StorageMetadataStore, StorageTaskKind, StorageTaskState } from './file-metadata'
|
|
8
|
+
import {
|
|
9
|
+
aggregateTaskState,
|
|
10
|
+
dispatchTasks,
|
|
11
|
+
followCopy,
|
|
12
|
+
followDelete,
|
|
13
|
+
followRename,
|
|
14
|
+
metadataFor as recordFor,
|
|
15
|
+
metadataUnder,
|
|
16
|
+
normalizeTags,
|
|
17
|
+
setFavorite as writeFavorite,
|
|
18
|
+
setTags as writeTags,
|
|
19
|
+
STORAGE_TASK_KINDS,
|
|
20
|
+
sweepMetadata,
|
|
21
|
+
tasksForContentType,
|
|
22
|
+
} from './file-metadata'
|
|
23
|
+
import { databaseMetadataStore } from './file-metadata-store'
|
|
6
24
|
|
|
7
25
|
const DEFAULT_DISK = 'public'
|
|
8
26
|
const DEFAULT_MAX_ENTRIES = 1000
|
|
9
27
|
const MAX_PATH_LENGTH = 2048
|
|
10
28
|
const MAX_COMPONENT_LENGTH = 255
|
|
11
29
|
const STAT_CONCURRENCY = 24
|
|
30
|
+
/** Bounds on what a caller may attach, so one file cannot fill the tag table. */
|
|
31
|
+
const MAX_TAGS = 32
|
|
32
|
+
const MAX_TAG_LENGTH = 60
|
|
12
33
|
|
|
13
34
|
export interface DashboardFileNode {
|
|
14
35
|
id: string
|
|
@@ -20,7 +41,23 @@ export interface DashboardFileNode {
|
|
|
20
41
|
mime_type?: string
|
|
21
42
|
url?: string
|
|
22
43
|
thumbnail?: string
|
|
23
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Whether somebody starred this, from `storage_items` rather than from the
|
|
46
|
+
* disk (stacksjs/stacks#2577). It was the literal `false` for as long as
|
|
47
|
+
* there was nowhere to record the answer.
|
|
48
|
+
*/
|
|
49
|
+
starred: boolean
|
|
50
|
+
/** Tags from the `taggable` vocabulary, empty for a file nobody has tagged. */
|
|
51
|
+
tags: string[]
|
|
52
|
+
/**
|
|
53
|
+
* The one word for this file's background work (stacksjs/stacks#2578), or
|
|
54
|
+
* `null` when none was ever dispatched - which is not the same as work that
|
|
55
|
+
* finished. Worst-first across the kinds, because a failure is what a viewer
|
|
56
|
+
* needs to see even when two other kinds succeeded.
|
|
57
|
+
*/
|
|
58
|
+
processing: StorageTaskState | null
|
|
59
|
+
/** Each kind separately, for a UI that wants to show which half failed. */
|
|
60
|
+
tasks: StorageItemTask[]
|
|
24
61
|
shared: boolean
|
|
25
62
|
items?: DashboardFileNode[]
|
|
26
63
|
}
|
|
@@ -280,6 +317,7 @@ export function normalizeDashboardFileLimit(value: unknown): number {
|
|
|
280
317
|
export async function getDashboardFileSnapshot(
|
|
281
318
|
options: { disk?: string, maxEntries?: number } = {},
|
|
282
319
|
manager: Manager = Storage,
|
|
320
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
283
321
|
): Promise<DashboardFileSnapshot> {
|
|
284
322
|
const selected = resolveDisk(manager, options.disk)
|
|
285
323
|
const maxEntries = normalizeDashboardFileLimit(options.maxEntries)
|
|
@@ -297,12 +335,18 @@ export async function getDashboardFileSnapshot(
|
|
|
297
335
|
entries.push({ path, type: rawEntry.type })
|
|
298
336
|
}
|
|
299
337
|
|
|
300
|
-
|
|
338
|
+
// One query for the whole subtree, not one per file: a folder of a thousand
|
|
339
|
+
// files would otherwise be a thousand round trips to answer a question about
|
|
340
|
+
// the handful of them anybody starred (stacksjs/stacks#2577).
|
|
341
|
+
const records = await metadataUnder(store, selected.name)
|
|
342
|
+
const tasks = await store.tasksUnder(selected.name, '')
|
|
343
|
+
|
|
344
|
+
const stated = await mapInBatches(
|
|
301
345
|
entries,
|
|
302
346
|
STAT_CONCURRENCY,
|
|
303
347
|
entry => metadataFor(selected.adapter, entry, selected.public, selected.name === 'public' && selected.config.driver === 'local'),
|
|
304
348
|
)
|
|
305
|
-
|
|
349
|
+
stated.sort((a, b) => {
|
|
306
350
|
const depth = a.path.split('/').length - b.path.split('/').length
|
|
307
351
|
if (depth)
|
|
308
352
|
return depth
|
|
@@ -318,7 +362,12 @@ export async function getDashboardFileSnapshot(
|
|
|
318
362
|
size: 0,
|
|
319
363
|
path: '',
|
|
320
364
|
lastModified: null,
|
|
365
|
+
// The disk root is not a file and has no row; a star on "everything" would
|
|
366
|
+
// not mean anything, and nothing processes a disk.
|
|
321
367
|
starred: false,
|
|
368
|
+
tags: [],
|
|
369
|
+
processing: null,
|
|
370
|
+
tasks: [],
|
|
322
371
|
shared: selected.public,
|
|
323
372
|
items: [],
|
|
324
373
|
}
|
|
@@ -332,6 +381,23 @@ export async function getDashboardFileSnapshot(
|
|
|
332
381
|
disk: capacity.stats,
|
|
333
382
|
}
|
|
334
383
|
|
|
384
|
+
/**
|
|
385
|
+
* The recorded metadata for a path, as the fields a node carries.
|
|
386
|
+
*
|
|
387
|
+
* A path with no row is a file nobody starred or tagged, which is most of
|
|
388
|
+
* them - so the absence of a row is the answer rather than a missing one.
|
|
389
|
+
*/
|
|
390
|
+
function pick(path: string): Pick<DashboardFileNode, 'starred' | 'tags' | 'processing' | 'tasks'> {
|
|
391
|
+
const record = recordFor(records, path)
|
|
392
|
+
const running = tasks.get(path) ?? []
|
|
393
|
+
return {
|
|
394
|
+
starred: record.favorite,
|
|
395
|
+
tags: record.tags,
|
|
396
|
+
processing: aggregateTaskState(running),
|
|
397
|
+
tasks: running,
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
335
401
|
function ensureFolder(path: string): DashboardFileNode {
|
|
336
402
|
const normalized = normalizeListedPath(path)
|
|
337
403
|
const existing = folders.get(normalized)
|
|
@@ -346,7 +412,7 @@ export async function getDashboardFileSnapshot(
|
|
|
346
412
|
size: 0,
|
|
347
413
|
path: normalized,
|
|
348
414
|
lastModified: null,
|
|
349
|
-
|
|
415
|
+
...pick(normalized),
|
|
350
416
|
shared: selected.public,
|
|
351
417
|
items: [],
|
|
352
418
|
}
|
|
@@ -356,7 +422,7 @@ export async function getDashboardFileSnapshot(
|
|
|
356
422
|
return folder
|
|
357
423
|
}
|
|
358
424
|
|
|
359
|
-
for (const entry of
|
|
425
|
+
for (const entry of stated) {
|
|
360
426
|
if (entry.type === 'directory') {
|
|
361
427
|
const folder = ensureFolder(entry.path)
|
|
362
428
|
folder.lastModified = entry.lastModified
|
|
@@ -375,7 +441,7 @@ export async function getDashboardFileSnapshot(
|
|
|
375
441
|
mime_type: entry.mimeType,
|
|
376
442
|
url: entry.url,
|
|
377
443
|
thumbnail: entry.thumbnail,
|
|
378
|
-
|
|
444
|
+
...pick(entry.path),
|
|
379
445
|
shared: selected.public,
|
|
380
446
|
}
|
|
381
447
|
parent.items!.push(file)
|
|
@@ -392,6 +458,23 @@ export async function getDashboardFileSnapshot(
|
|
|
392
458
|
})
|
|
393
459
|
}
|
|
394
460
|
|
|
461
|
+
// Rows for paths this walk did not see are orphans - a file deleted by
|
|
462
|
+
// something other than the dashboard, which is the normal case for a bucket
|
|
463
|
+
// several systems write to. The walk already enumerated every path, so
|
|
464
|
+
// removing them costs one delete rather than a second listing pass.
|
|
465
|
+
//
|
|
466
|
+
// Skipped when the listing was TRUNCATED, and that guard is the whole reason
|
|
467
|
+
// this takes the flag: a truncated walk has not shown that a path is absent,
|
|
468
|
+
// only that it stopped before reaching it, and sweeping on that would delete
|
|
469
|
+
// the metadata of every file past the limit (stacksjs/stacks#2577).
|
|
470
|
+
await sweepMetadata(
|
|
471
|
+
store,
|
|
472
|
+
selected.name,
|
|
473
|
+
'',
|
|
474
|
+
new Set(stated.map(entry => entry.path)),
|
|
475
|
+
{ truncated },
|
|
476
|
+
)
|
|
477
|
+
|
|
395
478
|
return {
|
|
396
479
|
disk: selected.name,
|
|
397
480
|
disks: manager.getConfiguredDisks().map((name) => {
|
|
@@ -403,7 +486,7 @@ export async function getDashboardFileSnapshot(
|
|
|
403
486
|
truncated,
|
|
404
487
|
warnings: [
|
|
405
488
|
...(capacity.warning ? [capacity.warning] : []),
|
|
406
|
-
...
|
|
489
|
+
...stated.flatMap(entry => entry.warnings),
|
|
407
490
|
],
|
|
408
491
|
}
|
|
409
492
|
}
|
|
@@ -427,29 +510,459 @@ export async function createDashboardDirectory(
|
|
|
427
510
|
export async function deleteDashboardFile(
|
|
428
511
|
input: { disk?: string, path: unknown },
|
|
429
512
|
manager: Manager = Storage,
|
|
513
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
430
514
|
): Promise<{ path: string, type: 'file' | 'directory' }> {
|
|
431
515
|
const selected = resolveDisk(manager, input.disk)
|
|
432
516
|
const path = normalizeDashboardFilePath(input.path)
|
|
433
517
|
|
|
518
|
+
// Metadata is forgotten AFTER the storage delete in both branches, never
|
|
519
|
+
// before: a delete that fails having already dropped the rows would leave a
|
|
520
|
+
// file that still exists with its stars and tags gone (stacksjs/stacks#2577).
|
|
434
521
|
if (await selected.adapter.fileExists(path)) {
|
|
435
522
|
await selected.adapter.deleteFile(path)
|
|
523
|
+
await followDelete(store, selected.name, path)
|
|
436
524
|
return { path, type: 'file' }
|
|
437
525
|
}
|
|
438
526
|
if (await selected.adapter.directoryExists(path)) {
|
|
439
527
|
await selected.adapter.deleteDirectory(path)
|
|
528
|
+
// The subtree, because deleting a folder deletes everything under it.
|
|
529
|
+
await followDelete(store, selected.name, path)
|
|
440
530
|
return { path, type: 'directory' }
|
|
441
531
|
}
|
|
442
532
|
|
|
443
533
|
throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
|
|
444
534
|
}
|
|
445
535
|
|
|
536
|
+
/**
|
|
537
|
+
* Rename a file or a folder in place.
|
|
538
|
+
*
|
|
539
|
+
* The parent stays put and only the last segment changes, which is what a
|
|
540
|
+
* rename in a file manager means - moving something elsewhere is a different
|
|
541
|
+
* gesture and would want a different endpoint.
|
|
542
|
+
*
|
|
543
|
+
* A directory is renamed by moving what is inside it rather than by moving the
|
|
544
|
+
* directory. On a local disk `moveFile` is `fs.rename` and would happily move a
|
|
545
|
+
* whole tree, but object storage has no directories at all: a folder there is a
|
|
546
|
+
* shared key prefix, and renaming it means rewriting the key of every object
|
|
547
|
+
* under it. Doing it the same way on both is the only version that is not
|
|
548
|
+
* quietly wrong on one of them.
|
|
549
|
+
*
|
|
550
|
+
* See stacksjs/stacks#245.
|
|
551
|
+
*/
|
|
552
|
+
export async function renameDashboardFile(
|
|
553
|
+
input: { disk?: string, path: unknown, name: unknown },
|
|
554
|
+
manager: Manager = Storage,
|
|
555
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
556
|
+
): Promise<{ from: string, to: string, type: 'file' | 'directory', moved: number }> {
|
|
557
|
+
const selected = resolveDisk(manager, input.disk)
|
|
558
|
+
const from = normalizeDashboardFilePath(input.path)
|
|
559
|
+
const name = normalizeDashboardFileName(input.name)
|
|
560
|
+
|
|
561
|
+
const separator = from.lastIndexOf('/')
|
|
562
|
+
const parent = separator === -1 ? '' : from.slice(0, separator)
|
|
563
|
+
const to = [parent, name].filter(Boolean).join('/')
|
|
564
|
+
|
|
565
|
+
if (to === from)
|
|
566
|
+
throw new DashboardFileError('The new name matches the current one.', 422, { name: 'Choose a different name.' })
|
|
567
|
+
|
|
568
|
+
if (await selected.adapter.fileExists(to) || await selected.adapter.directoryExists(to))
|
|
569
|
+
throw new DashboardFileError(`An item named "${name}" already exists.`, 409, { name: 'Choose a different name.' })
|
|
570
|
+
|
|
571
|
+
if (await selected.adapter.fileExists(from)) {
|
|
572
|
+
await selected.adapter.moveFile(from, to)
|
|
573
|
+
// After the move, so a failed move does not leave the metadata describing a
|
|
574
|
+
// path with no file at it.
|
|
575
|
+
await followRename(store, selected.name, from, to)
|
|
576
|
+
return { from, to, type: 'file', moved: 1 }
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
if (!(await selected.adapter.directoryExists(from)))
|
|
580
|
+
throw new DashboardFileError(`Storage item "${from}" was not found.`, 404)
|
|
581
|
+
|
|
582
|
+
// Collected before anything moves. Mutating a tree while iterating it is how
|
|
583
|
+
// a rename half-completes and leaves files under both names.
|
|
584
|
+
const files: string[] = []
|
|
585
|
+
for await (const entry of selected.adapter.list(from, { deep: true })) {
|
|
586
|
+
const path = normalizeListedPath(String(entry.path))
|
|
587
|
+
if (entry.type === 'file' && path)
|
|
588
|
+
files.push(path)
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
for (const file of files) {
|
|
592
|
+
// `list` may answer absolute-from-root or relative-to-`from` paths
|
|
593
|
+
// depending on the adapter; both end with the part that has to be kept.
|
|
594
|
+
const relative = file.startsWith(`${from}/`) ? file.slice(from.length + 1) : file
|
|
595
|
+
await selected.adapter.moveFile(file.startsWith(`${from}/`) ? file : `${from}/${relative}`, `${to}/${relative}`)
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// An empty source is what is left, and it should not be: a rename leaves one
|
|
599
|
+
// item, not two. Directories are implicit on object storage, so this is a
|
|
600
|
+
// no-op there rather than a failure.
|
|
601
|
+
await selected.adapter.deleteDirectory(from)
|
|
602
|
+
|
|
603
|
+
// A folder rename moves every file beneath it, so this is a prefix update
|
|
604
|
+
// rather than one row - the part #2577 flagged as the one that would bite.
|
|
605
|
+
await followRename(store, selected.name, from, to)
|
|
606
|
+
|
|
607
|
+
return { from, to, type: 'directory', moved: files.length }
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* Make a file, or everything in a folder, public or private.
|
|
612
|
+
*
|
|
613
|
+
* A folder is applied file by file rather than to the folder itself, for the
|
|
614
|
+
* same reason a rename is: object storage has no directories, only a shared key
|
|
615
|
+
* prefix, and an ACL belongs to an object. `directoryExists` there is a
|
|
616
|
+
* prefix-has-objects check with nothing at that key, so asking to change the
|
|
617
|
+
* "directory" would address something that does not exist.
|
|
618
|
+
*
|
|
619
|
+
* It is also the right answer on a local disk, where a mode on a directory
|
|
620
|
+
* controls listing and traversal but not whether a file inside it can be read.
|
|
621
|
+
* The files are what gate access on both, so the files are what this sets.
|
|
622
|
+
*
|
|
623
|
+
* See stacksjs/stacks#245.
|
|
624
|
+
*/
|
|
625
|
+
export async function setDashboardFileVisibility(
|
|
626
|
+
input: { disk?: string, path: unknown, visibility: unknown },
|
|
627
|
+
manager: Manager = Storage,
|
|
628
|
+
): Promise<{ path: string, visibility: Visibility, type: 'file' | 'directory', changed: number }> {
|
|
629
|
+
const selected = resolveDisk(manager, input.disk)
|
|
630
|
+
const path = normalizeDashboardFilePath(input.path)
|
|
631
|
+
|
|
632
|
+
if (input.visibility !== 'public' && input.visibility !== 'private') {
|
|
633
|
+
throw new DashboardFileError('Visibility must be "public" or "private".', 422, {
|
|
634
|
+
visibility: 'Choose either public or private.',
|
|
635
|
+
})
|
|
636
|
+
}
|
|
637
|
+
const visibility = input.visibility as Visibility
|
|
638
|
+
|
|
639
|
+
if (await selected.adapter.fileExists(path)) {
|
|
640
|
+
await selected.adapter.changeVisibility(path, visibility)
|
|
641
|
+
return { path, visibility, type: 'file', changed: 1 }
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
if (!(await selected.adapter.directoryExists(path)))
|
|
645
|
+
throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
|
|
646
|
+
|
|
647
|
+
let changed = 0
|
|
648
|
+
for await (const entry of selected.adapter.list(path, { deep: true })) {
|
|
649
|
+
if (entry.type !== 'file')
|
|
650
|
+
continue
|
|
651
|
+
const listed = normalizeListedPath(String(entry.path))
|
|
652
|
+
if (!listed)
|
|
653
|
+
continue
|
|
654
|
+
await selected.adapter.changeVisibility(listed.startsWith(`${path}/`) ? listed : `${path}/${listed}`, visibility)
|
|
655
|
+
changed++
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
return { path, visibility, type: 'directory', changed }
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* The name a duplicate gets when the caller does not choose one.
|
|
663
|
+
*
|
|
664
|
+
* `readme.txt` becomes `readme copy.txt`, then `readme copy 2.txt` - the
|
|
665
|
+
* suffix goes before the extension, because `readme.txt copy` is a file whose
|
|
666
|
+
* type the operating system, the browser and this dashboard's own type
|
|
667
|
+
* grouping all read as unknown.
|
|
668
|
+
*
|
|
669
|
+
* A directory has no extension to preserve, and `posix.extname` returns `''`
|
|
670
|
+
* for one, so the same code handles both.
|
|
671
|
+
*/
|
|
672
|
+
async function availableCopyName(path: string, exists: (candidate: string) => Promise<boolean>): Promise<string> {
|
|
673
|
+
const base = posix.basename(path)
|
|
674
|
+
const extension = posix.extname(base)
|
|
675
|
+
const stem = extension ? base.slice(0, -extension.length) : base
|
|
676
|
+
const parent = path.slice(0, Math.max(0, path.length - base.length - 1))
|
|
677
|
+
|
|
678
|
+
for (let attempt = 1; attempt <= 100; attempt++) {
|
|
679
|
+
const suffix = attempt === 1 ? 'copy' : `copy ${attempt}`
|
|
680
|
+
const candidate = `${stem} ${suffix}${extension}`
|
|
681
|
+
const full = [parent, candidate].filter(Boolean).join('/')
|
|
682
|
+
if (!(await exists(full)))
|
|
683
|
+
return candidate
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
throw new DashboardFileError('Too many copies of this item already exist.', 409, {
|
|
687
|
+
name: 'Rename some copies, or choose a name.',
|
|
688
|
+
})
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Copy a file, or a folder and everything in it, beside the original.
|
|
693
|
+
*
|
|
694
|
+
* Deep-walks a directory for the same reason rename and visibility do: object
|
|
695
|
+
* storage has no folder to copy, only a shared key prefix, so duplicating one
|
|
696
|
+
* means copying every object beneath it.
|
|
697
|
+
*
|
|
698
|
+
* The name is optional. A file manager's "Duplicate" is a single gesture that
|
|
699
|
+
* has to produce something, so an omitted name becomes `<name> copy`, then
|
|
700
|
+
* `<name> copy 2` - checked for availability rather than assumed, since the
|
|
701
|
+
* first copy is usually not the only one.
|
|
702
|
+
*
|
|
703
|
+
* See stacksjs/stacks#245.
|
|
704
|
+
*/
|
|
705
|
+
export async function duplicateDashboardFile(
|
|
706
|
+
input: { disk?: string, path: unknown, name?: unknown },
|
|
707
|
+
manager: Manager = Storage,
|
|
708
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
709
|
+
): Promise<{ from: string, to: string, type: 'file' | 'directory', copied: number }> {
|
|
710
|
+
const selected = resolveDisk(manager, input.disk)
|
|
711
|
+
const from = normalizeDashboardFilePath(input.path)
|
|
712
|
+
const taken = async (candidate: string): Promise<boolean> =>
|
|
713
|
+
await selected.adapter.fileExists(candidate) || await selected.adapter.directoryExists(candidate)
|
|
714
|
+
|
|
715
|
+
const name = input.name === undefined || input.name === null || input.name === ''
|
|
716
|
+
? await availableCopyName(from, taken)
|
|
717
|
+
: normalizeDashboardFileName(input.name)
|
|
718
|
+
|
|
719
|
+
const separator = from.lastIndexOf('/')
|
|
720
|
+
const parent = separator === -1 ? '' : from.slice(0, separator)
|
|
721
|
+
const to = [parent, name].filter(Boolean).join('/')
|
|
722
|
+
|
|
723
|
+
if (to === from)
|
|
724
|
+
throw new DashboardFileError('A copy needs a different name.', 422, { name: 'Choose a different name.' })
|
|
725
|
+
if (await taken(to))
|
|
726
|
+
throw new DashboardFileError(`An item named "${name}" already exists.`, 409, { name: 'Choose a different name.' })
|
|
727
|
+
|
|
728
|
+
if (await selected.adapter.fileExists(from)) {
|
|
729
|
+
await selected.adapter.copyFile(from, to)
|
|
730
|
+
// The star and tags come with the copy. A duplicate that silently lost them
|
|
731
|
+
// reads as the copy having half-failed.
|
|
732
|
+
await followCopy(store, selected.name, from, to)
|
|
733
|
+
return { from, to, type: 'file', copied: 1 }
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
if (!(await selected.adapter.directoryExists(from)))
|
|
737
|
+
throw new DashboardFileError(`Storage item "${from}" was not found.`, 404)
|
|
738
|
+
|
|
739
|
+
// Collected before anything is written, so a copy cannot pick up the files
|
|
740
|
+
// it is itself creating - `to` sits beside `from` under the same parent, and
|
|
741
|
+
// a deep listing that ran while copying could see them.
|
|
742
|
+
const files: string[] = []
|
|
743
|
+
for await (const entry of selected.adapter.list(from, { deep: true })) {
|
|
744
|
+
const listed = normalizeListedPath(String(entry.path))
|
|
745
|
+
if (entry.type === 'file' && listed)
|
|
746
|
+
files.push(listed.startsWith(`${from}/`) ? listed : `${from}/${listed}`)
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
for (const file of files) {
|
|
750
|
+
const destination = `${to}/${file.slice(from.length + 1)}`
|
|
751
|
+
await selected.adapter.copyFile(file, destination)
|
|
752
|
+
await followCopy(store, selected.name, file, destination)
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
// The folder's own record, separately: it is not among the files copied.
|
|
756
|
+
await followCopy(store, selected.name, from, to)
|
|
757
|
+
|
|
758
|
+
return { from, to, type: 'directory', copied: files.length }
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* Star or unstar a file or a folder (stacksjs/stacks#2577).
|
|
763
|
+
*
|
|
764
|
+
* Existence is checked against the DISK, not against the table: a star on a
|
|
765
|
+
* path that is not there would be an orphan the moment it was written, and the
|
|
766
|
+
* caller would get a 200 for it. Folders can be starred too - they are a path
|
|
767
|
+
* like any other to this table, even where the disk has no such thing.
|
|
768
|
+
*/
|
|
769
|
+
export async function setDashboardFileFavorite(
|
|
770
|
+
input: { disk?: string, path: unknown, favorite: unknown },
|
|
771
|
+
manager: Manager = Storage,
|
|
772
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
773
|
+
): Promise<{ path: string, favorite: boolean, tags: string[] }> {
|
|
774
|
+
const selected = resolveDisk(manager, input.disk)
|
|
775
|
+
const path = normalizeDashboardFilePath(input.path)
|
|
776
|
+
|
|
777
|
+
if (typeof input.favorite !== 'boolean') {
|
|
778
|
+
throw new DashboardFileError('Favorite must be true or false.', 422, {
|
|
779
|
+
favorite: 'Send a boolean.',
|
|
780
|
+
})
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
await assertExists(selected.adapter, path)
|
|
784
|
+
|
|
785
|
+
const record = await writeFavorite(store, selected.name, path, input.favorite)
|
|
786
|
+
return { path, favorite: record.favorite, tags: record.tags }
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Replace a file's tags (stacksjs/stacks#2577).
|
|
791
|
+
*
|
|
792
|
+
* The whole set, not a delta: a UI that can add a tag can also remove one, and
|
|
793
|
+
* there is no separate signal for a removal. Names are trimmed, deduplicated
|
|
794
|
+
* case-insensitively and sorted, so sending the same tags in a different order
|
|
795
|
+
* produces the same record.
|
|
796
|
+
*
|
|
797
|
+
* They go into the `taggable` vocabulary the CMS already uses, scoped by
|
|
798
|
+
* `taggable_type`, rather than a second tag table - which #2577 asks for
|
|
799
|
+
* explicitly, and which matters because a tag is a word somebody chose and
|
|
800
|
+
* having it mean two things in two places is how a tag list stops being useful.
|
|
801
|
+
*/
|
|
802
|
+
export async function setDashboardFileTags(
|
|
803
|
+
input: { disk?: string, path: unknown, tags: unknown },
|
|
804
|
+
manager: Manager = Storage,
|
|
805
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
806
|
+
): Promise<{ path: string, favorite: boolean, tags: string[] }> {
|
|
807
|
+
const selected = resolveDisk(manager, input.disk)
|
|
808
|
+
const path = normalizeDashboardFilePath(input.path)
|
|
809
|
+
|
|
810
|
+
if (!Array.isArray(input.tags))
|
|
811
|
+
throw new DashboardFileError('Tags must be an array.', 422, { tags: 'Send an array of tag names.' })
|
|
812
|
+
if (input.tags.length > MAX_TAGS)
|
|
813
|
+
throw new DashboardFileError(`A file may carry at most ${MAX_TAGS} tags.`, 422, { tags: 'Too many tags.' })
|
|
814
|
+
for (const tag of input.tags) {
|
|
815
|
+
if (typeof tag !== 'string')
|
|
816
|
+
throw new DashboardFileError('Tags must be strings.', 422, { tags: 'Every tag must be a string.' })
|
|
817
|
+
if (tag.trim().length > MAX_TAG_LENGTH)
|
|
818
|
+
throw new DashboardFileError(`A tag must not exceed ${MAX_TAG_LENGTH} characters.`, 422, { tags: 'A tag is too long.' })
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
await assertExists(selected.adapter, path)
|
|
822
|
+
|
|
823
|
+
const record = await writeTags(store, selected.name, path, normalizeTags(input.tags))
|
|
824
|
+
return { path, favorite: record.favorite, tags: record.tags }
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/** 404 unless the path is a file or a folder on the disk. */
|
|
828
|
+
async function assertExists(adapter: StorageAdapter, path: string): Promise<void> {
|
|
829
|
+
if (await adapter.fileExists(path) || await adapter.directoryExists(path))
|
|
830
|
+
return
|
|
831
|
+
throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
/**
|
|
835
|
+
* How a dispatched job reaches the queue.
|
|
836
|
+
*
|
|
837
|
+
* A parameter so the upload path can be tested without a queue, and so an app
|
|
838
|
+
* that wants a different dispatcher - a synchronous one in a test, a
|
|
839
|
+
* rate-limited one in front of a paid vision API - can supply it. The default
|
|
840
|
+
* reads `@stacksjs/queue` lazily: `uploadDashboardFiles` is on the request path
|
|
841
|
+
* and most uploads are not media.
|
|
842
|
+
*/
|
|
843
|
+
export type TaskDispatcher = (job: JobName, payload: Record<string, unknown>) => Promise<void>
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* The job each kind of work runs.
|
|
847
|
+
*
|
|
848
|
+
* Typed as the generated `JobName` union rather than `string`, so a job renamed
|
|
849
|
+
* or removed under this map is a compile error instead of a dispatch that fails
|
|
850
|
+
* at runtime for a file nobody is watching.
|
|
851
|
+
*/
|
|
852
|
+
const TASK_JOBS = {
|
|
853
|
+
optimize: 'OptimizeStorageImageJob',
|
|
854
|
+
transcode: 'TranscodeStorageVideoJob',
|
|
855
|
+
tag: 'TagStorageMediaJob',
|
|
856
|
+
} as const satisfies Record<StorageTaskKind, JobName>
|
|
857
|
+
|
|
858
|
+
const queueDispatcher: TaskDispatcher = async (job, payload) => {
|
|
859
|
+
const { Jobs } = await import('@stacksjs/queue')
|
|
860
|
+
await Jobs.dispatch(job as JobName, payload)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Queue the processing an uploaded file calls for (stacksjs/stacks#2578).
|
|
865
|
+
*
|
|
866
|
+
* Only ever dispatches for a content type that has work to do - most uploads
|
|
867
|
+
* are documents, and a queue entry per PDF that immediately finds nothing to do
|
|
868
|
+
* is noise in the one place somebody looks when a transcode is stuck.
|
|
869
|
+
*
|
|
870
|
+
* A video is dispatched only when the caller supplies a profile: the ladder is
|
|
871
|
+
* derived from the source dimensions, and a job that guesses builds renditions
|
|
872
|
+
* nobody asked for. The dashboard knows them from the upload; a caller that
|
|
873
|
+
* does not can dispatch the transcode later through
|
|
874
|
+
* {@link reprocessDashboardFile}.
|
|
875
|
+
*/
|
|
876
|
+
export async function dispatchDashboardFileTasks(
|
|
877
|
+
input: { disk?: string, path: unknown, contentType?: string, videoProfile?: Record<string, unknown>, only?: readonly StorageTaskKind[] },
|
|
878
|
+
manager: Manager = Storage,
|
|
879
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
880
|
+
dispatch: TaskDispatcher = queueDispatcher,
|
|
881
|
+
): Promise<StorageItemTask[]> {
|
|
882
|
+
const selected = resolveDisk(manager, input.disk)
|
|
883
|
+
const path = normalizeDashboardFilePath(input.path)
|
|
884
|
+
|
|
885
|
+
const applicable = tasksForContentType(input.contentType)
|
|
886
|
+
const wanted = input.only
|
|
887
|
+
? applicable.filter(kind => input.only!.includes(kind))
|
|
888
|
+
: applicable
|
|
889
|
+
|
|
890
|
+
const runnable = wanted.filter(kind => kind !== 'transcode' || input.videoProfile !== undefined)
|
|
891
|
+
if (runnable.length === 0)
|
|
892
|
+
return []
|
|
893
|
+
|
|
894
|
+
return await dispatchTasks(store, selected.name, path, runnable, async (kind) => {
|
|
895
|
+
await dispatch(TASK_JOBS[kind], {
|
|
896
|
+
disk: selected.name,
|
|
897
|
+
path,
|
|
898
|
+
...(kind === 'transcode' ? { profile: input.videoProfile } : {}),
|
|
899
|
+
})
|
|
900
|
+
})
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* Run a file's processing again (stacksjs/stacks#2578).
|
|
905
|
+
*
|
|
906
|
+
* The re-run path the issue asked for, and it is not optional: the first
|
|
907
|
+
* version of any of these produces output somebody wants regenerated - a
|
|
908
|
+
* better ladder, a model that has since improved, an optimization that ran
|
|
909
|
+
* before a preset changed.
|
|
910
|
+
*
|
|
911
|
+
* Existence is checked against the DISK, so re-running a file that has since
|
|
912
|
+
* been deleted is a 404 rather than a job that fails minutes later in a worker
|
|
913
|
+
* log nobody reads.
|
|
914
|
+
*/
|
|
915
|
+
export async function reprocessDashboardFile(
|
|
916
|
+
input: { disk?: string, path: unknown, kinds?: unknown, videoProfile?: Record<string, unknown> },
|
|
917
|
+
manager: Manager = Storage,
|
|
918
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
919
|
+
dispatch: TaskDispatcher = queueDispatcher,
|
|
920
|
+
): Promise<{ path: string, tasks: StorageItemTask[] }> {
|
|
921
|
+
const selected = resolveDisk(manager, input.disk)
|
|
922
|
+
const path = normalizeDashboardFilePath(input.path)
|
|
923
|
+
|
|
924
|
+
let only: StorageTaskKind[] | undefined
|
|
925
|
+
if (input.kinds !== undefined && input.kinds !== null) {
|
|
926
|
+
if (!Array.isArray(input.kinds))
|
|
927
|
+
throw new DashboardFileError('Kinds must be an array.', 422, { kinds: 'Send an array of task kinds.' })
|
|
928
|
+
for (const kind of input.kinds) {
|
|
929
|
+
if (!STORAGE_TASK_KINDS.includes(kind as StorageTaskKind)) {
|
|
930
|
+
throw new DashboardFileError(`Unknown task kind "${String(kind)}".`, 422, {
|
|
931
|
+
kinds: `Choose from ${STORAGE_TASK_KINDS.join(', ')}.`,
|
|
932
|
+
})
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
only = input.kinds as StorageTaskKind[]
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
if (!(await selected.adapter.fileExists(path)))
|
|
939
|
+
throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
|
|
940
|
+
|
|
941
|
+
const tasks = await dispatchDashboardFileTasks(
|
|
942
|
+
{
|
|
943
|
+
disk: selected.name,
|
|
944
|
+
path,
|
|
945
|
+
contentType: await selected.adapter.mimeType(path),
|
|
946
|
+
videoProfile: input.videoProfile,
|
|
947
|
+
only,
|
|
948
|
+
},
|
|
949
|
+
manager,
|
|
950
|
+
store,
|
|
951
|
+
dispatch,
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
return { path, tasks }
|
|
955
|
+
}
|
|
956
|
+
|
|
446
957
|
export async function uploadDashboardFiles(
|
|
447
958
|
input: { disk?: string, path?: string, files: UploadedFileLike[] },
|
|
448
959
|
manager: Manager = Storage,
|
|
449
|
-
|
|
960
|
+
store: StorageMetadataStore = databaseMetadataStore,
|
|
961
|
+
dispatch: TaskDispatcher = queueDispatcher,
|
|
962
|
+
): Promise<Array<{ path: string, url: string, size: number, tasks: StorageItemTask[] }>> {
|
|
450
963
|
const selected = resolveDisk(manager, input.disk)
|
|
451
964
|
const directory = normalizeDashboardFilePath(input.path ?? '', { allowEmpty: true })
|
|
452
|
-
const uploaded: Array<{ path: string, url: string, size: number }> = []
|
|
965
|
+
const uploaded: Array<{ path: string, url: string, size: number, tasks: StorageItemTask[] }> = []
|
|
453
966
|
|
|
454
967
|
try {
|
|
455
968
|
for (const file of input.files) {
|
|
@@ -459,7 +972,7 @@ export async function uploadDashboardFiles(
|
|
|
459
972
|
filename: 'original',
|
|
460
973
|
overwrite: false,
|
|
461
974
|
})
|
|
462
|
-
uploaded.push({ path: result.path, url: result.url, size: result.size })
|
|
975
|
+
uploaded.push({ path: result.path, url: result.url, size: result.size, tasks: [] })
|
|
463
976
|
}
|
|
464
977
|
}
|
|
465
978
|
catch (error) {
|
|
@@ -484,5 +997,25 @@ export async function uploadDashboardFiles(
|
|
|
484
997
|
throw error
|
|
485
998
|
}
|
|
486
999
|
|
|
1000
|
+
// Dispatched after every file is safely written, not inside the loop above:
|
|
1001
|
+
// the rollback path deletes what was uploaded, and a job already queued for a
|
|
1002
|
+
// file that is about to be deleted would run against nothing.
|
|
1003
|
+
//
|
|
1004
|
+
// A dispatch failure is recorded on the task row rather than thrown, so a
|
|
1005
|
+
// queue that is down does not fail an upload that succeeded - the file is
|
|
1006
|
+
// there, and the dashboard shows the processing as failed.
|
|
1007
|
+
for (const file of uploaded) {
|
|
1008
|
+
file.tasks = await dispatchDashboardFileTasks(
|
|
1009
|
+
{
|
|
1010
|
+
disk: selected.name,
|
|
1011
|
+
path: file.path,
|
|
1012
|
+
contentType: await selected.adapter.mimeType(file.path).catch(() => undefined),
|
|
1013
|
+
},
|
|
1014
|
+
manager,
|
|
1015
|
+
store,
|
|
1016
|
+
dispatch,
|
|
1017
|
+
)
|
|
1018
|
+
}
|
|
1019
|
+
|
|
487
1020
|
return uploaded
|
|
488
1021
|
}
|