@stacksjs/defaults 0.74.33 → 0.74.34

package/app/Actions/Dashboard/Content/file-manager.ts

@@ -1,14 +1,35 @@
 import type { ResponseStatus } from '@stacksjs/bun-router'
 import { statfs } from 'node:fs/promises'
 import { posix } from 'node:path'
+import type { JobName } from '@stacksjs/queue'
 import type { StorageAdapter, StorageManager, UploadedFileLike, Visibility } from '@stacksjs/storage'
 import { Storage } from '@stacksjs/storage'
+import type { StorageItemTask, StorageMetadataStore, StorageTaskKind, StorageTaskState } from './file-metadata'
+import {
+  aggregateTaskState,
+  dispatchTasks,
+  followCopy,
+  followDelete,
+  followRename,
+  metadataFor as recordFor,
+  metadataUnder,
+  normalizeTags,
+  setFavorite as writeFavorite,
+  setTags as writeTags,
+  STORAGE_TASK_KINDS,
+  sweepMetadata,
+  tasksForContentType,
+} from './file-metadata'
+import { databaseMetadataStore } from './file-metadata-store'
 
 const DEFAULT_DISK = 'public'
 const DEFAULT_MAX_ENTRIES = 1000
 const MAX_PATH_LENGTH = 2048
 const MAX_COMPONENT_LENGTH = 255
 const STAT_CONCURRENCY = 24
+/** Bounds on what a caller may attach, so one file cannot fill the tag table. */
+const MAX_TAGS = 32
+const MAX_TAG_LENGTH = 60
 
 export interface DashboardFileNode {
   id: string
@@ -20,7 +41,23 @@ export interface DashboardFileNode {
   mime_type?: string
   url?: string
   thumbnail?: string
-  starred: false
+  /**
+   * Whether somebody starred this, from `storage_items` rather than from the
+   * disk (stacksjs/stacks#2577). It was the literal `false` for as long as
+   * there was nowhere to record the answer.
+   */
+  starred: boolean
+  /** Tags from the `taggable` vocabulary, empty for a file nobody has tagged. */
+  tags: string[]
+  /**
+   * The one word for this file's background work (stacksjs/stacks#2578), or
+   * `null` when none was ever dispatched - which is not the same as work that
+   * finished. Worst-first across the kinds, because a failure is what a viewer
+   * needs to see even when two other kinds succeeded.
+   */
+  processing: StorageTaskState | null
+  /** Each kind separately, for a UI that wants to show which half failed. */
+  tasks: StorageItemTask[]
   shared: boolean
   items?: DashboardFileNode[]
 }
@@ -280,6 +317,7 @@ export function normalizeDashboardFileLimit(value: unknown): number {
 export async function getDashboardFileSnapshot(
   options: { disk?: string, maxEntries?: number } = {},
   manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
 ): Promise<DashboardFileSnapshot> {
   const selected = resolveDisk(manager, options.disk)
   const maxEntries = normalizeDashboardFileLimit(options.maxEntries)
@@ -297,12 +335,18 @@ export async function getDashboardFileSnapshot(
     entries.push({ path, type: rawEntry.type })
   }
 
-  const metadata = await mapInBatches(
+  // One query for the whole subtree, not one per file: a folder of a thousand
+  // files would otherwise be a thousand round trips to answer a question about
+  // the handful of them anybody starred (stacksjs/stacks#2577).
+  const records = await metadataUnder(store, selected.name)
+  const tasks = await store.tasksUnder(selected.name, '')
+
+  const stated = await mapInBatches(
     entries,
     STAT_CONCURRENCY,
     entry => metadataFor(selected.adapter, entry, selected.public, selected.name === 'public' && selected.config.driver === 'local'),
   )
-  metadata.sort((a, b) => {
+  stated.sort((a, b) => {
     const depth = a.path.split('/').length - b.path.split('/').length
     if (depth)
       return depth
@@ -318,7 +362,12 @@ export async function getDashboardFileSnapshot(
     size: 0,
     path: '',
     lastModified: null,
+    // The disk root is not a file and has no row; a star on "everything" would
+    // not mean anything, and nothing processes a disk.
     starred: false,
+    tags: [],
+    processing: null,
+    tasks: [],
     shared: selected.public,
     items: [],
   }
@@ -332,6 +381,23 @@ export async function getDashboardFileSnapshot(
     disk: capacity.stats,
   }
 
+  /**
+   * The recorded metadata for a path, as the fields a node carries.
+   *
+   * A path with no row is a file nobody starred or tagged, which is most of
+   * them - so the absence of a row is the answer rather than a missing one.
+   */
+  function pick(path: string): Pick<DashboardFileNode, 'starred' | 'tags' | 'processing' | 'tasks'> {
+    const record = recordFor(records, path)
+    const running = tasks.get(path) ?? []
+    return {
+      starred: record.favorite,
+      tags: record.tags,
+      processing: aggregateTaskState(running),
+      tasks: running,
+    }
+  }
+
   function ensureFolder(path: string): DashboardFileNode {
     const normalized = normalizeListedPath(path)
     const existing = folders.get(normalized)
@@ -346,7 +412,7 @@ export async function getDashboardFileSnapshot(
       size: 0,
       path: normalized,
       lastModified: null,
-      starred: false,
+      ...pick(normalized),
       shared: selected.public,
       items: [],
     }
@@ -356,7 +422,7 @@ export async function getDashboardFileSnapshot(
     return folder
   }
 
-  for (const entry of metadata) {
+  for (const entry of stated) {
     if (entry.type === 'directory') {
       const folder = ensureFolder(entry.path)
       folder.lastModified = entry.lastModified
@@ -375,7 +441,7 @@ export async function getDashboardFileSnapshot(
       mime_type: entry.mimeType,
       url: entry.url,
       thumbnail: entry.thumbnail,
-      starred: false,
+      ...pick(entry.path),
       shared: selected.public,
     }
     parent.items!.push(file)
@@ -392,6 +458,23 @@ export async function getDashboardFileSnapshot(
     })
   }
 
+  // Rows for paths this walk did not see are orphans - a file deleted by
+  // something other than the dashboard, which is the normal case for a bucket
+  // several systems write to. The walk already enumerated every path, so
+  // removing them costs one delete rather than a second listing pass.
+  //
+  // Skipped when the listing was TRUNCATED, and that guard is the whole reason
+  // this takes the flag: a truncated walk has not shown that a path is absent,
+  // only that it stopped before reaching it, and sweeping on that would delete
+  // the metadata of every file past the limit (stacksjs/stacks#2577).
+  await sweepMetadata(
+    store,
+    selected.name,
+    '',
+    new Set(stated.map(entry => entry.path)),
+    { truncated },
+  )
+
   return {
     disk: selected.name,
     disks: manager.getConfiguredDisks().map((name) => {
@@ -403,7 +486,7 @@ export async function getDashboardFileSnapshot(
     truncated,
     warnings: [
       ...(capacity.warning ? [capacity.warning] : []),
-      ...metadata.flatMap(entry => entry.warnings),
+      ...stated.flatMap(entry => entry.warnings),
     ],
   }
 }
@@ -427,16 +510,23 @@ export async function createDashboardDirectory(
 export async function deleteDashboardFile(
   input: { disk?: string, path: unknown },
   manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
 ): Promise<{ path: string, type: 'file' | 'directory' }> {
   const selected = resolveDisk(manager, input.disk)
   const path = normalizeDashboardFilePath(input.path)
 
+  // Metadata is forgotten AFTER the storage delete in both branches, never
+  // before: a delete that fails having already dropped the rows would leave a
+  // file that still exists with its stars and tags gone (stacksjs/stacks#2577).
   if (await selected.adapter.fileExists(path)) {
     await selected.adapter.deleteFile(path)
+    await followDelete(store, selected.name, path)
     return { path, type: 'file' }
   }
   if (await selected.adapter.directoryExists(path)) {
     await selected.adapter.deleteDirectory(path)
+    // The subtree, because deleting a folder deletes everything under it.
+    await followDelete(store, selected.name, path)
     return { path, type: 'directory' }
   }
 
@@ -462,6 +552,7 @@ export async function deleteDashboardFile(
 export async function renameDashboardFile(
   input: { disk?: string, path: unknown, name: unknown },
   manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
 ): Promise<{ from: string, to: string, type: 'file' | 'directory', moved: number }> {
   const selected = resolveDisk(manager, input.disk)
   const from = normalizeDashboardFilePath(input.path)
@@ -479,6 +570,9 @@ export async function renameDashboardFile(
 
   if (await selected.adapter.fileExists(from)) {
     await selected.adapter.moveFile(from, to)
+    // After the move, so a failed move does not leave the metadata describing a
+    // path with no file at it.
+    await followRename(store, selected.name, from, to)
     return { from, to, type: 'file', moved: 1 }
   }
 
@@ -506,6 +600,10 @@ export async function renameDashboardFile(
   // no-op there rather than a failure.
   await selected.adapter.deleteDirectory(from)
 
+  // A folder rename moves every file beneath it, so this is a prefix update
+  // rather than one row - the part #2577 flagged as the one that would bite.
+  await followRename(store, selected.name, from, to)
+
   return { from, to, type: 'directory', moved: files.length }
 }
 
@@ -607,6 +705,7 @@ async function availableCopyName(path: string, exists: (candidate: string) => Pr
 export async function duplicateDashboardFile(
   input: { disk?: string, path: unknown, name?: unknown },
   manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
 ): Promise<{ from: string, to: string, type: 'file' | 'directory', copied: number }> {
   const selected = resolveDisk(manager, input.disk)
   const from = normalizeDashboardFilePath(input.path)
@@ -628,6 +727,9 @@ export async function duplicateDashboardFile(
 
   if (await selected.adapter.fileExists(from)) {
     await selected.adapter.copyFile(from, to)
+    // The star and tags come with the copy. A duplicate that silently lost them
+    // reads as the copy having half-failed.
+    await followCopy(store, selected.name, from, to)
     return { from, to, type: 'file', copied: 1 }
   }
 
@@ -644,19 +746,223 @@ export async function duplicateDashboardFile(
       files.push(listed.startsWith(`${from}/`) ? listed : `${from}/${listed}`)
   }
 
-  for (const file of files)
-    await selected.adapter.copyFile(file, `${to}/${file.slice(from.length + 1)}`)
+  for (const file of files) {
+    const destination = `${to}/${file.slice(from.length + 1)}`
+    await selected.adapter.copyFile(file, destination)
+    await followCopy(store, selected.name, file, destination)
+  }
+
+  // The folder's own record, separately: it is not among the files copied.
+  await followCopy(store, selected.name, from, to)
 
   return { from, to, type: 'directory', copied: files.length }
 }
 
+/**
+ * Star or unstar a file or a folder (stacksjs/stacks#2577).
+ *
+ * Existence is checked against the DISK, not against the table: a star on a
+ * path that is not there would be an orphan the moment it was written, and the
+ * caller would get a 200 for it. Folders can be starred too - they are a path
+ * like any other to this table, even where the disk has no such thing.
+ */
+export async function setDashboardFileFavorite(
+  input: { disk?: string, path: unknown, favorite: unknown },
+  manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
+): Promise<{ path: string, favorite: boolean, tags: string[] }> {
+  const selected = resolveDisk(manager, input.disk)
+  const path = normalizeDashboardFilePath(input.path)
+
+  if (typeof input.favorite !== 'boolean') {
+    throw new DashboardFileError('Favorite must be true or false.', 422, {
+      favorite: 'Send a boolean.',
+    })
+  }
+
+  await assertExists(selected.adapter, path)
+
+  const record = await writeFavorite(store, selected.name, path, input.favorite)
+  return { path, favorite: record.favorite, tags: record.tags }
+}
+
+/**
+ * Replace a file's tags (stacksjs/stacks#2577).
+ *
+ * The whole set, not a delta: a UI that can add a tag can also remove one, and
+ * there is no separate signal for a removal. Names are trimmed, deduplicated
+ * case-insensitively and sorted, so sending the same tags in a different order
+ * produces the same record.
+ *
+ * They go into the `taggable` vocabulary the CMS already uses, scoped by
+ * `taggable_type`, rather than a second tag table - which #2577 asks for
+ * explicitly, and which matters because a tag is a word somebody chose and
+ * having it mean two things in two places is how a tag list stops being useful.
+ */
+export async function setDashboardFileTags(
+  input: { disk?: string, path: unknown, tags: unknown },
+  manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
+): Promise<{ path: string, favorite: boolean, tags: string[] }> {
+  const selected = resolveDisk(manager, input.disk)
+  const path = normalizeDashboardFilePath(input.path)
+
+  if (!Array.isArray(input.tags))
+    throw new DashboardFileError('Tags must be an array.', 422, { tags: 'Send an array of tag names.' })
+  if (input.tags.length > MAX_TAGS)
+    throw new DashboardFileError(`A file may carry at most ${MAX_TAGS} tags.`, 422, { tags: 'Too many tags.' })
+  for (const tag of input.tags) {
+    if (typeof tag !== 'string')
+      throw new DashboardFileError('Tags must be strings.', 422, { tags: 'Every tag must be a string.' })
+    if (tag.trim().length > MAX_TAG_LENGTH)
+      throw new DashboardFileError(`A tag must not exceed ${MAX_TAG_LENGTH} characters.`, 422, { tags: 'A tag is too long.' })
+  }
+
+  await assertExists(selected.adapter, path)
+
+  const record = await writeTags(store, selected.name, path, normalizeTags(input.tags))
+  return { path, favorite: record.favorite, tags: record.tags }
+}
+
+/** 404 unless the path is a file or a folder on the disk. */
+async function assertExists(adapter: StorageAdapter, path: string): Promise<void> {
+  if (await adapter.fileExists(path) || await adapter.directoryExists(path))
+    return
+  throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
+}
+
+/**
+ * How a dispatched job reaches the queue.
+ *
+ * A parameter so the upload path can be tested without a queue, and so an app
+ * that wants a different dispatcher - a synchronous one in a test, a
+ * rate-limited one in front of a paid vision API - can supply it. The default
+ * reads `@stacksjs/queue` lazily: `uploadDashboardFiles` is on the request path
+ * and most uploads are not media.
+ */
+export type TaskDispatcher = (job: JobName, payload: Record<string, unknown>) => Promise<void>
+
+/**
+ * The job each kind of work runs.
+ *
+ * Typed as the generated `JobName` union rather than `string`, so a job renamed
+ * or removed under this map is a compile error instead of a dispatch that fails
+ * at runtime for a file nobody is watching.
+ */
+const TASK_JOBS = {
+  optimize: 'OptimizeStorageImageJob',
+  transcode: 'TranscodeStorageVideoJob',
+  tag: 'TagStorageMediaJob',
+} as const satisfies Record<StorageTaskKind, JobName>
+
+const queueDispatcher: TaskDispatcher = async (job, payload) => {
+  const { Jobs } = await import('@stacksjs/queue')
+  await Jobs.dispatch(job as JobName, payload)
+}
+
+/**
+ * Queue the processing an uploaded file calls for (stacksjs/stacks#2578).
+ *
+ * Only ever dispatches for a content type that has work to do - most uploads
+ * are documents, and a queue entry per PDF that immediately finds nothing to do
+ * is noise in the one place somebody looks when a transcode is stuck.
+ *
+ * A video is dispatched only when the caller supplies a profile: the ladder is
+ * derived from the source dimensions, and a job that guesses builds renditions
+ * nobody asked for. The dashboard knows them from the upload; a caller that
+ * does not can dispatch the transcode later through
+ * {@link reprocessDashboardFile}.
+ */
+export async function dispatchDashboardFileTasks(
+  input: { disk?: string, path: unknown, contentType?: string, videoProfile?: Record<string, unknown>, only?: readonly StorageTaskKind[] },
+  manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
+  dispatch: TaskDispatcher = queueDispatcher,
+): Promise<StorageItemTask[]> {
+  const selected = resolveDisk(manager, input.disk)
+  const path = normalizeDashboardFilePath(input.path)
+
+  const applicable = tasksForContentType(input.contentType)
+  const wanted = input.only
+    ? applicable.filter(kind => input.only!.includes(kind))
+    : applicable
+
+  const runnable = wanted.filter(kind => kind !== 'transcode' || input.videoProfile !== undefined)
+  if (runnable.length === 0)
+    return []
+
+  return await dispatchTasks(store, selected.name, path, runnable, async (kind) => {
+    await dispatch(TASK_JOBS[kind], {
+      disk: selected.name,
+      path,
+      ...(kind === 'transcode' ? { profile: input.videoProfile } : {}),
+    })
+  })
+}
+
+/**
+ * Run a file's processing again (stacksjs/stacks#2578).
+ *
+ * The re-run path the issue asked for, and it is not optional: the first
+ * version of any of these produces output somebody wants regenerated - a
+ * better ladder, a model that has since improved, an optimization that ran
+ * before a preset changed.
+ *
+ * Existence is checked against the DISK, so re-running a file that has since
+ * been deleted is a 404 rather than a job that fails minutes later in a worker
+ * log nobody reads.
+ */
+export async function reprocessDashboardFile(
+  input: { disk?: string, path: unknown, kinds?: unknown, videoProfile?: Record<string, unknown> },
+  manager: Manager = Storage,
+  store: StorageMetadataStore = databaseMetadataStore,
+  dispatch: TaskDispatcher = queueDispatcher,
+): Promise<{ path: string, tasks: StorageItemTask[] }> {
+  const selected = resolveDisk(manager, input.disk)
+  const path = normalizeDashboardFilePath(input.path)
+
+  let only: StorageTaskKind[] | undefined
+  if (input.kinds !== undefined && input.kinds !== null) {
+    if (!Array.isArray(input.kinds))
+      throw new DashboardFileError('Kinds must be an array.', 422, { kinds: 'Send an array of task kinds.' })
+    for (const kind of input.kinds) {
+      if (!STORAGE_TASK_KINDS.includes(kind as StorageTaskKind)) {
+        throw new DashboardFileError(`Unknown task kind "${String(kind)}".`, 422, {
+          kinds: `Choose from ${STORAGE_TASK_KINDS.join(', ')}.`,
+        })
+      }
+    }
+    only = input.kinds as StorageTaskKind[]
+  }
+
+  if (!(await selected.adapter.fileExists(path)))
+    throw new DashboardFileError(`Storage item "${path}" was not found.`, 404)
+
+  const tasks = await dispatchDashboardFileTasks(
+    {
+      disk: selected.name,
+      path,
+      contentType: await selected.adapter.mimeType(path),
+      videoProfile: input.videoProfile,
+      only,
+    },
+    manager,
+    store,
+    dispatch,
+  )
+
+  return { path, tasks }
+}
+
 export async function uploadDashboardFiles(
   input: { disk?: string, path?: string, files: UploadedFileLike[] },
   manager: Manager = Storage,
-): Promise<Array<{ path: string, url: string, size: number }>> {
+  store: StorageMetadataStore = databaseMetadataStore,
+  dispatch: TaskDispatcher = queueDispatcher,
+): Promise<Array<{ path: string, url: string, size: number, tasks: StorageItemTask[] }>> {
   const selected = resolveDisk(manager, input.disk)
   const directory = normalizeDashboardFilePath(input.path ?? '', { allowEmpty: true })
-  const uploaded: Array<{ path: string, url: string, size: number }> = []
+  const uploaded: Array<{ path: string, url: string, size: number, tasks: StorageItemTask[] }> = []
 
   try {
     for (const file of input.files) {
@@ -666,7 +972,7 @@ export async function uploadDashboardFiles(
         filename: 'original',
         overwrite: false,
       })
-      uploaded.push({ path: result.path, url: result.url, size: result.size })
+      uploaded.push({ path: result.path, url: result.url, size: result.size, tasks: [] })
     }
   }
   catch (error) {
@@ -691,5 +997,25 @@ export async function uploadDashboardFiles(
     throw error
   }
 
+  // Dispatched after every file is safely written, not inside the loop above:
+  // the rollback path deletes what was uploaded, and a job already queued for a
+  // file that is about to be deleted would run against nothing.
+  //
+  // A dispatch failure is recorded on the task row rather than thrown, so a
+  // queue that is down does not fail an upload that succeeded - the file is
+  // there, and the dashboard shows the processing as failed.
+  for (const file of uploaded) {
+    file.tasks = await dispatchDashboardFileTasks(
+      {
+        disk: selected.name,
+        path: file.path,
+        contentType: await selected.adapter.mimeType(file.path).catch(() => undefined),
+      },
+      manager,
+      store,
+      dispatch,
+    )
+  }
+
   return uploaded
 }