@stacksjs/defaults 0.75.40 → 0.75.42

package/app/Actions/Dashboard/Content/file-manager.ts

@@ -19,8 +19,10 @@ import {
   STORAGE_TASK_KINDS,
   sweepMetadata,
   tasksForContentType,
+  variantsPathFor,
 } from './file-metadata'
 import { databaseMetadataStore } from './file-metadata-store'
+import { claimedContentType, hasPreview, previewPathFor, previewPlanFor, previewUrlFor } from './file-preview'
 
 const DEFAULT_DISK = 'public'
 const DEFAULT_MAX_ENTRIES = 1000
@@ -41,6 +43,20 @@ export interface DashboardFileNode {
   mime_type?: string
   url?: string
   thumbnail?: string
+  /**
+   * The URL of this file's generated preview image (stacksjs/stacks#308), when
+   * one was made and the disk can serve it. Absent for a file with no preview,
+   * and for every file on a private disk - which has no URL for the original
+   * either.
+   */
+  preview?: string
+  /**
+   * The first lines of a text file, as plain text, when its preview was made.
+   * Text gets a snippet rather than an image: drawing text needs a font, and a
+   * card can show twelve lines as text. Read on any disk, private included,
+   * because the server reads it rather than the browser.
+   */
+  previewText?: string
   /**
    * Whether somebody starred this, from `storage_items` rather than from the
    * disk (stacksjs/stacks#2577). It was the literal `false` for as long as
@@ -372,6 +388,7 @@ export async function getDashboardFileSnapshot(
     items: [],
   }
   const folders = new Map<string, DashboardFileNode>([['', root]])
+  const previewed: DashboardFileNode[] = []
   const capacity = await diskCapacity(selected.name, selected.config)
   const stats: DashboardFileStats = {
     files: 0,
@@ -444,12 +461,37 @@ export async function getDashboardFileSnapshot(
       ...pick(entry.path),
       shared: selected.public,
     }
+    if (hasPreview(file.tasks))
+      previewed.push(file)
     parent.items!.push(file)
     stats.files++
     stats.contentBytes += entry.size
     stats.byType[categoryFor(entry)] += entry.size
   }
 
+  // Previews resolve after the walk, in the same batches as the stats: an image
+  // preview is a URL and costs nothing, but a text preview is a read, and a
+  // folder of markdown is a read per file (stacksjs/stacks#308).
+  await mapInBatches(previewed, STAT_CONCURRENCY, async (file) => {
+    const plan = previewPlanFor(claimedContentType(file.path, file.mime_type))
+    if (!plan || !('renderer' in plan))
+      return
+    if (plan.format === 'png') {
+      file.preview = await previewUrlFor(selected.adapter, file.path, {
+        public: selected.public,
+        servesProjectPublic: selected.name === 'public' && selected.config.driver === 'local',
+      })
+      return
+    }
+    try {
+      file.previewText = await selected.adapter.readToString(previewPathFor(file.path, 'txt'))
+    }
+    catch {
+      // A snippet that has gone missing is a file without a preview, not a
+      // listing that fails. Reprocessing makes it again.
+    }
+  })
+
   for (const folder of folders.values()) {
     folder.items!.sort((a, b) => {
       if ((a.type === 'folder') !== (b.type === 'folder'))
@@ -521,12 +563,14 @@ export async function deleteDashboardFile(
   if (await selected.adapter.fileExists(path)) {
     await selected.adapter.deleteFile(path)
     await followDelete(store, selected.name, path)
+    await deleteVariants(selected.adapter, path)
     return { path, type: 'file' }
   }
   if (await selected.adapter.directoryExists(path)) {
     await selected.adapter.deleteDirectory(path)
     // The subtree, because deleting a folder deletes everything under it.
     await followDelete(store, selected.name, path)
+    await deleteVariants(selected.adapter, path)
     return { path, type: 'directory' }
   }
 
@@ -573,6 +617,7 @@ export async function renameDashboardFile(
     // After the move, so a failed move does not leave the metadata describing a
     // path with no file at it.
     await followRename(store, selected.name, from, to)
+    await moveVariants(selected.adapter, from, to)
     return { from, to, type: 'file', moved: 1 }
   }
 
@@ -603,6 +648,7 @@ export async function renameDashboardFile(
   // A folder rename moves every file beneath it, so this is a prefix update
   // rather than one row - the part #2577 flagged as the one that would bite.
   await followRename(store, selected.name, from, to)
+  await moveVariants(selected.adapter, from, to)
 
   return { from, to, type: 'directory', moved: files.length }
 }
@@ -824,6 +870,54 @@ export async function setDashboardFileTags(
   return { path, favorite: record.favorite, tags: record.tags }
 }
 
+/**
+ * Move a path's derivatives with it (stacksjs/stacks#308).
+ *
+ * The task rows already follow a rename, so without this a finished preview
+ * would report `done` at the new path while its file sat under the old one.
+ * The variants folder mirrors the original's path, so a folder rename is a
+ * prefix move here too.
+ *
+ * Best effort, after the rename has succeeded: the rename is what was asked
+ * for, and a derivative that failed to follow is regenerated by reprocessing
+ * rather than worth failing the request over.
+ */
+async function moveVariants(adapter: StorageAdapter, from: string, to: string): Promise<void> {
+  const source = variantsPathFor(from)
+  const target = variantsPathFor(to)
+
+  try {
+    if (!(await adapter.directoryExists(source)))
+      return
+
+    const files: string[] = []
+    for await (const entry of adapter.list(source, { deep: true })) {
+      const listed = normalizeListedPath(String(entry.path))
+      if (entry.type === 'file' && listed)
+        files.push(listed.startsWith(`${source}/`) ? listed : `${source}/${listed}`)
+    }
+
+    for (const file of files)
+      await adapter.moveFile(file, `${target}/${file.slice(source.length + 1)}`)
+
+    await adapter.deleteDirectory(source)
+  }
+  catch {
+    // See above: the rename succeeded, and that is the answer the caller gets.
+  }
+}
+
+/** Remove a path's derivatives once the path itself is gone. Best effort, like {@link moveVariants}. */
+async function deleteVariants(adapter: StorageAdapter, path: string): Promise<void> {
+  try {
+    if (await adapter.directoryExists(variantsPathFor(path)))
+      await adapter.deleteDirectory(variantsPathFor(path))
+  }
+  catch {
+    // An orphaned derivative is hidden from the listing and costs only space.
+  }
+}
+
 /** 404 unless the path is a file or a folder on the disk. */
 async function assertExists(adapter: StorageAdapter, path: string): Promise<void> {
   if (await adapter.fileExists(path) || await adapter.directoryExists(path))
@@ -853,6 +947,7 @@ const TASK_JOBS = {
   optimize: 'OptimizeStorageImageJob',
   transcode: 'TranscodeStorageVideoJob',
   tag: 'TagStorageMediaJob',
+  preview: 'GenerateStoragePreviewJob',
 } as const satisfies Record<StorageTaskKind, JobName>
 
 const queueDispatcher: TaskDispatcher = async (job, payload) => {
@@ -872,6 +967,12 @@ const queueDispatcher: TaskDispatcher = async (job, payload) => {
  * nobody asked for. The dashboard knows them from the upload; a caller that
  * does not can dispatch the transcode later through
  * {@link reprocessDashboardFile}.
+ *
+ * A preview is decided separately, by `file-preview.ts`, and from the type the
+ * file CLAIMS - the reported type, or the extension when the disk reports
+ * nothing useful - because far more kinds have an answer there (#308). A kind
+ * with no renderer here is recorded `skipped` with its reason and never
+ * queued.
  */
 export async function dispatchDashboardFileTasks(
   input: { disk?: string, path: unknown, contentType?: string, videoProfile?: Record<string, unknown>, only?: readonly StorageTaskKind[] },
@@ -882,7 +983,11 @@ export async function dispatchDashboardFileTasks(
   const selected = resolveDisk(manager, input.disk)
   const path = normalizeDashboardFilePath(input.path)
 
-  const applicable = tasksForContentType(input.contentType)
+  const preview = previewPlanFor(claimedContentType(path, input.contentType))
+  const applicable: StorageTaskKind[] = [
+    ...tasksForContentType(input.contentType),
+    ...(preview ? ['preview' as const] : []),
+  ]
   const wanted = input.only
     ? applicable.filter(kind => input.only!.includes(kind))
     : applicable
@@ -891,13 +996,20 @@ export async function dispatchDashboardFileTasks(
   if (runnable.length === 0)
     return []
 
-  return await dispatchTasks(store, selected.name, path, runnable, async (kind) => {
-    await dispatch(TASK_JOBS[kind], {
-      disk: selected.name,
-      path,
-      ...(kind === 'transcode' ? { profile: input.videoProfile } : {}),
-    })
-  })
+  return await dispatchTasks(
+    store,
+    selected.name,
+    path,
+    runnable,
+    async (kind) => {
+      await dispatch(TASK_JOBS[kind], {
+        disk: selected.name,
+        path,
+        ...(kind === 'transcode' ? { profile: input.videoProfile } : {}),
+      })
+    },
+    kind => kind === 'preview' && preview && 'skip' in preview ? preview.skip : undefined,
+  )
 }
 
 /**

package/app/Actions/Dashboard/Content/file-metadata.ts

@@ -42,28 +42,79 @@ export interface StorageItemMetadata {
 }
 
 /**
- * The three kinds of background work a file can carry (stacksjs/stacks#2578).
+ * The kinds of background work a file can carry (stacksjs/stacks#2578, #308).
  *
  * `optimize` builds the image variants, `transcode` the mp4 and HLS renditions,
- * `tag` asks a vision model what is in the file. Independent: a video whose
- * transcode finished and whose tagging failed is a normal state.
+ * `tag` asks a vision model what is in the file, `preview` renders the small
+ * image a file manager shows in place of a generic icon. Independent: a video
+ * whose transcode finished and whose tagging failed is a normal state.
  */
-export type StorageTaskKind = 'optimize' | 'transcode' | 'tag'
+export type StorageTaskKind = 'optimize' | 'transcode' | 'tag' | 'preview'
 
-export type StorageTaskState = 'queued' | 'running' | 'done' | 'failed'
+/**
+ * Where one task is.
+ *
+ * `skipped` is not a failure and not a success: the work does not apply to this
+ * file HERE - a PDF with no homegrown renderer to draw it, an SVG that is not
+ * rasterized because it is markup somebody else wrote. It is recorded rather
+ * than left absent so a file manager can say why a file has no preview, and it
+ * is terminal, so the queue does not retry something that cannot succeed.
+ */
+export type StorageTaskState = 'queued' | 'running' | 'done' | 'failed' | 'skipped'
 
 /** One unit of background work, as the dashboard sees it. */
 export interface StorageItemTask {
   kind: StorageTaskKind
   state: StorageTaskState
   attempts: number
+  /** Why a `failed` task failed, or why a `skipped` one did not apply. */
   error?: string
   startedAt?: string
   finishedAt?: string
 }
 
 /** Every task kind, in the order a file would run them. */
-export const STORAGE_TASK_KINDS: readonly StorageTaskKind[] = ['optimize', 'transcode', 'tag']
+export const STORAGE_TASK_KINDS: readonly StorageTaskKind[] = ['optimize', 'transcode', 'tag', 'preview']
+
+/**
+ * Thrown from a task's work to record it as `skipped` rather than `failed`.
+ *
+ * For the case only the bytes can reveal: a `.png` whose bytes are a PDF, a
+ * `.ttf` that turns out to be a compressed WOFF. That is not a fault to retry -
+ * a second attempt reads the same bytes - so {@link runTask} records the reason
+ * and returns instead of rethrowing.
+ */
+export class StorageTaskSkipped extends Error {
+  constructor(reason: string) {
+    super(reason)
+    this.name = 'StorageTaskSkipped'
+  }
+}
+
+/**
+ * Where derivatives live, relative to the disk root (stacksjs/stacks#2578).
+ *
+ * Hidden, so the file manager's listing skips it. Defined here rather than in
+ * a job, because the request path needs it too - a rename has to move a file's
+ * derivatives with it - and importing a job would load its encoder.
+ */
+export const VARIANT_PREFIX = '.variants'
+
+/** The folder holding one file's derivatives. */
+export function variantsPathFor(path: string): string {
+  return `${VARIANT_PREFIX}/${path}`
+}
+
+/**
+ * Where a file's preview is stored (stacksjs/stacks#308).
+ *
+ * Derived from the path rather than recorded in a column, so it cannot go
+ * stale: the task row says whether one was made, and this says where. A rename
+ * through the dashboard moves the folder it sits in along with the task row.
+ */
+export function previewPathFor(path: string): string {
+  return `${variantsPathFor(path)}/preview.png`
+}
 
 /** The `taggable_type` these rows use, which keeps them apart from the CMS's. */
 export const STORAGE_ITEM_TYPE = 'storage_items'
@@ -313,25 +364,40 @@ export async function sweepMetadata(
 export function aggregateTaskState(tasks: readonly StorageItemTask[]): StorageTaskState | null {
   if (tasks.length === 0)
     return null
-  if (tasks.some(task => task.state === 'failed'))
+  // A skipped task says nothing about the work that DID apply, so it does not
+  // count towards the word - a photo whose preview was skipped and whose
+  // variants finished is done. Only a file where nothing applied reads as
+  // skipped, which is still more than `null`: somebody looked.
+  const applied = tasks.filter(task => task.state !== 'skipped')
+  if (applied.length === 0)
+    return 'skipped'
+  if (applied.some(task => task.state === 'failed'))
     return 'failed'
-  if (tasks.some(task => task.state === 'running'))
+  if (applied.some(task => task.state === 'running'))
     return 'running'
-  if (tasks.some(task => task.state === 'queued'))
+  if (applied.some(task => task.state === 'queued'))
     return 'queued'
   return 'done'
 }
 
+/** The bare, lower-cased MIME type, without parameters. */
+export function bareMime(contentType: string | undefined): string {
+  return (contentType ?? '').toLowerCase().split(';')[0]?.trim() ?? ''
+}
+
 /**
- * Which kinds of work a file's content type calls for.
+ * Which kinds of media work a file's content type calls for.
  *
  * Decided from the MIME type rather than the extension, when one is known: a
  * `.mp4` that is actually a PDF should not be handed to a transcoder. Tagging
  * applies to images and video alike, because a vision model can describe a
  * frame as readily as a photograph.
+ *
+ * `preview` is not decided here: it applies to far more types than these, and
+ * `file-preview.ts` owns which (stacksjs/stacks#308).
  */
 export function tasksForContentType(contentType: string | undefined): StorageTaskKind[] {
-  const mime = (contentType ?? '').toLowerCase().split(';')[0]?.trim() ?? ''
+  const mime = bareMime(contentType)
 
   if (mime.startsWith('image/')) {
     // SVG is markup, not a raster: there is nothing to re-encode, and running
@@ -361,10 +427,24 @@ export async function dispatchTasks(
   path: string,
   kinds: readonly StorageTaskKind[],
   dispatch: (kind: StorageTaskKind) => Promise<void>,
+  /**
+   * The reason a kind does not apply, when that is already known from the
+   * content type. Such a kind is recorded `skipped` and never queued: a queue
+   * entry that can only find out what the dispatcher already knew is noise.
+   */
+  skip: (kind: StorageTaskKind) => string | undefined = () => undefined,
 ): Promise<StorageItemTask[]> {
   const queued: StorageItemTask[] = []
 
   for (const kind of kinds) {
+    const reason = skip(kind)
+    if (reason !== undefined) {
+      const skipped: StorageItemTask = { kind, state: 'skipped', attempts: 0, error: reason, finishedAt: new Date().toISOString() }
+      await store.writeTask(disk, path, skipped)
+      queued.push(skipped)
+      continue
+    }
+
     const task: StorageItemTask = { kind, state: 'queued', attempts: 0 }
     await store.writeTask(disk, path, task)
 
@@ -400,6 +480,10 @@ export async function dispatchTasks(
  * and applies its backoff. The row and the queue are answering different
  * questions: the queue decides whether to try again, the row is what somebody
  * looking at the file sees.
+ *
+ * The one exception is {@link StorageTaskSkipped}: recorded as `skipped` with
+ * its reason, and NOT rethrown, because a retry would read the same bytes and
+ * reach the same answer. The result is `undefined` in that case.
  */
 export async function runTask<T>(
   store: StorageMetadataStore,
@@ -407,7 +491,7 @@ export async function runTask<T>(
   path: string,
   kind: StorageTaskKind,
   work: () => Promise<T>,
-): Promise<T> {
+): Promise<T | undefined> {
   const previous = (await store.tasksUnder(disk, path)).get(path)?.find(task => task.kind === kind)
   const attempts = (previous?.attempts ?? 0) + 1
   const startedAt = new Date().toISOString()
@@ -428,6 +512,18 @@ export async function runTask<T>(
     return result
   }
   catch (error) {
+    if (error instanceof StorageTaskSkipped) {
+      await store.writeTask(disk, path, {
+        kind,
+        state: 'skipped',
+        attempts,
+        startedAt,
+        finishedAt: new Date().toISOString(),
+        error: describeError(error),
+      })
+      return undefined
+    }
+
     await store.writeTask(disk, path, {
       kind,
       state: 'failed',

package/app/Actions/Dashboard/Content/file-pipeline.test.ts

@@ -127,8 +127,9 @@ describe('dispatchDashboardFileTasks', () => {
       dispatch,
     )
 
-    expect(tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
-    expect(dispatched.map(entry => entry.job)).toEqual(['OptimizeStorageImageJob', 'TagStorageMediaJob'])
+    // Plus the preview, which `file-preview.test.ts` covers (stacksjs/stacks#308).
+    expect(tasks.map(task => task.kind)).toEqual(['optimize', 'tag', 'preview'])
+    expect(dispatched.map(entry => entry.job)).toEqual(['OptimizeStorageImageJob', 'TagStorageMediaJob', 'GenerateStoragePreviewJob'])
     expect(dispatched[0]?.payload).toEqual({ disk: 'public', path: 'photo.png' })
 
     const recorded = (await store.tasksUnder('public', '')).get('photo.png')
@@ -136,9 +137,11 @@ describe('dispatchDashboardFileTasks', () => {
   })
 
   test('dispatches nothing for a content type with no work', async () => {
-    await manager.disk('public').write('notes.pdf', 'x')
+    // An archive rather than a PDF since #308: a PDF now records a skipped
+    // preview, which is a row but still not a queue entry.
+    await manager.disk('public').write('bundle.zip', 'x')
 
-    expect(await dispatchDashboardFileTasks({ path: 'notes.pdf', contentType: 'application/pdf' }, manager, store, dispatch)).toEqual([])
+    expect(await dispatchDashboardFileTasks({ path: 'bundle.zip', contentType: 'application/zip' }, manager, store, dispatch)).toEqual([])
     expect(dispatched).toEqual([])
     expect(await store.tasksUnder('public', '')).toEqual(new Map())
   })
@@ -149,7 +152,8 @@ describe('dispatchDashboardFileTasks', () => {
     await manager.disk('public').write('clip.mp4', 'x')
 
     const withoutProfile = await dispatchDashboardFileTasks({ path: 'clip.mp4', contentType: 'video/mp4' }, manager, store, dispatch)
-    expect(withoutProfile.map(task => task.kind)).toEqual(['tag'])
+    // The preview is recorded skipped: nothing here decodes a video frame.
+    expect(withoutProfile.map(task => [task.kind, task.state])).toEqual([['tag', 'queued'], ['preview', 'skipped']])
 
     dispatched = []
     const withProfile = await dispatchDashboardFileTasks(
@@ -158,7 +162,7 @@ describe('dispatchDashboardFileTasks', () => {
       store,
       dispatch,
     )
-    expect(withProfile.map(task => task.kind)).toEqual(['transcode', 'tag'])
+    expect(withProfile.map(task => task.kind)).toEqual(['transcode', 'tag', 'preview'])
     expect(dispatched[0]?.payload.profile).toEqual({ width: 1920, height: 1080 })
   })
 
@@ -251,7 +255,7 @@ describe('the snapshot reports processing', () => {
     const node = nodeAt(snapshot.root, 'photo.png')
 
     expect(node?.processing).toBe('queued')
-    expect(node?.tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
+    expect(node?.tasks.map(task => task.kind)).toEqual(['optimize', 'tag', 'preview'])
   })
 
   test('a file nobody processed reports null, not done', async () => {
@@ -309,7 +313,7 @@ describe('reprocessDashboardFile', () => {
 
     const result = await reprocessDashboardFile({ path: 'photo.png' }, manager, store, dispatch)
 
-    expect(result.tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
+    expect(result.tasks.map(task => task.kind)).toEqual(['optimize', 'tag', 'preview'])
     // The failed task is reset to queued rather than left showing its old
     // error beside a job that is about to run.
     const optimize = (await store.tasksUnder('public', '')).get('photo.png')?.find(task => task.kind === 'optimize')
@@ -333,7 +337,7 @@ describe('reprocessDashboardFile', () => {
       .rejects.toThrow(/was not found/)
   })
 
-  test('rejects a kind that is not one of the three', async () => {
+  test('rejects a kind that is not one of the four', async () => {
     await manager.disk('public').write('photo.png', 'x')
 
     expect(reprocessDashboardFile({ path: 'photo.png', kinds: ['reticulate'] }, manager, store, dispatch))