@stacksjs/defaults 0.74.33 → 0.74.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,344 @@
1
+ // The media pipeline behind uploads (stacksjs/stacks#2578).
2
+ //
3
+ // What is tested here is the dispatch and the state, not the encoders. Whether
4
+ // `ts-images` produces a correct WebP is that package's business; what this
5
+ // layer has to get right is which work an upload calls for, that a queue being
6
+ // down leaves a visible failure rather than a silently unprocessed file, and
7
+ // that the state survives the file being renamed.
8
+ //
9
+ // The dispatcher is a recording function rather than a real queue, so these run
10
+ // without a worker.
11
+
12
+ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
13
+ import { mkdtemp, mkdir, rm } from 'node:fs/promises'
14
+ import { tmpdir } from 'node:os'
15
+ import { join } from 'node:path'
16
+ import { StorageManager } from '@stacksjs/storage'
17
+ import type { DashboardFileNode } from './file-manager'
18
+ import {
19
+ deleteDashboardFile,
20
+ dispatchDashboardFileTasks,
21
+ getDashboardFileSnapshot,
22
+ renameDashboardFile,
23
+ reprocessDashboardFile,
24
+ } from './file-manager'
25
+ import {
26
+ aggregateTaskState,
27
+ createMemoryMetadataStore,
28
+ describeError,
29
+ runTask,
30
+ tasksForContentType,
31
+ } from './file-metadata'
32
+
33
+ let root = ''
34
+ let manager: StorageManager
35
+ let store = createMemoryMetadataStore()
36
+ let dispatched: Array<{ job: string, payload: Record<string, unknown> }> = []
37
+ const dispatch = async (job: string, payload: Record<string, unknown>): Promise<void> => {
38
+ dispatched.push({ job, payload })
39
+ }
40
+
41
+ beforeEach(async () => {
42
+ store = createMemoryMetadataStore()
43
+ dispatched = []
44
+ root = await mkdtemp(join(tmpdir(), 'stacks-file-pipeline-'))
45
+ await mkdir(join(root, 'public'), { recursive: true })
46
+ manager = new StorageManager().init({
47
+ default: 'public',
48
+ disks: {
49
+ public: { driver: 'local', root: join(root, 'public'), url: '/storage', visibility: 'public' },
50
+ },
51
+ })
52
+ })
53
+
54
+ afterEach(async () => {
55
+ manager.reset()
56
+ await rm(root, { force: true, recursive: true })
57
+ })
58
+
59
+ function nodeAt(node: DashboardFileNode, path: string): DashboardFileNode | undefined {
60
+ if (node.path === path)
61
+ return node
62
+ for (const child of node.items ?? []) {
63
+ const found = nodeAt(child, path)
64
+ if (found)
65
+ return found
66
+ }
67
+ return undefined
68
+ }
69
+
70
+ describe('tasksForContentType', () => {
71
+ test('an image is optimized and tagged', () => {
72
+ expect(tasksForContentType('image/png')).toEqual(['optimize', 'tag'])
73
+ expect(tasksForContentType('image/jpeg; charset=binary')).toEqual(['optimize', 'tag'])
74
+ })
75
+
76
+ test('a video is transcoded and tagged', () => {
77
+ expect(tasksForContentType('video/mp4')).toEqual(['transcode', 'tag'])
78
+ })
79
+
80
+ test('SVG is left alone, because it is markup rather than a raster', () => {
81
+ // Nothing to re-encode, and running markup through an image decoder is a
82
+ // parser attack surface for no benefit.
83
+ expect(tasksForContentType('image/svg+xml')).toEqual([])
84
+ })
85
+
86
+ test('everything else calls for nothing', () => {
87
+ // Most uploads are documents. A queue entry per PDF that immediately finds
88
+ // nothing to do is noise in the one place somebody looks when a transcode
89
+ // is stuck.
90
+ expect(tasksForContentType('application/pdf')).toEqual([])
91
+ expect(tasksForContentType('text/plain')).toEqual([])
92
+ expect(tasksForContentType(undefined)).toEqual([])
93
+ expect(tasksForContentType('')).toEqual([])
94
+ })
95
+ })
96
+
97
+ describe('aggregateTaskState', () => {
98
+ test('is null when nothing was ever dispatched', () => {
99
+ // Not the same as work that finished: an image uploaded before
100
+ // optimization existed should not claim to have been optimized.
101
+ expect(aggregateTaskState([])).toBeNull()
102
+ })
103
+
104
+ test('reports the worst state, because a failure is what needs surfacing', () => {
105
+ const done = { kind: 'optimize', state: 'done', attempts: 1 } as const
106
+ const failed = { kind: 'tag', state: 'failed', attempts: 2 } as const
107
+ const running = { kind: 'transcode', state: 'running', attempts: 1 } as const
108
+ const queued = { kind: 'tag', state: 'queued', attempts: 0 } as const
109
+
110
+ expect(aggregateTaskState([done, failed])).toBe('failed')
111
+ expect(aggregateTaskState([done, running])).toBe('running')
112
+ expect(aggregateTaskState([done, queued])).toBe('queued')
113
+ expect(aggregateTaskState([done])).toBe('done')
114
+ // A failure outranks work still in flight.
115
+ expect(aggregateTaskState([running, failed])).toBe('failed')
116
+ })
117
+ })
118
+
119
+ describe('dispatchDashboardFileTasks', () => {
120
+ test('queues the jobs an image calls for, and records them', async () => {
121
+ await manager.disk('public').write('photo.png', 'x')
122
+
123
+ const tasks = await dispatchDashboardFileTasks(
124
+ { path: 'photo.png', contentType: 'image/png' },
125
+ manager,
126
+ store,
127
+ dispatch,
128
+ )
129
+
130
+ expect(tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
131
+ expect(dispatched.map(entry => entry.job)).toEqual(['OptimizeStorageImageJob', 'TagStorageMediaJob'])
132
+ expect(dispatched[0]?.payload).toEqual({ disk: 'public', path: 'photo.png' })
133
+
134
+ const recorded = (await store.tasksUnder('public', '')).get('photo.png')
135
+ expect(recorded?.every(task => task.state === 'queued')).toBeTrue()
136
+ })
137
+
138
+ test('dispatches nothing for a content type with no work', async () => {
139
+ await manager.disk('public').write('notes.pdf', 'x')
140
+
141
+ expect(await dispatchDashboardFileTasks({ path: 'notes.pdf', contentType: 'application/pdf' }, manager, store, dispatch)).toEqual([])
142
+ expect(dispatched).toEqual([])
143
+ expect(await store.tasksUnder('public', '')).toEqual(new Map())
144
+ })
145
+
146
+ test('holds a transcode back until it has a profile to build a ladder from', async () => {
147
+ // A job that guesses the source dimensions builds renditions nobody asked
148
+ // for. Tagging still runs, because it needs nothing.
149
+ await manager.disk('public').write('clip.mp4', 'x')
150
+
151
+ const withoutProfile = await dispatchDashboardFileTasks({ path: 'clip.mp4', contentType: 'video/mp4' }, manager, store, dispatch)
152
+ expect(withoutProfile.map(task => task.kind)).toEqual(['tag'])
153
+
154
+ dispatched = []
155
+ const withProfile = await dispatchDashboardFileTasks(
156
+ { path: 'clip.mp4', contentType: 'video/mp4', videoProfile: { width: 1920, height: 1080 } },
157
+ manager,
158
+ store,
159
+ dispatch,
160
+ )
161
+ expect(withProfile.map(task => task.kind)).toEqual(['transcode', 'tag'])
162
+ expect(dispatched[0]?.payload.profile).toEqual({ width: 1920, height: 1080 })
163
+ })
164
+
165
+ test('records a failure when the queue refuses, rather than losing the work', async () => {
166
+ // The whole reason the state is in a table the dashboard reads: a queue
167
+ // that is down should leave a visible failure, not a file that silently
168
+ // never gets processed.
169
+ await manager.disk('public').write('photo.png', 'x')
170
+
171
+ const tasks = await dispatchDashboardFileTasks(
172
+ { path: 'photo.png', contentType: 'image/png' },
173
+ manager,
174
+ store,
175
+ async () => { throw new Error('queue unavailable') },
176
+ )
177
+
178
+ expect(tasks.every(task => task.state === 'failed')).toBeTrue()
179
+ expect(tasks[0]?.error).toBe('queue unavailable')
180
+ })
181
+
182
+ test('narrows to the kinds asked for', async () => {
183
+ await manager.disk('public').write('photo.png', 'x')
184
+
185
+ const tasks = await dispatchDashboardFileTasks(
186
+ { path: 'photo.png', contentType: 'image/png', only: ['tag'] },
187
+ manager,
188
+ store,
189
+ dispatch,
190
+ )
191
+
192
+ expect(tasks.map(task => task.kind)).toEqual(['tag'])
193
+ expect(dispatched.map(entry => entry.job)).toEqual(['TagStorageMediaJob'])
194
+ })
195
+ })
196
+
197
+ describe('runTask', () => {
198
+ test('moves a task through running to done, counting the attempt', async () => {
199
+ await store.writeTask('public', 'a.png', { kind: 'optimize', state: 'queued', attempts: 0 })
200
+
201
+ const result = await runTask(store, 'public', 'a.png', 'optimize', async () => 'built')
202
+ expect(result).toBe('built')
203
+
204
+ const task = (await store.tasksUnder('public', '')).get('a.png')?.[0]
205
+ expect(task?.state).toBe('done')
206
+ expect(task?.attempts).toBe(1)
207
+ expect(task?.startedAt).toBeTruthy()
208
+ expect(task?.finishedAt).toBeTruthy()
209
+ })
210
+
211
+ test('records the failure and rethrows, so the queue still retries', async () => {
212
+ // The row and the queue answer different questions: the queue decides
213
+ // whether to try again, the row is what somebody looking at the file sees.
214
+ expect(runTask(store, 'public', 'a.png', 'optimize', async () => {
215
+ throw new Error('decode failed')
216
+ })).rejects.toThrow('decode failed')
217
+
218
+ await Bun.sleep(0)
219
+ const task = (await store.tasksUnder('public', '')).get('a.png')?.[0]
220
+ expect(task?.state).toBe('failed')
221
+ expect(task?.error).toBe('decode failed')
222
+ })
223
+
224
+ test('a retry that succeeds clears the previous error', async () => {
225
+ // A green file carrying a red message from two attempts ago is worse than
226
+ // no message.
227
+ await runTask(store, 'public', 'a.png', 'optimize', async () => { throw new Error('transient') }).catch(() => {})
228
+ await runTask(store, 'public', 'a.png', 'optimize', async () => 'ok')
229
+
230
+ const task = (await store.tasksUnder('public', '')).get('a.png')?.[0]
231
+ expect(task?.state).toBe('done')
232
+ expect(task?.error).toBeUndefined()
233
+ expect(task?.attempts).toBe(2)
234
+ })
235
+ })
236
+
237
+ describe('describeError', () => {
238
+ test('bounds what can reach the column', () => {
239
+ expect(describeError(new Error('boom'))).toBe('boom')
240
+ expect(describeError('a string')).toBe('a string')
241
+ expect(describeError(new Error('x'.repeat(5000)))).toHaveLength(2000)
242
+ })
243
+ })
244
+
245
+ describe('the snapshot reports processing', () => {
246
+ test('a queued file shows as queued, and names its tasks', async () => {
247
+ await manager.disk('public').write('photo.png', 'x')
248
+ await dispatchDashboardFileTasks({ path: 'photo.png', contentType: 'image/png' }, manager, store, dispatch)
249
+
250
+ const snapshot = await getDashboardFileSnapshot({}, manager, store)
251
+ const node = nodeAt(snapshot.root, 'photo.png')
252
+
253
+ expect(node?.processing).toBe('queued')
254
+ expect(node?.tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
255
+ })
256
+
257
+ test('a file nobody processed reports null, not done', async () => {
258
+ await manager.disk('public').write('notes.txt', 'x')
259
+
260
+ const snapshot = await getDashboardFileSnapshot({}, manager, store)
261
+ expect(nodeAt(snapshot.root, 'notes.txt')?.processing).toBeNull()
262
+ })
263
+ })
264
+
265
+ describe('tasks follow the file', () => {
266
+ test('a rename carries them, so a finished transcode is not lost', async () => {
267
+ await manager.disk('public').write('clip.mp4', 'x')
268
+ await store.writeTask('public', 'clip.mp4', { kind: 'transcode', state: 'done', attempts: 1 })
269
+
270
+ await renameDashboardFile({ path: 'clip.mp4', name: 'final.mp4' }, manager, store)
271
+
272
+ const tasks = await store.tasksUnder('public', '')
273
+ expect(tasks.get('clip.mp4')).toBeUndefined()
274
+ expect(tasks.get('final.mp4')?.[0]?.state).toBe('done')
275
+ })
276
+
277
+ test('a folder rename carries the tasks beneath it', async () => {
278
+ await manager.disk('public').write('media/clip.mp4', 'x')
279
+ await store.writeTask('public', 'media/clip.mp4', { kind: 'transcode', state: 'done', attempts: 1 })
280
+
281
+ await renameDashboardFile({ path: 'media', name: 'video' }, manager, store)
282
+
283
+ expect((await store.tasksUnder('public', '')).has('video/clip.mp4')).toBeTrue()
284
+ })
285
+
286
+ test('a delete forgets them', async () => {
287
+ await manager.disk('public').write('clip.mp4', 'x')
288
+ await store.writeTask('public', 'clip.mp4', { kind: 'transcode', state: 'done', attempts: 1 })
289
+
290
+ await deleteDashboardFile({ path: 'clip.mp4' }, manager, store)
291
+
292
+ expect(await store.tasksUnder('public', '')).toEqual(new Map())
293
+ })
294
+
295
+ test('a completed listing sweeps the ones whose file is gone', async () => {
296
+ await manager.disk('public').write('a.txt', 'x')
297
+ await store.writeTask('public', 'gone.png', { kind: 'optimize', state: 'done', attempts: 1 })
298
+
299
+ await getDashboardFileSnapshot({}, manager, store)
300
+
301
+ expect((await store.tasksUnder('public', '')).has('gone.png')).toBeFalse()
302
+ })
303
+ })
304
+
305
+ describe('reprocessDashboardFile', () => {
306
+ test('re-runs every kind the file calls for', async () => {
307
+ await manager.disk('public').write('photo.png', 'x')
308
+ await store.writeTask('public', 'photo.png', { kind: 'optimize', state: 'failed', attempts: 3, error: 'old' })
309
+
310
+ const result = await reprocessDashboardFile({ path: 'photo.png' }, manager, store, dispatch)
311
+
312
+ expect(result.tasks.map(task => task.kind)).toEqual(['optimize', 'tag'])
313
+ // The failed task is reset to queued rather than left showing its old
314
+ // error beside a job that is about to run.
315
+ const optimize = (await store.tasksUnder('public', '')).get('photo.png')?.find(task => task.kind === 'optimize')
316
+ expect(optimize?.state).toBe('queued')
317
+ expect(optimize?.error).toBeUndefined()
318
+ })
319
+
320
+ test('re-runs one kind when asked, leaving the others as they were', async () => {
321
+ await manager.disk('public').write('photo.png', 'x')
322
+ await store.writeTask('public', 'photo.png', { kind: 'optimize', state: 'done', attempts: 1 })
323
+
324
+ await reprocessDashboardFile({ path: 'photo.png', kinds: ['tag'] }, manager, store, dispatch)
325
+
326
+ expect(dispatched.map(entry => entry.job)).toEqual(['TagStorageMediaJob'])
327
+ const optimize = (await store.tasksUnder('public', '')).get('photo.png')?.find(task => task.kind === 'optimize')
328
+ expect(optimize?.state).toBe('done')
329
+ })
330
+
331
+ test('404s for a file that is gone, rather than queueing work against nothing', async () => {
332
+ expect(reprocessDashboardFile({ path: 'missing.png' }, manager, store, dispatch))
333
+ .rejects.toThrow(/was not found/)
334
+ })
335
+
336
+ test('rejects a kind that is not one of the three', async () => {
337
+ await manager.disk('public').write('photo.png', 'x')
338
+
339
+ expect(reprocessDashboardFile({ path: 'photo.png', kinds: ['reticulate'] }, manager, store, dispatch))
340
+ .rejects.toThrow(/Unknown task kind/)
341
+ expect(reprocessDashboardFile({ path: 'photo.png', kinds: 'tag' }, manager, store, dispatch))
342
+ .rejects.toThrow(/must be an array/)
343
+ })
344
+ })
@@ -0,0 +1,74 @@
1
+ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
2
+ import { tmpdir } from 'node:os'
3
+ import { basename, join } from 'node:path'
4
+ import { image } from '@stacksjs/image'
5
+ import { log } from '@stacksjs/logging'
6
+ import { Job } from '@stacksjs/queue'
7
+ import { Storage } from '@stacksjs/storage'
8
+ import { runTask } from '../Actions/Dashboard/Content/file-metadata'
9
+ import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
10
+
11
+ /**
12
+ * Build the responsive variants for an uploaded image (stacksjs/stacks#2578).
13
+ *
14
+ * Dispatched by the file manager's upload handler, not run inside it: encoding
15
+ * four widths in three formats is seconds of CPU, and an upload handler that
16
+ * waits for it is an upload handler that times out on a slow phone connection.
17
+ *
18
+ * Variants are written back to the SAME disk under `.variants/`, so they travel
19
+ * with the original and a disk that is swapped out does not leave them behind.
20
+ * The leading dot keeps them out of the file manager's listing, which skips
21
+ * hidden components - a folder of thirty derivatives beside every photo would
22
+ * make the browser useless.
23
+ *
24
+ * `ts-images` reads from the local filesystem, so a remote disk is staged
25
+ * through a temp file. That is the whole reason for the download: an S3 object
26
+ * has no path to hand a decoder.
27
+ */
28
+ interface OptimizeStorageImagePayload {
29
+ disk: string
30
+ path: string
31
+ }
32
+
33
+ /** Where derivatives live, relative to the disk root. Hidden, so listings skip it. */
34
+ export const VARIANT_PREFIX = '.variants'
35
+
36
+ export default new Job({
37
+ name: 'OptimizeStorageImage',
38
+ description: 'Build responsive variants for an uploaded image (background)',
39
+ queue: 'media',
40
+ tries: 3,
41
+ backoff: [10, 30, 90],
42
+
43
+ async handle(payload: OptimizeStorageImagePayload) {
44
+ if (!payload?.disk || !payload?.path)
45
+ throw new Error('[OptimizeStorageImage] payload.disk and payload.path are required')
46
+
47
+ return await runTask(databaseMetadataStore, payload.disk, payload.path, 'optimize', async () => {
48
+ const adapter = Storage.disk(payload.disk)
49
+ const staging = await mkdtemp(join(tmpdir(), 'stacks-optimize-'))
50
+
51
+ try {
52
+ const source = join(staging, basename(payload.path))
53
+ await writeFile(source, await adapter.readToBuffer(payload.path))
54
+
55
+ const manifest = await image(source)
56
+ .preset('content')
57
+ .storage(adapter, `${VARIANT_PREFIX}/${payload.path}`)
58
+ .generate()
59
+
60
+ log.debug(`[OptimizeStorageImage] ${payload.path}: ${manifest.variants.length} variants`)
61
+
62
+ return {
63
+ variants: manifest.variants.length,
64
+ placeholder: manifest.placeholder,
65
+ }
66
+ }
67
+ finally {
68
+ // Always, including on the failure path: a staging directory left
69
+ // behind on every retry is how a worker fills its disk overnight.
70
+ await rm(staging, { force: true, recursive: true })
71
+ }
72
+ })
73
+ },
74
+ })
@@ -0,0 +1,122 @@
1
+ import type { ConfiguredAIOptions } from '@stacksjs/ai'
2
+ import { buildMessageWithImages, createAIClient } from '@stacksjs/ai'
3
+ import { config } from '@stacksjs/config'
4
+ import { log } from '@stacksjs/logging'
5
+ import { Job } from '@stacksjs/queue'
6
+ import { Storage } from '@stacksjs/storage'
7
+ import { runTask, setTags } from '../Actions/Dashboard/Content/file-metadata'
8
+ import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
9
+
10
+ /**
11
+ * Ask a vision model what is in an uploaded image, and record the answer as
12
+ * tags (stacksjs/stacks#2578).
13
+ *
14
+ * A background job because it is a network round trip to a third party, and one
15
+ * that can take ten seconds under load. It also costs money per call, which is
16
+ * the reason for the guards below: this runs once per upload, and a retry storm
17
+ * against a misconfigured key would be an invoice rather than an outage.
18
+ *
19
+ * Tags are MERGED with whatever a person already put on the file rather than
20
+ * replacing them. A human tag is a decision and a model's tag is a suggestion,
21
+ * and having the suggestion overwrite the decision is the version of this
22
+ * feature nobody wants twice.
23
+ */
24
+ interface TagStorageMediaPayload {
25
+ disk: string
26
+ path: string
27
+ /**
28
+ * How many tags to ask for. Bounded low on purpose: a model asked for twenty
29
+ * will produce twenty, and the last fifteen are noise that make the tag list
30
+ * less useful rather than more.
31
+ */
32
+ limit?: number
33
+ }
34
+
35
+ /** What the model is asked to return, so the answer is parsed rather than scraped. */
36
+ const TAG_SCHEMA = {
37
+ type: 'object',
38
+ properties: {
39
+ tags: {
40
+ type: 'array',
41
+ items: { type: 'string' },
42
+ },
43
+ },
44
+ required: ['tags'],
45
+ additionalProperties: false,
46
+ } as const
47
+
48
+ const DEFAULT_LIMIT = 5
49
+ const MAX_LIMIT = 12
50
+
51
+ /** Beyond this a base64 image is a request body the provider will refuse anyway. */
52
+ const MAX_IMAGE_BYTES = 8 * 1024 * 1024
53
+
54
+ export default new Job({
55
+ name: 'TagStorageMedia',
56
+ description: 'Ask a vision model to tag an uploaded image (background)',
57
+ queue: 'media',
58
+ // Two, not three: each attempt is a paid call, and the failures worth
59
+ // retrying here are transient network ones rather than a rejected image.
60
+ tries: 2,
61
+ backoff: [30, 120],
62
+
63
+ async handle(payload: TagStorageMediaPayload) {
64
+ if (!payload?.disk || !payload?.path)
65
+ throw new Error('[TagStorageMedia] payload.disk and payload.path are required')
66
+
67
+ const limit = Math.max(1, Math.min(payload.limit ?? DEFAULT_LIMIT, MAX_LIMIT))
68
+
69
+ return await runTask(databaseMetadataStore, payload.disk, payload.path, 'tag', async () => {
70
+ const adapter = Storage.disk(payload.disk)
71
+ const bytes = await adapter.readToBuffer(payload.path)
72
+
73
+ if (bytes.byteLength > MAX_IMAGE_BYTES) {
74
+ throw new Error(
75
+ `[TagStorageMedia] ${payload.path} is ${bytes.byteLength} bytes; the vision path is capped at ${MAX_IMAGE_BYTES}. `
76
+ + 'Tag a generated variant instead of the original.',
77
+ )
78
+ }
79
+
80
+ const mediaType = await adapter.mimeType(payload.path)
81
+ if (!mediaType.startsWith('image/'))
82
+ throw new Error(`[TagStorageMedia] ${payload.path} is ${mediaType}, which is not something a vision model reads`)
83
+
84
+ const aiConfig = (config.ai || {}) as ConfiguredAIOptions
85
+ const client = createAIClient(aiConfig)
86
+
87
+ // Sent as base64 rather than a URL: the file may be on a private disk,
88
+ // and handing a provider a signed URL would mean minting one that
89
+ // outlives the call.
90
+ const content = buildMessageWithImages(
91
+ `Give up to ${limit} short lowercase tags describing what is in this image. `
92
+ + 'Prefer concrete nouns over judgements. Return only the JSON object.',
93
+ [{ dataBase64: bytes.toString('base64'), mediaType }],
94
+ )
95
+
96
+ const { data } = await client.generateObject<{ tags: string[] }>(
97
+ [{ role: 'user', content }],
98
+ TAG_SCHEMA as unknown as Record<string, unknown>,
99
+ { maxTokens: 300, temperature: 0 },
100
+ )
101
+
102
+ const suggested = Array.isArray(data?.tags) ? data.tags.slice(0, limit) : []
103
+ if (suggested.length === 0)
104
+ return { tags: [] }
105
+
106
+ // Merged, not replaced. `setTags` normalizes and deduplicates, so a model
107
+ // suggesting a tag somebody already applied is a no-op rather than a
108
+ // duplicate.
109
+ const existing = (await databaseMetadataStore.under(payload.disk, payload.path)).get(payload.path)
110
+ const record = await setTags(
111
+ databaseMetadataStore,
112
+ payload.disk,
113
+ payload.path,
114
+ [...(existing?.tags ?? []), ...suggested],
115
+ )
116
+
117
+ log.debug(`[TagStorageMedia] ${payload.path}: ${suggested.length} suggested, ${record.tags.length} total`)
118
+
119
+ return { tags: record.tags }
120
+ })
121
+ },
122
+ })
@@ -0,0 +1,88 @@
1
+ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
2
+ import { tmpdir } from 'node:os'
3
+ import { basename, join } from 'node:path'
4
+ import { log } from '@stacksjs/logging'
5
+ import { Job } from '@stacksjs/queue'
6
+ import { Storage } from '@stacksjs/storage'
7
+ import { video } from '@stacksjs/video'
8
+ import { runTask } from '../Actions/Dashboard/Content/file-metadata'
9
+ import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
10
+ import { VARIANT_PREFIX } from './OptimizeStorageImageJob'
11
+
12
+ /**
13
+ * Transcode an uploaded video to mp4 and HLS (stacksjs/stacks#2578).
14
+ *
15
+ * #2578 asked whether video was in scope at all, on the grounds that mp4 and
16
+ * HLS mean ffmpeg - a heavyweight external binary, its licensing, and its
17
+ * provisioning across every deploy target. That question is already answered
18
+ * here: `@stacksjs/video` is built on `ts-videos`, which does the encoding
19
+ * itself, so there is no external binary to provision and no licence to take a
20
+ * view on. The decision the issue wanted made was made before it was written.
21
+ *
22
+ * Renditions and the HLS playlist go to the same disk under `.variants/`,
23
+ * alongside the image variants and hidden from the listing for the same reason.
24
+ *
25
+ * The profile has to be supplied by the caller. `ts-videos` can inspect a file,
26
+ * but the dashboard already knows the dimensions it needs from the upload, and
27
+ * a job that guesses wrong builds a ladder nobody asked for.
28
+ */
29
+ interface TranscodeStorageVideoPayload {
30
+ disk: string
31
+ path: string
32
+ profile: Parameters<ReturnType<typeof video>['profile']>[0]
33
+ /** Cap the ladder at this height. Omit for every rung up to the source. */
34
+ maxHeight?: number
35
+ }
36
+
37
+ export default new Job({
38
+ name: 'TranscodeStorageVideo',
39
+ description: 'Transcode an uploaded video to mp4 and HLS (background)',
40
+ queue: 'media',
41
+ // Fewer tries than the image job: a transcode is minutes of CPU, and a video
42
+ // that fails three times is not going to succeed on a fourth.
43
+ tries: 2,
44
+ backoff: [60, 300],
45
+
46
+ async handle(payload: TranscodeStorageVideoPayload) {
47
+ if (!payload?.disk || !payload?.path)
48
+ throw new Error('[TranscodeStorageVideo] payload.disk and payload.path are required')
49
+ if (!payload?.profile)
50
+ throw new Error('[TranscodeStorageVideo] payload.profile is required - the ladder is derived from it')
51
+
52
+ return await runTask(databaseMetadataStore, payload.disk, payload.path, 'transcode', async () => {
53
+ const adapter = Storage.disk(payload.disk)
54
+ const staging = await mkdtemp(join(tmpdir(), 'stacks-transcode-'))
55
+
56
+ try {
57
+ const source = join(staging, basename(payload.path))
58
+ await writeFile(source, await adapter.readToBuffer(payload.path))
59
+
60
+ const delivery = await video(source)
61
+ .profile(payload.profile)
62
+ .ladder(payload.maxHeight ?? 'auto')
63
+ .output(['mp4'])
64
+ .streaming(['hls'])
65
+ .process()
66
+
67
+ // `files` is the whole delivery - renditions, playlists and segments -
68
+ // keyed by its path within the output. Written one at a time rather
69
+ // than in parallel: a transcode already saturated the box, and a
70
+ // hundred concurrent uploads to object storage is how the last step of
71
+ // a ten-minute job fails on a rate limit.
72
+ const prefix = `${VARIANT_PREFIX}/${payload.path}`
73
+ for (const [name, bytes] of Object.entries(delivery.files))
74
+ await adapter.write(`${prefix}/${name}`, bytes)
75
+
76
+ log.debug(`[TranscodeStorageVideo] ${payload.path}: ${Object.keys(delivery.files).length} files`)
77
+
78
+ return {
79
+ files: Object.keys(delivery.files).length,
80
+ renditions: delivery.derivatives.length,
81
+ }
82
+ }
83
+ finally {
84
+ await rm(staging, { force: true, recursive: true })
85
+ }
86
+ })
87
+ },
88
+ })