@stacksjs/defaults 0.74.33 → 0.74.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/skills/stacks-auto-imports/SKILL.md +1 -1
- package/ai/skills/stacks-dashboard/SKILL.md +90 -0
- package/ai/skills/stacks-orm/SKILL.md +1 -1
- package/ai/skills/stacks-storage/SKILL.md +58 -4
- package/app/Actions/Dashboard/Content/FileFavoriteAction.ts +25 -0
- package/app/Actions/Dashboard/Content/FileReprocessAction.ts +31 -0
- package/app/Actions/Dashboard/Content/FileTagsAction.ts +27 -0
- package/app/Actions/Dashboard/Content/file-manager.test.ts +47 -27
- package/app/Actions/Dashboard/Content/file-manager.ts +338 -12
- package/app/Actions/Dashboard/Content/file-metadata-store.ts +432 -0
- package/app/Actions/Dashboard/Content/file-metadata.test.ts +357 -0
- package/app/Actions/Dashboard/Content/file-metadata.ts +550 -0
- package/app/Actions/Dashboard/Content/file-pipeline.test.ts +344 -0
- package/app/Actions/Dashboard/Remote/RemoteCommandIndexAction.ts +29 -0
- package/app/Actions/Dashboard/Remote/RemoteCommandRunAction.ts +57 -0
- package/app/Actions/Dashboard/Remote/remote-commands.test.ts +272 -0
- package/app/Actions/Dashboard/Remote/remote-commands.ts +297 -0
- package/app/Actions/Dashboard/Remote/remote-routes.test.ts +58 -0
- package/app/Actions/Dashboard/Remote/ssh-runner.ts +94 -0
- package/app/Jobs/OptimizeStorageImageJob.ts +74 -0
- package/app/Jobs/TagStorageMediaJob.ts +122 -0
- package/app/Jobs/TranscodeStorageVideoJob.ts +88 -0
- package/app/Models/StorageItem.ts +123 -0
- package/app/Models/StorageItemTask.ts +134 -0
- package/ide/vscode/package.json +1 -1
- package/package.json +2 -2
- package/routes/dashboard-api.ts +25 -0
- package/vcs/github/workflows/buddy-bot.yml +109 -0
- package/vcs/github/workflows/ci.yml +3 -3
- package/vcs/github/workflows/release.yml +1 -1
- package/vcs/github/renovate.json +0 -5
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { RemoteAuditSink, RemoteCommand, RemoteHost, RemoteRunner } from './remote-commands'
|
|
2
|
+
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { log } from '@stacksjs/logging'
|
|
6
|
+
import { sshArgv } from './remote-commands'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The SSH transport and audit sink behind `runRemoteCommand`
|
|
10
|
+
* (stacksjs/stacks#960).
|
|
11
|
+
*
|
|
12
|
+
* Split from `remote-commands.ts` so the resolution, authorization and audit
|
|
13
|
+
* ordering can be tested without a network or a database - the same split
|
|
14
|
+
* `file-manager.ts` makes with its `Manager` parameter.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A runner for one host, with that host's pinned keys written to a file only
|
|
19
|
+
* this process can read.
|
|
20
|
+
*
|
|
21
|
+
* The known-hosts file is created per run and removed afterwards rather than
|
|
22
|
+
* kept in `~/.ssh`: the app server may reach several hosts with different
|
|
23
|
+
* operators, and a shared file is one where a stale entry for a decommissioned
|
|
24
|
+
* box silently authorizes whoever picked up its address.
|
|
25
|
+
*/
|
|
26
|
+
export function createSshRunner(host: RemoteHost, command: RemoteCommand): RemoteRunner {
|
|
27
|
+
return async (_argv, options) => {
|
|
28
|
+
const dir = mkdtempSync(join(tmpdir(), 'stacks-remote-'))
|
|
29
|
+
const knownHostsPath = join(dir, 'known_hosts')
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
writeFileSync(knownHostsPath, host.knownHosts.endsWith('\n') ? host.knownHosts : `${host.knownHosts}\n`)
|
|
33
|
+
// ssh refuses a known-hosts file others can write.
|
|
34
|
+
chmodSync(knownHostsPath, 0o600)
|
|
35
|
+
|
|
36
|
+
const proc = Bun.spawn(sshArgv(host, command, knownHostsPath), {
|
|
37
|
+
stdin: 'ignore',
|
|
38
|
+
stdout: 'pipe',
|
|
39
|
+
stderr: 'pipe',
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
// A run that has not finished is killed rather than left. Without this a
|
|
43
|
+
// hung ssh holds a process and a request until the server restarts.
|
|
44
|
+
let timedOut = false
|
|
45
|
+
const timer = setTimeout(() => {
|
|
46
|
+
timedOut = true
|
|
47
|
+
proc.kill()
|
|
48
|
+
}, options.timeoutMs)
|
|
49
|
+
|
|
50
|
+
try {
|
|
51
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
52
|
+
new Response(proc.stdout).text(),
|
|
53
|
+
new Response(proc.stderr).text(),
|
|
54
|
+
proc.exited,
|
|
55
|
+
])
|
|
56
|
+
|
|
57
|
+
return { exitCode, stdout, stderr, timedOut }
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
clearTimeout(timer)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
// Always, including on the failure path: a pinned-key file left in the
|
|
65
|
+
// temp directory on every run is both a leak and a growing surface.
|
|
66
|
+
rmSync(dir, { force: true, recursive: true })
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The audit sink that writes to the application log.
|
|
73
|
+
*
|
|
74
|
+
* The log rather than a table, deliberately. A remote-command record is
|
|
75
|
+
* append-only evidence about who did what, and the dashboard's own database is
|
|
76
|
+
* the thing an operator with dashboard access could edit. Shipping to whatever
|
|
77
|
+
* the app's logging is already configured to ship to keeps it outside the blast
|
|
78
|
+
* radius of the surface it audits.
|
|
79
|
+
*
|
|
80
|
+
* An application that wants these queryable can pass its own sink; the
|
|
81
|
+
* interface is two functions.
|
|
82
|
+
*/
|
|
83
|
+
export const loggingAuditSink: RemoteAuditSink = {
|
|
84
|
+
async started(entry) {
|
|
85
|
+
// Not `log.debug`: this is the record, and a level nobody ships in
|
|
86
|
+
// production is the same as not recording it.
|
|
87
|
+
await log.info(`[remote] ${entry.user} started ${entry.commandKey} on ${entry.hostKey} at ${entry.at}`)
|
|
88
|
+
},
|
|
89
|
+
|
|
90
|
+
async finished(entry) {
|
|
91
|
+
const outcome = entry.timedOut ? 'timed out' : `exited ${entry.exitCode}`
|
|
92
|
+
await log.info(`[remote] ${entry.user} finished ${entry.commandKey} on ${entry.hostKey}: ${outcome} in ${entry.durationMs}ms`)
|
|
93
|
+
},
|
|
94
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { basename, join } from 'node:path'
|
|
4
|
+
import { image } from '@stacksjs/image'
|
|
5
|
+
import { log } from '@stacksjs/logging'
|
|
6
|
+
import { Job } from '@stacksjs/queue'
|
|
7
|
+
import { Storage } from '@stacksjs/storage'
|
|
8
|
+
import { runTask } from '../Actions/Dashboard/Content/file-metadata'
|
|
9
|
+
import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Build the responsive variants for an uploaded image (stacksjs/stacks#2578).
|
|
13
|
+
*
|
|
14
|
+
* Dispatched by the file manager's upload handler, not run inside it: encoding
|
|
15
|
+
* four widths in three formats is seconds of CPU, and an upload handler that
|
|
16
|
+
* waits for it is an upload handler that times out on a slow phone connection.
|
|
17
|
+
*
|
|
18
|
+
* Variants are written back to the SAME disk under `.variants/`, so they travel
|
|
19
|
+
* with the original and a disk that is swapped out does not leave them behind.
|
|
20
|
+
* The leading dot keeps them out of the file manager's listing, which skips
|
|
21
|
+
* hidden components - a folder of thirty derivatives beside every photo would
|
|
22
|
+
* make the browser useless.
|
|
23
|
+
*
|
|
24
|
+
* `ts-images` reads from the local filesystem, so a remote disk is staged
|
|
25
|
+
* through a temp file. That is the whole reason for the download: an S3 object
|
|
26
|
+
* has no path to hand a decoder.
|
|
27
|
+
*/
|
|
28
|
+
interface OptimizeStorageImagePayload {
|
|
29
|
+
disk: string
|
|
30
|
+
path: string
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Where derivatives live, relative to the disk root. Hidden, so listings skip it. */
|
|
34
|
+
export const VARIANT_PREFIX = '.variants'
|
|
35
|
+
|
|
36
|
+
export default new Job({
|
|
37
|
+
name: 'OptimizeStorageImage',
|
|
38
|
+
description: 'Build responsive variants for an uploaded image (background)',
|
|
39
|
+
queue: 'media',
|
|
40
|
+
tries: 3,
|
|
41
|
+
backoff: [10, 30, 90],
|
|
42
|
+
|
|
43
|
+
async handle(payload: OptimizeStorageImagePayload) {
|
|
44
|
+
if (!payload?.disk || !payload?.path)
|
|
45
|
+
throw new Error('[OptimizeStorageImage] payload.disk and payload.path are required')
|
|
46
|
+
|
|
47
|
+
return await runTask(databaseMetadataStore, payload.disk, payload.path, 'optimize', async () => {
|
|
48
|
+
const adapter = Storage.disk(payload.disk)
|
|
49
|
+
const staging = await mkdtemp(join(tmpdir(), 'stacks-optimize-'))
|
|
50
|
+
|
|
51
|
+
try {
|
|
52
|
+
const source = join(staging, basename(payload.path))
|
|
53
|
+
await writeFile(source, await adapter.readToBuffer(payload.path))
|
|
54
|
+
|
|
55
|
+
const manifest = await image(source)
|
|
56
|
+
.preset('content')
|
|
57
|
+
.storage(adapter, `${VARIANT_PREFIX}/${payload.path}`)
|
|
58
|
+
.generate()
|
|
59
|
+
|
|
60
|
+
log.debug(`[OptimizeStorageImage] ${payload.path}: ${manifest.variants.length} variants`)
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
variants: manifest.variants.length,
|
|
64
|
+
placeholder: manifest.placeholder,
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
// Always, including on the failure path: a staging directory left
|
|
69
|
+
// behind on every retry is how a worker fills its disk overnight.
|
|
70
|
+
await rm(staging, { force: true, recursive: true })
|
|
71
|
+
}
|
|
72
|
+
})
|
|
73
|
+
},
|
|
74
|
+
})
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { ConfiguredAIOptions } from '@stacksjs/ai'
|
|
2
|
+
import { buildMessageWithImages, createAIClient } from '@stacksjs/ai'
|
|
3
|
+
import { config } from '@stacksjs/config'
|
|
4
|
+
import { log } from '@stacksjs/logging'
|
|
5
|
+
import { Job } from '@stacksjs/queue'
|
|
6
|
+
import { Storage } from '@stacksjs/storage'
|
|
7
|
+
import { runTask, setTags } from '../Actions/Dashboard/Content/file-metadata'
|
|
8
|
+
import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Ask a vision model what is in an uploaded image, and record the answer as
|
|
12
|
+
* tags (stacksjs/stacks#2578).
|
|
13
|
+
*
|
|
14
|
+
* A background job because it is a network round trip to a third party, and one
|
|
15
|
+
* that can take ten seconds under load. It also costs money per call, which is
|
|
16
|
+
* the reason for the guards below: this runs once per upload, and a retry storm
|
|
17
|
+
* against a misconfigured key would be an invoice rather than an outage.
|
|
18
|
+
*
|
|
19
|
+
* Tags are MERGED with whatever a person already put on the file rather than
|
|
20
|
+
* replacing them. A human tag is a decision and a model's tag is a suggestion,
|
|
21
|
+
* and having the suggestion overwrite the decision is the version of this
|
|
22
|
+
* feature nobody wants twice.
|
|
23
|
+
*/
|
|
24
|
+
interface TagStorageMediaPayload {
|
|
25
|
+
disk: string
|
|
26
|
+
path: string
|
|
27
|
+
/**
|
|
28
|
+
* How many tags to ask for. Bounded low on purpose: a model asked for twenty
|
|
29
|
+
* will produce twenty, and the last fifteen are noise that make the tag list
|
|
30
|
+
* less useful rather than more.
|
|
31
|
+
*/
|
|
32
|
+
limit?: number
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** What the model is asked to return, so the answer is parsed rather than scraped. */
|
|
36
|
+
const TAG_SCHEMA = {
|
|
37
|
+
type: 'object',
|
|
38
|
+
properties: {
|
|
39
|
+
tags: {
|
|
40
|
+
type: 'array',
|
|
41
|
+
items: { type: 'string' },
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
required: ['tags'],
|
|
45
|
+
additionalProperties: false,
|
|
46
|
+
} as const
|
|
47
|
+
|
|
48
|
+
const DEFAULT_LIMIT = 5
|
|
49
|
+
const MAX_LIMIT = 12
|
|
50
|
+
|
|
51
|
+
/** Beyond this a base64 image is a request body the provider will refuse anyway. */
|
|
52
|
+
const MAX_IMAGE_BYTES = 8 * 1024 * 1024
|
|
53
|
+
|
|
54
|
+
export default new Job({
|
|
55
|
+
name: 'TagStorageMedia',
|
|
56
|
+
description: 'Ask a vision model to tag an uploaded image (background)',
|
|
57
|
+
queue: 'media',
|
|
58
|
+
// Two, not three: each attempt is a paid call, and the failures worth
|
|
59
|
+
// retrying here are transient network ones rather than a rejected image.
|
|
60
|
+
tries: 2,
|
|
61
|
+
backoff: [30, 120],
|
|
62
|
+
|
|
63
|
+
async handle(payload: TagStorageMediaPayload) {
|
|
64
|
+
if (!payload?.disk || !payload?.path)
|
|
65
|
+
throw new Error('[TagStorageMedia] payload.disk and payload.path are required')
|
|
66
|
+
|
|
67
|
+
const limit = Math.max(1, Math.min(payload.limit ?? DEFAULT_LIMIT, MAX_LIMIT))
|
|
68
|
+
|
|
69
|
+
return await runTask(databaseMetadataStore, payload.disk, payload.path, 'tag', async () => {
|
|
70
|
+
const adapter = Storage.disk(payload.disk)
|
|
71
|
+
const bytes = await adapter.readToBuffer(payload.path)
|
|
72
|
+
|
|
73
|
+
if (bytes.byteLength > MAX_IMAGE_BYTES) {
|
|
74
|
+
throw new Error(
|
|
75
|
+
`[TagStorageMedia] ${payload.path} is ${bytes.byteLength} bytes; the vision path is capped at ${MAX_IMAGE_BYTES}. `
|
|
76
|
+
+ 'Tag a generated variant instead of the original.',
|
|
77
|
+
)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const mediaType = await adapter.mimeType(payload.path)
|
|
81
|
+
if (!mediaType.startsWith('image/'))
|
|
82
|
+
throw new Error(`[TagStorageMedia] ${payload.path} is ${mediaType}, which is not something a vision model reads`)
|
|
83
|
+
|
|
84
|
+
const aiConfig = (config.ai || {}) as ConfiguredAIOptions
|
|
85
|
+
const client = createAIClient(aiConfig)
|
|
86
|
+
|
|
87
|
+
// Sent as base64 rather than a URL: the file may be on a private disk,
|
|
88
|
+
// and handing a provider a signed URL would mean minting one that
|
|
89
|
+
// outlives the call.
|
|
90
|
+
const content = buildMessageWithImages(
|
|
91
|
+
`Give up to ${limit} short lowercase tags describing what is in this image. `
|
|
92
|
+
+ 'Prefer concrete nouns over judgements. Return only the JSON object.',
|
|
93
|
+
[{ dataBase64: bytes.toString('base64'), mediaType }],
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
const { data } = await client.generateObject<{ tags: string[] }>(
|
|
97
|
+
[{ role: 'user', content }],
|
|
98
|
+
TAG_SCHEMA as unknown as Record<string, unknown>,
|
|
99
|
+
{ maxTokens: 300, temperature: 0 },
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
const suggested = Array.isArray(data?.tags) ? data.tags.slice(0, limit) : []
|
|
103
|
+
if (suggested.length === 0)
|
|
104
|
+
return { tags: [] }
|
|
105
|
+
|
|
106
|
+
// Merged, not replaced. `setTags` normalizes and deduplicates, so a model
|
|
107
|
+
// suggesting a tag somebody already applied is a no-op rather than a
|
|
108
|
+
// duplicate.
|
|
109
|
+
const existing = (await databaseMetadataStore.under(payload.disk, payload.path)).get(payload.path)
|
|
110
|
+
const record = await setTags(
|
|
111
|
+
databaseMetadataStore,
|
|
112
|
+
payload.disk,
|
|
113
|
+
payload.path,
|
|
114
|
+
[...(existing?.tags ?? []), ...suggested],
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
log.debug(`[TagStorageMedia] ${payload.path}: ${suggested.length} suggested, ${record.tags.length} total`)
|
|
118
|
+
|
|
119
|
+
return { tags: record.tags }
|
|
120
|
+
})
|
|
121
|
+
},
|
|
122
|
+
})
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { basename, join } from 'node:path'
|
|
4
|
+
import { log } from '@stacksjs/logging'
|
|
5
|
+
import { Job } from '@stacksjs/queue'
|
|
6
|
+
import { Storage } from '@stacksjs/storage'
|
|
7
|
+
import { video } from '@stacksjs/video'
|
|
8
|
+
import { runTask } from '../Actions/Dashboard/Content/file-metadata'
|
|
9
|
+
import { databaseMetadataStore } from '../Actions/Dashboard/Content/file-metadata-store'
|
|
10
|
+
import { VARIANT_PREFIX } from './OptimizeStorageImageJob'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Transcode an uploaded video to mp4 and HLS (stacksjs/stacks#2578).
|
|
14
|
+
*
|
|
15
|
+
* #2578 asked whether video was in scope at all, on the grounds that mp4 and
|
|
16
|
+
* HLS mean ffmpeg - a heavyweight external binary, its licensing, and its
|
|
17
|
+
* provisioning across every deploy target. That question is already answered
|
|
18
|
+
* here: `@stacksjs/video` is built on `ts-videos`, which does the encoding
|
|
19
|
+
* itself, so there is no external binary to provision and no licence to take a
|
|
20
|
+
* view on. The decision the issue wanted made was made before it was written.
|
|
21
|
+
*
|
|
22
|
+
* Renditions and the HLS playlist go to the same disk under `.variants/`,
|
|
23
|
+
* alongside the image variants and hidden from the listing for the same reason.
|
|
24
|
+
*
|
|
25
|
+
* The profile has to be supplied by the caller. `ts-videos` can inspect a file,
|
|
26
|
+
* but the dashboard already knows the dimensions it needs from the upload, and
|
|
27
|
+
* a job that guesses wrong builds a ladder nobody asked for.
|
|
28
|
+
*/
|
|
29
|
+
interface TranscodeStorageVideoPayload {
|
|
30
|
+
disk: string
|
|
31
|
+
path: string
|
|
32
|
+
profile: Parameters<ReturnType<typeof video>['profile']>[0]
|
|
33
|
+
/** Cap the ladder at this height. Omit for every rung up to the source. */
|
|
34
|
+
maxHeight?: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export default new Job({
|
|
38
|
+
name: 'TranscodeStorageVideo',
|
|
39
|
+
description: 'Transcode an uploaded video to mp4 and HLS (background)',
|
|
40
|
+
queue: 'media',
|
|
41
|
+
// Fewer tries than the image job: a transcode is minutes of CPU, and a video
|
|
42
|
+
// that fails three times is not going to succeed on a fourth.
|
|
43
|
+
tries: 2,
|
|
44
|
+
backoff: [60, 300],
|
|
45
|
+
|
|
46
|
+
async handle(payload: TranscodeStorageVideoPayload) {
|
|
47
|
+
if (!payload?.disk || !payload?.path)
|
|
48
|
+
throw new Error('[TranscodeStorageVideo] payload.disk and payload.path are required')
|
|
49
|
+
if (!payload?.profile)
|
|
50
|
+
throw new Error('[TranscodeStorageVideo] payload.profile is required - the ladder is derived from it')
|
|
51
|
+
|
|
52
|
+
return await runTask(databaseMetadataStore, payload.disk, payload.path, 'transcode', async () => {
|
|
53
|
+
const adapter = Storage.disk(payload.disk)
|
|
54
|
+
const staging = await mkdtemp(join(tmpdir(), 'stacks-transcode-'))
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
const source = join(staging, basename(payload.path))
|
|
58
|
+
await writeFile(source, await adapter.readToBuffer(payload.path))
|
|
59
|
+
|
|
60
|
+
const delivery = await video(source)
|
|
61
|
+
.profile(payload.profile)
|
|
62
|
+
.ladder(payload.maxHeight ?? 'auto')
|
|
63
|
+
.output(['mp4'])
|
|
64
|
+
.streaming(['hls'])
|
|
65
|
+
.process()
|
|
66
|
+
|
|
67
|
+
// `files` is the whole delivery - renditions, playlists and segments -
|
|
68
|
+
// keyed by its path within the output. Written one at a time rather
|
|
69
|
+
// than in parallel: a transcode already saturated the box, and a
|
|
70
|
+
// hundred concurrent uploads to object storage is how the last step of
|
|
71
|
+
// a ten-minute job fails on a rate limit.
|
|
72
|
+
const prefix = `${VARIANT_PREFIX}/${payload.path}`
|
|
73
|
+
for (const [name, bytes] of Object.entries(delivery.files))
|
|
74
|
+
await adapter.write(`${prefix}/${name}`, bytes)
|
|
75
|
+
|
|
76
|
+
log.debug(`[TranscodeStorageVideo] ${payload.path}: ${Object.keys(delivery.files).length} files`)
|
|
77
|
+
|
|
78
|
+
return {
|
|
79
|
+
files: Object.keys(delivery.files).length,
|
|
80
|
+
renditions: delivery.derivatives.length,
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
await rm(staging, { force: true, recursive: true })
|
|
85
|
+
}
|
|
86
|
+
})
|
|
87
|
+
},
|
|
88
|
+
})
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { defineModel } from '@stacksjs/orm'
|
|
2
|
+
import { schema } from '@stacksjs/validation'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Dashboard metadata for one file on one disk (stacksjs/stacks#2577).
|
|
6
|
+
*
|
|
7
|
+
* Every file-manager operation shipped before this maps to a `StorageAdapter`
|
|
8
|
+
* method - `write`, `list`, `moveFile`, `changeVisibility`, `deleteFile`. A disk
|
|
9
|
+
* knows a path, some bytes, a size and an ACL. It does not know that somebody
|
|
10
|
+
* starred a file, and there is nowhere on a disk to write that down: extended
|
|
11
|
+
* attributes do not survive a copy and cost a syscall per file, and S3 object
|
|
12
|
+
* metadata is set at write time, so starring a 2 GB video would rewrite 2 GB.
|
|
13
|
+
* Both are a database row wearing a disguise, so this is the row.
|
|
14
|
+
*
|
|
15
|
+
* ## The disk is authoritative, this table is advisory
|
|
16
|
+
*
|
|
17
|
+
* That is the question #2577 asks to settle first, and it decides everything
|
|
18
|
+
* else. A bucket several systems write to changes without the dashboard's
|
|
19
|
+
* knowledge, so a table that claimed to describe its contents would be wrong
|
|
20
|
+
* within a day of being right. Instead:
|
|
21
|
+
*
|
|
22
|
+
* - The listing comes from the disk and is joined to these rows. A path with no
|
|
23
|
+
* row is a file with no stars, which is the normal case and needs no row.
|
|
24
|
+
* - A row whose path no longer exists is an orphan and is simply not shown. The
|
|
25
|
+
* listing pass already enumerates every path in the subtree, so it sweeps the
|
|
26
|
+
* orphans it can prove are orphans - the ones under a prefix it just walked -
|
|
27
|
+
* at no extra cost. Nothing walks the whole disk to garbage collect.
|
|
28
|
+
* - Renames and deletes made THROUGH the dashboard reconcile eagerly, so a
|
|
29
|
+
* starred file keeps its star the moment it moves rather than waiting for a
|
|
30
|
+
* sweep. A folder rename is a prefix update, because moving a folder moves
|
|
31
|
+
* every file beneath it.
|
|
32
|
+
*
|
|
33
|
+
* The consequence worth stating: a file renamed outside the dashboard loses its
|
|
34
|
+
* metadata, because nothing connects the old path to the new one. That is not
|
|
35
|
+
* a gap to close later - a copy and a rename are indistinguishable to a bucket
|
|
36
|
+
* listing, so any reconciliation would be guessing.
|
|
37
|
+
*
|
|
38
|
+
* Keyed by `(disk, path)` rather than by a file id, because a disk has no file
|
|
39
|
+
* ids to key by.
|
|
40
|
+
*
|
|
41
|
+
* ## Tags are the existing vocabulary, reached the way the CMS reaches it
|
|
42
|
+
*
|
|
43
|
+
* #2577 asks for the existing vocabulary rather than a second one. That is the
|
|
44
|
+
* `tags` table, joined through the `taggable_models` pivot, with
|
|
45
|
+
* `taggable_type` keeping one model's attachments away from another's - which
|
|
46
|
+
* is what the dashboard's own tag manager writes and reads.
|
|
47
|
+
*
|
|
48
|
+
* `file-metadata-store.ts` queries it directly rather than declaring a
|
|
49
|
+
* `belongsToMany` here, because every operation is a set operation over a
|
|
50
|
+
* subtree and the relation would do them a row at a time. The relation would
|
|
51
|
+
* resolve correctly, though: `taggable_models.tag_id` is a `tags` id, which
|
|
52
|
+
* took stacksjs/stacks#2579 to establish - the migration comment and four
|
|
53
|
+
* queries in `@stacksjs/cms` said `taggables`, a different table belonging to a
|
|
54
|
+
* different mechanism.
|
|
55
|
+
*/
|
|
56
|
+
export default defineModel({
|
|
57
|
+
name: 'StorageItem',
|
|
58
|
+
table: 'storage_items',
|
|
59
|
+
primaryKey: 'id',
|
|
60
|
+
autoIncrement: true,
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* One row per file, enforced rather than assumed.
|
|
64
|
+
*
|
|
65
|
+
* Every write here is a get-or-create keyed on this pair, and a duplicate
|
|
66
|
+
* would not be a visible bug - it would be a file that is starred and also
|
|
67
|
+
* not starred, depending which row the query happened to read first.
|
|
68
|
+
*/
|
|
69
|
+
indexes: [
|
|
70
|
+
{
|
|
71
|
+
name: 'storage_items_disk_path_unique',
|
|
72
|
+
columns: ['disk', 'path'],
|
|
73
|
+
unique: true,
|
|
74
|
+
},
|
|
75
|
+
],
|
|
76
|
+
|
|
77
|
+
traits: {
|
|
78
|
+
useUuid: true,
|
|
79
|
+
useTimestamps: true,
|
|
80
|
+
|
|
81
|
+
useSearch: {
|
|
82
|
+
displayable: ['id', 'disk', 'path', 'favorite'],
|
|
83
|
+
searchable: ['path'],
|
|
84
|
+
sortable: ['path', 'createdAt'],
|
|
85
|
+
filterable: ['disk', 'favorite'],
|
|
86
|
+
},
|
|
87
|
+
|
|
88
|
+
useSeeder: { count: 0 },
|
|
89
|
+
},
|
|
90
|
+
|
|
91
|
+
attributes: {
|
|
92
|
+
disk: {
|
|
93
|
+
order: 1,
|
|
94
|
+
fillable: true,
|
|
95
|
+
// The disk NAME, not its driver: two disks can point at one bucket with
|
|
96
|
+
// different prefixes, and they are different namespaces to the dashboard.
|
|
97
|
+
validation: { rule: schema.string().required().min(1).max(64) },
|
|
98
|
+
factory: () => 'public',
|
|
99
|
+
},
|
|
100
|
+
|
|
101
|
+
path: {
|
|
102
|
+
order: 2,
|
|
103
|
+
fillable: true,
|
|
104
|
+
// Disk-relative and without a leading slash, exactly as the file manager
|
|
105
|
+
// reports it, so a lookup is an equality match rather than a normalization
|
|
106
|
+
// problem at every call site.
|
|
107
|
+
validation: { rule: schema.string().required().min(1).max(2048) },
|
|
108
|
+
factory: faker => `${faker.system.fileName()}`,
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
favorite: {
|
|
112
|
+
order: 3,
|
|
113
|
+
fillable: true,
|
|
114
|
+
default: false,
|
|
115
|
+
validation: { rule: schema.boolean() },
|
|
116
|
+
factory: () => false,
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
|
|
120
|
+
// Managed from the file manager, not as a top-level model row: a list of
|
|
121
|
+
// (disk, path, favorite) tuples is not a thing anybody wants to browse.
|
|
122
|
+
dashboard: { enabled: false },
|
|
123
|
+
} as const)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { defineModel } from '@stacksjs/orm'
|
|
2
|
+
import { schema } from '@stacksjs/validation'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* One unit of background processing for one file (stacksjs/stacks#2578).
|
|
6
|
+
*
|
|
7
|
+
* The three things #245 asked for that could not happen inside a request -
|
|
8
|
+
* image optimization, video transcode, AI tagging - share the same shape:
|
|
9
|
+
* dispatch work and show what happened to it. Transcoding is minutes, not
|
|
10
|
+
* milliseconds; a vision call is a round trip to a third party. An upload
|
|
11
|
+
* handler that waits for either is an upload handler that times out.
|
|
12
|
+
*
|
|
13
|
+
* ## Why this is not a column on `StorageItem`
|
|
14
|
+
*
|
|
15
|
+
* A file can have three of these at once and they succeed and fail
|
|
16
|
+
* independently - a video whose transcode finished and whose tagging failed is
|
|
17
|
+
* a normal state, and a single `processing` column cannot say it. The file
|
|
18
|
+
* manager aggregates these rows into the one word a UI shows; the rows are what
|
|
19
|
+
* a retry and an error message need.
|
|
20
|
+
*
|
|
21
|
+
* It is also keyed by `(disk, path, kind)` rather than by a `storage_items` id,
|
|
22
|
+
* deliberately. A `storage_items` row exists only while somebody has starred or
|
|
23
|
+
* tagged the file - it is deleted when it has nothing left to say - and most
|
|
24
|
+
* uploads are neither. Hanging a task off it would mean creating a metadata row
|
|
25
|
+
* for every upload just to have somewhere to put the task.
|
|
26
|
+
*
|
|
27
|
+
* Same reconciliation rules as `storage_items`, through the same store: a
|
|
28
|
+
* rename moves these rows, a delete forgets them, and a completed listing
|
|
29
|
+
* sweeps the ones whose file is gone.
|
|
30
|
+
*/
|
|
31
|
+
export default defineModel({
|
|
32
|
+
name: 'StorageItemTask',
|
|
33
|
+
table: 'storage_item_tasks',
|
|
34
|
+
primaryKey: 'id',
|
|
35
|
+
autoIncrement: true,
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* One task per kind per file.
|
|
39
|
+
*
|
|
40
|
+
* Re-running replaces the row rather than adding a second, so "what happened
|
|
41
|
+
* to the transcode" has one answer. The history of previous attempts is the
|
|
42
|
+
* queue's to keep, not this table's.
|
|
43
|
+
*/
|
|
44
|
+
indexes: [
|
|
45
|
+
{
|
|
46
|
+
name: 'storage_item_tasks_disk_path_kind_unique',
|
|
47
|
+
columns: ['disk', 'path', 'kind'],
|
|
48
|
+
unique: true,
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
|
|
52
|
+
traits: {
|
|
53
|
+
useUuid: true,
|
|
54
|
+
useTimestamps: true,
|
|
55
|
+
|
|
56
|
+
useSearch: {
|
|
57
|
+
displayable: ['id', 'disk', 'path', 'kind', 'state'],
|
|
58
|
+
searchable: ['path'],
|
|
59
|
+
sortable: ['path', 'createdAt'],
|
|
60
|
+
filterable: ['disk', 'kind', 'state'],
|
|
61
|
+
},
|
|
62
|
+
|
|
63
|
+
useSeeder: { count: 0 },
|
|
64
|
+
},
|
|
65
|
+
|
|
66
|
+
attributes: {
|
|
67
|
+
disk: {
|
|
68
|
+
order: 1,
|
|
69
|
+
fillable: true,
|
|
70
|
+
validation: { rule: schema.string().required().min(1).max(64) },
|
|
71
|
+
factory: () => 'public',
|
|
72
|
+
},
|
|
73
|
+
|
|
74
|
+
path: {
|
|
75
|
+
order: 2,
|
|
76
|
+
fillable: true,
|
|
77
|
+
validation: { rule: schema.string().required().min(1).max(2048) },
|
|
78
|
+
factory: faker => faker.system.fileName(),
|
|
79
|
+
},
|
|
80
|
+
|
|
81
|
+
kind: {
|
|
82
|
+
order: 3,
|
|
83
|
+
fillable: true,
|
|
84
|
+
// `optimize` (images), `transcode` (video), `tag` (AI). An enum rather
|
|
85
|
+
// than free text so a typo in a dispatch is a validation error instead of
|
|
86
|
+
// a task nothing will ever run.
|
|
87
|
+
validation: { rule: schema.enum(['optimize', 'transcode', 'tag']) },
|
|
88
|
+
factory: () => 'optimize',
|
|
89
|
+
},
|
|
90
|
+
|
|
91
|
+
state: {
|
|
92
|
+
order: 4,
|
|
93
|
+
fillable: true,
|
|
94
|
+
default: 'queued',
|
|
95
|
+
validation: { rule: schema.enum(['queued', 'running', 'done', 'failed']) },
|
|
96
|
+
factory: () => 'queued',
|
|
97
|
+
},
|
|
98
|
+
|
|
99
|
+
attempts: {
|
|
100
|
+
order: 5,
|
|
101
|
+
fillable: true,
|
|
102
|
+
default: 0,
|
|
103
|
+
// Counted here as well as in the queue, because the queue's count is gone
|
|
104
|
+
// once the job leaves it and this is what the dashboard reads.
|
|
105
|
+
validation: { rule: schema.number() },
|
|
106
|
+
factory: () => 0,
|
|
107
|
+
},
|
|
108
|
+
|
|
109
|
+
error: {
|
|
110
|
+
order: 6,
|
|
111
|
+
fillable: true,
|
|
112
|
+
// The failure as the worker saw it. Truncated by the writer rather than
|
|
113
|
+
// by the column, so a stack trace does not silently lose its first line.
|
|
114
|
+
validation: { rule: schema.string().max(2000) },
|
|
115
|
+
factory: () => '',
|
|
116
|
+
},
|
|
117
|
+
|
|
118
|
+
startedAt: {
|
|
119
|
+
order: 7,
|
|
120
|
+
fillable: true,
|
|
121
|
+
validation: { rule: schema.string() },
|
|
122
|
+
factory: () => '',
|
|
123
|
+
},
|
|
124
|
+
|
|
125
|
+
finishedAt: {
|
|
126
|
+
order: 8,
|
|
127
|
+
fillable: true,
|
|
128
|
+
validation: { rule: schema.string() },
|
|
129
|
+
factory: () => '',
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
|
|
133
|
+
dashboard: { enabled: false },
|
|
134
|
+
} as const)
|
package/ide/vscode/package.json
CHANGED
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@stacksjs/defaults",
|
|
3
3
|
"type": "module",
|
|
4
4
|
"sideEffects": false,
|
|
5
|
-
"version": "0.74.
|
|
5
|
+
"version": "0.74.35",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "git+https://github.com/stacksjs/stacks.git",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@iconify-json/f7": "^1.2.2",
|
|
57
57
|
"@iconify-json/hugeicons": "^1.2.27",
|
|
58
|
-
"@stacksjs/mobile": "^0.74.
|
|
58
|
+
"@stacksjs/mobile": "^0.74.35",
|
|
59
59
|
"@stacksjs/sanitizer": "^0.2.113",
|
|
60
60
|
"ts-qr-codes": "^0.1.8"
|
|
61
61
|
}
|