@gotcos/glasses-server 6.24.5 → 6.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/package.json +1 -1
- package/server/index.ts +45 -3
- package/server/lib/media-store.ts +241 -12
- package/server/lib/rich-media-safety.ts +206 -44
- package/server/lib/upload-session.ts +409 -0
- package/server/lib/video-compression.ts +376 -0
- package/server/routes/health.ts +20 -2
- package/server/routes/media.ts +471 -16
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,59 @@
|
|
|
1
|
+
## 6.26.0
|
|
2
|
+
|
|
3
|
+
Chunked, resumable upload: video is no longer limited by what fits in one request.
|
|
4
|
+
|
|
5
|
+
- A video can now be uploaded in pieces, so length is bounded by storage rather than by a
|
|
6
|
+
single request. A 3-minute 4K clip is roughly 570 MB and could never fit a one-shot
|
|
7
|
+
limit; it now transfers as a sequence of 8 MiB chunks, losslessly, and is compressed
|
|
8
|
+
afterwards for storage.
|
|
9
|
+
- Interrupted uploads resume. The phone asks the server what it actually received and
|
|
10
|
+
continues from there rather than trusting its own count, because a chunk whose
|
|
11
|
+
acknowledgement was lost makes the client's number wrong. Resume covers network drops,
|
|
12
|
+
which is the common case; a server restart clears in-flight uploads and the phone starts
|
|
13
|
+
over cleanly rather than resuming onto nothing.
|
|
14
|
+
- Cancelling or giving up releases the server's slot and staging disk immediately instead
|
|
15
|
+
of holding them for four hours. Without this, a handful of give-ups on a poor connection
|
|
16
|
+
could make new uploads unavailable until the sessions expired.
|
|
17
|
+
- Assembly is verified before anything is published: the reassembled size must match what
|
|
18
|
+
the phone declared, a chunk that arrives out of order is refused rather than appended,
|
|
19
|
+
and a partial write is rejected rather than producing a correctly-sized file with a hole
|
|
20
|
+
in it.
|
|
21
|
+
- Finalizing a chunked upload runs the same validation, size cap, atomic publish and
|
|
22
|
+
background compression as a single-shot upload — one path, so the safety rules cannot
|
|
23
|
+
drift between them. Only video is allowed the larger chunked ceiling; documents and
|
|
24
|
+
images keep the existing limit, because reading a multi-gigabyte text file into memory
|
|
25
|
+
would fail in a far worse way than refusing it.
|
|
26
|
+
- `GET /api/health` advertises chunked availability and the chunk size, so the phone
|
|
27
|
+
decides from what this server actually supports rather than from a built-in assumption.
|
|
28
|
+
|
|
29
|
+
## 6.25.0
|
|
30
|
+
|
|
31
|
+
Large video uploads: a 100 MiB cap, streamed to disk, compressed in the background.
|
|
32
|
+
|
|
33
|
+
- Video attachments may now be up to 100 MiB. Images and documents stay at 64 MiB, and
|
|
34
|
+
the kind is decided from the file's magic bytes rather than its declared Content-Type,
|
|
35
|
+
so a declared video type cannot buy the larger ceiling.
|
|
36
|
+
- Uploads no longer buffer in memory. The body streams into the existing staging
|
|
37
|
+
directory and moves into place through the hardened atomic rename, with the byte
|
|
38
|
+
ceiling enforced during the stream so an oversized body is refused about one chunk
|
|
39
|
+
past the limit instead of after landing in full. `GET /api/media/:id/content` is
|
|
40
|
+
streamed for the same reason — raising the cap had otherwise taken that route's peak
|
|
41
|
+
allocation from 64 MiB to 100 MiB per concurrent download.
|
|
42
|
+
- `requestTimeout` is now explicit at 900s on both listeners. Node's 300s default was
|
|
43
|
+
invisible at 64 MiB but would have destroyed a 100 MiB upload's socket roughly 140
|
|
44
|
+
seconds before the client's own deadline, breaking exactly the size band this enables.
|
|
45
|
+
900s is the client's own ceiling, so the client always gives up first and can report a
|
|
46
|
+
real diagnostic instead of an opaque network error.
|
|
47
|
+
- Stored videos are compressed in the background with `libx265 -crf 30`, measured at
|
|
48
|
+
2.7x smaller and SSIM 0.969 on a real 4K 30fps upload. Resolution and frame rate are
|
|
49
|
+
never reduced, because that is what later frame-by-frame review depends on and
|
|
50
|
+
upscaling cannot recover it. An encode that is not smaller, or that fails, or that
|
|
51
|
+
changes the geometry, leaves the original in place — there is no path where the only
|
|
52
|
+
copy is lost. Requires ffmpeg and ffprobe; without them the original is simply kept.
|
|
53
|
+
- `GET /api/health` publishes `mediaLimits`, so the phone no longer hardcodes a byte cap
|
|
54
|
+
that can drift from what this server will actually accept. Chunked upload is
|
|
55
|
+
advertised as unavailable because its endpoints are not mounted yet.
|
|
56
|
+
|
|
1
57
|
## 6.24.5
|
|
2
58
|
|
|
3
59
|
COS Control provider proofs now isolate themselves from project customizations.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.26.0",
|
|
4
4
|
"description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/server/index.ts
CHANGED
|
@@ -192,6 +192,20 @@ app.use('/api', (req, res, next) => {
|
|
|
192
192
|
})
|
|
193
193
|
|
|
194
194
|
try {
|
|
195
|
+
// KNOWN HAZARD, deliberately not fixed here (2026-08-11). This lease is held for
|
|
196
|
+
// the WHOLE request, including the body transfer. Every other mutation is
|
|
197
|
+
// sub-second, but POST /api/media/file now accepts up to 100 MiB, which the
|
|
198
|
+
// client itself budgets ~7.3 minutes for — far past COS Control's 90s drain
|
|
199
|
+
// timeout (main.swift waitForRestartProof). A drain that catches a large upload
|
|
200
|
+
// in flight will therefore hard-fail to Repair.
|
|
201
|
+
//
|
|
202
|
+
// Why no fix in this change: there is no per-kind budget map to declare a longer
|
|
203
|
+
// allowance against, and blocksRestart belongs to the meeting-sync surface rather
|
|
204
|
+
// than a generic active-work registry, so a new 'media_upload' kind would be a
|
|
205
|
+
// label with no behaviour — a false signal that the case is handled. The correct
|
|
206
|
+
// fix is to scope this lease to the INGEST (fast: validate + index write) instead
|
|
207
|
+
// of the network transfer, which means changing a fail-closed middleware that
|
|
208
|
+
// guards every mutation. That needs its own pass and its own tests.
|
|
195
209
|
const lease = acquireMaintenanceWork('api_mutation', {
|
|
196
210
|
allowDuringDrain: controllerProof,
|
|
197
211
|
})
|
|
@@ -333,20 +347,48 @@ process.on('unhandledRejection', (reason: any) => {
|
|
|
333
347
|
// Start HTTPS alongside HTTP — Even Hub WebView prefers HTTPS (iOS ATS).
|
|
334
348
|
// Optional: drop cert.pem + key.pem in server/certs/ (e.g. via mkcert) to enable.
|
|
335
349
|
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
|
350
|
+
/**
|
|
351
|
+
* Let a slow large upload finish instead of killing its socket mid-body.
|
|
352
|
+
*
|
|
353
|
+
* Node's default `requestTimeout` is 300s. That was invisible while the cap was
|
|
354
|
+
* 64 MiB, which needs ~262s at the throughput the CLIENT itself budgets for
|
|
355
|
+
* (UPLOAD_FLOOR_BYTES_PER_SEC = 250 KiB/s, cos-glasses-app shared/media-attachment.ts).
|
|
356
|
+
* Raising the video cap to 100 MiB pushes the worst case to ~7.3 minutes, so the default
|
|
357
|
+
* would have destroyed the socket roughly 140 seconds BEFORE the client's own deadline
|
|
358
|
+
* expired — breaking exactly the size band the raise exists to enable, and surfacing as
|
|
359
|
+
* an opaque network error instead of a timeout that states its budget.
|
|
360
|
+
*
|
|
361
|
+
* 900_000 is deliberately the client's UPLOAD_TIMEOUT_CEILING_MS, so the two repos agree
|
|
362
|
+
* by construction: the client always gives up first and gets to report the diagnostic.
|
|
363
|
+
*
|
|
364
|
+
* Wrapped at each createServer call rather than looped over `listeners`, because
|
|
365
|
+
* RequiredListener types `server` as the base net.Server, which has no requestTimeout.
|
|
366
|
+
* The generic constraint makes a server that cannot carry the timeout a compile error
|
|
367
|
+
* instead of a silent no-op or a cast that hides the HTTP-specific dependency.
|
|
368
|
+
*
|
|
369
|
+
* headersTimeout stays at its default — headers arrive immediately even on a slow body,
|
|
370
|
+
* so shortening their window is the wrong lever.
|
|
371
|
+
*/
|
|
372
|
+
const MAX_REQUEST_MS = 900_000
|
|
373
|
+
function withRequestTimeout<T extends { requestTimeout: number }>(server: T): T {
|
|
374
|
+
server.requestTimeout = MAX_REQUEST_MS
|
|
375
|
+
return server
|
|
376
|
+
}
|
|
377
|
+
|
|
336
378
|
const HTTPS_PORT = parseInt(process.env.HTTPS_PORT ?? '3143', 10)
|
|
337
379
|
const certDir = path.join(__dirname, 'certs')
|
|
338
380
|
const listeners: RequiredListener[] = []
|
|
339
381
|
if (existsSync(path.join(certDir, 'cert.pem'))) {
|
|
340
|
-
const httpsServer = createHttpsServer({
|
|
382
|
+
const httpsServer = withRequestTimeout(createHttpsServer({
|
|
341
383
|
cert: readFileSync(path.join(certDir, 'cert.pem')),
|
|
342
384
|
key: readFileSync(path.join(certDir, 'key.pem')),
|
|
343
|
-
}, app)
|
|
385
|
+
}, app))
|
|
344
386
|
listeners.push({ server: httpsServer, port: HTTPS_PORT, host: BIND_HOST, label: 'HTTPS' })
|
|
345
387
|
} else {
|
|
346
388
|
console.log('[COS API] No certs found — HTTPS disabled (drop cert.pem/key.pem in server/certs to enable)')
|
|
347
389
|
}
|
|
348
390
|
|
|
349
|
-
const httpServer = createHttpServer(app)
|
|
391
|
+
const httpServer = withRequestTimeout(createHttpServer(app))
|
|
350
392
|
listeners.push({ server: httpServer, port: PORT, host: BIND_HOST, label: 'HTTP' })
|
|
351
393
|
|
|
352
394
|
listenRequiredServers(listeners).then(() => {
|
|
@@ -20,12 +20,14 @@
|
|
|
20
20
|
|
|
21
21
|
import { createHash, randomBytes } from 'node:crypto'
|
|
22
22
|
import {
|
|
23
|
+
createReadStream,
|
|
23
24
|
existsSync,
|
|
24
25
|
mkdirSync,
|
|
25
26
|
readdirSync,
|
|
26
27
|
readFileSync,
|
|
27
28
|
renameSync,
|
|
28
29
|
rmSync,
|
|
30
|
+
statSync,
|
|
29
31
|
writeFileSync,
|
|
30
32
|
} from 'node:fs'
|
|
31
33
|
import { join, resolve, sep } from 'node:path'
|
|
@@ -51,7 +53,12 @@ import {
|
|
|
51
53
|
sniffImageType,
|
|
52
54
|
validateSourceImage,
|
|
53
55
|
} from './image-safety.js'
|
|
54
|
-
import {
|
|
56
|
+
import {
|
|
57
|
+
prepareRichMediaFromFile,
|
|
58
|
+
type MediaTransferMode,
|
|
59
|
+
type PreparedRichMediaFile,
|
|
60
|
+
} from './rich-media-safety.js'
|
|
61
|
+
import { VIDEO_COMPRESSION_LABEL, compressVideoFile } from './video-compression.js'
|
|
55
62
|
|
|
56
63
|
// Standalone state belongs under the same durable data root as conversations,
|
|
57
64
|
// archives, and run ledgers. COS_MEDIA_ROOT remains an explicit escape hatch
|
|
@@ -108,6 +115,24 @@ async function renameWithTransientRetry(
|
|
|
108
115
|
}
|
|
109
116
|
}
|
|
110
117
|
|
|
118
|
+
/** The compressor's contract is enforced by the compiler, not mirrored here:
|
|
119
|
+
* this alias is what the constructor's test seam has to satisfy. */
|
|
120
|
+
export type CompressVideoFile = typeof compressVideoFile
|
|
121
|
+
export type { CompressionResult, CompressionStatus } from './video-compression.js'
|
|
122
|
+
|
|
123
|
+
/** Hash without holding the file in memory — a 100 MB video would otherwise
|
|
124
|
+
* reintroduce exactly the allocation the streaming upload path removed. */
|
|
125
|
+
async function sha256OfFile(path: string): Promise<string> {
|
|
126
|
+
const hash = createHash('sha256')
|
|
127
|
+
await new Promise<void>((resolveHash, rejectHash) => {
|
|
128
|
+
const stream = createReadStream(path)
|
|
129
|
+
stream.on('data', chunk => hash.update(chunk))
|
|
130
|
+
stream.once('error', rejectHash)
|
|
131
|
+
stream.once('end', () => resolveHash())
|
|
132
|
+
})
|
|
133
|
+
return hash.digest('hex')
|
|
134
|
+
}
|
|
135
|
+
|
|
111
136
|
/** Test seam for the File Provider rename recovery contract. */
|
|
112
137
|
export async function _renameWithTransientRetryForTests(
|
|
113
138
|
source: string,
|
|
@@ -120,6 +145,17 @@ export async function _renameWithTransientRetryForTests(
|
|
|
120
145
|
|
|
121
146
|
export type MediaLifecycle = 'staged' | 'reserved' | 'associated' | 'expired' | 'deleted'
|
|
122
147
|
|
|
148
|
+
export interface VideoCompressionRecord {
|
|
149
|
+
label: string
|
|
150
|
+
/** Byte count as uploaded, before the encode. */
|
|
151
|
+
originalBytes: number
|
|
152
|
+
/** Byte count now stored. Always smaller — the module returns
|
|
153
|
+
* `skipped_not_smaller` otherwise, and two measured settings really did
|
|
154
|
+
* produce files LARGER than the source. */
|
|
155
|
+
bytes: number
|
|
156
|
+
atMs: number
|
|
157
|
+
}
|
|
158
|
+
|
|
123
159
|
export interface MediaRecord {
|
|
124
160
|
ref: MediaAttachmentRef
|
|
125
161
|
/** Relative to the media root. Never exposed through the API. */
|
|
@@ -131,6 +167,9 @@ export interface MediaRecord {
|
|
|
131
167
|
bytes: number
|
|
132
168
|
sha256: string
|
|
133
169
|
lifecycle: MediaLifecycle
|
|
170
|
+
/** Set once the background x265 pass replaced the stored original with a
|
|
171
|
+
* smaller file. Kept so the size drop is explainable and never retried. */
|
|
172
|
+
videoCompression?: VideoCompressionRecord
|
|
134
173
|
/** True once asset bytes were removed (content TTL or GC) while the
|
|
135
174
|
* metadata record remains (e.g. expired traffic frames). */
|
|
136
175
|
contentRemoved?: boolean
|
|
@@ -183,12 +222,46 @@ export interface IngestRichMediaInput {
|
|
|
183
222
|
sessionId?: string
|
|
184
223
|
}
|
|
185
224
|
|
|
225
|
+
/** A streamed upload that already landed in tmp/. Ingest MOVES this file into
|
|
226
|
+
* the asset directory, so the bytes are written exactly once. */
|
|
227
|
+
export interface IngestRichMediaFileInput {
|
|
228
|
+
sourcePath: string
|
|
229
|
+
byteLength: number
|
|
230
|
+
label?: string
|
|
231
|
+
declaredMime?: string
|
|
232
|
+
capturedAt?: string
|
|
233
|
+
sessionId?: string
|
|
234
|
+
/** How the bytes arrived. Omitted means single_shot, so /api/media/file keeps its
|
|
235
|
+
* existing ceiling; chunked finalize passes 'chunked' so a multi-hundred-MB video
|
|
236
|
+
* is judged against the chunked cap instead of being refused AFTER transfer. */
|
|
237
|
+
transfer?: MediaTransferMode
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Handle for a streaming upload's staging file. `dispose()` is idempotent and
|
|
241
|
+
* becomes a no-op once ingest has moved the file out. */
|
|
242
|
+
export interface MediaStagingFile {
|
|
243
|
+
path: string
|
|
244
|
+
dispose: () => void
|
|
245
|
+
}
|
|
246
|
+
|
|
186
247
|
export type MediaContentResult =
|
|
187
248
|
| { status: 'ok'; path: string; mime: MediaMime; bytes: number }
|
|
188
249
|
| { status: 'not_found' }
|
|
189
250
|
| { status: 'expired' }
|
|
190
251
|
| { status: 'unavailable' }
|
|
191
252
|
|
|
253
|
+
/** Provenance only. A malformed value is dropped rather than invalidating the
|
|
254
|
+
* whole record — losing the note is survivable, losing the asset is not. */
|
|
255
|
+
function sanitizeVideoCompression(raw: unknown): VideoCompressionRecord | null {
|
|
256
|
+
if (!raw || typeof raw !== 'object') return null
|
|
257
|
+
const r = raw as Record<string, unknown>
|
|
258
|
+
const positiveInt = (value: unknown): value is number =>
|
|
259
|
+
typeof value === 'number' && Number.isSafeInteger(value) && value > 0
|
|
260
|
+
if (typeof r.label !== 'string' || r.label.length === 0 || r.label.length > 64) return null
|
|
261
|
+
if (!positiveInt(r.originalBytes) || !positiveInt(r.bytes) || !positiveInt(r.atMs)) return null
|
|
262
|
+
return { label: r.label.slice(0, 64), originalBytes: r.originalBytes, bytes: r.bytes, atMs: r.atMs }
|
|
263
|
+
}
|
|
264
|
+
|
|
192
265
|
function sanitizeRecord(raw: unknown): MediaRecord | null {
|
|
193
266
|
if (!raw || typeof raw !== 'object') return null
|
|
194
267
|
const r = raw as Record<string, unknown>
|
|
@@ -208,6 +281,7 @@ function sanitizeRecord(raw: unknown): MediaRecord | null {
|
|
|
208
281
|
const derivativePaths = Array.isArray(r.derivativePaths)
|
|
209
282
|
? r.derivativePaths.filter(isOwnedPath).slice(0, 8)
|
|
210
283
|
: undefined
|
|
284
|
+
const videoCompression = sanitizeVideoCompression(r.videoCompression)
|
|
211
285
|
return {
|
|
212
286
|
ref,
|
|
213
287
|
storagePath: r.storagePath,
|
|
@@ -217,6 +291,7 @@ function sanitizeRecord(raw: unknown): MediaRecord | null {
|
|
|
217
291
|
bytes: typeof r.bytes === 'number' && r.bytes >= 0 ? r.bytes : 0,
|
|
218
292
|
sha256: typeof r.sha256 === 'string' ? r.sha256 : '',
|
|
219
293
|
lifecycle,
|
|
294
|
+
...(videoCompression ? { videoCompression } : {}),
|
|
220
295
|
contentRemoved: r.contentRemoved === true,
|
|
221
296
|
...(typeof r.sessionId === 'string' ? { sessionId: r.sessionId } : {}),
|
|
222
297
|
...(typeof r.clientQueueItemId === 'string' ? { clientQueueItemId: r.clientQueueItemId } : {}),
|
|
@@ -236,8 +311,11 @@ export class MediaStore {
|
|
|
236
311
|
private readonly root: string
|
|
237
312
|
private readonly records = new Map<string, MediaRecord>()
|
|
238
313
|
private readonly renderLensVariant: typeof renderG2Variant
|
|
314
|
+
private readonly compressVideo: CompressVideoFile
|
|
239
315
|
/** One cold-cache render per media id. Callers share the same promise. */
|
|
240
316
|
private readonly g2InFlight = new Map<string, Promise<MediaContentResult>>()
|
|
317
|
+
/** Background x265 passes, keyed by media id. Never awaited by a request. */
|
|
318
|
+
private readonly compressionJobs = new Map<string, Promise<void>>()
|
|
241
319
|
/** A corrupt/unreadable index makes the asset directory authoritative only
|
|
242
320
|
* for recovery. Never classify its entries as disposable orphans that boot. */
|
|
243
321
|
private allowOrphanCleanup = true
|
|
@@ -247,10 +325,16 @@ export class MediaStore {
|
|
|
247
325
|
|
|
248
326
|
constructor(
|
|
249
327
|
root: string = DEFAULT_MEDIA_ROOT,
|
|
250
|
-
dependencies: {
|
|
328
|
+
dependencies: {
|
|
329
|
+
renderG2Variant?: typeof renderG2Variant
|
|
330
|
+
/** Injected in tests so each compression outcome can be driven without
|
|
331
|
+
* running a real x265 encode. */
|
|
332
|
+
compressVideoFile?: CompressVideoFile
|
|
333
|
+
} = {},
|
|
251
334
|
) {
|
|
252
335
|
this.root = root
|
|
253
336
|
this.renderLensVariant = dependencies.renderG2Variant ?? renderG2Variant
|
|
337
|
+
this.compressVideo = dependencies.compressVideoFile ?? compressVideoFile
|
|
254
338
|
this.ensureDirs()
|
|
255
339
|
this.loadIndex()
|
|
256
340
|
this.reconcile()
|
|
@@ -417,19 +501,61 @@ export class MediaStore {
|
|
|
417
501
|
return this.publishNormalizedImage(input, normalized)
|
|
418
502
|
}
|
|
419
503
|
|
|
420
|
-
/**
|
|
421
|
-
*
|
|
422
|
-
|
|
423
|
-
|
|
504
|
+
/** Reserve a private staging file for a streaming upload. Uploads have always
|
|
505
|
+
* staged in tmp/ before the atomic move into assets/; a streamed body uses
|
|
506
|
+
* the same convention rather than inventing a second one. */
|
|
507
|
+
createStagingFile(): MediaStagingFile {
|
|
508
|
+
// tmp/ is emptied by boot reconcile, so recreate defensively rather than
|
|
509
|
+
// trusting a directory that existed at construction time.
|
|
510
|
+
this.ensureDirs()
|
|
511
|
+
const path = join(this.root, 'tmp', `upload-${randomBytes(12).toString('hex')}.bin`)
|
|
512
|
+
return {
|
|
513
|
+
path,
|
|
514
|
+
dispose: () => {
|
|
515
|
+
try { rmSync(path, { force: true }) } catch { /* already moved or gone */ }
|
|
516
|
+
},
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/** Authenticated user document/video ingress from a streamed staging file.
|
|
521
|
+
* Validation and derivative generation happen before the serialized index
|
|
522
|
+
* publication; the staged file is MOVED into the asset dir, never copied. */
|
|
523
|
+
async ingestRichMediaFromFile(input: IngestRichMediaFileInput): Promise<MediaAttachmentRef> {
|
|
524
|
+
const prepared = await prepareRichMediaFromFile(input.sourcePath, {
|
|
424
525
|
label: input.label,
|
|
425
526
|
declaredMime: input.declaredMime,
|
|
527
|
+
byteLength: input.byteLength,
|
|
528
|
+
// Chunked finalize must be judged against the chunked ceiling, not the
|
|
529
|
+
// single-shot one it used to inherit. Defaults to single_shot, so
|
|
530
|
+
// /api/media/file is unchanged.
|
|
531
|
+
transfer: input.transfer,
|
|
426
532
|
})
|
|
427
533
|
return this.publishPreparedRichMedia(input, prepared)
|
|
428
534
|
}
|
|
429
535
|
|
|
536
|
+
/** In-memory ingress for callers that already hold the bytes. Stages them in
|
|
537
|
+
* tmp/ and joins the single file-based path above — a second publish path
|
|
538
|
+
* would be a second place for the invariants to rot. */
|
|
539
|
+
async ingestRichMedia(input: IngestRichMediaInput): Promise<MediaAttachmentRef> {
|
|
540
|
+
const staged = this.createStagingFile()
|
|
541
|
+
try {
|
|
542
|
+
writeFileSync(staged.path, input.bytes, { mode: 0o600 })
|
|
543
|
+
return await this.ingestRichMediaFromFile({
|
|
544
|
+
sourcePath: staged.path,
|
|
545
|
+
byteLength: input.bytes.length,
|
|
546
|
+
label: input.label,
|
|
547
|
+
declaredMime: input.declaredMime,
|
|
548
|
+
capturedAt: input.capturedAt,
|
|
549
|
+
sessionId: input.sessionId,
|
|
550
|
+
})
|
|
551
|
+
} finally {
|
|
552
|
+
staged.dispose()
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
430
556
|
private async publishPreparedRichMedia(
|
|
431
|
-
input:
|
|
432
|
-
prepared:
|
|
557
|
+
input: { label?: string; capturedAt?: string; sessionId?: string },
|
|
558
|
+
prepared: PreparedRichMediaFile,
|
|
433
559
|
): Promise<MediaAttachmentRef> {
|
|
434
560
|
const id = `m_${randomBytes(12).toString('hex')}`
|
|
435
561
|
const now = Date.now()
|
|
@@ -449,7 +575,7 @@ export class MediaStore {
|
|
|
449
575
|
width: prepared.category === 'video' ? prepared.width : 1,
|
|
450
576
|
height: prepared.category === 'video' ? prepared.height : 1,
|
|
451
577
|
createdAt: nowIso,
|
|
452
|
-
bytes: prepared.
|
|
578
|
+
bytes: prepared.originalBytes,
|
|
453
579
|
...(prepared.category === 'video'
|
|
454
580
|
? { durationMs: prepared.durationMs, frameCount: prepared.frames.length }
|
|
455
581
|
: {
|
|
@@ -464,8 +590,13 @@ export class MediaStore {
|
|
|
464
590
|
mkdirSync(stageDir, { recursive: true, mode: 0o700 })
|
|
465
591
|
const originalName = `original.${extension}`
|
|
466
592
|
const derivativeNames: string[] = []
|
|
593
|
+
let sha256: string
|
|
467
594
|
try {
|
|
468
|
-
|
|
595
|
+
// Hash by streaming, then MOVE the staged upload in. Both live under
|
|
596
|
+
// tmp/, so this is a same-volume rename — the alternative is a second
|
|
597
|
+
// full write of a body that can be 100 MB.
|
|
598
|
+
sha256 = await sha256OfFile(prepared.originalPath)
|
|
599
|
+
await renameWithTransientRetry(prepared.originalPath, join(stageDir, originalName))
|
|
469
600
|
if (prepared.category === 'document') {
|
|
470
601
|
writeFileSync(join(stageDir, 'content.txt'), prepared.extractedText, { mode: 0o600 })
|
|
471
602
|
}
|
|
@@ -487,8 +618,8 @@ export class MediaStore {
|
|
|
487
618
|
thumbPath: derivativePaths[0] ?? storagePath,
|
|
488
619
|
...(prepared.category === 'document' ? { textPath: join('assets', id, 'content.txt') } : {}),
|
|
489
620
|
...(derivativePaths.length ? { derivativePaths } : {}),
|
|
490
|
-
bytes: prepared.
|
|
491
|
-
sha256
|
|
621
|
+
bytes: prepared.originalBytes,
|
|
622
|
+
sha256,
|
|
492
623
|
lifecycle: 'staged',
|
|
493
624
|
...(input.sessionId ? { sessionId: input.sessionId } : {}),
|
|
494
625
|
createdAtMs: now,
|
|
@@ -498,9 +629,107 @@ export class MediaStore {
|
|
|
498
629
|
this.records.set(id, record)
|
|
499
630
|
this.saveIndex()
|
|
500
631
|
})
|
|
632
|
+
// AFTER publication and outside the lock: x265 encodes at roughly real
|
|
633
|
+
// time, so awaiting it here would hold the upload response open for the
|
|
634
|
+
// length of the video.
|
|
635
|
+
if (prepared.category === 'video') this.scheduleVideoCompression(id)
|
|
501
636
|
return ref
|
|
502
637
|
}
|
|
503
638
|
|
|
639
|
+
// ── Background video compression ───────────────────────────────────────────
|
|
640
|
+
|
|
641
|
+
private scheduleVideoCompression(id: string): void {
|
|
642
|
+
const job = (async () => {
|
|
643
|
+
try {
|
|
644
|
+
await this.compressVideoAsset(id)
|
|
645
|
+
} catch (err) {
|
|
646
|
+
// A failed encode is a non-event: the original is still the asset.
|
|
647
|
+
console.error(`[media-store] video compression failed for ${id}:`, err)
|
|
648
|
+
}
|
|
649
|
+
})()
|
|
650
|
+
this.compressionJobs.set(id, job)
|
|
651
|
+
void job.then(() => {
|
|
652
|
+
if (this.compressionJobs.get(id) === job) this.compressionJobs.delete(id)
|
|
653
|
+
})
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** Encode a published video smaller in place. Every exit other than a
|
|
657
|
+
* validated smaller output leaves the original untouched, and the swap is a
|
|
658
|
+
* rename, so there is no window in which the only copy is missing. */
|
|
659
|
+
private async compressVideoAsset(id: string): Promise<void> {
|
|
660
|
+
const before = this.getRecord(id)
|
|
661
|
+
if (!before || before.videoCompression) return
|
|
662
|
+
if (mediaCategoryOf(before.ref) !== 'video') return
|
|
663
|
+
if (before.lifecycle === 'deleted' || before.lifecycle === 'expired' || before.contentRemoved) return
|
|
664
|
+
const compress = this.compressVideo
|
|
665
|
+
const inputPath = this.absPath(before.storagePath)
|
|
666
|
+
if (!existsSync(inputPath)) return
|
|
667
|
+
|
|
668
|
+
// workDir sits under the media root so the output rename into assets/ is a
|
|
669
|
+
// same-volume atomic replace; a cross-device rename fails EXDEV.
|
|
670
|
+
const workDir = join(this.root, 'tmp', `compress-${id}-${randomBytes(6).toString('hex')}`)
|
|
671
|
+
mkdirSync(workDir, { recursive: true, mode: 0o700 })
|
|
672
|
+
try {
|
|
673
|
+
const result = await compress(inputPath, workDir)
|
|
674
|
+
if (result.status !== 'compressed' || !result.outputPath) {
|
|
675
|
+
console.log(
|
|
676
|
+
`[media-store] video compression ${result.status} for ${id}` +
|
|
677
|
+
`${result.reason ? ` (${result.reason})` : ''}`,
|
|
678
|
+
)
|
|
679
|
+
return
|
|
680
|
+
}
|
|
681
|
+
if (!existsSync(result.outputPath)) {
|
|
682
|
+
console.error(`[media-store] video compression reported 'compressed' for ${id} with no output file`)
|
|
683
|
+
return
|
|
684
|
+
}
|
|
685
|
+
const compressedBytes = statSync(result.outputPath).size
|
|
686
|
+
// Trust-but-verify the module's own contract. An encode that is not
|
|
687
|
+
// smaller is a failure, not a result — two measured settings inflated
|
|
688
|
+
// the file — and swapping one in would cost storage for nothing.
|
|
689
|
+
if (compressedBytes <= 0 || compressedBytes >= before.bytes) {
|
|
690
|
+
console.warn(
|
|
691
|
+
`[media-store] discarding compression output for ${id}: ` +
|
|
692
|
+
`${compressedBytes} bytes vs original ${before.bytes}`,
|
|
693
|
+
)
|
|
694
|
+
return
|
|
695
|
+
}
|
|
696
|
+
const sha256 = await sha256OfFile(result.outputPath)
|
|
697
|
+
const outputPath = result.outputPath
|
|
698
|
+
await this.withLock(async () => {
|
|
699
|
+
// Compression is asynchronous, so a delete/release/GC may have landed
|
|
700
|
+
// while it ran. Re-check before publishing so removed bytes are never
|
|
701
|
+
// resurrected.
|
|
702
|
+
const rec = this.records.get(id)
|
|
703
|
+
if (!rec || rec.lifecycle === 'deleted' || rec.lifecycle === 'expired' || rec.contentRemoved) return
|
|
704
|
+
if (rec.storagePath !== before.storagePath || !existsSync(inputPath)) return
|
|
705
|
+
await renameWithTransientRetry(outputPath, inputPath)
|
|
706
|
+
rec.videoCompression = {
|
|
707
|
+
label: VIDEO_COMPRESSION_LABEL,
|
|
708
|
+
originalBytes: rec.bytes,
|
|
709
|
+
bytes: compressedBytes,
|
|
710
|
+
atMs: Date.now(),
|
|
711
|
+
}
|
|
712
|
+
rec.bytes = compressedBytes
|
|
713
|
+
rec.sha256 = sha256
|
|
714
|
+
// Keep the public size honest about what is actually stored.
|
|
715
|
+
rec.ref = { ...rec.ref, bytes: compressedBytes }
|
|
716
|
+
rec.updatedAtMs = Date.now()
|
|
717
|
+
this.saveIndex()
|
|
718
|
+
})
|
|
719
|
+
} finally {
|
|
720
|
+
try { rmSync(workDir, { recursive: true, force: true }) } catch { /* best effort */ }
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** Test seam: background compression is deliberately not awaited by ingest,
|
|
725
|
+
* so tests need a handle to settle it. */
|
|
726
|
+
async _awaitVideoCompressionForTests(id?: string): Promise<void> {
|
|
727
|
+
const jobs = id
|
|
728
|
+
? [this.compressionJobs.get(id)].filter((job): job is Promise<void> => job != null)
|
|
729
|
+
: [...this.compressionJobs.values()]
|
|
730
|
+
await Promise.all(jobs)
|
|
731
|
+
}
|
|
732
|
+
|
|
504
733
|
/** Trusted-local agent artifact ingress. Unlike the public upload path,
|
|
505
734
|
* this accepts a bounded larger JPEG/PNG/WebP/HEIC/AVIF and immediately
|
|
506
735
|
* converts it into the same normalized JPEG contract before publication. */
|