@jami-studio/core 0.92.30 → 0.92.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +20 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +7360 -7302
- package/corpus/core/src/deploy/build.ts +4193 -4127
- package/corpus/core/src/file-upload/types.ts +9 -0
- package/corpus/templates/analytics/changelog/2026-07-11-tracking-and-session-replay-ingest-endpoints-are-reachable-f.md +6 -0
- package/corpus/templates/analytics/package.json +9 -0
- package/corpus/templates/clips/actions/create-recording.ts +16 -1
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +60 -14
- package/corpus/templates/clips/app/routes/record.tsx +35 -9
- package/corpus/templates/clips/changelog/2026-07-11-uploads-now-stream-straight-to-s3-compatible-storage-r2-s3-m.md +6 -0
- package/corpus/templates/clips/server/lib/s3-upload-provider.ts +256 -0
- package/corpus/templates/clips/server/lib/streaming-upload-mode.ts +9 -3
- package/corpus/templates/clips/server/lib/upload-chunk-limits.ts +19 -0
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts +15 -4
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/reset-chunks.post.ts +49 -1
- package/dist/agent/production-agent.d.ts +15 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +61 -15
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/deploy/build.d.ts +21 -1
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +736 -676
- package/dist/deploy/build.js.map +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/file-upload/types.d.ts +9 -0
- package/dist/file-upload/types.d.ts.map +1 -1
- package/dist/file-upload/types.js.map +1 -1
- package/dist/provider-api/corpus-jobs.d.ts +2 -2
- package/dist/resources/handlers.d.ts +2 -2
- package/package.json +1 -1
|
@@ -67,6 +67,15 @@ export interface FileUploadProvider {
|
|
|
67
67
|
* during recording instead of assembling the full blob after stop().
|
|
68
68
|
*/
|
|
69
69
|
resumable?: {
|
|
70
|
+
/**
|
|
71
|
+
* Provider-preferred chunk size in bytes. Servers surface this to upload
|
|
72
|
+
* clients so each relayed chunk maps to one provider part. S3-compatible
|
|
73
|
+
* multipart uploads require every part except the last to be >= 5 MiB
|
|
74
|
+
* (and R2 additionally requires uniform part sizes), so an S3-backed
|
|
75
|
+
* provider sets 5 MiB here while GCS-style providers can omit it and
|
|
76
|
+
* accept the client default.
|
|
77
|
+
*/
|
|
78
|
+
preferredChunkBytes?: number;
|
|
70
79
|
startSession(
|
|
71
80
|
filename: string,
|
|
72
81
|
mimeType: string,
|
|
@@ -3,6 +3,15 @@
|
|
|
3
3
|
"displayName": "Agent-Native Analytics",
|
|
4
4
|
"private": true,
|
|
5
5
|
"type": "module",
|
|
6
|
+
"agent-native": {
|
|
7
|
+
"workspaceApp": {
|
|
8
|
+
"publicPaths": [
|
|
9
|
+
"/track",
|
|
10
|
+
"/api/analytics/track",
|
|
11
|
+
"/api/analytics/replay"
|
|
12
|
+
]
|
|
13
|
+
}
|
|
14
|
+
},
|
|
6
15
|
"scripts": {
|
|
7
16
|
"dev": "DATABASE_URL=${ANALYTICS_DATABASE_URL:-file:./data/app.db} agent-native dev",
|
|
8
17
|
"build": "agent-native build && tsx scripts/emit-netlify-dashboard-report-cron.ts",
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from "../server/lib/recordings.js";
|
|
25
25
|
import { setResumableSession } from "../server/lib/resumable-session.js";
|
|
26
26
|
import { shouldEnableStreamingUpload } from "../server/lib/streaming-upload-mode.js";
|
|
27
|
+
import { maxChunkUploadBytes } from "../server/lib/upload-chunk-limits.js";
|
|
27
28
|
import { createRecordingSchema } from "./lib/create-recording-schema.js";
|
|
28
29
|
import { DEFAULT_RECORDING_TITLE } from "./lib/title-source.js";
|
|
29
30
|
|
|
@@ -86,14 +87,24 @@ export default defineAction({
|
|
|
86
87
|
// to the SQL chunk path when no provider supports resumable uploads or the
|
|
87
88
|
// init fails.
|
|
88
89
|
let uploadMode: UploadMode = "buffered";
|
|
90
|
+
let uploadChunkBytes: number | undefined;
|
|
89
91
|
const uploadProvider = await getActiveFileUploadProviderForRequest();
|
|
92
|
+
const providerChunkBytes = uploadProvider?.resumable?.preferredChunkBytes;
|
|
93
|
+
const chunkBytesViable =
|
|
94
|
+
!providerChunkBytes || providerChunkBytes <= maxChunkUploadBytes();
|
|
95
|
+
if (providerChunkBytes && !chunkBytesViable) {
|
|
96
|
+
console.warn(
|
|
97
|
+
`[create-recording] provider ${uploadProvider?.id} needs ${providerChunkBytes}-byte chunks but this platform caps chunk POSTs at ${maxChunkUploadBytes()} bytes — falling back to buffered`,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
90
100
|
if (
|
|
91
101
|
args.requestStreaming === true &&
|
|
92
102
|
shouldEnableStreamingUpload({
|
|
93
103
|
client: args.streamingUploadClient,
|
|
94
104
|
mimeType: args.mimeType,
|
|
95
105
|
}) &&
|
|
96
|
-
uploadProvider?.resumable
|
|
106
|
+
uploadProvider?.resumable &&
|
|
107
|
+
chunkBytesViable
|
|
97
108
|
) {
|
|
98
109
|
try {
|
|
99
110
|
const recordingMimeType =
|
|
@@ -120,6 +131,7 @@ export default defineAction({
|
|
|
120
131
|
lastCommittedIndex: -1,
|
|
121
132
|
});
|
|
122
133
|
uploadMode = "streaming";
|
|
134
|
+
uploadChunkBytes = providerChunkBytes;
|
|
123
135
|
console.log(
|
|
124
136
|
`[create-recording] resumable session ready for ${id}: provider=${uploadProvider.id}`,
|
|
125
137
|
);
|
|
@@ -140,6 +152,9 @@ export default defineAction({
|
|
|
140
152
|
// Frontend substitutes {index}/{total}/{isFinal}
|
|
141
153
|
uploadChunkUrlTemplate: `/api/uploads/${id}/chunk?index={index}&total={total}&isFinal={isFinal}`,
|
|
142
154
|
uploadMode,
|
|
155
|
+
// Provider-preferred chunk size for streaming mode (e.g. S3 multipart
|
|
156
|
+
// needs uniform >=5 MiB parts). Clients slice on this boundary.
|
|
157
|
+
...(uploadChunkBytes ? { uploadChunkBytes } : {}),
|
|
143
158
|
};
|
|
144
159
|
},
|
|
145
160
|
});
|
|
@@ -548,6 +548,12 @@ export class RecorderEngine {
|
|
|
548
548
|
*/
|
|
549
549
|
private uploadAbort: AbortController | null = null;
|
|
550
550
|
private uploadMode: UploadMode = "buffered";
|
|
551
|
+
/**
|
|
552
|
+
* Server/provider-negotiated chunk size (from create-recording's
|
|
553
|
+
* `uploadChunkBytes`, e.g. 5 MiB for S3 multipart parts). Null = use the
|
|
554
|
+
* GCS-aligned STREAM_CHUNK_BYTES default.
|
|
555
|
+
*/
|
|
556
|
+
private streamChunkBytes: number | null = null;
|
|
551
557
|
/**
|
|
552
558
|
* Streaming-path buffer. MediaRecorder blobs accumulate here until at least
|
|
553
559
|
* STREAM_CHUNK_BYTES is available, then a 256 KiB-aligned slice is PUT to API
|
|
@@ -1124,11 +1130,17 @@ export class RecorderEngine {
|
|
|
1124
1130
|
uploadUrl: string;
|
|
1125
1131
|
abortUrl: string;
|
|
1126
1132
|
uploadMode?: UploadMode;
|
|
1133
|
+
uploadChunkBytes?: number;
|
|
1127
1134
|
}): void {
|
|
1128
1135
|
this.opts.recordingId = target.recordingId;
|
|
1129
1136
|
this.opts.uploadUrl = target.uploadUrl;
|
|
1130
1137
|
this.opts.abortUrl = target.abortUrl;
|
|
1131
1138
|
this.opts.uploadMode = target.uploadMode ?? "buffered";
|
|
1139
|
+
this.streamChunkBytes =
|
|
1140
|
+
typeof target.uploadChunkBytes === "number" &&
|
|
1141
|
+
target.uploadChunkBytes > 0
|
|
1142
|
+
? target.uploadChunkBytes
|
|
1143
|
+
: null;
|
|
1132
1144
|
}
|
|
1133
1145
|
|
|
1134
1146
|
// -------------------------------------------------------------------------
|
|
@@ -1550,6 +1562,7 @@ export class RecorderEngine {
|
|
|
1550
1562
|
private async resetUploadedChunks(
|
|
1551
1563
|
compression: CompressionUploadMeta | null,
|
|
1552
1564
|
signal?: AbortSignal,
|
|
1565
|
+
mimeType?: string,
|
|
1553
1566
|
): Promise<void> {
|
|
1554
1567
|
const resetUrl = `${appBasePath()}/api/uploads/${
|
|
1555
1568
|
this.opts.recordingId
|
|
@@ -1559,7 +1572,10 @@ export class RecorderEngine {
|
|
|
1559
1572
|
resetRes = await fetch(resetUrl, {
|
|
1560
1573
|
method: "POST",
|
|
1561
1574
|
headers: { "Content-Type": "application/json" },
|
|
1562
|
-
body: JSON.stringify({
|
|
1575
|
+
body: JSON.stringify({
|
|
1576
|
+
compression,
|
|
1577
|
+
mimeType: mimeType ?? this.mimeType ?? undefined,
|
|
1578
|
+
}),
|
|
1563
1579
|
signal,
|
|
1564
1580
|
});
|
|
1565
1581
|
} catch (err) {
|
|
@@ -1583,6 +1599,26 @@ export class RecorderEngine {
|
|
|
1583
1599
|
}). ${text || resetRes.statusText}`,
|
|
1584
1600
|
);
|
|
1585
1601
|
}
|
|
1602
|
+
// The server may have re-initialized a fresh provider streaming session
|
|
1603
|
+
// for the re-upload (remote-DB deployments cannot buffer chunks in SQL).
|
|
1604
|
+
// Honor its negotiated mode/chunk size for the slicing pass that follows.
|
|
1605
|
+
try {
|
|
1606
|
+
const resetInfo = (await resetRes.json()) as {
|
|
1607
|
+
uploadMode?: UploadMode;
|
|
1608
|
+
uploadChunkBytes?: number;
|
|
1609
|
+
} | null;
|
|
1610
|
+
if (resetInfo?.uploadMode) {
|
|
1611
|
+
this.opts.uploadMode = resetInfo.uploadMode;
|
|
1612
|
+
this.uploadMode = resetInfo.uploadMode;
|
|
1613
|
+
this.streamChunkBytes =
|
|
1614
|
+
typeof resetInfo.uploadChunkBytes === "number" &&
|
|
1615
|
+
resetInfo.uploadChunkBytes > 0
|
|
1616
|
+
? resetInfo.uploadChunkBytes
|
|
1617
|
+
: null;
|
|
1618
|
+
}
|
|
1619
|
+
} catch {
|
|
1620
|
+
// Older servers return no JSON body — keep the current mode.
|
|
1621
|
+
}
|
|
1586
1622
|
}
|
|
1587
1623
|
|
|
1588
1624
|
// -------------------------------------------------------------------------
|
|
@@ -1670,7 +1706,11 @@ export class RecorderEngine {
|
|
|
1670
1706
|
outputMimeType: compression.outputMimeType,
|
|
1671
1707
|
}
|
|
1672
1708
|
: null;
|
|
1673
|
-
await this.resetUploadedChunks(
|
|
1709
|
+
await this.resetUploadedChunks(
|
|
1710
|
+
compressionPayload,
|
|
1711
|
+
abort.signal,
|
|
1712
|
+
compression.outputMimeType,
|
|
1713
|
+
);
|
|
1674
1714
|
|
|
1675
1715
|
if (compressionError) {
|
|
1676
1716
|
// Compression itself failed — we still hold the original assembled
|
|
@@ -1872,22 +1912,21 @@ export class RecorderEngine {
|
|
|
1872
1912
|
}
|
|
1873
1913
|
|
|
1874
1914
|
/**
|
|
1875
|
-
* Drain the streaming buffer in
|
|
1915
|
+
* Drain the streaming buffer in aligned chunks.
|
|
1876
1916
|
* GCS rejects non-final resumable chunks that are not a multiple of 256 KiB,
|
|
1877
|
-
* so
|
|
1878
|
-
*
|
|
1917
|
+
* so the default slices on STREAM_CHUNK_BYTES boundaries and carries the
|
|
1918
|
+
* remainder; providers with their own part-size rules (S3 multipart needs
|
|
1919
|
+
* uniform >=5 MiB parts) negotiate `streamChunkBytes` via create-recording.
|
|
1920
|
+
* The leftover is uploaded as the final chunk on stop().
|
|
1879
1921
|
*/
|
|
1880
1922
|
private flushAlignedStreamChunks(): void {
|
|
1881
|
-
|
|
1923
|
+
const chunkBytes = this.streamChunkBytes ?? STREAM_CHUNK_BYTES;
|
|
1924
|
+
while (this.pendingStreamBytes >= chunkBytes) {
|
|
1882
1925
|
const combined = new Blob(this.pendingStreamBlobs, {
|
|
1883
1926
|
type: this.mimeType,
|
|
1884
1927
|
});
|
|
1885
|
-
const head = combined.slice(0,
|
|
1886
|
-
const tail = combined.slice(
|
|
1887
|
-
STREAM_CHUNK_BYTES,
|
|
1888
|
-
combined.size,
|
|
1889
|
-
this.mimeType,
|
|
1890
|
-
);
|
|
1928
|
+
const head = combined.slice(0, chunkBytes, this.mimeType);
|
|
1929
|
+
const tail = combined.slice(chunkBytes, combined.size, this.mimeType);
|
|
1891
1930
|
this.pendingStreamBlobs = tail.size > 0 ? [tail] : [];
|
|
1892
1931
|
this.pendingStreamBytes = tail.size;
|
|
1893
1932
|
const index = this.chunkIndex++;
|
|
@@ -1945,8 +1984,15 @@ export class RecorderEngine {
|
|
|
1945
1984
|
|
|
1946
1985
|
// Keep binary uploads comfortably under Netlify's effective function
|
|
1947
1986
|
// payload limit. This mirrors the local-file upload path in record.tsx.
|
|
1948
|
-
|
|
1949
|
-
|
|
1987
|
+
// Streaming mode uses the provider-negotiated part size instead (S3
|
|
1988
|
+
// multipart needs uniform >=5 MiB parts) and MUST upload sequentially —
|
|
1989
|
+
// the resumable chunk route advances provider offsets in index order.
|
|
1990
|
+
const streaming = (this.opts.uploadMode ?? this.uploadMode) === "streaming";
|
|
1991
|
+
const UPLOAD_SLICE_BYTES =
|
|
1992
|
+
streaming && this.streamChunkBytes
|
|
1993
|
+
? this.streamChunkBytes
|
|
1994
|
+
: 3 * 1024 * 1024;
|
|
1995
|
+
const PARALLELISM = streaming ? 1 : 4;
|
|
1950
1996
|
const totalSlices = Math.max(1, Math.ceil(blob.size / UPLOAD_SLICE_BYTES));
|
|
1951
1997
|
|
|
1952
1998
|
const slices = Array.from({ length: totalSlices }, (_, i) => {
|
|
@@ -1580,7 +1580,10 @@ export default function RecordRoute() {
|
|
|
1580
1580
|
spaceIds: spaceIdFromUrl ? [spaceIdFromUrl] : undefined,
|
|
1581
1581
|
folderId: folderIdFromUrl ?? undefined,
|
|
1582
1582
|
mimeType: uploadMimeType,
|
|
1583
|
-
|
|
1583
|
+
// Streaming lets deployments without SQL chunk scratch (remote
|
|
1584
|
+
// DB + S3-compatible storage) relay chunks straight to the
|
|
1585
|
+
// provider. The server decides; buffered stays the fallback.
|
|
1586
|
+
requestStreaming: true,
|
|
1584
1587
|
}),
|
|
1585
1588
|
},
|
|
1586
1589
|
);
|
|
@@ -1596,10 +1599,18 @@ export default function RecordRoute() {
|
|
|
1596
1599
|
);
|
|
1597
1600
|
}
|
|
1598
1601
|
const created = (await res.json()) as {
|
|
1599
|
-
result?: {
|
|
1602
|
+
result?: {
|
|
1603
|
+
id: string;
|
|
1604
|
+
uploadChunkUrl: string;
|
|
1605
|
+
abortUrl?: string;
|
|
1606
|
+
uploadMode?: UploadMode;
|
|
1607
|
+
uploadChunkBytes?: number;
|
|
1608
|
+
};
|
|
1600
1609
|
id?: string;
|
|
1601
1610
|
uploadChunkUrl?: string;
|
|
1602
1611
|
abortUrl?: string;
|
|
1612
|
+
uploadMode?: UploadMode;
|
|
1613
|
+
uploadChunkBytes?: number;
|
|
1603
1614
|
};
|
|
1604
1615
|
const info =
|
|
1605
1616
|
created.result ??
|
|
@@ -1607,6 +1618,8 @@ export default function RecordRoute() {
|
|
|
1607
1618
|
id: string;
|
|
1608
1619
|
uploadChunkUrl: string;
|
|
1609
1620
|
abortUrl?: string;
|
|
1621
|
+
uploadMode?: UploadMode;
|
|
1622
|
+
uploadChunkBytes?: number;
|
|
1610
1623
|
});
|
|
1611
1624
|
if (!info?.id) {
|
|
1612
1625
|
throw new Error("create-recording did not return an id");
|
|
@@ -1616,14 +1629,22 @@ export default function RecordRoute() {
|
|
|
1616
1629
|
if (isStale()) throw makeAbortError("Upload cancelled");
|
|
1617
1630
|
const uploadBase = `${appBasePath()}${info.uploadChunkUrl}`;
|
|
1618
1631
|
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1632
|
+
// Streaming mode relays each chunk to the provider as one part — the
|
|
1633
|
+
// provider dictates the part size (S3 multipart: uniform >=5 MiB) and
|
|
1634
|
+
// parts must arrive in order, so the parallel pool drops to 1.
|
|
1635
|
+
const streamingUpload = info.uploadMode === "streaming";
|
|
1636
|
+
const chunkBytes =
|
|
1637
|
+
streamingUpload &&
|
|
1638
|
+
typeof info.uploadChunkBytes === "number" &&
|
|
1639
|
+
info.uploadChunkBytes > 0
|
|
1640
|
+
? info.uploadChunkBytes
|
|
1641
|
+
: UPLOAD_CHUNK_BYTES;
|
|
1642
|
+
|
|
1643
|
+
const totalChunks = Math.max(1, Math.ceil(uploadBlob.size / chunkBytes));
|
|
1623
1644
|
|
|
1624
1645
|
const chunkDescs = Array.from({ length: totalChunks }, (_, i) => {
|
|
1625
|
-
const start = i *
|
|
1626
|
-
const end = Math.min(start +
|
|
1646
|
+
const start = i * chunkBytes;
|
|
1647
|
+
const end = Math.min(start + chunkBytes, uploadBlob.size);
|
|
1627
1648
|
const isFinal = i === totalChunks - 1;
|
|
1628
1649
|
return {
|
|
1629
1650
|
index: i,
|
|
@@ -1707,7 +1728,12 @@ export default function RecordRoute() {
|
|
|
1707
1728
|
|
|
1708
1729
|
await Promise.all(
|
|
1709
1730
|
Array.from(
|
|
1710
|
-
{
|
|
1731
|
+
{
|
|
1732
|
+
length: Math.min(
|
|
1733
|
+
streamingUpload ? 1 : UPLOAD_PARALLELISM,
|
|
1734
|
+
parallelChunks.length,
|
|
1735
|
+
),
|
|
1736
|
+
},
|
|
1711
1737
|
worker,
|
|
1712
1738
|
),
|
|
1713
1739
|
);
|
|
@@ -394,6 +394,155 @@ export async function deleteS3ObjectByUrl(url: string): Promise<boolean> {
|
|
|
394
394
|
return true;
|
|
395
395
|
}
|
|
396
396
|
|
|
397
|
+
// ── Multipart (resumable) upload ──────────────────────────────────────
|
|
398
|
+
//
|
|
399
|
+
// Maps the framework's GCS-shaped resumable seam onto S3 multipart uploads:
|
|
400
|
+
// startSession → CreateMultipartUpload, relayChunk → UploadPart (one part per
|
|
401
|
+
// relayed chunk, ETags carried in session meta), completeSession →
|
|
402
|
+
// CompleteMultipartUpload. S3/R2 require every part except the last to be at
|
|
403
|
+
// least 5 MiB (R2 additionally requires uniform part sizes), so the provider
|
|
404
|
+
// advertises `preferredChunkBytes` and upload clients slice on that boundary.
|
|
405
|
+
|
|
406
|
+
/** S3/R2 multipart minimum part size (all parts except the last). */
|
|
407
|
+
export const S3_MULTIPART_PART_BYTES = 5 * 1024 * 1024;
|
|
408
|
+
|
|
409
|
+
interface S3MultipartMeta {
|
|
410
|
+
key: string;
|
|
411
|
+
uploadId: string;
|
|
412
|
+
parts: Array<{ partNumber: number; etag: string }>;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function multipartMetaFromSession(meta: Record<string, unknown>): S3MultipartMeta {
|
|
416
|
+
const key = typeof meta.key === "string" ? meta.key : "";
|
|
417
|
+
const uploadId = typeof meta.uploadId === "string" ? meta.uploadId : "";
|
|
418
|
+
const parts = Array.isArray(meta.parts)
|
|
419
|
+
? (meta.parts as Array<{ partNumber: number; etag: string }>).filter(
|
|
420
|
+
(p) =>
|
|
421
|
+
p &&
|
|
422
|
+
typeof p.partNumber === "number" &&
|
|
423
|
+
typeof p.etag === "string",
|
|
424
|
+
)
|
|
425
|
+
: [];
|
|
426
|
+
if (!key || !uploadId) {
|
|
427
|
+
throw new Error("S3 resumable session meta is missing key/uploadId");
|
|
428
|
+
}
|
|
429
|
+
return { key, uploadId, parts };
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Sign and send one S3 request (SigV4, Web Crypto) with query-string support.
|
|
434
|
+
* Multipart operations need canonical query strings (`uploads=`,
|
|
435
|
+
* `partNumber=…&uploadId=…`), which the single-object helpers above never
|
|
436
|
+
* used.
|
|
437
|
+
*/
|
|
438
|
+
async function s3SignedRequest(
|
|
439
|
+
cfg: S3Config,
|
|
440
|
+
input: {
|
|
441
|
+
method: string;
|
|
442
|
+
key: string;
|
|
443
|
+
query?: Record<string, string>;
|
|
444
|
+
body?: Uint8Array;
|
|
445
|
+
contentType?: string;
|
|
446
|
+
timeoutMs?: number;
|
|
447
|
+
},
|
|
448
|
+
): Promise<Response> {
|
|
449
|
+
const now = new Date();
|
|
450
|
+
const amzDate =
|
|
451
|
+
now
|
|
452
|
+
.toISOString()
|
|
453
|
+
.replace(/[:-]|\.\d{3}/g, "")
|
|
454
|
+
.slice(0, 15) + "Z";
|
|
455
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
456
|
+
const credentialScope = `${dateStamp}/${cfg.region}/s3/aws4_request`;
|
|
457
|
+
|
|
458
|
+
const hostUrl = new URL(cfg.endpoint);
|
|
459
|
+
const host = hostUrl.host;
|
|
460
|
+
const canonicalUri = `/${cfg.bucket}/${input.key.split("/").map(rfc3986).join("/")}`;
|
|
461
|
+
const canonicalQuery = Object.entries(input.query ?? {})
|
|
462
|
+
.map(([k, v]) => [rfc3986(k), rfc3986(v)] as const)
|
|
463
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
464
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
465
|
+
.join("&");
|
|
466
|
+
|
|
467
|
+
const body = input.body ?? new Uint8Array(0);
|
|
468
|
+
const payloadHash = await sha256(body);
|
|
469
|
+
|
|
470
|
+
const headers: Record<string, string> = {
|
|
471
|
+
host,
|
|
472
|
+
...(input.contentType ? { "content-type": input.contentType } : {}),
|
|
473
|
+
"x-amz-content-sha256": payloadHash,
|
|
474
|
+
"x-amz-date": amzDate,
|
|
475
|
+
};
|
|
476
|
+
|
|
477
|
+
const signedHeaderKeys = Object.keys(headers).sort();
|
|
478
|
+
const signedHeaders = signedHeaderKeys.join(";");
|
|
479
|
+
const canonicalHeaders =
|
|
480
|
+
signedHeaderKeys.map((k) => `${k}:${headers[k]}`).join("\n") + "\n";
|
|
481
|
+
|
|
482
|
+
const canonicalRequest = [
|
|
483
|
+
input.method,
|
|
484
|
+
canonicalUri,
|
|
485
|
+
canonicalQuery,
|
|
486
|
+
canonicalHeaders,
|
|
487
|
+
signedHeaders,
|
|
488
|
+
payloadHash,
|
|
489
|
+
].join("\n");
|
|
490
|
+
|
|
491
|
+
const crHash = await sha256(new TextEncoder().encode(canonicalRequest));
|
|
492
|
+
const stringToSign = [
|
|
493
|
+
"AWS4-HMAC-SHA256",
|
|
494
|
+
amzDate,
|
|
495
|
+
credentialScope,
|
|
496
|
+
crHash,
|
|
497
|
+
].join("\n");
|
|
498
|
+
|
|
499
|
+
const signingKey = await deriveSigningKey(
|
|
500
|
+
cfg.secretAccessKey,
|
|
501
|
+
dateStamp,
|
|
502
|
+
cfg.region,
|
|
503
|
+
);
|
|
504
|
+
const signature = toHex(await hmac(signingKey, stringToSign));
|
|
505
|
+
|
|
506
|
+
const authorization =
|
|
507
|
+
`AWS4-HMAC-SHA256 Credential=${cfg.accessKeyId}/${credentialScope}, ` +
|
|
508
|
+
`SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
509
|
+
|
|
510
|
+
const url = `${cfg.endpoint}${canonicalUri}${canonicalQuery ? `?${canonicalQuery}` : ""}`;
|
|
511
|
+
return fetchWithTimeout(
|
|
512
|
+
url,
|
|
513
|
+
{
|
|
514
|
+
method: input.method,
|
|
515
|
+
headers: {
|
|
516
|
+
...headers,
|
|
517
|
+
Authorization: authorization,
|
|
518
|
+
...(input.body ? { "Content-Length": String(body.byteLength) } : {}),
|
|
519
|
+
},
|
|
520
|
+
...(input.body
|
|
521
|
+
? {
|
|
522
|
+
body: body.buffer.slice(
|
|
523
|
+
body.byteOffset,
|
|
524
|
+
body.byteOffset + body.byteLength,
|
|
525
|
+
) as BodyInit,
|
|
526
|
+
}
|
|
527
|
+
: {}),
|
|
528
|
+
},
|
|
529
|
+
input.timeoutMs ?? S3_PUT_TIMEOUT_MS,
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function publicObjectUrl(cfg: S3Config, key: string): string {
|
|
534
|
+
return cfg.publicBaseUrl
|
|
535
|
+
? `${cfg.publicBaseUrl}/${key}`
|
|
536
|
+
: `${cfg.endpoint}/${cfg.bucket}/${key}`;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function xmlEscape(value: string): string {
|
|
540
|
+
return value
|
|
541
|
+
.replace(/&/g, "&")
|
|
542
|
+
.replace(/</g, "<")
|
|
543
|
+
.replace(/>/g, ">");
|
|
544
|
+
}
|
|
545
|
+
|
|
397
546
|
// ── Provider ──────────────────────────────────────────────────────────
|
|
398
547
|
|
|
399
548
|
export const s3FileUploadProvider: FileUploadProvider = {
|
|
@@ -419,4 +568,111 @@ export const s3FileUploadProvider: FileUploadProvider = {
|
|
|
419
568
|
const publicUrl = await putObject(cfg, objectKey, bytes, contentType);
|
|
420
569
|
return { url: publicUrl, provider: "s3" };
|
|
421
570
|
},
|
|
571
|
+
resumable: {
|
|
572
|
+
preferredChunkBytes: S3_MULTIPART_PART_BYTES,
|
|
573
|
+
startSession: async (filename, mimeType) => {
|
|
574
|
+
const cfg = await readS3Config();
|
|
575
|
+
if (!cfg) throw new Error("S3 credentials are not configured");
|
|
576
|
+
|
|
577
|
+
const ext = filename?.split(".").pop() ?? "bin";
|
|
578
|
+
const stamp = Date.now();
|
|
579
|
+
const rand = Math.random().toString(36).slice(2, 10);
|
|
580
|
+
const objectKey = `clips/${stamp}-${rand}.${ext}`;
|
|
581
|
+
const contentType = mimeType || "application/octet-stream";
|
|
582
|
+
|
|
583
|
+
const res = await s3SignedRequest(cfg, {
|
|
584
|
+
method: "POST",
|
|
585
|
+
key: objectKey,
|
|
586
|
+
query: { uploads: "" },
|
|
587
|
+
contentType,
|
|
588
|
+
timeoutMs: S3_DELETE_TIMEOUT_MS,
|
|
589
|
+
});
|
|
590
|
+
const text = await res.text().catch(() => "");
|
|
591
|
+
if (!res.ok) {
|
|
592
|
+
throw new Error(
|
|
593
|
+
`S3 CreateMultipartUpload failed (${res.status}): ${text || res.statusText}`,
|
|
594
|
+
);
|
|
595
|
+
}
|
|
596
|
+
const uploadId = /<UploadId>([^<]+)<\/UploadId>/.exec(text)?.[1];
|
|
597
|
+
if (!uploadId) {
|
|
598
|
+
throw new Error(
|
|
599
|
+
"S3 CreateMultipartUpload succeeded but no UploadId was returned",
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
sessionId: uploadId,
|
|
604
|
+
meta: { key: objectKey, uploadId, contentType, parts: [] },
|
|
605
|
+
};
|
|
606
|
+
},
|
|
607
|
+
relayChunk: async (session, contentRange, bytes) => {
|
|
608
|
+
// Recorder close sentinel ("bytes */<total>", empty body): all data
|
|
609
|
+
// parts were already uploaded; CompleteMultipartUpload happens in
|
|
610
|
+
// completeSession. Nothing to relay.
|
|
611
|
+
if (/^bytes \*\//.test(contentRange)) {
|
|
612
|
+
return { ok: true, status: 200 };
|
|
613
|
+
}
|
|
614
|
+
const cfg = await readS3Config();
|
|
615
|
+
if (!cfg) return { ok: false, status: 500 };
|
|
616
|
+
const meta = multipartMetaFromSession(session.meta);
|
|
617
|
+
const partNumber = meta.parts.length + 1;
|
|
618
|
+
const res = await s3SignedRequest(cfg, {
|
|
619
|
+
method: "PUT",
|
|
620
|
+
key: meta.key,
|
|
621
|
+
query: {
|
|
622
|
+
partNumber: String(partNumber),
|
|
623
|
+
uploadId: meta.uploadId,
|
|
624
|
+
},
|
|
625
|
+
body: bytes,
|
|
626
|
+
});
|
|
627
|
+
if (!res.ok) {
|
|
628
|
+
const text = await res.text().catch(() => "");
|
|
629
|
+
console.error(
|
|
630
|
+
`[s3-resumable] UploadPart ${partNumber} failed (${res.status}): ${text.slice(0, 300)}`,
|
|
631
|
+
);
|
|
632
|
+
return { ok: false, status: res.status };
|
|
633
|
+
}
|
|
634
|
+
await res.body?.cancel().catch(() => {});
|
|
635
|
+
const etag = res.headers.get("etag") ?? "";
|
|
636
|
+
if (!etag) return { ok: false, status: 502 };
|
|
637
|
+
return {
|
|
638
|
+
ok: true,
|
|
639
|
+
status: 200,
|
|
640
|
+
updatedMeta: {
|
|
641
|
+
parts: [...meta.parts, { partNumber, etag }],
|
|
642
|
+
},
|
|
643
|
+
};
|
|
644
|
+
},
|
|
645
|
+
completeSession: async (session) => {
|
|
646
|
+
const cfg = await readS3Config();
|
|
647
|
+
if (!cfg) throw new Error("S3 credentials are not configured");
|
|
648
|
+
const meta = multipartMetaFromSession(session.meta);
|
|
649
|
+
if (meta.parts.length === 0) {
|
|
650
|
+
throw new Error("S3 resumable session has no uploaded parts");
|
|
651
|
+
}
|
|
652
|
+
const xml =
|
|
653
|
+
`<CompleteMultipartUpload>` +
|
|
654
|
+
meta.parts
|
|
655
|
+
.map(
|
|
656
|
+
(p) =>
|
|
657
|
+
`<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${xmlEscape(p.etag)}</ETag></Part>`,
|
|
658
|
+
)
|
|
659
|
+
.join("") +
|
|
660
|
+
`</CompleteMultipartUpload>`;
|
|
661
|
+
const res = await s3SignedRequest(cfg, {
|
|
662
|
+
method: "POST",
|
|
663
|
+
key: meta.key,
|
|
664
|
+
query: { uploadId: meta.uploadId },
|
|
665
|
+
body: new TextEncoder().encode(xml),
|
|
666
|
+
contentType: "application/xml",
|
|
667
|
+
});
|
|
668
|
+
const text = await res.text().catch(() => "");
|
|
669
|
+
// S3 can return 200 with an <Error> body for CompleteMultipartUpload.
|
|
670
|
+
if (!res.ok || /<Error>/.test(text)) {
|
|
671
|
+
throw new Error(
|
|
672
|
+
`S3 CompleteMultipartUpload failed (${res.status}): ${text.slice(0, 300) || res.statusText}`,
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
return publicObjectUrl(cfg, meta.key);
|
|
676
|
+
},
|
|
677
|
+
},
|
|
422
678
|
};
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { enabledFlag } from "./env-flags.js";
|
|
2
|
+
import { requiresConfiguredVideoStorage } from "./video-storage.js";
|
|
2
3
|
|
|
3
4
|
// Streaming resumable uploads are deployment opt-in while the provider/finalize
|
|
4
|
-
// path hardens
|
|
5
|
-
//
|
|
5
|
+
// path hardens — EXCEPT when the deployment cannot buffer chunks in SQL at all
|
|
6
|
+
// (production / remote database), where streaming to the provider is the only
|
|
7
|
+
// viable upload path and gating it behind the opt-in flag guarantees every
|
|
8
|
+
// upload dies with a 409. Set CLIPS_ENABLE_STREAMING_UPLOAD=1 to opt in on
|
|
9
|
+
// SQL-scratch-capable deployments; CLIPS_DISABLE_STREAMING_UPLOAD=1 still
|
|
10
|
+
// forces the buffered fallback everywhere.
|
|
6
11
|
export function isStreamingUploadDisabled(): boolean {
|
|
7
12
|
return enabledFlag(process.env.CLIPS_DISABLE_STREAMING_UPLOAD);
|
|
8
13
|
}
|
|
@@ -12,7 +17,8 @@ export function shouldEnableStreamingUpload(args: {
|
|
|
12
17
|
mimeType?: string | null;
|
|
13
18
|
}): boolean {
|
|
14
19
|
if (isStreamingUploadDisabled()) return false;
|
|
15
|
-
|
|
20
|
+
const optedIn = enabledFlag(process.env.CLIPS_ENABLE_STREAMING_UPLOAD);
|
|
21
|
+
if (!optedIn && !requiresConfiguredVideoStorage()) return false;
|
|
16
22
|
|
|
17
23
|
const mimeType = (args.mimeType ?? "").split(";")[0]?.trim().toLowerCase();
|
|
18
24
|
return !mimeType || mimeType.startsWith("video/");
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platform-aware per-request chunk upload cap.
|
|
3
|
+
*
|
|
4
|
+
* Netlify functions buffer request bodies with a 6 MB cap, and binary bodies
|
|
5
|
+
* are base64-encoded by the gateway (effective cap ~4.5 MB) — so Netlify
|
|
6
|
+
* deployments must keep chunks at 4 MiB. Every other runtime (local dev,
|
|
7
|
+
* Cloudflare workerd, Vercel) accepts larger bodies, and S3-compatible
|
|
8
|
+
* multipart uploads NEED 5 MiB parts (see s3-upload-provider), so the cap is
|
|
9
|
+
* lifted to 5 MiB + slack there.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const NETLIFY_MAX_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
13
|
+
const DEFAULT_MAX_CHUNK_BYTES = 5 * 1024 * 1024 + 256 * 1024;
|
|
14
|
+
|
|
15
|
+
export function maxChunkUploadBytes(): number {
|
|
16
|
+
return process.env.NETLIFY
|
|
17
|
+
? NETLIFY_MAX_CHUNK_BYTES
|
|
18
|
+
: DEFAULT_MAX_CHUNK_BYTES;
|
|
19
|
+
}
|
|
@@ -57,13 +57,14 @@ import {
|
|
|
57
57
|
shouldRejectVideoUploadWithoutStorage,
|
|
58
58
|
STORAGE_SETUP_REQUIRED_REASON,
|
|
59
59
|
} from "../../../../lib/video-storage.js";
|
|
60
|
+
import { maxChunkUploadBytes } from "../../../../lib/upload-chunk-limits.js";
|
|
60
61
|
|
|
61
62
|
const RECORDING_TOO_LARGE_REASON = `Recording exceeds the ${Math.round(MAX_RECORDING_UPLOAD_BYTES / (1024 * 1024))} MB size limit. Please record a shorter clip.`;
|
|
62
63
|
|
|
63
|
-
// Netlify
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
const MAX_CHUNK_BYTES =
|
|
64
|
+
// Platform-aware: Netlify's buffered-request gateway caps chunks at 4 MiB;
|
|
65
|
+
// other runtimes allow the 5 MiB parts S3-compatible multipart needs. See
|
|
66
|
+
// server/lib/upload-chunk-limits.ts.
|
|
67
|
+
const MAX_CHUNK_BYTES = maxChunkUploadBytes();
|
|
67
68
|
|
|
68
69
|
const ALLOWED_RECORDING_MIME_TYPES = new Set([
|
|
69
70
|
"video/webm",
|
|
@@ -760,6 +761,16 @@ async function handleResumableChunk(
|
|
|
760
761
|
};
|
|
761
762
|
}
|
|
762
763
|
} else {
|
|
764
|
+
// Enforce the recording size ceiling server-side — the provider session
|
|
765
|
+
// was opened with this budget but S3-style multipart uploads do not
|
|
766
|
+
// enforce it themselves.
|
|
767
|
+
if (
|
|
768
|
+
session.bytesUploaded + bytes.byteLength >
|
|
769
|
+
MAX_RECORDING_UPLOAD_BYTES
|
|
770
|
+
) {
|
|
771
|
+
setResponseStatus(event, 413);
|
|
772
|
+
return { ok: false, error: RECORDING_TOO_LARGE_REASON };
|
|
773
|
+
}
|
|
763
774
|
// Forward the data chunk to the provider and advance offsets only after
|
|
764
775
|
// the provider confirms receipt (308 Resume Incomplete for non-final, 2xx for final).
|
|
765
776
|
const start = session.bytesUploaded;
|