@taprootio/site-authoring 0.12.7 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/package.json +3 -2
- package/src/api.js +81 -2
- package/src/cli.js +1 -1
- package/src/constants.js +3 -0
- package/src/content/components.js +159 -2
- package/src/content/free-form-section-registry.json +11 -1
- package/src/content/markdown.js +12 -2
- package/src/content/media-presentation.js +123 -0
- package/src/content/tiptap-prosemirror.ts +12 -64
- package/src/content/video-embed-url.js +104 -0
- package/src/content/vocabulary.js +9 -32
- package/src/field-schema.json +11 -1
- package/src/field-validation.js +8 -2
- package/src/fixture-contract.js +13 -2
- package/src/fixture-init.js +3 -0
- package/src/image-metadata.js +1 -1
- package/src/reference-help.js +18 -4
- package/src/session.js +22 -1
- package/src/transport.js +77 -3
- package/src/typed-pages.js +83 -13
- package/src/verbs/media-upload.js +352 -28
- package/src/verbs/pages-push.js +97 -2
- package/src/verbs/validate.js +37 -2
- package/src/video-metadata.js +90 -0
- package/src/video-prepare.js +136 -0
- package/src/workspace.js +113 -1
package/src/typed-pages.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FREE_FORM_TEMPLATE_VERSION, TEMPLATE_TYPE_FREE_FORM } from "./api.js";
|
|
2
|
+
import { isVideoEmbedId } from "./content/video-embed-url.js";
|
|
2
3
|
import { SiteAuthoringError } from "./errors.js";
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -596,6 +597,26 @@ export const COVER_IMAGE_UNUSED_MESSAGE = "The selected cover image must be used
|
|
|
596
597
|
* wrongly thinks unused would be refused for nothing, and the server has the
|
|
597
598
|
* final say.
|
|
598
599
|
*/
|
|
600
|
+
/**
|
|
601
|
+
* The parsed JSON of a `componentData` string, or `undefined` when it is not
|
|
602
|
+
* JSON. The server decodes HTML entities (up to three rounds) before reading
|
|
603
|
+
* component JSON, so a string that only parses once decoded is read the way the
|
|
604
|
+
* server reads it.
|
|
605
|
+
*/
|
|
606
|
+
function parseComponentData(text) {
|
|
607
|
+
let decoded = text;
|
|
608
|
+
for (let round = 0; round < 3 && /&(?:quot|amp|#39|lt|gt);/u.test(decoded); round += 1) {
|
|
609
|
+
decoded = decoded.replace(/"/gu, "\"").replace(/'/gu, "'").replace(/</gu, "<").replace(/>/gu, ">")
|
|
610
|
+
.replace(/&/gu, "&");
|
|
611
|
+
}
|
|
612
|
+
for (const candidate of [text, decoded]) {
|
|
613
|
+
try {
|
|
614
|
+
return JSON.parse(candidate);
|
|
615
|
+
} catch { /* not JSON: try the decoded form, then nothing to read */ }
|
|
616
|
+
}
|
|
617
|
+
return undefined;
|
|
618
|
+
}
|
|
619
|
+
|
|
599
620
|
export function usedImageIds(document_) {
|
|
600
621
|
const found = new Set();
|
|
601
622
|
const visit = (value) => {
|
|
@@ -607,19 +628,8 @@ export function usedImageIds(document_) {
|
|
|
607
628
|
else if (key === "imageIds" && Array.isArray(child)) {
|
|
608
629
|
child.filter((id) => typeof id === "string").forEach((id) => found.add(id.toLowerCase()));
|
|
609
630
|
} else if (key === "componentData" && typeof child === "string") {
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
for (let round = 0; round < 3 && /&(?:quot|amp|#39|lt|gt);/u.test(text); round += 1) {
|
|
613
|
-
text = text.replace(/"/gu, "\"").replace(/'/gu, "'").replace(/</gu, "<").replace(/>/gu, ">")
|
|
614
|
-
.replace(/&/gu, "&");
|
|
615
|
-
}
|
|
616
|
-
try {
|
|
617
|
-
visit(JSON.parse(child));
|
|
618
|
-
} catch {
|
|
619
|
-
try {
|
|
620
|
-
visit(JSON.parse(text));
|
|
621
|
-
} catch { /* not JSON: nothing to read */ }
|
|
622
|
-
}
|
|
631
|
+
const parsed = parseComponentData(child);
|
|
632
|
+
if (parsed !== undefined) visit(parsed);
|
|
623
633
|
} else visit(child);
|
|
624
634
|
}
|
|
625
635
|
}
|
|
@@ -629,6 +639,66 @@ export function usedImageIds(document_) {
|
|
|
629
639
|
return found;
|
|
630
640
|
}
|
|
631
641
|
|
|
642
|
+
/**
|
|
643
|
+
* Every video id the page places: the `videoId` of each `video` component block
|
|
644
|
+
* in any of its documents. Block-scoped on purpose — a `videoId` anywhere else
|
|
645
|
+
* is author prose, not a reference the publisher resolves.
|
|
646
|
+
*/
|
|
647
|
+
export function placedVideoIds(document_) {
|
|
648
|
+
const found = new Set();
|
|
649
|
+
const visit = (value) => {
|
|
650
|
+
if (Array.isArray(value)) {
|
|
651
|
+
value.forEach(visit);
|
|
652
|
+
} else if (isPlainObject(value)) {
|
|
653
|
+
if (value.type === "componentBlock" && isPlainObject(value.attrs) && value.attrs.componentType === "video") {
|
|
654
|
+
const data = typeof value.attrs.componentData === "string"
|
|
655
|
+
? parseComponentData(value.attrs.componentData)
|
|
656
|
+
: undefined;
|
|
657
|
+
if (isPlainObject(data) && typeof data.videoId === "string") found.add(data.videoId.toLowerCase());
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
Object.values(value).forEach(visit);
|
|
661
|
+
}
|
|
662
|
+
};
|
|
663
|
+
for (const { doc } of contentDocuments(document_)) visit(doc);
|
|
664
|
+
return found;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/**
|
|
668
|
+
* The `video-embed` blocks in any of the page's documents that name a video and
|
|
669
|
+
* hold no poster, each with a way to set one. `pages push` copies the provider's
|
|
670
|
+
* thumbnail into the site's images for these (TR01109). Block-scoped like
|
|
671
|
+
* `placedVideoIds`.
|
|
672
|
+
*/
|
|
673
|
+
export function posterlessVideoEmbeds(document_) {
|
|
674
|
+
const found = [];
|
|
675
|
+
const visit = (value) => {
|
|
676
|
+
if (Array.isArray(value)) {
|
|
677
|
+
value.forEach(visit);
|
|
678
|
+
} else if (isPlainObject(value)) {
|
|
679
|
+
if (value.type === "componentBlock" && isPlainObject(value.attrs) && value.attrs.componentType === "video-embed") {
|
|
680
|
+
const data = typeof value.attrs.componentData === "string"
|
|
681
|
+
? parseComponentData(value.attrs.componentData)
|
|
682
|
+
: undefined;
|
|
683
|
+
if (isPlainObject(data) && (data.poster === undefined || data.poster === null)
|
|
684
|
+
&& isVideoEmbedId(data.provider, data.videoId)) {
|
|
685
|
+
found.push({
|
|
686
|
+
provider: data.provider,
|
|
687
|
+
videoId: data.videoId,
|
|
688
|
+
setPoster: (poster) => {
|
|
689
|
+
value.attrs.componentData = JSON.stringify({ ...data, poster });
|
|
690
|
+
},
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
Object.values(value).forEach(visit);
|
|
696
|
+
}
|
|
697
|
+
};
|
|
698
|
+
for (const { doc } of contentDocuments(document_)) visit(doc);
|
|
699
|
+
return found;
|
|
700
|
+
}
|
|
701
|
+
|
|
632
702
|
/** The selected cover image when the page does not use it, else `undefined`. */
|
|
633
703
|
export function unusedCoverImageId(document_) {
|
|
634
704
|
if (documentTemplate(document_) === TEMPLATE_FREE_FORM) return undefined;
|
|
@@ -1,17 +1,25 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
|
|
1
3
|
import {
|
|
2
4
|
confirmImageUpload,
|
|
5
|
+
confirmVideoUpload,
|
|
3
6
|
IMAGE_OWNERSHIP_SCOPE_SITE,
|
|
4
7
|
IMAGE_PROCESSING_STATE_COMPLETE,
|
|
5
8
|
IMAGE_PROCESSING_STATE_FAILED,
|
|
6
9
|
listSiteImages,
|
|
10
|
+
listVideos,
|
|
7
11
|
normalizeImage,
|
|
8
12
|
poll,
|
|
9
13
|
requestImageUpload,
|
|
14
|
+
requestVideoUpload,
|
|
10
15
|
withRefusalGuidance,
|
|
11
16
|
} from "../api.js";
|
|
12
17
|
import { LIMITS, VERB_MEDIA_UPLOAD } from "../constants.js";
|
|
13
|
-
import { SiteAuthoringError } from "../errors.js";
|
|
18
|
+
import { SiteAuthoringError, sanitizeDiagnostic } from "../errors.js";
|
|
14
19
|
import { contentHash, inspectImageBytes } from "../image-metadata.js";
|
|
20
|
+
import { ApiError } from "../transport.js";
|
|
21
|
+
import { sniffVideoContentType, VIDEO_EXTENSIONS, videoTooLarge } from "../video-metadata.js";
|
|
22
|
+
import { mp4Name, prepareVideoFile } from "../video-prepare.js";
|
|
15
23
|
import { boundedList, openSession, successResult, warnIfExternalWritesPaused } from "../session.js";
|
|
16
24
|
import {
|
|
17
25
|
ensureWorkspaceRoot,
|
|
@@ -19,8 +27,10 @@ import {
|
|
|
19
27
|
MEDIA_DIRECTORY,
|
|
20
28
|
MEDIA_MANIFEST_FILE_NAME,
|
|
21
29
|
MEDIA_MANIFEST_VERSION,
|
|
30
|
+
openWorkspaceFileStream,
|
|
22
31
|
readMediaManifest,
|
|
23
32
|
readWorkspaceFile,
|
|
33
|
+
resolveWorkspacePath,
|
|
24
34
|
SAFE_MEDIA_SEGMENT,
|
|
25
35
|
walkWorkspaceFiles,
|
|
26
36
|
WORKSPACE_LIMITS,
|
|
@@ -30,6 +40,14 @@ import {
|
|
|
30
40
|
/**
|
|
31
41
|
* `media upload` — hash, request, PUT, confirm, then wait for processing.
|
|
32
42
|
*
|
|
43
|
+
* A file whose container is MP4, MOV or WebM takes the video path instead. Taproot does
|
|
44
|
+
* not encode, so the file is read with mediabunny first: an H.264 and AAC file that is not
|
|
45
|
+
* an MP4 with its index first is rewritten as one (copying the tracks, never re-encoding),
|
|
46
|
+
* and the real codec strings are declared. `RequestVideoUpload`, a presigned PUT, and
|
|
47
|
+
* `ConfirmVideoUpload` follow; the video is ready when confirm returns, with nothing to
|
|
48
|
+
* wait for. The server decides what is accepted and its one-line refusal is printed as it
|
|
49
|
+
* came. The container is read from the file's bytes, not its extension.
|
|
50
|
+
*
|
|
33
51
|
* The client computes the SHA-256 and the pixel dimensions before it asks for
|
|
34
52
|
* anything, because `RequestImageUpload` needs all three: the hash drives the
|
|
35
53
|
* dedup short-circuit, and the dimensions are recorded with the image. When the
|
|
@@ -56,7 +74,7 @@ import {
|
|
|
56
74
|
|
|
57
75
|
const MAXIMUM_REPORTED = 200;
|
|
58
76
|
const MAXIMUM_FILES = 500;
|
|
59
|
-
const MEDIA_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
|
|
77
|
+
const MEDIA_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp", ...VIDEO_EXTENSIONS];
|
|
60
78
|
const MEDIA_WALK_OPTIONS = Object.freeze({
|
|
61
79
|
segmentPattern: SAFE_MEDIA_SEGMENT,
|
|
62
80
|
segmentDescription: "letters, digits, '.', '_', '-', and '@'",
|
|
@@ -78,6 +96,14 @@ function requireUploadCapability(response, fileName) {
|
|
|
78
96
|
return { url: response.presignedUrl, requiredHeaders: response.requiredHeaders };
|
|
79
97
|
}
|
|
80
98
|
|
|
99
|
+
function videoComponent(videoId) {
|
|
100
|
+
const componentData = JSON.stringify({ videoId });
|
|
101
|
+
return {
|
|
102
|
+
markdown: `\`\`\`component:video\n${componentData}\n\`\`\``,
|
|
103
|
+
block: { type: "componentBlock", attrs: { componentType: "video", componentData } },
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
81
107
|
function fileNameOf(relativePath) {
|
|
82
108
|
return relativePath.slice(relativePath.lastIndexOf("/") + 1);
|
|
83
109
|
}
|
|
@@ -134,35 +160,255 @@ async function resolveUploadTargets(workspaceDir, positionals) {
|
|
|
134
160
|
return files;
|
|
135
161
|
}
|
|
136
162
|
|
|
137
|
-
|
|
138
|
-
|
|
163
|
+
/**
|
|
164
|
+
* Waits until every uploaded item has reached a terminal state in its library
|
|
165
|
+
* listing, and returns what was last observed for each. One wait serves images
|
|
166
|
+
* and videos: only what is read, how an entry is identified and which states
|
|
167
|
+
* end the wait differ.
|
|
168
|
+
*/
|
|
169
|
+
async function waitForTerminalStates(client, ids, {
|
|
170
|
+
read,
|
|
171
|
+
idOf,
|
|
172
|
+
stateOf,
|
|
173
|
+
terminalStates,
|
|
174
|
+
timeoutMilliseconds,
|
|
175
|
+
timeoutCode,
|
|
176
|
+
waitingMessage,
|
|
177
|
+
onProgress,
|
|
178
|
+
now,
|
|
179
|
+
}) {
|
|
180
|
+
const pending = new Set(ids);
|
|
139
181
|
if (pending.size === 0) return new Map();
|
|
140
182
|
const observed = new Map();
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
183
|
+
try {
|
|
184
|
+
return await poll({
|
|
185
|
+
client,
|
|
186
|
+
now,
|
|
187
|
+
onProgress,
|
|
188
|
+
timeoutMilliseconds,
|
|
189
|
+
// The poller's `{ deadline, now }` travels into every page of the listing,
|
|
190
|
+
// so the bound holds *within* one read of a large library and not only
|
|
191
|
+
// between reads.
|
|
192
|
+
read,
|
|
193
|
+
evaluate: (entries) => {
|
|
194
|
+
for (const entry of entries) {
|
|
195
|
+
if (pending.has(idOf(entry))) observed.set(idOf(entry), entry);
|
|
196
|
+
}
|
|
197
|
+
const waiting = [...pending].filter((id) => !terminalStates.includes(stateOf(observed.get(id))));
|
|
198
|
+
if (waiting.length === 0) return { done: true, value: observed };
|
|
199
|
+
return { done: false, progress: waitingMessage(waiting.length) };
|
|
200
|
+
},
|
|
201
|
+
timeoutCode,
|
|
202
|
+
});
|
|
203
|
+
} catch (error) {
|
|
204
|
+
// What was observed before the deadline travels with the refusal, so a
|
|
205
|
+
// caller can still record the items that did reach a terminal state.
|
|
206
|
+
if (error instanceof SiteAuthoringError) error.observed = observed;
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* `byId` reads the named images directly instead of paging the library, for an image
|
|
213
|
+
* the library never lists (a copied embed poster).
|
|
214
|
+
*/
|
|
215
|
+
export function waitForProcessing(client, siteId, imageIds, { onProgress, now, byId = false }) {
|
|
216
|
+
return waitForTerminalStates(client, imageIds, {
|
|
217
|
+
read: async (requestOptions) =>
|
|
218
|
+
(await listSiteImages(client, siteId, requestOptions, byId ? { imageIds } : {})).images,
|
|
219
|
+
idOf: (image) => image.imageId,
|
|
220
|
+
stateOf: (image) => image?.processingState,
|
|
221
|
+
terminalStates: [IMAGE_PROCESSING_STATE_COMPLETE, IMAGE_PROCESSING_STATE_FAILED],
|
|
145
222
|
timeoutMilliseconds: LIMITS.uploadMilliseconds,
|
|
146
|
-
// The poller's `{ deadline, now }` travels into every page of the listing,
|
|
147
|
-
// so the bound holds *within* one read of a large library and not only
|
|
148
|
-
// between reads.
|
|
149
|
-
read: async (requestOptions) => (await listSiteImages(client, siteId, requestOptions)).images,
|
|
150
|
-
evaluate: (images) => {
|
|
151
|
-
for (const image of images) {
|
|
152
|
-
if (!pending.has(image.imageId)) continue;
|
|
153
|
-
observed.set(image.imageId, image);
|
|
154
|
-
}
|
|
155
|
-
const waiting = [...pending].filter((imageId) => {
|
|
156
|
-
const state = observed.get(imageId)?.processingState;
|
|
157
|
-
return state !== IMAGE_PROCESSING_STATE_COMPLETE && state !== IMAGE_PROCESSING_STATE_FAILED;
|
|
158
|
-
});
|
|
159
|
-
if (waiting.length === 0) return { done: true, value: observed };
|
|
160
|
-
return { done: false, progress: `Waiting for ${waiting.length} image(s) to finish processing.` };
|
|
161
|
-
},
|
|
162
223
|
timeoutCode: "media.processing_timeout",
|
|
224
|
+
waitingMessage: (count) => `Waiting for ${count} image(s) to finish processing.`,
|
|
225
|
+
onProgress,
|
|
226
|
+
now,
|
|
163
227
|
});
|
|
164
228
|
}
|
|
165
229
|
|
|
230
|
+
/** Where the server words a refusal of a video upload; the line is printed as it came. */
|
|
231
|
+
const VIDEO_REFUSAL_FIELDS = Object.freeze([
|
|
232
|
+
"VideoCodec",
|
|
233
|
+
"AudioCodec",
|
|
234
|
+
"FileName",
|
|
235
|
+
"SiteId",
|
|
236
|
+
"UploadId",
|
|
237
|
+
"ContentType",
|
|
238
|
+
"SizeBytes",
|
|
239
|
+
"Video",
|
|
240
|
+
"UpgradePrompt",
|
|
241
|
+
"VideoNotIncluded",
|
|
242
|
+
"VideoPendingDowngrade",
|
|
243
|
+
]);
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The server is the single authority for refusing a video, so its one-line message is shown
|
|
247
|
+
* rather than the generic "rejected the request field" the transport would give. Anything
|
|
248
|
+
* that is not such a refusal is rethrown unchanged.
|
|
249
|
+
*/
|
|
250
|
+
function surfaceVideoRefusal(error, file) {
|
|
251
|
+
if (error instanceof ApiError) {
|
|
252
|
+
for (const field of VIDEO_REFUSAL_FIELDS) {
|
|
253
|
+
const description = error.descriptionFor?.(field);
|
|
254
|
+
if (typeof description === "string" && description !== "") {
|
|
255
|
+
return new SiteAuthoringError(
|
|
256
|
+
"media.video_refused",
|
|
257
|
+
`'${file}': ${sanitizeDiagnostic(description, "Taproot refused this video.")}`,
|
|
258
|
+
{ field: file },
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return error;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Uploads one video: prepare it, request, a PUT, confirm. Returns null when the file is not a
|
|
268
|
+
* video container, so the caller takes the image path. The file is opened once for the size
|
|
269
|
+
* and the prefix and held open across the PUT's retries, each of which streams it again from
|
|
270
|
+
* the start; a file that had to be remuxed is sent from memory (it is at most the largest
|
|
271
|
+
* licence cap).
|
|
272
|
+
*
|
|
273
|
+
* Unlike an image, a video has no content hash to short-circuit on, and a re-run would upload
|
|
274
|
+
* every file again, each a new library entry and a new quota reservation. So an unchanged file
|
|
275
|
+
* this workspace already uploaded (the recorded size and modification time match) is reused when
|
|
276
|
+
* the site's library still holds that video.
|
|
277
|
+
*
|
|
278
|
+
* The upload id is recorded before the bytes are sent, and confirming an upload id twice returns
|
|
279
|
+
* the same video. So a PUT or a confirm whose answer was lost is resumed by the next run (the
|
|
280
|
+
* confirm says whether the object arrived; if it did not, the file is sent again), not repeated
|
|
281
|
+
* as a second reservation. A 412 on a retried create-only PUT means it had already landed.
|
|
282
|
+
*/
|
|
283
|
+
async function uploadVideoFile({
|
|
284
|
+
client,
|
|
285
|
+
config,
|
|
286
|
+
siteId,
|
|
287
|
+
file,
|
|
288
|
+
fileName,
|
|
289
|
+
recorded,
|
|
290
|
+
readLibrary,
|
|
291
|
+
recordPending,
|
|
292
|
+
onProgress,
|
|
293
|
+
}) {
|
|
294
|
+
// Opened with no size bound of its own: the video limit is checked below, once
|
|
295
|
+
// the container says this is a video, so an oversized video is named as one and
|
|
296
|
+
// not as a generic file that is too large.
|
|
297
|
+
const source = await openWorkspaceFileStream(config.workspaceDir, file, Number.MAX_SAFE_INTEGER);
|
|
298
|
+
try {
|
|
299
|
+
if (sniffVideoContentType(source.header) === undefined) return null;
|
|
300
|
+
if (source.byteLength > WORKSPACE_LIMITS.videoBytes) {
|
|
301
|
+
throw videoTooLarge(file, WORKSPACE_LIMITS.videoBytes);
|
|
302
|
+
}
|
|
303
|
+
const contentType = "video/mp4";
|
|
304
|
+
const base = {
|
|
305
|
+
contentType,
|
|
306
|
+
byteLength: source.byteLength,
|
|
307
|
+
modifiedMilliseconds: source.modifiedMilliseconds,
|
|
308
|
+
};
|
|
309
|
+
if (
|
|
310
|
+
recorded?.byteLength === source.byteLength
|
|
311
|
+
&& recorded?.modifiedMilliseconds === source.modifiedMilliseconds
|
|
312
|
+
&& typeof recorded?.videoId === "string"
|
|
313
|
+
) {
|
|
314
|
+
if (recorded.pendingConfirm === true) {
|
|
315
|
+
// The recorded id is an upload that may never have become a video, so
|
|
316
|
+
// the library cannot be asked about it: confirming is the only check.
|
|
317
|
+
try {
|
|
318
|
+
const video = await confirmVideoUpload(client, siteId, recorded.videoId);
|
|
319
|
+
onProgress(`Confirmed the earlier upload of '${file}'; the video is ready.`);
|
|
320
|
+
return { video, deduplicated: false, ...base };
|
|
321
|
+
} catch (error) {
|
|
322
|
+
// A client-side refusal (the reservation expired, say) means that upload
|
|
323
|
+
// cannot be confirmed any more, so the file is sent again; a server
|
|
324
|
+
// error, a throttle or a lost connection is transient or ambiguous and
|
|
325
|
+
// is left for the next run.
|
|
326
|
+
const refused = error instanceof ApiError
|
|
327
|
+
&& error.httpStatus >= 400
|
|
328
|
+
&& error.httpStatus < 500
|
|
329
|
+
&& ![408, 425, 429].includes(error.httpStatus);
|
|
330
|
+
if (!refused) throw error;
|
|
331
|
+
}
|
|
332
|
+
} else {
|
|
333
|
+
const { byId, truncated } = await readLibrary();
|
|
334
|
+
const existing = byId.get(recorded.videoId);
|
|
335
|
+
if (existing !== undefined) {
|
|
336
|
+
onProgress(`'${file}' is unchanged since it was uploaded as video ${existing.videoId}; skipping the upload.`);
|
|
337
|
+
return { video: existing, deduplicated: true, ...base };
|
|
338
|
+
}
|
|
339
|
+
// A listing cut short cannot say the video is gone, and uploading again
|
|
340
|
+
// would spend storage on a video that may well be there.
|
|
341
|
+
if (truncated) {
|
|
342
|
+
throw new SiteAuthoringError(
|
|
343
|
+
"media.video_library_unverifiable",
|
|
344
|
+
`'${file}' was uploaded as video ${recorded.videoId}, which is not among the first ${byId.size} videos of this `
|
|
345
|
+
+ "site's library, and the library is larger than one run can check. Remove the file's entry from "
|
|
346
|
+
+ `${MEDIA_MANIFEST_FILE_NAME} to upload it again, or leave it.`,
|
|
347
|
+
{ field: file },
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
onProgress(`Checking '${file}' (${source.byteLength} bytes).`);
|
|
353
|
+
const prepared = await prepareVideoFile({
|
|
354
|
+
filePath: resolveWorkspacePath(config.workspaceDir, file),
|
|
355
|
+
byteLength: source.byteLength,
|
|
356
|
+
});
|
|
357
|
+
// The prepare step reopened the file by path, so prove it is still the file that was opened
|
|
358
|
+
// before anything is requested or sent.
|
|
359
|
+
await source.verifyUnchanged();
|
|
360
|
+
const sent = prepared.remuxed === null
|
|
361
|
+
? { byteLength: source.byteLength, stream: source.stream }
|
|
362
|
+
: { byteLength: prepared.remuxed.byteLength, stream: () => Readable.from([prepared.remuxed]) };
|
|
363
|
+
if (prepared.remuxed !== null) {
|
|
364
|
+
onProgress(`Rewrote '${file}' as an MP4 with its index first, without re-encoding (${sent.byteLength} bytes).`);
|
|
365
|
+
}
|
|
366
|
+
let response;
|
|
367
|
+
try {
|
|
368
|
+
response = await requestVideoUpload(client, siteId, {
|
|
369
|
+
fileName: mp4Name(fileName),
|
|
370
|
+
contentType,
|
|
371
|
+
sizeBytes: sent.byteLength,
|
|
372
|
+
videoCodec: prepared.videoCodec,
|
|
373
|
+
audioCodec: prepared.audioCodec,
|
|
374
|
+
});
|
|
375
|
+
} catch (error) {
|
|
376
|
+
throw surfaceVideoRefusal(error, file);
|
|
377
|
+
}
|
|
378
|
+
const capability = requireUploadCapability(response, file);
|
|
379
|
+
// Recorded before the bytes are sent: a PUT can land while its answer is lost, and the
|
|
380
|
+
// next run must find this upload id and confirm it, not reserve and send the file again.
|
|
381
|
+
await recordPending(response.uploadId, base);
|
|
382
|
+
try {
|
|
383
|
+
await client.upload(capability, {
|
|
384
|
+
byteLength: sent.byteLength,
|
|
385
|
+
stream: sent.stream,
|
|
386
|
+
timeoutMilliseconds: LIMITS.videoUploadMilliseconds,
|
|
387
|
+
});
|
|
388
|
+
} catch (error) {
|
|
389
|
+
// The upload is create-only, so a retry after a PUT that landed is refused with 412: the
|
|
390
|
+
// object is in place, and confirm checks it.
|
|
391
|
+
if (!(error instanceof SiteAuthoringError && error.code === "upload.rejected" && error.status === "http:412")) {
|
|
392
|
+
throw error;
|
|
393
|
+
}
|
|
394
|
+
onProgress(`The upload of '${file}' had already landed; confirming it.`);
|
|
395
|
+
}
|
|
396
|
+
// The PUT streamed the opened length. If the file grew or was rewritten
|
|
397
|
+
// meanwhile, what Taproot holds is not the file on disk, so it is not confirmed.
|
|
398
|
+
await source.verifyUnchanged();
|
|
399
|
+
let video;
|
|
400
|
+
try {
|
|
401
|
+
video = await confirmVideoUpload(client, siteId, response.uploadId);
|
|
402
|
+
} catch (error) {
|
|
403
|
+
throw surfaceVideoRefusal(error, file);
|
|
404
|
+
}
|
|
405
|
+
onProgress(`Uploaded and confirmed '${file}'; the video is ready.`);
|
|
406
|
+
return { video, deduplicated: false, ...base };
|
|
407
|
+
} finally {
|
|
408
|
+
await source.close();
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
166
412
|
export async function mediaUpload(invocation) {
|
|
167
413
|
const session = await openSession(invocation);
|
|
168
414
|
const { client, config, siteId, now, onProgress } = session;
|
|
@@ -188,8 +434,10 @@ export async function mediaUpload(invocation) {
|
|
|
188
434
|
throw new SiteAuthoringError(
|
|
189
435
|
"media.none_found",
|
|
190
436
|
selected.length > 0
|
|
191
|
-
? `No PNG, JPEG, GIF, or
|
|
192
|
-
|
|
437
|
+
? `No PNG, JPEG, GIF, WebP, MP4, MOV, or WebM files were found under ${
|
|
438
|
+
selected.map((value) => `'${value}'`).join(", ")
|
|
439
|
+
}.`
|
|
440
|
+
: `No PNG, JPEG, GIF, WebP, MP4, MOV, or WebM files were found under '${MEDIA_DIRECTORY}/' in the workspace.`,
|
|
193
441
|
{ field: selected.length > 0 ? normalizePositional(selected[0]) : MEDIA_DIRECTORY },
|
|
194
442
|
);
|
|
195
443
|
}
|
|
@@ -202,11 +450,66 @@ export async function mediaUpload(invocation) {
|
|
|
202
450
|
}
|
|
203
451
|
|
|
204
452
|
const uploaded = [];
|
|
453
|
+
const uploadedVideos = [];
|
|
454
|
+
mediaManifest.videos ??= {};
|
|
455
|
+
// One read of the site's videos serves every unchanged file in the run, and
|
|
456
|
+
// only when there is a recorded upload to check.
|
|
457
|
+
let library;
|
|
458
|
+
const readLibrary = async () => {
|
|
459
|
+
if (library === undefined) {
|
|
460
|
+
const listing = await listVideos(client, siteId);
|
|
461
|
+
library = {
|
|
462
|
+
byId: new Map(listing.videos.map((video) => [video.videoId, video])),
|
|
463
|
+
truncated: listing.truncated,
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
return library;
|
|
467
|
+
};
|
|
205
468
|
|
|
206
469
|
return await withRefusalGuidance(onProgress, "upload", async () => {
|
|
207
470
|
try {
|
|
208
471
|
for (const file of files) {
|
|
209
472
|
const fileName = fileNameOf(file);
|
|
473
|
+
const videoUpload = await uploadVideoFile({
|
|
474
|
+
client,
|
|
475
|
+
config,
|
|
476
|
+
siteId,
|
|
477
|
+
file,
|
|
478
|
+
fileName,
|
|
479
|
+
recorded: mediaManifest.videos[file],
|
|
480
|
+
readLibrary,
|
|
481
|
+
recordPending: async (videoId, base) => {
|
|
482
|
+
mediaManifest.videos[file] = { videoId, ...base, pendingConfirm: true };
|
|
483
|
+
mediaManifest.mediaManifestVersion = MEDIA_MANIFEST_VERSION;
|
|
484
|
+
mediaManifest.siteId = siteId;
|
|
485
|
+
await writeMediaManifest(config.workspaceDir, mediaManifest);
|
|
486
|
+
},
|
|
487
|
+
onProgress,
|
|
488
|
+
});
|
|
489
|
+
if (videoUpload !== null) {
|
|
490
|
+
const { video, contentType, byteLength, modifiedMilliseconds, deduplicated } = videoUpload;
|
|
491
|
+
mediaManifest.videos[file] = {
|
|
492
|
+
videoId: video.videoId,
|
|
493
|
+
contentType,
|
|
494
|
+
byteLength,
|
|
495
|
+
modifiedMilliseconds,
|
|
496
|
+
};
|
|
497
|
+
uploadedVideos.push({
|
|
498
|
+
file,
|
|
499
|
+
videoId: video.videoId,
|
|
500
|
+
contentType,
|
|
501
|
+
byteLength,
|
|
502
|
+
deduplicated,
|
|
503
|
+
durationMilliseconds: video.durationMilliseconds,
|
|
504
|
+
});
|
|
505
|
+
// Recorded as soon as it is confirmed: a later original can take an
|
|
506
|
+
// hour to send, and an interrupt then must not forget this id, which a
|
|
507
|
+
// re-run would answer by uploading the video again.
|
|
508
|
+
mediaManifest.mediaManifestVersion = MEDIA_MANIFEST_VERSION;
|
|
509
|
+
mediaManifest.siteId = siteId;
|
|
510
|
+
await writeMediaManifest(config.workspaceDir, mediaManifest);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
210
513
|
const bytes = await readWorkspaceFile(config.workspaceDir, file, WORKSPACE_LIMITS.mediaBytes);
|
|
211
514
|
const { contentType, width, height } = inspectImageBytes(bytes, file);
|
|
212
515
|
const hash = contentHash(bytes);
|
|
@@ -260,7 +563,9 @@ export async function mediaUpload(invocation) {
|
|
|
260
563
|
// with it, because an image id means nothing without one.
|
|
261
564
|
mediaManifest.mediaManifestVersion = MEDIA_MANIFEST_VERSION;
|
|
262
565
|
mediaManifest.siteId = siteId;
|
|
263
|
-
if (uploaded.length > 0
|
|
566
|
+
if (uploaded.length > 0 || uploadedVideos.length > 0) {
|
|
567
|
+
await writeMediaManifest(config.workspaceDir, mediaManifest);
|
|
568
|
+
}
|
|
264
569
|
}
|
|
265
570
|
|
|
266
571
|
onProgress("Waiting for image processing to finish.");
|
|
@@ -291,7 +596,6 @@ export async function mediaUpload(invocation) {
|
|
|
291
596
|
manifestEntry.urls = image.responsiveUrls;
|
|
292
597
|
}
|
|
293
598
|
await writeMediaManifest(config.workspaceDir, mediaManifest);
|
|
294
|
-
|
|
295
599
|
const reported = boundedList(
|
|
296
600
|
uploaded.map((entry) => ({
|
|
297
601
|
file: entry.file,
|
|
@@ -311,6 +615,21 @@ export async function mediaUpload(invocation) {
|
|
|
311
615
|
})),
|
|
312
616
|
MAXIMUM_REPORTED,
|
|
313
617
|
);
|
|
618
|
+
const reportedVideos = boundedList(
|
|
619
|
+
uploadedVideos.map((entry) => ({
|
|
620
|
+
file: entry.file,
|
|
621
|
+
videoId: entry.videoId,
|
|
622
|
+
contentType: entry.contentType,
|
|
623
|
+
byteLength: entry.byteLength,
|
|
624
|
+
deduplicated: entry.deduplicated,
|
|
625
|
+
durationMilliseconds: entry.durationMilliseconds,
|
|
626
|
+
// Both forms that place it on a page, each accepted as written: a
|
|
627
|
+
// `component:video` fence for Markdown sources, and the ProseMirror
|
|
628
|
+
// block for `.pm.json` sources.
|
|
629
|
+
component: videoComponent(entry.videoId),
|
|
630
|
+
})),
|
|
631
|
+
MAXIMUM_REPORTED,
|
|
632
|
+
);
|
|
314
633
|
return successResult(VERB_MEDIA_UPLOAD, siteId, {
|
|
315
634
|
mediaManifestFile: MEDIA_MANIFEST_FILE_NAME,
|
|
316
635
|
media: {
|
|
@@ -319,6 +638,11 @@ export async function mediaUpload(invocation) {
|
|
|
319
638
|
items: reported.items,
|
|
320
639
|
...(reported.truncated ? { itemsTruncated: true } : {}),
|
|
321
640
|
},
|
|
641
|
+
videos: {
|
|
642
|
+
total: uploadedVideos.length,
|
|
643
|
+
items: reportedVideos.items,
|
|
644
|
+
...(reportedVideos.truncated ? { itemsTruncated: true } : {}),
|
|
645
|
+
},
|
|
322
646
|
});
|
|
323
647
|
});
|
|
324
648
|
}
|