@bubo-squared/gyroview 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +20 -11
- package/dist/index.d.ts +56 -2
- package/dist/index.js +2 -2
- package/dist/packages/adapters/mediabunny/src/MediabunnyAudioPackager.js +49 -15
- package/dist/packages/adapters/mediabunny/src/MediabunnyCodecReader.js +3 -1
- package/dist/packages/adapters/mediabunny/src/decoderConfigurations.js +2 -2
- package/dist/packages/adapters/mediabunny/src/trackColour.js +21 -0
- package/dist/packages/adapters/mse-audio/src/SourceBufferFeeder.js +12 -1
- package/dist/packages/adapters/three/src/ThreeFrameRenderer.js +3 -0
- package/dist/packages/adapters/three/src/conversionUniforms.js +61 -0
- package/dist/packages/adapters/three/src/glslLiterals.js +10 -0
- package/dist/packages/adapters/three/src/matrixCorrections.js +35 -0
- package/dist/packages/adapters/three/src/rendererUniforms.js +33 -12
- package/dist/packages/adapters/three/src/shaderPrograms.js +4 -2
- package/dist/packages/adapters/three/src/shaders/analysis.frag.js +1 -1
- package/dist/packages/adapters/three/src/shaders/displayConversion.js +4 -0
- package/dist/packages/adapters/three/src/shaders/lensModels.js +1 -1
- package/dist/packages/adapters/three/src/shaders/lensSampling.js +1 -1
- package/dist/packages/adapters/three/src/shaders/rawLenses.frag.js +1 -1
- package/dist/packages/adapters/three/src/shaders/stitch.frag.js +1 -1
- package/dist/packages/adapters/three/src/threeMatrix.js +10 -0
- package/dist/packages/adapters/webcodecs/src/WebCodecsVideoDecoderPort.js +16 -5
- package/dist/packages/adapters/webcodecs/src/colorSpaceOf.js +32 -0
- package/dist/packages/core/src/application/playback/probeDecoding.js +2 -2
- package/dist/packages/core/src/application/recording/frameTimesOf.js +0 -1
- package/dist/packages/core/src/domain/colour/DisplayConversion.js +137 -0
- package/dist/packages/core/src/domain/colour/TrackColour.js +38 -0
- package/dist/packages/core/src/domain/colour/matrixCorrection.js +76 -0
- package/dist/packages/core/src/domain/format/calibration/CalibrationVersion.js +7 -4
- package/dist/packages/core/src/domain/format/calibration/layouts/CalibrationStringLayout.js +1 -1
- package/dist/packages/core/src/domain/format/calibration/layouts/LegacyCalibrationLayout.js +1 -0
- package/dist/packages/core/src/domain/format/calibration/layouts/MeiCalibrationLayout.js +109 -35
- package/dist/packages/core/src/domain/format/calibration/layouts/PolynomialCalibrationLayout.js +1 -0
- package/dist/packages/core/src/domain/format/calibration/offsetTokens.js +33 -4
- package/dist/packages/core/src/domain/format/calibration/parseOffsetString.js +22 -17
- package/dist/packages/core/src/domain/format/calibration/selectCalibration.js +28 -15
- package/dist/packages/core/src/domain/format/calibration/v6TermReading.js +20 -0
- package/dist/packages/core/src/domain/format/info/calibrationSources.js +27 -0
- package/dist/packages/core/src/domain/format/info/infoFields.js +8 -2
- package/dist/packages/core/src/domain/format/info/parseInfoRecord.js +11 -6
- package/dist/packages/core/src/domain/motion/imu/ImuFrame.js +17 -8
- package/dist/packages/core/src/domain/motion/timing/FrameTimes.js +1 -3
- package/dist/packages/core/src/domain/motion/timing/frameTimeSources.js +0 -3
- package/dist/packages/core/src/domain/optics/MeiDistortion.js +43 -0
- package/dist/packages/core/src/domain/optics/MeiModel.js +6 -14
- package/dist/packages/core/src/domain/optics/scaledProjection.js +30 -0
- package/dist/packages/core/src/domain/stitching/StitchingSetup.js +13 -4
- package/dist/packages/core/src/index.js +7 -3
- package/dist/packages/core/src/shared/errors/GyroViewError.js +55 -3
- package/dist/packages/core/src/shared/index.js +11 -0
- package/dist/packages/core/src/shared/mapRecord.js +11 -0
- package/dist/packages/core/src/shared/protobuf/ProtobufMessage.js +12 -26
- package/dist/packages/core/src/shared/protobuf/wireFormat.js +16 -0
- package/dist/packages/player/src/composition/buildPipeline.js +2 -1
- package/dist/packages/player/src/composition/openInputs.js +4 -1
- package/dist/packages/player/src/controls/messages.js +2 -1
- package/dist/standalone.js +211 -100
- package/package.json +4 -4
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,55 @@
|
|
|
3
3
|
What changed for a page using the package, newest first. Until 1.0, a minor version may change
|
|
4
4
|
the API.
|
|
5
5
|
|
|
6
|
+
## 0.4.0 (2026-10-01)
|
|
7
|
+
|
|
8
|
+
New:
|
|
9
|
+
|
|
10
|
+
- Insta360 X6 recordings play. Their only calibration is a v6 string, now read as the Mei model
|
|
11
|
+
with more terms (ADR 0032); the X6's IMU frame and the radial scale its lenses want are
|
|
12
|
+
measured (ADR 0009, ADR 0023); and its 10-bit HLG video is shown as SDR, as Insta360 Studio
|
|
13
|
+
shows it (ADR 0033).
|
|
14
|
+
- Each of the `ready` event's `tracks` has a `colour`: its primaries, transfer, matrix and range
|
|
15
|
+
as the track's bitstream says (`TrackColour`, with the types `ColourPrimaries`,
|
|
16
|
+
`TransferCharacteristics`, `MatrixCoefficients` and `ColourRange`).
|
|
17
|
+
- `inspectRecording`'s calibration strings include `offsetV6`, and `calibrationVersion` may be
|
|
18
|
+
6, the v6 string's.
|
|
19
|
+
|
|
20
|
+
What a page may notice:
|
|
21
|
+
|
|
22
|
+
- A recording whose only calibration is a v6 string plays where it failed with
|
|
23
|
+
`no-calibration`.
|
|
24
|
+
- Stabilization samples the gyro half way through each frame's shutter, no longer half a
|
|
25
|
+
readout later. An X5's stabilized picture shifts by that half readout (4 ms at 5.7K60, 10.6 ms
|
|
26
|
+
at 8K30), and a swinging camera's world holds stiller (ADR 0034).
|
|
27
|
+
- A track whose colour the player cannot show as it should is drawn with a
|
|
28
|
+
`recording-degraded` warning: a PQ or linear-light transfer is drawn as recorded, SDR of wider
|
|
29
|
+
primaries than BT.709's is shown as BT.709, HLG of primaries other than BT.709's or BT.2020's
|
|
30
|
+
keeps its gamut.
|
|
31
|
+
- The decoder is told the track's whole colour, not only its range (the range alone where the
|
|
32
|
+
browser's WebCodecs does not know one of its values).
|
|
33
|
+
- Where a browser's decoder converts a track's colour through another matrix than the track's
|
|
34
|
+
own, as Safari's does with the X6's BT.2020, the picture is brought back to the track's.
|
|
35
|
+
- Seeking again and again, or dragging the seek bar, no longer stops playback now and then with
|
|
36
|
+
`decode` ("the audio element failed (media error 3)"): the sound is handed to the browser in
|
|
37
|
+
whole segments, so a seek can no longer cut one in half.
|
|
38
|
+
|
|
39
|
+
## 0.3.1 (2026-09-30)
|
|
40
|
+
|
|
41
|
+
New:
|
|
42
|
+
|
|
43
|
+
- Every error has a `category`, derived from its code, that says whose side the failure is on:
|
|
44
|
+
`browser`, `recording`, `source`, `usage` or `internal`; `GYRO_VIEW_ERROR_CATEGORIES` and the
|
|
45
|
+
type `GyroViewErrorCategory` list them. The README's "When a recording cannot play" gives the
|
|
46
|
+
codes of each, and what a `<video>` fallback can and cannot do (ADR 0030).
|
|
47
|
+
- The `webcodecs-unavailable` error: the browser has no WebCodecs, on a page not served over
|
|
48
|
+
HTTPS or in an old browser.
|
|
49
|
+
|
|
50
|
+
What a page may notice:
|
|
51
|
+
|
|
52
|
+
- A page on plain HTTP, or a browser without WebCodecs, hears `webcodecs-unavailable` where it
|
|
53
|
+
heard `codec-unsupported`, whose message blamed the codec.
|
|
54
|
+
|
|
6
55
|
## 0.3.0 (2026-09-30)
|
|
7
56
|
|
|
8
57
|
What a page may notice:
|
package/README.md
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
# @bubo-squared/gyroview
|
|
2
2
|
|
|
3
|
-
Play raw Insta360 `.insv` recordings (X3, X4, X5) in the browser. `<gyro-view>` reads the
|
|
3
|
+
Play raw Insta360 `.insv` recordings (X3, X4, X5, X6) in the browser. `<gyro-view>` reads the
|
|
4
4
|
camera's dual-fisheye file directly, over HTTP byte ranges or from a local file, decodes both
|
|
5
5
|
lenses in hardware with WebCodecs, and stitches and gyro-stabilizes them on the GPU. No Insta360
|
|
6
6
|
Studio export step.
|
|
7
7
|
|
|
8
|
+
**Try it first:** [insv-player.com](https://insv-player.com/) plays `.insv` files with this
|
|
9
|
+
package. Drop your own recording to check that your camera and browser work before you install.
|
|
10
|
+
|
|
8
11
|
## Install
|
|
9
12
|
|
|
10
13
|
```sh
|
|
@@ -188,7 +191,8 @@ loads a recording only for the player in view, removing `src` from the others.
|
|
|
188
191
|
|
|
189
192
|
## Requirements
|
|
190
193
|
|
|
191
|
-
- **A secure page.** WebCodecs exists only on `https://` pages, or `http://localhost
|
|
194
|
+
- **A secure page.** WebCodecs exists only on `https://` pages, or `http://localhost`; elsewhere
|
|
195
|
+
the player fails with `webcodecs-unavailable`.
|
|
192
196
|
- **Recordings served in byte ranges.** The server answers `Range` requests with `206`, and
|
|
193
197
|
sends CORS headers when the recordings live on another origin than the page:
|
|
194
198
|
|
|
@@ -205,17 +209,20 @@ loads a recording only for the player in view, removing `src` from the others.
|
|
|
205
209
|
`Access-Control-Allow-Credentials: true`.
|
|
206
210
|
|
|
207
211
|
- **A hardware HEVC decoder.** 5.7K plays on recent laptops and phones; 8K needs a Level 6
|
|
208
|
-
decoder (Apple Silicon, recent NVIDIA and Intel).
|
|
209
|
-
|
|
212
|
+
decoder (Apple Silicon, recent NVIDIA and Intel). On Linux, Chrome reaches the decoder only
|
|
213
|
+
through VA-API: an Intel or AMD GPU whose driver offers HEVC, not NVIDIA's own driver or a
|
|
214
|
+
virtual machine (`chrome://gpu` lists `Decode hevc main` under Video Acceleration
|
|
215
|
+
Information when it can). H.264 recordings do not need it. A recording the browser cannot
|
|
216
|
+
decode fails with the `codec-unsupported` error.
|
|
210
217
|
|
|
211
218
|
The supported browsers, with the oldest versions that have what the player uses (WebCodecs,
|
|
212
219
|
WebGL 2, container queries, and on iPhone `ManagedMediaSource` for the sound):
|
|
213
220
|
|
|
214
|
-
| Browser | From | Notes
|
|
215
|
-
| --------------------- | ---- |
|
|
216
|
-
| Chrome, Edge desktop | 107 | HEVC is decoded in hardware from this version on.
|
|
217
|
-
| Safari on macOS | 16.4 |
|
|
218
|
-
| Safari on iPhone/iPad | 17.1 | 16.4 to 17.0 play without sound, with a `warning` event.
|
|
221
|
+
| Browser | From | Notes |
|
|
222
|
+
| --------------------- | ---- | ----------------------------------------------------------------- |
|
|
223
|
+
| Chrome, Edge desktop | 107 | HEVC is decoded in hardware from this version on (Linux: VA-API). |
|
|
224
|
+
| Safari on macOS | 16.4 | |
|
|
225
|
+
| Safari on iPhone/iPad | 17.1 | 16.4 to 17.0 play without sound, with a `warning` event. |
|
|
219
226
|
|
|
220
227
|
Firefox and Chrome on Android are untested: they play what their decoders accept.
|
|
221
228
|
|
|
@@ -224,8 +231,10 @@ Until 1.0, a minor version may change the API; [CHANGELOG.md](./CHANGELOG.md) sa
|
|
|
224
231
|
## Reference
|
|
225
232
|
|
|
226
233
|
Every attribute, method, event and keyboard shortcut is in the
|
|
227
|
-
[project README](https://github.com/bubo-squared/GyroView#using-the-player)
|
|
228
|
-
|
|
234
|
+
[project README](https://github.com/bubo-squared/GyroView#using-the-player), and what a failure
|
|
235
|
+
tells a page, with what a `<video>` fallback can do, under
|
|
236
|
+
[When a recording cannot play](https://github.com/bubo-squared/GyroView#when-a-recording-cannot-play);
|
|
237
|
+
hosting and the error codes are in [docs/DEPLOYMENT.md](https://github.com/bubo-squared/GyroView/blob/main/docs/DEPLOYMENT.md).
|
|
229
238
|
|
|
230
239
|
## License
|
|
231
240
|
|
package/dist/index.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ declare const CalibrationVersion: {
|
|
|
42
42
|
readonly Legacy: 1;
|
|
43
43
|
readonly Polynomial: 2;
|
|
44
44
|
readonly Mei: 3;
|
|
45
|
+
readonly ExtendedMei: 6;
|
|
45
46
|
};
|
|
46
47
|
type CalibrationVersion = (typeof CalibrationVersion)[keyof typeof CalibrationVersion];
|
|
47
48
|
type Milliseconds = Brand<number, "Milliseconds">;
|
|
@@ -67,6 +68,7 @@ interface CalibrationStrings {
|
|
|
67
68
|
readonly offset: string | undefined;
|
|
68
69
|
readonly offsetV2: string | undefined;
|
|
69
70
|
readonly offsetV3: string | undefined;
|
|
71
|
+
readonly offsetV6: string | undefined;
|
|
70
72
|
}
|
|
71
73
|
/**
|
|
72
74
|
* Everything the player learns from the info record. Absent fields are `undefined`; nothing here
|
|
@@ -225,6 +227,39 @@ export interface RecordingInspection {
|
|
|
225
227
|
readonly gyro: GyroSummary | UnreadableGyro | undefined;
|
|
226
228
|
readonly exposure: ExposureReport | undefined;
|
|
227
229
|
}
|
|
230
|
+
declare const COLOUR_PRIMARIES: readonly [
|
|
231
|
+
"bt709",
|
|
232
|
+
"bt470bg",
|
|
233
|
+
"smpte170m",
|
|
234
|
+
"bt2020",
|
|
235
|
+
"smpte432"
|
|
236
|
+
];
|
|
237
|
+
declare const TRANSFER_CHARACTERISTICS: readonly [
|
|
238
|
+
"bt709",
|
|
239
|
+
"smpte170m",
|
|
240
|
+
"iec61966-2-1",
|
|
241
|
+
"linear",
|
|
242
|
+
"pq",
|
|
243
|
+
"hlg"
|
|
244
|
+
];
|
|
245
|
+
declare const MATRIX_COEFFICIENTS: readonly [
|
|
246
|
+
"rgb",
|
|
247
|
+
"bt709",
|
|
248
|
+
"bt470bg",
|
|
249
|
+
"smpte170m",
|
|
250
|
+
"bt2020-ncl"
|
|
251
|
+
];
|
|
252
|
+
type Unspecified = "unspecified";
|
|
253
|
+
export type ColourPrimaries = (typeof COLOUR_PRIMARIES)[number] | Unspecified;
|
|
254
|
+
export type TransferCharacteristics = (typeof TRANSFER_CHARACTERISTICS)[number] | Unspecified;
|
|
255
|
+
export type MatrixCoefficients = (typeof MATRIX_COEFFICIENTS)[number] | Unspecified;
|
|
256
|
+
export type ColourRange = "full" | "limited" | Unspecified;
|
|
257
|
+
export interface TrackColour {
|
|
258
|
+
readonly primaries: ColourPrimaries;
|
|
259
|
+
readonly transfer: TransferCharacteristics;
|
|
260
|
+
readonly matrix: MatrixCoefficients;
|
|
261
|
+
readonly range: ColourRange;
|
|
262
|
+
}
|
|
228
263
|
interface VideoTrackDescription {
|
|
229
264
|
readonly trackIndex: number;
|
|
230
265
|
readonly codedWidth: number;
|
|
@@ -233,6 +268,7 @@ interface VideoTrackDescription {
|
|
|
233
268
|
* WebCodecs codec string, for example `hvc1.1.6.L153.B0` or `avc1.640033`.
|
|
234
269
|
*/
|
|
235
270
|
readonly codec: string;
|
|
271
|
+
readonly colour: TrackColour;
|
|
236
272
|
}
|
|
237
273
|
export type StabilizationMode = "off" | "lock" | "horizon" | "follow";
|
|
238
274
|
export declare const STABILIZATION_MODES: readonly StabilizationMode[];
|
|
@@ -261,7 +297,7 @@ interface DragDelta {
|
|
|
261
297
|
export type ViewMode = "raw-lenses" | "equirectangular" | "normal";
|
|
262
298
|
export declare const VIEW_MODES: readonly ViewMode[];
|
|
263
299
|
/**
|
|
264
|
-
* Stable machine-readable failure
|
|
300
|
+
* Stable machine-readable failure codes. Embedders switch on these; messages are for humans.
|
|
265
301
|
*/
|
|
266
302
|
export declare const GYRO_VIEW_ERROR_CODES: readonly [
|
|
267
303
|
"binary-out-of-bounds",
|
|
@@ -291,12 +327,30 @@ export declare const GYRO_VIEW_ERROR_CODES: readonly [
|
|
|
291
327
|
"unsupported-container",
|
|
292
328
|
"unsupported-gyro-record",
|
|
293
329
|
"unsupported-info-format",
|
|
294
|
-
"unsupported-layout"
|
|
330
|
+
"unsupported-layout",
|
|
331
|
+
"webcodecs-unavailable"
|
|
295
332
|
];
|
|
296
333
|
export type GyroViewErrorCode = (typeof GYRO_VIEW_ERROR_CODES)[number];
|
|
334
|
+
/**
|
|
335
|
+
* Whose side a failure is on, for embedders that handle failures by kind: the browser cannot
|
|
336
|
+
* decode or draw the recording, the file is not one the player can play, its bytes could not be
|
|
337
|
+
* read, the page misused the API, or the player failed in a way it did not expect.
|
|
338
|
+
*/
|
|
339
|
+
export declare const GYRO_VIEW_ERROR_CATEGORIES: readonly [
|
|
340
|
+
"browser",
|
|
341
|
+
"recording",
|
|
342
|
+
"source",
|
|
343
|
+
"usage",
|
|
344
|
+
"internal"
|
|
345
|
+
];
|
|
346
|
+
export type GyroViewErrorCategory = (typeof GYRO_VIEW_ERROR_CATEGORIES)[number];
|
|
297
347
|
export declare function isGyroViewErrorCode(value: unknown): value is GyroViewErrorCode;
|
|
298
348
|
export declare class GyroViewError extends Error {
|
|
299
349
|
readonly code: GyroViewErrorCode;
|
|
350
|
+
/**
|
|
351
|
+
* Its own property rather than a getter, so a serialized or logged copy of the error keeps it.
|
|
352
|
+
*/
|
|
353
|
+
readonly category: GyroViewErrorCategory;
|
|
300
354
|
constructor(code: GyroViewErrorCode, message: string, options?: ErrorOptions);
|
|
301
355
|
}
|
|
302
356
|
export declare function hasErrorCode(error: unknown, code: GyroViewErrorCode): boolean;
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GYRO_VIEW_ERROR_CODES, GyroViewError, hasErrorCode, isGyroViewErrorCode } from "./packages/core/src/shared/errors/GyroViewError.js";
|
|
1
|
+
import { GYRO_VIEW_ERROR_CATEGORIES, GYRO_VIEW_ERROR_CODES, GyroViewError, hasErrorCode, isGyroViewErrorCode } from "./packages/core/src/shared/errors/GyroViewError.js";
|
|
2
2
|
import { STABILIZATION_MODES } from "./packages/core/src/domain/motion/stabilization/Stabilizer.js";
|
|
3
3
|
import { PICTURE_QUALITIES } from "./packages/core/src/domain/view/PictureQuality.js";
|
|
4
4
|
import { VIEW_MODES } from "./packages/core/src/domain/view/ViewMode.js";
|
|
@@ -10,4 +10,4 @@ import { inspectRecording } from "./packages/player/src/inspectRecording.js";
|
|
|
10
10
|
import { GyroViewElement } from "./packages/player/src/element/GyroViewElement.js";
|
|
11
11
|
import { GYRO_VIEW_TAG, defineGyroView } from "./packages/player/src/element/defineGyroView.js";
|
|
12
12
|
import "./packages/player/src/index.js";
|
|
13
|
-
export { GYRO_VIEW_ERROR_CODES, GYRO_VIEW_TAG, GyroViewElement, GyroViewError, PICTURE_QUALITIES, Player, STABILIZATION_MODES, VIEW_MODES, attachKeyboard, attachViewGestures, createBrowserPlayer, defineGyroView, hasErrorCode, inspectRecording, isGyroViewErrorCode };
|
|
13
|
+
export { GYRO_VIEW_ERROR_CATEGORIES, GYRO_VIEW_ERROR_CODES, GYRO_VIEW_TAG, GyroViewElement, GyroViewError, PICTURE_QUALITIES, Player, STABILIZATION_MODES, VIEW_MODES, attachKeyboard, attachViewGestures, createBrowserPlayer, defineGyroView, hasErrorCode, inspectRecording, isGyroViewErrorCode };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GyroViewError } from "../../../core/src/shared/errors/GyroViewError.js";
|
|
1
|
+
import { GyroViewError, ensureInvariant } from "../../../core/src/shared/errors/GyroViewError.js";
|
|
2
2
|
import "../../../core/src/index.js";
|
|
3
3
|
import { SegmentChannel } from "./SegmentChannel.js";
|
|
4
4
|
import { EncodedAudioPacketSource, EncodedPacket, Mp4OutputFormat, NullTarget, Output } from "mediabunny";
|
|
@@ -9,10 +9,9 @@ import { EncodedAudioPacketSource, EncodedPacket, Mp4OutputFormat, NullTarget, O
|
|
|
9
9
|
*/
|
|
10
10
|
var FRAGMENT_DURATION_SECONDS = 1;
|
|
11
11
|
/**
|
|
12
|
-
*
|
|
13
|
-
* re-packaging pauses.
|
|
12
|
+
* Media segments that may wait for the consumer before re-packaging pauses.
|
|
14
13
|
*/
|
|
15
|
-
var SEGMENTS_AHEAD =
|
|
14
|
+
var SEGMENTS_AHEAD = 2;
|
|
16
15
|
/**
|
|
17
16
|
* The codec strings of the audio the cameras record, AAC (`mp4a.40.2` for AAC-LC), and mediabunny's
|
|
18
17
|
* name for it. Other codecs are not re-packaged: the picture then plays on a silent clock.
|
|
@@ -21,10 +20,10 @@ var AAC_CODEC_STRING_PREFIX = "mp4a.40.";
|
|
|
21
20
|
var AAC = "aac";
|
|
22
21
|
/**
|
|
23
22
|
* AudioPackager over mediabunny: the samples re-packaged, unchanged, into fragmented MP4.
|
|
24
|
-
* Segments are
|
|
25
|
-
* yields whole
|
|
26
|
-
*
|
|
27
|
-
*
|
|
23
|
+
* Segments are assembled from the muxer's box callbacks rather than taken from its byte stream:
|
|
24
|
+
* that yields whole segments and leaves out the `mfra` index the muxer appends at the end, which
|
|
25
|
+
* Media Source Extensions do not accept. Fragmented output keeps the track's own timestamps, so
|
|
26
|
+
* segments started mid-track land at their true time.
|
|
28
27
|
*/
|
|
29
28
|
var MediabunnyAudioPackager = class {
|
|
30
29
|
segmentsOf(samples, configuration) {
|
|
@@ -112,18 +111,53 @@ function decoderConfigOf(configuration) {
|
|
|
112
111
|
...configuration.description && { description: configuration.description }
|
|
113
112
|
};
|
|
114
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* The muxer's boxes handed on as whole segments: the initialization segment (`ftyp` and `moov`)
|
|
116
|
+
* and each media segment (`moof` and its `mdat`). A consumer that stops between two segments, as
|
|
117
|
+
* a seek does, then never leaves a source buffer's parser inside one, where Chromium reads the
|
|
118
|
+
* next segments' bytes as the samples of the `moof` it holds and fails to decode them.
|
|
119
|
+
*/
|
|
115
120
|
function fragmentedMp4Into(channel) {
|
|
116
|
-
const
|
|
117
|
-
channel.push(new Uint8Array(data));
|
|
118
|
-
};
|
|
121
|
+
const segment = new SegmentAssembly(channel);
|
|
119
122
|
return new Mp4OutputFormat({
|
|
120
123
|
fastStart: "fragmented",
|
|
121
124
|
minimumFragmentDuration: FRAGMENT_DURATION_SECONDS,
|
|
122
|
-
onFtyp:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
125
|
+
onFtyp: (data) => {
|
|
126
|
+
segment.begin(data);
|
|
127
|
+
},
|
|
128
|
+
onMoov: (data) => {
|
|
129
|
+
segment.end(data);
|
|
130
|
+
},
|
|
131
|
+
onMoof: (data) => {
|
|
132
|
+
segment.begin(data);
|
|
133
|
+
},
|
|
134
|
+
onMdat: (data) => {
|
|
135
|
+
segment.end(data);
|
|
136
|
+
}
|
|
126
137
|
});
|
|
127
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* A segment's first box held until the box that ends it arrives, then both pushed as one.
|
|
141
|
+
*/
|
|
142
|
+
var SegmentAssembly = class {
|
|
143
|
+
channel;
|
|
144
|
+
opening;
|
|
145
|
+
constructor(channel) {
|
|
146
|
+
this.channel = channel;
|
|
147
|
+
}
|
|
148
|
+
begin(box) {
|
|
149
|
+
ensureInvariant(this.opening === void 0, "a segment began before the last one ended");
|
|
150
|
+
this.opening = new Uint8Array(box);
|
|
151
|
+
}
|
|
152
|
+
end(box) {
|
|
153
|
+
const { opening } = this;
|
|
154
|
+
ensureInvariant(opening !== void 0, "a segment ended that never began");
|
|
155
|
+
const whole = new Uint8Array(opening.byteLength + box.byteLength);
|
|
156
|
+
whole.set(opening);
|
|
157
|
+
whole.set(box, opening.byteLength);
|
|
158
|
+
this.opening = void 0;
|
|
159
|
+
this.channel.push(whole);
|
|
160
|
+
}
|
|
161
|
+
};
|
|
128
162
|
//#endregion
|
|
129
163
|
export { MediabunnyAudioPackager };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { GyroViewError, isAbortError } from "../../../core/src/shared/errors/GyroViewError.js";
|
|
2
2
|
import "../../../core/src/index.js";
|
|
3
3
|
import { audioConfigurationOf, videoConfigurationOf } from "./decoderConfigurations.js";
|
|
4
|
+
import { trackColourOf } from "./trackColour.js";
|
|
4
5
|
import { ALL_FORMATS, BufferSource, Input } from "mediabunny";
|
|
5
6
|
//#region ../../packages/adapters/mediabunny/src/MediabunnyCodecReader.ts
|
|
6
7
|
/**
|
|
@@ -39,7 +40,8 @@ async function videoCodecOf(track, trackIndex) {
|
|
|
39
40
|
trackIndex,
|
|
40
41
|
codedWidth,
|
|
41
42
|
codedHeight,
|
|
42
|
-
codec: config.codec
|
|
43
|
+
codec: config.codec,
|
|
44
|
+
colour: trackColourOf(config.colorSpace)
|
|
43
45
|
};
|
|
44
46
|
return {
|
|
45
47
|
trackId: track.id,
|
|
@@ -2,7 +2,7 @@ import { copyOfBytes } from "./bufferSources.js";
|
|
|
2
2
|
//#region ../../packages/adapters/mediabunny/src/decoderConfigurations.ts
|
|
3
3
|
/**
|
|
4
4
|
* A WebCodecs video decoder configuration in the core's terms; the coded size comes from the
|
|
5
|
-
* track where the configuration leaves it out.
|
|
5
|
+
* track where the configuration leaves it out, the colour from the track's description.
|
|
6
6
|
*/
|
|
7
7
|
function videoConfigurationOf(config, description) {
|
|
8
8
|
return {
|
|
@@ -10,7 +10,7 @@ function videoConfigurationOf(config, description) {
|
|
|
10
10
|
codedWidth: config.codedWidth ?? description.codedWidth,
|
|
11
11
|
codedHeight: config.codedHeight ?? description.codedHeight,
|
|
12
12
|
description: config.description === void 0 ? void 0 : copyOfBytes(config.description),
|
|
13
|
-
|
|
13
|
+
colour: description.colour
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { COLOUR_PRIMARIES, MATRIX_COEFFICIENTS, TRANSFER_CHARACTERISTICS, namedOrUnspecified } from "../../../core/src/domain/colour/TrackColour.js";
|
|
2
|
+
import "../../../core/src/index.js";
|
|
3
|
+
//#region ../../packages/adapters/mediabunny/src/trackColour.ts
|
|
4
|
+
/**
|
|
5
|
+
* A track's colour in the core's terms, from the colour space mediabunny reads off its sample
|
|
6
|
+
* entry or, without a `colr` box, its SPS: a value the core does not name is unspecified.
|
|
7
|
+
*/
|
|
8
|
+
function trackColourOf(colorSpace) {
|
|
9
|
+
return {
|
|
10
|
+
primaries: namedOrUnspecified(COLOUR_PRIMARIES, colorSpace?.primaries),
|
|
11
|
+
transfer: namedOrUnspecified(TRANSFER_CHARACTERISTICS, colorSpace?.transfer),
|
|
12
|
+
matrix: namedOrUnspecified(MATRIX_COEFFICIENTS, colorSpace?.matrix),
|
|
13
|
+
range: rangeOf(colorSpace?.fullRange)
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function rangeOf(fullRange) {
|
|
17
|
+
if (fullRange === true) return "full";
|
|
18
|
+
return fullRange === false ? "limited" : "unspecified";
|
|
19
|
+
}
|
|
20
|
+
//#endregion
|
|
21
|
+
export { trackColourOf };
|
|
@@ -73,7 +73,7 @@ var SourceBufferFeeder = class {
|
|
|
73
73
|
async feed(from, stop) {
|
|
74
74
|
try {
|
|
75
75
|
await this.settlePendingAppend();
|
|
76
|
-
if (this.hasAllAudioFrom(from)) return;
|
|
76
|
+
if (stop.wasStopped || this.hasElementFailed() || this.hasAllAudioFrom(from)) return;
|
|
77
77
|
await this.feedFrom(this.startOf(from), stop);
|
|
78
78
|
} catch (error) {
|
|
79
79
|
this.failureValue ??= asGyroViewError(error, "decode", "feeding the audio buffer failed");
|
|
@@ -106,11 +106,15 @@ var SourceBufferFeeder = class {
|
|
|
106
106
|
}
|
|
107
107
|
/**
|
|
108
108
|
* Appends segments as the playhead needs them. True when the track ended, false when stopped.
|
|
109
|
+
* A run stops only between two whole segments (the port's promise) and never cancels an
|
|
110
|
+
* append, so the source buffer's parser is always at a segment's start when the next run
|
|
111
|
+
* appends its initialization segment.
|
|
109
112
|
*/
|
|
110
113
|
async pump(segments, stop) {
|
|
111
114
|
let next = await this.nextWhenNeeded(segments, stop);
|
|
112
115
|
while (next !== STOPPED) {
|
|
113
116
|
if (next.done === true) return true;
|
|
117
|
+
if (this.hasElementFailed()) return false;
|
|
114
118
|
this.evictBehind();
|
|
115
119
|
await this.append(next.value);
|
|
116
120
|
next = await this.nextWhenNeeded(segments, stop);
|
|
@@ -162,6 +166,13 @@ var SourceBufferFeeder = class {
|
|
|
162
166
|
}
|
|
163
167
|
if (await outcome === "error") throw new GyroViewError("decode", "the audio buffer could not parse a segment");
|
|
164
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* The element failed: every append now throws, and the element's own error says why, not the
|
|
171
|
+
* feeder's.
|
|
172
|
+
*/
|
|
173
|
+
hasElementFailed() {
|
|
174
|
+
return this.parts.element.error !== null;
|
|
175
|
+
}
|
|
165
176
|
async settlePendingAppend() {
|
|
166
177
|
if (this.parts.sourceBuffer.updating) await nextOfEvents(this.parts.sourceBuffer, ["updateend"]);
|
|
167
178
|
}
|
|
@@ -5,6 +5,7 @@ import { aspectOf } from "../../../core/src/domain/view/screenLayout.js";
|
|
|
5
5
|
import { DEFAULT_FRAMING } from "../../../core/src/domain/view/Framing.js";
|
|
6
6
|
import { viewModeRulesFor } from "../../../core/src/domain/view/viewModes.js";
|
|
7
7
|
import "../../../core/src/index.js";
|
|
8
|
+
import { MatrixCorrections } from "./matrixCorrections.js";
|
|
8
9
|
import { SAMPLING_STRATEGIES } from "./samplingStrategies.js";
|
|
9
10
|
import { applyLensGain, applyPicture, applyShaderSampling, applyStabilization, createRendererUniforms } from "./rendererUniforms.js";
|
|
10
11
|
import { createFullscreenTriangle } from "./fullscreenPass.js";
|
|
@@ -127,6 +128,7 @@ var ThreeFrameRenderer = class {
|
|
|
127
128
|
const frame = frames[index];
|
|
128
129
|
if (frame) texture.setFrame(frame.handle);
|
|
129
130
|
}
|
|
131
|
+
this.parts.matrixCorrections.follow(frames.map((frame) => frame.handle));
|
|
130
132
|
this.hasFrames = true;
|
|
131
133
|
this.render();
|
|
132
134
|
}
|
|
@@ -250,6 +252,7 @@ function assembleParts(renderer, setup, pictures) {
|
|
|
250
252
|
materials,
|
|
251
253
|
textures,
|
|
252
254
|
uniforms,
|
|
255
|
+
matrixCorrections: new MatrixCorrections(setup.lenses, uniforms.uLensMatrixCorrection),
|
|
253
256
|
lensCount: setup.lenses.length,
|
|
254
257
|
seamProof
|
|
255
258
|
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { IDENTITY_MATRIX3 } from "../../../core/src/shared/math/Matrix3.js";
|
|
2
|
+
import { BT709_LUMINANCE, HLG_OETF } from "../../../core/src/domain/colour/DisplayConversion.js";
|
|
3
|
+
import "../../../core/src/index.js";
|
|
4
|
+
import { toThreeMatrix } from "./threeMatrix.js";
|
|
5
|
+
import { glslFloat } from "./glslLiterals.js";
|
|
6
|
+
import { Vector4 } from "three";
|
|
7
|
+
//#region ../../packages/adapters/three/src/conversionUniforms.ts
|
|
8
|
+
/**
|
|
9
|
+
* The kinds of conversion as the shader tells them apart; it shows any kind but HLG's as
|
|
10
|
+
* recorded.
|
|
11
|
+
*/
|
|
12
|
+
var CONVERSION_AS_RECORDED = 0;
|
|
13
|
+
var CONVERSION_HLG_TO_SDR_BT709 = 1;
|
|
14
|
+
/**
|
|
15
|
+
* The constants `displayConversion.glsl` refers to: HLG's kind, BT.2100 HLG's constants and
|
|
16
|
+
* BT.709's luminance, from the core.
|
|
17
|
+
*/
|
|
18
|
+
var CONVERSION_DEFINES = [
|
|
19
|
+
["CONVERSION_HLG_TO_SDR_BT709", CONVERSION_HLG_TO_SDR_BT709],
|
|
20
|
+
["HLG_A", glslFloat(HLG_OETF.a)],
|
|
21
|
+
["HLG_B", glslFloat(HLG_OETF.b)],
|
|
22
|
+
["HLG_C", glslFloat(HLG_OETF.c)],
|
|
23
|
+
["HLG_SEGMENT_JOIN", glslFloat(HLG_OETF.segmentJoin)],
|
|
24
|
+
["HLG_SQUARE_SEGMENT_DIVISOR", glslFloat(HLG_OETF.squareSegmentDivisor)],
|
|
25
|
+
["HLG_LOG_SEGMENT_DIVISOR", glslFloat(HLG_OETF.logSegmentDivisor)],
|
|
26
|
+
["BT709_LUMINANCE_RED", glslFloat(BT709_LUMINANCE[0])],
|
|
27
|
+
["BT709_LUMINANCE_GREEN", glslFloat(BT709_LUMINANCE[1])],
|
|
28
|
+
["BT709_LUMINANCE_BLUE", glslFloat(BT709_LUMINANCE[2])]
|
|
29
|
+
];
|
|
30
|
+
function conversionUniforms(lenses) {
|
|
31
|
+
const slots = lenses.map((lens) => conversionSlotOf(lens.displayConversion));
|
|
32
|
+
return {
|
|
33
|
+
uLensConversion: { value: slots.map((slot) => slot.kind) },
|
|
34
|
+
uLensMatrixCorrection: { value: lenses.map(() => toThreeMatrix(IDENTITY_MATRIX3)) },
|
|
35
|
+
uLensGamut: { value: slots.map((slot) => slot.gamut) },
|
|
36
|
+
uLensTone: { value: slots.map((slot) => slot.tone) }
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Exhaustive over the conversion kinds: a new kind does not compile until it is packed. What a
|
|
41
|
+
* kind does not use stays neutral: the identity gamut, a tone of zeros.
|
|
42
|
+
*/
|
|
43
|
+
function conversionSlotOf(conversion) {
|
|
44
|
+
switch (conversion.kind) {
|
|
45
|
+
case "as-recorded": return {
|
|
46
|
+
kind: CONVERSION_AS_RECORDED,
|
|
47
|
+
gamut: toThreeMatrix(IDENTITY_MATRIX3),
|
|
48
|
+
tone: new Vector4()
|
|
49
|
+
};
|
|
50
|
+
case "hlg-to-sdr-bt709": {
|
|
51
|
+
const { exposure, kneeStart, ceiling, exponent } = conversion.tone;
|
|
52
|
+
return {
|
|
53
|
+
kind: CONVERSION_HLG_TO_SDR_BT709,
|
|
54
|
+
gamut: toThreeMatrix(conversion.gamut),
|
|
55
|
+
tone: new Vector4(exposure, kneeStart, ceiling, exponent)
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
//#endregion
|
|
61
|
+
export { CONVERSION_DEFINES, conversionUniforms };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
//#region ../../packages/adapters/three/src/glslLiterals.ts
|
|
2
|
+
/**
|
|
3
|
+
* A number as a GLSL float literal, with its decimal point: three.js writes a define's number as
|
|
4
|
+
* JavaScript prints it, so 3 would be an int, and GLSL ES 3.00 converts no int to a float.
|
|
5
|
+
*/
|
|
6
|
+
function glslFloat(value) {
|
|
7
|
+
return Number.isSafeInteger(value) ? value.toFixed(1) : String(value);
|
|
8
|
+
}
|
|
9
|
+
//#endregion
|
|
10
|
+
export { glslFloat };
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { MATRIX_COEFFICIENTS, namedOrUnspecified } from "../../../core/src/domain/colour/TrackColour.js";
|
|
2
|
+
import { matrixCorrectionOf } from "../../../core/src/domain/colour/matrixCorrection.js";
|
|
3
|
+
import "../../../core/src/index.js";
|
|
4
|
+
import { toThreeMatrix } from "./threeMatrix.js";
|
|
5
|
+
//#region ../../packages/adapters/three/src/matrixCorrections.ts
|
|
6
|
+
/**
|
|
7
|
+
* Follows the matrix each lens's frames name, and brings their texels back to the matrix the
|
|
8
|
+
* track was recorded with where the two differ (ADR 0033): WebKit's decoders convert Y′CbCr with
|
|
9
|
+
* BT.709 whatever a stream says, and name BT.709 on the frames they make.
|
|
10
|
+
*/
|
|
11
|
+
var MatrixCorrections = class {
|
|
12
|
+
lenses;
|
|
13
|
+
uniform;
|
|
14
|
+
named;
|
|
15
|
+
constructor(lenses, uniform) {
|
|
16
|
+
this.lenses = lenses;
|
|
17
|
+
this.uniform = uniform;
|
|
18
|
+
this.named = Array.from({ length: lenses.length });
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Updates a lens's correction when its frames name another matrix than before.
|
|
22
|
+
*/
|
|
23
|
+
follow(frames) {
|
|
24
|
+
for (const [index, lens] of this.lenses.entries()) {
|
|
25
|
+
const frame = frames[lens.frameSlot];
|
|
26
|
+
if (!frame) continue;
|
|
27
|
+
const named = namedOrUnspecified(MATRIX_COEFFICIENTS, frame.colorSpace.matrix);
|
|
28
|
+
if (named === this.named[index]) continue;
|
|
29
|
+
this.named[index] = named;
|
|
30
|
+
this.uniform.value[index] = toThreeMatrix(matrixCorrectionOf(lens.displayConversion.matrix, named));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
//#endregion
|
|
35
|
+
export { MatrixCorrections };
|