@decartai/sdk 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/files/types.d.ts +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.react-native.d.ts +3 -3
- package/dist/package.js +1 -1
- package/dist/realtime/client.d.ts +9 -1
- package/dist/realtime/client.js +18 -3
- package/dist/realtime/config-realtime.js +8 -0
- package/dist/realtime/stream-session.js +97 -7
- package/dist/realtime/types.d.ts +5 -1
- package/dist/shared/model.d.ts +14 -1
- package/dist/shared/model.js +16 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -82,6 +82,27 @@ Options:
|
|
|
82
82
|
- `"auto"` — mirror when the input track reports `facingMode: "user"` (mobile front cameras).
|
|
83
83
|
- `true` — always mirror (e.g. desktop webcams).
|
|
84
84
|
|
|
85
|
+
#### Fast mode
|
|
86
|
+
|
|
87
|
+
Fast mode (`speed: "fast"`) serves the session from a higher-compute tier for lower latency and higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` / `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at 2x the standard realtime rate for those models. Other models ignore the option. Omit it (the default) for standard mode.
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
const realtimeClient = await client.realtime.connect(stream, {
|
|
91
|
+
model: models.realtime("lucy-2.5"),
|
|
92
|
+
speed: "fast",
|
|
93
|
+
// ...
|
|
94
|
+
});
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Models that offer fast mode list it in their definition's `supportedSpeeds`:
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
models.realtime("lucy-2.5").supportedSpeeds; // ["fast"]
|
|
101
|
+
models.realtime("lucy-2.1").supportedSpeeds; // undefined
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Passing `speed` for a model without the capability logs a warning through the client's logger and the session runs at standard speed.
|
|
105
|
+
|
|
85
106
|
### Watch a Stream
|
|
86
107
|
|
|
87
108
|
A connected realtime session exposes an SDK `subscribeToken` once it reaches a
|
package/dist/files/types.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { ReactNativeFile } from "../process/types.js";
|
|
|
12
12
|
interface FileReference {
|
|
13
13
|
id: string;
|
|
14
14
|
filename: string | null;
|
|
15
|
+
/** Type of the stored file. The API re-encodes non-JPEG images to JPEG, so a PNG upload reports `image/jpeg`. */
|
|
15
16
|
mime_type: string;
|
|
16
17
|
size_bytes: number;
|
|
17
18
|
/**
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelKind, RealTimeModels, VideoModelDefinition, VideoModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias } from "./shared/model.js";
|
|
1
|
+
import { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelKind, RealTimeModels, RealtimeSpeed, VideoModelDefinition, VideoModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias } from "./shared/model.js";
|
|
2
2
|
import { FileInput, ProcessOptions, ReactNativeFile } from "./process/types.js";
|
|
3
3
|
import { FileReference, FileUploadInput } from "./files/types.js";
|
|
4
4
|
import { FilesClient, UploadFileOptions } from "./files/client.js";
|
|
@@ -11,7 +11,7 @@ import { G2GMetrics } from "./realtime/observability/g2g.js";
|
|
|
11
11
|
import { WebRTCStats } from "./realtime/observability/webrtc-stats.js";
|
|
12
12
|
import { ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport } from "./realtime/observability/connection-quality.js";
|
|
13
13
|
import { ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ReconnectEvent, VideoStallEvent } from "./realtime/observability/diagnostics.js";
|
|
14
|
-
import { ConnectionState, GenerationEndedMessage, QueuePosition, QueuePositionMessage } from "./realtime/types.js";
|
|
14
|
+
import { ConnectionState, GenerationEndedMessage, QueuePosition, QueuePositionMessage, SessionEnded } from "./realtime/types.js";
|
|
15
15
|
import { SetInput } from "./realtime/methods.js";
|
|
16
16
|
import { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, RealtimeMediaStream } from "./realtime/client.js";
|
|
17
17
|
import { CheckConnectivityOptions, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport } from "./realtime/preflight-types.js";
|
|
@@ -33,4 +33,4 @@ declare const createDecartClient: (options?: DecartClientOptions) => {
|
|
|
33
33
|
files: FilesClient;
|
|
34
34
|
};
|
|
35
35
|
//#endregion
|
|
36
|
-
export { CanonicalModel, CheckConnectivityOptions, ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport, ConnectionState, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport, CreateTokenOptions, CreateTokenResponse, CustomModelDefinition, type DecartClientOptions, DecartSDKError, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ERROR_CODES, FileInput, FileReference, FileUploadInput, FilesClient, G2GMetrics, GenerationEndedMessage, ImageModelDefinition, ImageModels, JobStatus, JobStatusResponse, JobSubmitResponse, ListedModelDefinition, LogLevel, Logger, Model, ModelDefinition, ModelFps, ModelKind, ModelState, ProcessClient, ProcessOptions, QueueClient, QueueJobResult, QueuePosition, QueuePositionMessage, QueueSubmitAndPollOptions, QueueSubmitOptions, ReactNativeFile, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, Events as RealTimeEvents, RealTimeModels, RealTimeSubscribeClient, RealtimeMediaStream, ReconnectEvent, SetInput, SubscribeEvents, SubscribeOptions, TokensClient, UploadFileOptions, VideoModelDefinition, VideoModels, VideoStallEvent, WebRTCStats, createConsoleLogger, createDecartClient, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, noopLogger, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias };
|
|
36
|
+
export { CanonicalModel, CheckConnectivityOptions, ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport, ConnectionState, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport, CreateTokenOptions, CreateTokenResponse, CustomModelDefinition, type DecartClientOptions, DecartSDKError, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ERROR_CODES, FileInput, FileReference, FileUploadInput, FilesClient, G2GMetrics, GenerationEndedMessage, ImageModelDefinition, ImageModels, JobStatus, JobStatusResponse, JobSubmitResponse, ListedModelDefinition, LogLevel, Logger, Model, ModelDefinition, ModelFps, ModelKind, ModelState, ProcessClient, ProcessOptions, QueueClient, QueueJobResult, QueuePosition, QueuePositionMessage, QueueSubmitAndPollOptions, QueueSubmitOptions, ReactNativeFile, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, Events as RealTimeEvents, RealTimeModels, RealTimeSubscribeClient, RealtimeMediaStream, RealtimeSpeed, ReconnectEvent, SessionEnded, SetInput, SubscribeEvents, SubscribeOptions, TokensClient, UploadFileOptions, VideoModelDefinition, VideoModels, VideoStallEvent, WebRTCStats, createConsoleLogger, createDecartClient, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, noopLogger, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelKind, RealTimeModels, VideoModelDefinition, VideoModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias } from "./shared/model.js";
|
|
1
|
+
import { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelKind, RealTimeModels, RealtimeSpeed, VideoModelDefinition, VideoModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias } from "./shared/model.js";
|
|
2
2
|
import { FileInput, ProcessOptions, ReactNativeFile } from "./process/types.js";
|
|
3
3
|
import { FileReference, FileUploadInput } from "./files/types.js";
|
|
4
4
|
import { FilesClient, UploadFileOptions } from "./files/client.js";
|
|
@@ -11,7 +11,7 @@ import { G2GMetrics } from "./realtime/observability/g2g.js";
|
|
|
11
11
|
import { WebRTCStats } from "./realtime/observability/webrtc-stats.js";
|
|
12
12
|
import { ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport } from "./realtime/observability/connection-quality.js";
|
|
13
13
|
import { ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ReconnectEvent, VideoStallEvent } from "./realtime/observability/diagnostics.js";
|
|
14
|
-
import { ConnectionState, GenerationEndedMessage, QueuePosition, QueuePositionMessage } from "./realtime/types.js";
|
|
14
|
+
import { ConnectionState, GenerationEndedMessage, QueuePosition, QueuePositionMessage, SessionEnded } from "./realtime/types.js";
|
|
15
15
|
import { SetInput } from "./realtime/methods.js";
|
|
16
16
|
import { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, RealtimeMediaStream } from "./realtime/client.js";
|
|
17
17
|
import { CheckConnectivityOptions, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport } from "./realtime/preflight-types.js";
|
|
@@ -33,4 +33,4 @@ declare const createDecartClient: (options?: DecartClientOptions) => {
|
|
|
33
33
|
files: FilesClient;
|
|
34
34
|
};
|
|
35
35
|
//#endregion
|
|
36
|
-
export { CanonicalModel, CheckConnectivityOptions, ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport, ConnectionState, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport, CreateTokenOptions, CreateTokenResponse, CustomModelDefinition, type DecartClientOptions, DecartSDKError, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ERROR_CODES, FileInput, FileReference, FileUploadInput, FilesClient, G2GMetrics, GenerationEndedMessage, ImageModelDefinition, ImageModels, JobStatus, JobStatusResponse, JobSubmitResponse, ListedModelDefinition, LogLevel, Logger, Model, ModelDefinition, ModelFps, ModelKind, ModelState, ProcessClient, ProcessOptions, QueueClient, QueueJobResult, QueuePosition, QueuePositionMessage, QueueSubmitAndPollOptions, QueueSubmitOptions, ReactNativeFile, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, Events as RealTimeEvents, RealTimeModels, RealTimeSubscribeClient, RealtimeMediaStream, ReconnectEvent, SetInput, SubscribeEvents, SubscribeOptions, TokensClient, UploadFileOptions, VideoModelDefinition, VideoModels, VideoStallEvent, WebRTCStats, createConsoleLogger, createDecartClient, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, noopLogger, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias };
|
|
36
|
+
export { CanonicalModel, CheckConnectivityOptions, ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, ConnectionQuality, ConnectionQualityLimitingFactor, ConnectionQualityMetrics, ConnectionQualityReport, ConnectionState, ConnectivityMetrics, ConnectivityReport, ConnectivityTransport, CreateTokenOptions, CreateTokenResponse, CustomModelDefinition, type DecartClientOptions, DecartSDKError, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ERROR_CODES, FileInput, FileReference, FileUploadInput, FilesClient, G2GMetrics, GenerationEndedMessage, ImageModelDefinition, ImageModels, JobStatus, JobStatusResponse, JobSubmitResponse, ListedModelDefinition, LogLevel, Logger, Model, ModelDefinition, ModelFps, ModelKind, ModelState, ProcessClient, ProcessOptions, QueueClient, QueueJobResult, QueuePosition, QueuePositionMessage, QueueSubmitAndPollOptions, QueueSubmitOptions, ReactNativeFile, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState, Events as RealTimeEvents, RealTimeModels, RealTimeSubscribeClient, RealtimeMediaStream, RealtimeSpeed, ReconnectEvent, SessionEnded, SetInput, SubscribeEvents, SubscribeOptions, TokensClient, UploadFileOptions, VideoModelDefinition, VideoModels, VideoStallEvent, WebRTCStats, createConsoleLogger, createDecartClient, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, models, noopLogger, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias };
|
package/dist/package.js
CHANGED
|
@@ -3,7 +3,7 @@ import { DecartSDKError } from "../utils/errors.js";
|
|
|
3
3
|
import { WebRTCStats } from "./observability/webrtc-stats.js";
|
|
4
4
|
import { ConnectionQualityReport } from "./observability/connection-quality.js";
|
|
5
5
|
import { DiagnosticEvent } from "./observability/diagnostics.js";
|
|
6
|
-
import { ConnectionState, GenerationEnded, GenerationTick, ImageSetOptions, QueuePosition } from "./types.js";
|
|
6
|
+
import { ConnectionState, GenerationEnded, GenerationTick, ImageSetOptions, QueuePosition, SessionEnded } from "./types.js";
|
|
7
7
|
import { SetInput } from "./methods.js";
|
|
8
8
|
import { z } from "zod";
|
|
9
9
|
|
|
@@ -36,6 +36,9 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
36
36
|
width: z.ZodNumber;
|
|
37
37
|
height: z.ZodNumber;
|
|
38
38
|
inputSchema: z.ZodOptional<z.ZodAny>;
|
|
39
|
+
supportedSpeeds: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodEnum<{
|
|
40
|
+
fast: "fast";
|
|
41
|
+
}>>>>;
|
|
39
42
|
}, z.core.$strip>;
|
|
40
43
|
onRemoteStream: z.ZodCustom<OnRemoteStreamFn, OnRemoteStreamFn>;
|
|
41
44
|
onConnectionChange: z.ZodOptional<z.ZodCustom<OnConnectionChangeFn, OnConnectionChangeFn>>;
|
|
@@ -55,6 +58,9 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
55
58
|
"720p": "720p";
|
|
56
59
|
"1080p": "1080p";
|
|
57
60
|
}>>;
|
|
61
|
+
speed: z.ZodOptional<z.ZodEnum<{
|
|
62
|
+
fast: "fast";
|
|
63
|
+
}>>;
|
|
58
64
|
preferredVideoCodec: z.ZodOptional<z.ZodEnum<{
|
|
59
65
|
h264: "h264";
|
|
60
66
|
vp8: "vp8";
|
|
@@ -78,6 +84,8 @@ type Events = {
|
|
|
78
84
|
error: DecartSDKError;
|
|
79
85
|
generationTick: GenerationTick;
|
|
80
86
|
generationEnded: GenerationEnded;
|
|
87
|
+
/** Terminal: the server ended the session and no reconnect is coming. */
|
|
88
|
+
sessionEnded: SessionEnded;
|
|
81
89
|
diagnostic: DiagnosticEvent;
|
|
82
90
|
stats: WebRTCStats;
|
|
83
91
|
};
|
package/dist/realtime/client.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { createConsoleLogger } from "../utils/logger.js";
|
|
3
3
|
import { isFileRefId } from "../files/types.js";
|
|
4
|
-
import { modelDefinitionSchema, resolveFpsNumber } from "../shared/model.js";
|
|
4
|
+
import { modelDefinitionSchema, realtimeSpeedSchema, resolveFpsNumber } from "../shared/model.js";
|
|
5
5
|
import { modelStateSchema } from "../shared/types.js";
|
|
6
6
|
import { imageToBase64 } from "../utils/media.js";
|
|
7
7
|
import { createEventBuffer } from "./event-buffer.js";
|
|
@@ -20,6 +20,14 @@ const realTimeClientConnectOptionsSchema = z.object({
|
|
|
20
20
|
queryParams: z.record(z.string(), z.string()).optional(),
|
|
21
21
|
mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
|
|
22
22
|
resolution: z.enum(["720p", "1080p"]).optional(),
|
|
23
|
+
/**
|
|
24
|
+
* Realtime speed tier. Fast mode (`"fast"`) serves the session from a higher-compute tier for lower
|
|
25
|
+
* latency and higher throughput; output quality is unchanged. It is currently available for `lucy-2.5` /
|
|
26
|
+
* `lucy-latest` and `lucy-vton-3.5` / `lucy-vton-latest`, in the US region only, and is billed at 2x the
|
|
27
|
+
* standard realtime rate for those models. Other models ignore the option (the SDK logs a warning when the
|
|
28
|
+
* model does not list it in `supportedSpeeds`). Omit it (the default) for standard mode.
|
|
29
|
+
*/
|
|
30
|
+
speed: realtimeSpeedSchema.optional(),
|
|
23
31
|
/** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
|
|
24
32
|
preferredVideoCodec: z.enum([
|
|
25
33
|
"h264",
|
|
@@ -39,7 +47,7 @@ const createRealTimeClient = (opts) => {
|
|
|
39
47
|
const connect = async (stream, options) => {
|
|
40
48
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
41
49
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
42
|
-
const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
|
|
50
|
+
const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, speed, preferredVideoCodec } = parsedOptions.data;
|
|
43
51
|
const mirror = parsedOptions.data.mirror ?? false;
|
|
44
52
|
let session;
|
|
45
53
|
let observability;
|
|
@@ -74,13 +82,19 @@ const createRealTimeClient = (opts) => {
|
|
|
74
82
|
}
|
|
75
83
|
});
|
|
76
84
|
observability = preparedConnection.observability;
|
|
85
|
+
if (speed && !options.model.supportedSpeeds?.includes(speed)) logger.warn("realtime: model does not support the requested speed tier; the server serves it at standard speed", {
|
|
86
|
+
model: options.model.name,
|
|
87
|
+
speed,
|
|
88
|
+
supportedSpeeds: options.model.supportedSpeeds ?? []
|
|
89
|
+
});
|
|
77
90
|
session = new StreamSession({
|
|
78
91
|
url: `${url}?${new URLSearchParams({
|
|
79
92
|
...preparedConnection.queryParams ?? {},
|
|
80
93
|
...options.queryParams ?? {},
|
|
81
94
|
api_key: apiKey,
|
|
82
95
|
model: options.model.name,
|
|
83
|
-
...resolution ? { resolution } : {}
|
|
96
|
+
...resolution ? { resolution } : {},
|
|
97
|
+
...speed ? { speed } : {}
|
|
84
98
|
}).toString()}`,
|
|
85
99
|
integration,
|
|
86
100
|
observability,
|
|
@@ -112,6 +126,7 @@ const createRealTimeClient = (opts) => {
|
|
|
112
126
|
});
|
|
113
127
|
session.on("generationTick", (e) => emitOrBuffer("generationTick", e));
|
|
114
128
|
session.on("generationEnded", (e) => emitOrBuffer("generationEnded", e));
|
|
129
|
+
session.on("sessionEnded", (e) => emitOrBuffer("sessionEnded", e));
|
|
115
130
|
session.on("error", (error) => {
|
|
116
131
|
logger.error("Realtime error", { error: error.message });
|
|
117
132
|
emitOrBuffer("error", classifyWebrtcError(error));
|
|
@@ -25,6 +25,14 @@ const REALTIME_CONFIG = {
|
|
|
25
25
|
minTimeout: 1e3,
|
|
26
26
|
maxTimeout: 1e4
|
|
27
27
|
},
|
|
28
|
+
/**
|
|
29
|
+
* `generation_ended` reasons the server will not change its mind about.
|
|
30
|
+
* The rest of its vocabulary (`error`, `disconnect`, `timeout`) is generic
|
|
31
|
+
* or overloaded, so those still retry.
|
|
32
|
+
*/
|
|
33
|
+
terminalEndReasons: ["moderation_violation", "insufficient_credits"],
|
|
34
|
+
/** Close code for a policy termination. */
|
|
35
|
+
terminalCloseCode: 1008,
|
|
28
36
|
permanentErrorSubstrings: [
|
|
29
37
|
"permission denied",
|
|
30
38
|
"not allowed",
|
|
@@ -4,6 +4,22 @@ import { SignalingChannel } from "./signaling-channel.js";
|
|
|
4
4
|
import mitt from "mitt";
|
|
5
5
|
import pRetry, { AbortError } from "p-retry";
|
|
6
6
|
//#region src/realtime/stream-session.ts
|
|
7
|
+
function isTerminalEndReason(reason) {
|
|
8
|
+
return reason !== void 0 && REALTIME_CONFIG.session.terminalEndReasons.includes(reason);
|
|
9
|
+
}
|
|
10
|
+
/** The reason is only sent once generation has started; earlier kills carry just the code. */
|
|
11
|
+
function terminalReasonFromClose(cause) {
|
|
12
|
+
return cause.code === REALTIME_CONFIG.session.terminalCloseCode ? "policy_violation" : null;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* SignalingChannel only emits `closed` after the handshake, so a close during the
|
|
16
|
+
* join reaches us as connect-failure text rather than an event.
|
|
17
|
+
*/
|
|
18
|
+
function terminalReasonFromError(error) {
|
|
19
|
+
if (!(error instanceof Error)) return null;
|
|
20
|
+
const marker = `websocket closed: ${REALTIME_CONFIG.session.terminalCloseCode}`;
|
|
21
|
+
return error.message.toLowerCase().includes(marker) ? "policy_violation" : null;
|
|
22
|
+
}
|
|
7
23
|
function encodeSubscribeToken(roomName, options = {}) {
|
|
8
24
|
return btoa(JSON.stringify({
|
|
9
25
|
room_name: roomName,
|
|
@@ -28,6 +44,13 @@ var StreamSession = class {
|
|
|
28
44
|
disposed = false;
|
|
29
45
|
currentAttempt = 0;
|
|
30
46
|
teardownGeneration = 0;
|
|
47
|
+
terminalEndReason = null;
|
|
48
|
+
/**
|
|
49
|
+
* Replayed as a reconnect's initial state. Callers usually connect first and
|
|
50
|
+
* send the image after, so without this the new session joins with nothing
|
|
51
|
+
* applied and never starts generating.
|
|
52
|
+
*/
|
|
53
|
+
appliedState = null;
|
|
31
54
|
logger;
|
|
32
55
|
constructor(config) {
|
|
33
56
|
this.config = config;
|
|
@@ -54,6 +77,7 @@ var StreamSession = class {
|
|
|
54
77
|
}
|
|
55
78
|
async connect() {
|
|
56
79
|
this.disposed = false;
|
|
80
|
+
this.terminalEndReason = null;
|
|
57
81
|
const attempt = ++this.currentAttempt;
|
|
58
82
|
this.setState("connecting");
|
|
59
83
|
this.logger.info("realtime connect: starting", { attemptCycle: attempt });
|
|
@@ -61,6 +85,14 @@ var StreamSession = class {
|
|
|
61
85
|
await pRetry(() => this.runOneConnect(attempt), this.retryOptionsFor(attempt));
|
|
62
86
|
} catch (error) {
|
|
63
87
|
const message = error instanceof Error ? error.message : String(error);
|
|
88
|
+
const terminal = this.terminalEndReason ?? terminalReasonFromError(error);
|
|
89
|
+
if (terminal && !this.disposed) {
|
|
90
|
+
this.finishTerminally(terminal, {
|
|
91
|
+
source: "connect",
|
|
92
|
+
error: message
|
|
93
|
+
});
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
64
96
|
this.logger.error("realtime connect: exhausted all retries", { error: message });
|
|
65
97
|
if (this.currentAttempt === attempt && !this.disposed) this.setState("disconnected");
|
|
66
98
|
throw error;
|
|
@@ -68,11 +100,25 @@ var StreamSession = class {
|
|
|
68
100
|
}
|
|
69
101
|
async sendPrompt(text, opts) {
|
|
70
102
|
this.assertConnected();
|
|
71
|
-
|
|
103
|
+
await this.signaling.sendPrompt(text, opts);
|
|
104
|
+
this.appliedState = {
|
|
105
|
+
...this.getInitialState(),
|
|
106
|
+
prompt: text,
|
|
107
|
+
enhance: opts?.enhance
|
|
108
|
+
};
|
|
72
109
|
}
|
|
73
110
|
async setImage(payload, opts) {
|
|
74
111
|
this.assertConnected();
|
|
75
|
-
|
|
112
|
+
await this.signaling.setImage(payload, opts);
|
|
113
|
+
this.appliedState = payload.kind === "ref" ? {
|
|
114
|
+
imageRef: payload.ref,
|
|
115
|
+
prompt: opts?.prompt ?? null,
|
|
116
|
+
enhance: opts?.enhance
|
|
117
|
+
} : {
|
|
118
|
+
image: payload.data,
|
|
119
|
+
prompt: opts?.prompt ?? null,
|
|
120
|
+
enhance: opts?.enhance
|
|
121
|
+
};
|
|
76
122
|
}
|
|
77
123
|
async replaceVideoTrack(track) {
|
|
78
124
|
this.assertConnected();
|
|
@@ -96,6 +142,12 @@ var StreamSession = class {
|
|
|
96
142
|
},
|
|
97
143
|
shouldRetry: (error) => {
|
|
98
144
|
if (this.disposed || this.currentAttempt !== attempt) return false;
|
|
145
|
+
const terminal = this.terminalEndReason ?? terminalReasonFromError(error);
|
|
146
|
+
if (terminal) {
|
|
147
|
+
this.terminalEndReason = terminal;
|
|
148
|
+
this.logger.error("realtime connect: session refused, not retrying", { reason: terminal });
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
99
151
|
const msg = error.message.toLowerCase();
|
|
100
152
|
const permanent = REALTIME_CONFIG.session.permanentErrorSubstrings.some((err) => msg.includes(err));
|
|
101
153
|
if (permanent) this.logger.error("realtime connect: permanent error, not retrying", { error: error.message });
|
|
@@ -158,6 +210,9 @@ var StreamSession = class {
|
|
|
158
210
|
});
|
|
159
211
|
}
|
|
160
212
|
getInitialState() {
|
|
213
|
+
return this.appliedState ?? this.configInitialState();
|
|
214
|
+
}
|
|
215
|
+
configInitialState() {
|
|
161
216
|
if (this.config.initialImageRef !== void 0) return {
|
|
162
217
|
imageRef: this.config.initialImageRef,
|
|
163
218
|
prompt: this.config.initialPrompt?.text,
|
|
@@ -183,12 +238,18 @@ var StreamSession = class {
|
|
|
183
238
|
this.markGenerating();
|
|
184
239
|
this.events.emit("generationTick", e);
|
|
185
240
|
});
|
|
186
|
-
this.signaling.on("generationEnded", (e) =>
|
|
241
|
+
this.signaling.on("generationEnded", (e) => {
|
|
242
|
+
if (isTerminalEndReason(e.reason)) this.terminalEndReason = e.reason;
|
|
243
|
+
this.events.emit("generationEnded", e);
|
|
244
|
+
});
|
|
187
245
|
this.signaling.on("serverError", (err) => this.events.emit("error", err));
|
|
188
|
-
this.signaling.on("closed", (info) =>
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
246
|
+
this.signaling.on("closed", (info) => {
|
|
247
|
+
this.terminalEndReason ??= terminalReasonFromClose({ ...info });
|
|
248
|
+
this.handleConnectionLoss({
|
|
249
|
+
source: "signaling",
|
|
250
|
+
...info
|
|
251
|
+
});
|
|
252
|
+
});
|
|
192
253
|
}
|
|
193
254
|
markGenerating() {
|
|
194
255
|
if (this.state === "connected") this.setState("generating");
|
|
@@ -202,6 +263,11 @@ var StreamSession = class {
|
|
|
202
263
|
}
|
|
203
264
|
handleConnectionLoss(cause) {
|
|
204
265
|
if (this.disposed) return;
|
|
266
|
+
const terminalReason = this.terminalEndReason ?? terminalReasonFromClose(cause);
|
|
267
|
+
if (terminalReason) {
|
|
268
|
+
this.finishTerminally(terminalReason, cause);
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
205
271
|
if (this.state !== "connected" && this.state !== "generating") {
|
|
206
272
|
this.logger.debug("connection loss ignored (not connected)", {
|
|
207
273
|
state: this.state,
|
|
@@ -215,6 +281,22 @@ var StreamSession = class {
|
|
|
215
281
|
});
|
|
216
282
|
this.scheduleReconnect();
|
|
217
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* The server ended this session on purpose: retrying would get the same answer
|
|
286
|
+
* and bill another session, and "reconnecting" would hide the reason.
|
|
287
|
+
*/
|
|
288
|
+
finishTerminally(reason, cause) {
|
|
289
|
+
this.logger.warn("realtime session ended by the server; not reconnecting", {
|
|
290
|
+
reason,
|
|
291
|
+
state: this.state,
|
|
292
|
+
...cause
|
|
293
|
+
});
|
|
294
|
+
this.terminalEndReason = reason;
|
|
295
|
+
this.disposed = true;
|
|
296
|
+
this.tearDown();
|
|
297
|
+
this.setState("disconnected");
|
|
298
|
+
this.events.emit("sessionEnded", { reason });
|
|
299
|
+
}
|
|
218
300
|
scheduleReconnect() {
|
|
219
301
|
const attempt = ++this.currentAttempt;
|
|
220
302
|
this.setState("reconnecting");
|
|
@@ -229,6 +311,14 @@ var StreamSession = class {
|
|
|
229
311
|
}).catch((error) => {
|
|
230
312
|
if (this.disposed || this.currentAttempt !== attempt) return;
|
|
231
313
|
const message = error instanceof Error ? error.message : String(error);
|
|
314
|
+
const terminal = this.terminalEndReason ?? terminalReasonFromError(error);
|
|
315
|
+
if (terminal) {
|
|
316
|
+
this.finishTerminally(terminal, {
|
|
317
|
+
source: "reconnect",
|
|
318
|
+
error: message
|
|
319
|
+
});
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
232
322
|
this.logger.error("realtime reconnect: failed permanently", { error: message });
|
|
233
323
|
this.tearDown();
|
|
234
324
|
this.setState("disconnected");
|
package/dist/realtime/types.d.ts
CHANGED
|
@@ -20,10 +20,14 @@ type GenerationEnded = {
|
|
|
20
20
|
seconds: number;
|
|
21
21
|
reason: string;
|
|
22
22
|
};
|
|
23
|
+
/** The server ended the session on purpose; the SDK will not reconnect. */
|
|
24
|
+
type SessionEnded = {
|
|
25
|
+
reason: string;
|
|
26
|
+
};
|
|
23
27
|
type ImageSetOptions = {
|
|
24
28
|
prompt?: string | null;
|
|
25
29
|
enhance?: boolean;
|
|
26
30
|
timeout?: number;
|
|
27
31
|
};
|
|
28
32
|
//#endregion
|
|
29
|
-
export { ConnectionState, GenerationEnded, GenerationEndedMessage, GenerationTick, ImageSetOptions, QueuePosition, QueuePositionMessage };
|
|
33
|
+
export { ConnectionState, GenerationEnded, GenerationEndedMessage, GenerationTick, ImageSetOptions, QueuePosition, QueuePositionMessage, SessionEnded };
|
package/dist/shared/model.d.ts
CHANGED
|
@@ -289,6 +289,14 @@ type ModelFps = number | {
|
|
|
289
289
|
ideal?: number;
|
|
290
290
|
exact?: number;
|
|
291
291
|
};
|
|
292
|
+
/**
|
|
293
|
+
* Realtime speed tiers selectable with the `speed` connect option.
|
|
294
|
+
* `"fast"` is the only tier today; the standard tier has no value (omit the option).
|
|
295
|
+
*/
|
|
296
|
+
declare const realtimeSpeedSchema: z.ZodEnum<{
|
|
297
|
+
fast: "fast";
|
|
298
|
+
}>;
|
|
299
|
+
type RealtimeSpeed = z.infer<typeof realtimeSpeedSchema>;
|
|
292
300
|
type ModelDefinition<T extends Model = Model> = {
|
|
293
301
|
name: T;
|
|
294
302
|
urlPath: string;
|
|
@@ -297,6 +305,11 @@ type ModelDefinition<T extends Model = Model> = {
|
|
|
297
305
|
width: number;
|
|
298
306
|
height: number;
|
|
299
307
|
inputSchema: T extends keyof ModelInputSchemas ? ModelInputSchemas[T] : z.ZodTypeAny;
|
|
308
|
+
/**
|
|
309
|
+
* Realtime speed tiers this model can be served from via the `speed` connect option.
|
|
310
|
+
* Absent when the model only offers the standard tier (the server ignores `speed` for it).
|
|
311
|
+
*/
|
|
312
|
+
supportedSpeeds?: readonly RealtimeSpeed[];
|
|
300
313
|
};
|
|
301
314
|
declare function resolveFpsNumber(fps: ModelFps, fallback?: number): number;
|
|
302
315
|
/**
|
|
@@ -375,4 +388,4 @@ declare const models: {
|
|
|
375
388
|
};
|
|
376
389
|
};
|
|
377
390
|
//#endregion
|
|
378
|
-
export { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelInputSchemas, ModelKind, RealTimeModels, VideoModelDefinition, VideoModels, canonicalModelSchema, imageModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, modelInputSchemas, modelSchema, models, realtimeModels, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias, videoModels };
|
|
391
|
+
export { CanonicalModel, CustomModelDefinition, ImageModelDefinition, ImageModels, ListedModelDefinition, Model, ModelDefinition, ModelFps, ModelInputSchemas, ModelKind, RealTimeModels, RealtimeSpeed, VideoModelDefinition, VideoModels, canonicalModelSchema, imageModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, modelInputSchemas, modelSchema, models, realtimeModels, realtimeSpeedSchema, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias, videoModels };
|
package/dist/shared/model.js
CHANGED
|
@@ -185,6 +185,11 @@ const modelInputSchemas = {
|
|
|
185
185
|
"lucy-pro-i2i": imageEditSchema,
|
|
186
186
|
"lucy-restyle-v2v": restyleSchema
|
|
187
187
|
};
|
|
188
|
+
/**
|
|
189
|
+
* Realtime speed tiers selectable with the `speed` connect option.
|
|
190
|
+
* `"fast"` is the only tier today; the standard tier has no value (omit the option).
|
|
191
|
+
*/
|
|
192
|
+
const realtimeSpeedSchema = z.enum(["fast"]);
|
|
188
193
|
function resolveFpsNumber(fps, fallback = 30) {
|
|
189
194
|
if (typeof fps === "number") return fps;
|
|
190
195
|
return fps.ideal ?? fps.max ?? fps.exact ?? fps.min ?? fallback;
|
|
@@ -201,7 +206,8 @@ const modelDefinitionSchema = z.object({
|
|
|
201
206
|
})]),
|
|
202
207
|
width: z.number().min(1),
|
|
203
208
|
height: z.number().min(1),
|
|
204
|
-
inputSchema: z.any().optional()
|
|
209
|
+
inputSchema: z.any().optional(),
|
|
210
|
+
supportedSpeeds: z.array(realtimeSpeedSchema).readonly().optional()
|
|
205
211
|
});
|
|
206
212
|
const _models = {
|
|
207
213
|
realtime: {
|
|
@@ -225,7 +231,8 @@ const _models = {
|
|
|
225
231
|
},
|
|
226
232
|
width: 1280,
|
|
227
233
|
height: 720,
|
|
228
|
-
inputSchema: z.object({})
|
|
234
|
+
inputSchema: z.object({}),
|
|
235
|
+
supportedSpeeds: ["fast"]
|
|
229
236
|
},
|
|
230
237
|
"lucy-vton-3.5": {
|
|
231
238
|
urlPath: "/v1/stream",
|
|
@@ -236,7 +243,8 @@ const _models = {
|
|
|
236
243
|
},
|
|
237
244
|
width: 1280,
|
|
238
245
|
height: 720,
|
|
239
|
-
inputSchema: z.object({})
|
|
246
|
+
inputSchema: z.object({}),
|
|
247
|
+
supportedSpeeds: ["fast"]
|
|
240
248
|
},
|
|
241
249
|
"lucy-restyle-2": {
|
|
242
250
|
urlPath: "/v1/stream",
|
|
@@ -258,7 +266,8 @@ const _models = {
|
|
|
258
266
|
},
|
|
259
267
|
width: 1088,
|
|
260
268
|
height: 624,
|
|
261
|
-
inputSchema: z.object({})
|
|
269
|
+
inputSchema: z.object({}),
|
|
270
|
+
supportedSpeeds: ["fast"]
|
|
262
271
|
},
|
|
263
272
|
"lucy-vton-latest": {
|
|
264
273
|
urlPath: "/v1/stream",
|
|
@@ -269,7 +278,8 @@ const _models = {
|
|
|
269
278
|
},
|
|
270
279
|
width: 1280,
|
|
271
280
|
height: 720,
|
|
272
|
-
inputSchema: z.object({})
|
|
281
|
+
inputSchema: z.object({}),
|
|
282
|
+
supportedSpeeds: ["fast"]
|
|
273
283
|
},
|
|
274
284
|
"lucy-restyle-latest": {
|
|
275
285
|
urlPath: "/v1/stream",
|
|
@@ -483,4 +493,4 @@ const models = {
|
|
|
483
493
|
}
|
|
484
494
|
};
|
|
485
495
|
//#endregion
|
|
486
|
-
export { canonicalImageModels, canonicalModelSchema, canonicalRealtimeModels, canonicalVideoModels, imageModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, modelDefinitionSchema, modelInputSchemas, modelSchema, models, realtimeModels, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias, videoModels };
|
|
496
|
+
export { canonicalImageModels, canonicalModelSchema, canonicalRealtimeModels, canonicalVideoModels, imageModels, isCanonicalModel, isImageModel, isModel, isRealtimeModel, isVideoModel, listModels, modelAliases, modelDefinitionSchema, modelInputSchemas, modelSchema, models, realtimeModels, realtimeSpeedSchema, resolveCanonicalModelAlias, resolveFpsNumber, resolveModelAlias, videoModels };
|