@warlock.js/ai-live 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/LICENSE +21 -0
- package/README.md +76 -0
- package/cjs/index.cjs +257 -0
- package/cjs/index.cjs.map +1 -0
- package/esm/contracts/realtime.contract.d.mts +104 -0
- package/esm/contracts/realtime.contract.d.mts.map +1 -0
- package/esm/contracts/video.contract.d.mts +72 -0
- package/esm/contracts/video.contract.d.mts.map +1 -0
- package/esm/index.d.mts +33 -0
- package/esm/index.d.mts.map +1 -0
- package/esm/index.mjs +12 -0
- package/esm/index.mjs.map +1 -0
- package/esm/mock/index.d.mts +52 -0
- package/esm/mock/index.d.mts.map +1 -0
- package/esm/mock/index.mjs +74 -0
- package/esm/mock/index.mjs.map +1 -0
- package/esm/realtime/realtime.d.mts +29 -0
- package/esm/realtime/realtime.d.mts.map +1 -0
- package/esm/realtime/realtime.mjs +61 -0
- package/esm/realtime/realtime.mjs.map +1 -0
- package/esm/video/video.d.mts +59 -0
- package/esm/video/video.d.mts.map +1 -0
- package/esm/video/video.mjs +118 -0
- package/esm/video/video.mjs.map +1 -0
- package/llms-full.txt +210 -0
- package/llms.txt +9 -0
- package/package.json +34 -0
- package/skills/use-ai-live/SKILL.md +200 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { RealtimeConnectConfig, RealtimeConnection, RealtimeEvent, RealtimeTransport } from "../contracts/realtime.contract.mjs";
|
|
2
|
+
import { GeneratedVideo, VideoGenerationResponse, VideoModelContract, VideoModelPricing, VideoOptions } from "../contracts/video.contract.mjs";
|
|
3
|
+
import { Usage } from "@warlock.js/ai";
|
|
4
|
+
|
|
5
|
+
//#region ../@warlock.js/ai-live/src/mock/index.d.ts
|
|
6
|
+
/** One scripted response for a {@link MockVideoModel}. */
|
|
7
|
+
type MockVideoResponse = {
|
|
8
|
+
video?: GeneratedVideo;
|
|
9
|
+
usage?: Usage;
|
|
10
|
+
durationSeconds?: number;
|
|
11
|
+
error?: Error;
|
|
12
|
+
};
|
|
13
|
+
/** Deterministic {@link VideoModelContract} double for tests — no HTTP, no polling. */
|
|
14
|
+
declare class MockVideoModel implements VideoModelContract {
|
|
15
|
+
readonly name: string;
|
|
16
|
+
private readonly responses;
|
|
17
|
+
readonly pricing?: VideoModelPricing | undefined;
|
|
18
|
+
readonly provider = "mock";
|
|
19
|
+
readonly calls: {
|
|
20
|
+
prompt: string;
|
|
21
|
+
options: VideoOptions | undefined;
|
|
22
|
+
}[];
|
|
23
|
+
private callIndex;
|
|
24
|
+
constructor(name: string, responses: MockVideoResponse[], pricing?: VideoModelPricing | undefined);
|
|
25
|
+
generate(prompt: string, options?: VideoOptions): Promise<VideoGenerationResponse>;
|
|
26
|
+
}
|
|
27
|
+
/** A recording {@link RealtimeConnection} the mock transport hands back. */
|
|
28
|
+
declare class MockRealtimeConnection implements RealtimeConnection {
|
|
29
|
+
private readonly scripted;
|
|
30
|
+
readonly sentAudio: {
|
|
31
|
+
base64: string;
|
|
32
|
+
mediaType: string;
|
|
33
|
+
}[];
|
|
34
|
+
readonly sentText: string[];
|
|
35
|
+
closed: boolean;
|
|
36
|
+
constructor(scripted: RealtimeEvent[]);
|
|
37
|
+
sendAudio(base64: string, mediaType: string): void;
|
|
38
|
+
sendText(text: string): void;
|
|
39
|
+
events(): AsyncIterable<RealtimeEvent>;
|
|
40
|
+
close(): Promise<void>;
|
|
41
|
+
}
|
|
42
|
+
/** Deterministic {@link RealtimeTransport} double — scripts the event stream, records sends. */
|
|
43
|
+
declare class MockRealtimeTransport implements RealtimeTransport {
|
|
44
|
+
private readonly scriptedEvents;
|
|
45
|
+
readonly connectConfigs: RealtimeConnectConfig[];
|
|
46
|
+
lastConnection?: MockRealtimeConnection;
|
|
47
|
+
constructor(scriptedEvents?: RealtimeEvent[]);
|
|
48
|
+
connect(config: RealtimeConnectConfig): Promise<RealtimeConnection>;
|
|
49
|
+
}
|
|
50
|
+
//#endregion
|
|
51
|
+
export { MockRealtimeConnection, MockRealtimeTransport, MockVideoModel, MockVideoResponse };
|
|
52
|
+
//# sourceMappingURL=index.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/mock/index.ts"],"mappings":";;;;;;KAgBY,iBAAA;EACV,KAAA,GAAQ,cAAA;EACR,KAAA,GAAQ,KAAA;EACR,eAAA;EACA,KAAA,GAAQ,KAAA;AAAA;;cAIG,cAAA,YAA0B,kBAAA;EAAA,SAOnB,IAAA;EAAA,iBACC,SAAA;EAAA,SACD,OAAA,GAAU,iBAAA;EAAA,SARZ,QAAA;EAAA,SACA,KAAA;IAAS,MAAA;IAAgB,OAAA,EAAS,YAAA;EAAA;EAAA,QAE1C,SAAA;cAGU,IAAA,UACC,SAAA,EAAW,iBAAA,IACZ,OAAA,GAAU,iBAAA;EAGf,QAAA,CACX,MAAA,UACA,OAAA,GAAU,YAAA,GACT,OAAA,CAAQ,uBAAA;AAAA;;cAmBA,sBAAA,YAAkC,kBAAA;EAAA,iBAKT,QAAA;EAAA,SAJpB,SAAA;IAAa,MAAA;IAAgB,SAAA;EAAA;EAAA,SAC7B,QAAA;EACT,MAAA;cAE6B,QAAA,EAAU,aAAA;EAEvC,SAAA,CAAU,MAAA,UAAgB,SAAA;EAI1B,QAAA,CAAS,IAAA;EAIF,MAAA,IAAU,aAAA,CAAc,aAAA;EAMzB,KAAA,IAAS,OAAA;AAAA;;cAMX,qBAAA,YAAiC,iBAAA;EAAA,iBAIR,cAAA;EAAA,SAHpB,cAAA,EAAgB,qBAAA;EACzB,cAAA,GAAiB,sBAAA;cAEY,cAAA,GAAgB,aAAA;EAEvC,OAAA,CAAQ,MAAA,EAAQ,qBAAA,GAAwB,OAAA,CAAQ,kBAAA;AAAA"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
//#region ../@warlock.js/ai-live/src/mock/index.ts
|
|
2
|
+
/** Deterministic {@link VideoModelContract} double for tests — no HTTP, no polling. */
|
|
3
|
+
var MockVideoModel = class {
|
|
4
|
+
constructor(name, responses, pricing) {
|
|
5
|
+
this.name = name;
|
|
6
|
+
this.responses = responses;
|
|
7
|
+
this.pricing = pricing;
|
|
8
|
+
this.provider = "mock";
|
|
9
|
+
this.calls = [];
|
|
10
|
+
this.callIndex = 0;
|
|
11
|
+
}
|
|
12
|
+
async generate(prompt, options) {
|
|
13
|
+
this.calls.push({
|
|
14
|
+
prompt,
|
|
15
|
+
options
|
|
16
|
+
});
|
|
17
|
+
const response = this.responses[Math.min(this.callIndex, this.responses.length - 1)] ?? {};
|
|
18
|
+
this.callIndex += 1;
|
|
19
|
+
if (response.error) throw response.error;
|
|
20
|
+
return {
|
|
21
|
+
video: response.video ?? {
|
|
22
|
+
type: "url",
|
|
23
|
+
url: "https://mock/video.mp4",
|
|
24
|
+
mediaType: "video/mp4"
|
|
25
|
+
},
|
|
26
|
+
usage: response.usage ?? {
|
|
27
|
+
input: 0,
|
|
28
|
+
output: 0,
|
|
29
|
+
total: 0
|
|
30
|
+
},
|
|
31
|
+
durationSeconds: response.durationSeconds ?? options?.durationSeconds ?? 5
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
/** A recording {@link RealtimeConnection} the mock transport hands back. */
|
|
36
|
+
var MockRealtimeConnection = class {
|
|
37
|
+
constructor(scripted) {
|
|
38
|
+
this.scripted = scripted;
|
|
39
|
+
this.sentAudio = [];
|
|
40
|
+
this.sentText = [];
|
|
41
|
+
this.closed = false;
|
|
42
|
+
}
|
|
43
|
+
sendAudio(base64, mediaType) {
|
|
44
|
+
this.sentAudio.push({
|
|
45
|
+
base64,
|
|
46
|
+
mediaType
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
sendText(text) {
|
|
50
|
+
this.sentText.push(text);
|
|
51
|
+
}
|
|
52
|
+
async *events() {
|
|
53
|
+
for (const event of this.scripted) yield event;
|
|
54
|
+
}
|
|
55
|
+
async close() {
|
|
56
|
+
this.closed = true;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
/** Deterministic {@link RealtimeTransport} double — scripts the event stream, records sends. */
|
|
60
|
+
var MockRealtimeTransport = class {
|
|
61
|
+
constructor(scriptedEvents = []) {
|
|
62
|
+
this.scriptedEvents = scriptedEvents;
|
|
63
|
+
this.connectConfigs = [];
|
|
64
|
+
}
|
|
65
|
+
async connect(config) {
|
|
66
|
+
this.connectConfigs.push(config);
|
|
67
|
+
this.lastConnection = new MockRealtimeConnection(this.scriptedEvents);
|
|
68
|
+
return this.lastConnection;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
//#endregion
|
|
73
|
+
export { MockRealtimeConnection, MockRealtimeTransport, MockVideoModel };
|
|
74
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/mock/index.ts"],"sourcesContent":["import type { Usage } from \"@warlock.js/ai\";\nimport type {\n RealtimeConnectConfig,\n RealtimeConnection,\n RealtimeEvent,\n RealtimeTransport,\n} from \"../contracts/realtime.contract\";\nimport type {\n GeneratedVideo,\n VideoGenerationResponse,\n VideoModelContract,\n VideoModelPricing,\n VideoOptions,\n} from \"../contracts/video.contract\";\n\n/** One scripted response for a {@link MockVideoModel}. */\nexport type MockVideoResponse = {\n video?: GeneratedVideo;\n usage?: Usage;\n durationSeconds?: number;\n error?: Error;\n};\n\n/** Deterministic {@link VideoModelContract} double for tests — no HTTP, no polling. */\nexport class MockVideoModel implements VideoModelContract {\n public readonly provider = \"mock\";\n public readonly calls: { prompt: string; options: VideoOptions | undefined }[] = [];\n\n private callIndex = 0;\n\n public constructor(\n public readonly name: string,\n private readonly responses: MockVideoResponse[],\n public readonly pricing?: VideoModelPricing,\n ) {}\n\n public async generate(\n prompt: string,\n options?: VideoOptions,\n ): Promise<VideoGenerationResponse> {\n this.calls.push({ prompt, options });\n\n const response = this.responses[Math.min(this.callIndex, this.responses.length - 1)] ?? {};\n this.callIndex += 1;\n\n if (response.error) {\n throw response.error;\n }\n\n return {\n video: response.video ?? { type: \"url\", url: \"https://mock/video.mp4\", mediaType: \"video/mp4\" },\n usage: response.usage ?? { input: 0, output: 0, total: 0 },\n durationSeconds: response.durationSeconds ?? options?.durationSeconds ?? 5,\n };\n }\n}\n\n/** A recording {@link RealtimeConnection} the mock transport hands back. */\nexport class MockRealtimeConnection implements RealtimeConnection {\n public readonly sentAudio: { base64: string; mediaType: string }[] = [];\n public readonly sentText: string[] = [];\n public closed = false;\n\n public constructor(private readonly scripted: RealtimeEvent[]) {}\n\n public sendAudio(base64: string, mediaType: string): void {\n this.sentAudio.push({ base64, mediaType });\n }\n\n public sendText(text: string): void {\n this.sentText.push(text);\n }\n\n public async *events(): AsyncIterable<RealtimeEvent> {\n for (const event of this.scripted) {\n yield event;\n }\n }\n\n public async close(): Promise<void> {\n this.closed = true;\n }\n}\n\n/** Deterministic {@link RealtimeTransport} double — scripts the event stream, records sends. */\nexport class MockRealtimeTransport implements RealtimeTransport {\n public readonly connectConfigs: RealtimeConnectConfig[] = [];\n public lastConnection?: MockRealtimeConnection;\n\n public constructor(private readonly scriptedEvents: RealtimeEvent[] = []) {}\n\n public async connect(config: RealtimeConnectConfig): Promise<RealtimeConnection> {\n this.connectConfigs.push(config);\n this.lastConnection = new MockRealtimeConnection(this.scriptedEvents);\n return this.lastConnection;\n }\n}\n"],"mappings":";;AAwBA,IAAa,iBAAb,MAA0D;CAMxD,AAAO,YACL,AAAgB,MAChB,AAAiB,WACjB,AAAgB,SAChB;EAHgB;EACC;EACD;kBARS;eACsD,CAAC;mBAE9D;CAMjB;CAEH,MAAa,SACX,QACA,SACkC;EAClC,KAAK,MAAM,KAAK;GAAE;GAAQ;EAAQ,CAAC;EAEnC,MAAM,WAAW,KAAK,UAAU,KAAK,IAAI,KAAK,WAAW,KAAK,UAAU,SAAS,CAAC,MAAM,CAAC;EACzF,KAAK,aAAa;EAElB,IAAI,SAAS,OACX,MAAM,SAAS;EAGjB,OAAO;GACL,OAAO,SAAS,SAAS;IAAE,MAAM;IAAO,KAAK;IAA0B,WAAW;GAAY;GAC9F,OAAO,SAAS,SAAS;IAAE,OAAO;IAAG,QAAQ;IAAG,OAAO;GAAE;GACzD,iBAAiB,SAAS,mBAAmB,SAAS,mBAAmB;EAC3E;CACF;AACF;;AAGA,IAAa,yBAAb,MAAkE;CAKhE,AAAO,YAAY,AAAiB,UAA2B;EAA3B;mBAJiC,CAAC;kBACjC,CAAC;gBACtB;CAEgD;CAEhE,AAAO,UAAU,QAAgB,WAAyB;EACxD,KAAK,UAAU,KAAK;GAAE;GAAQ;EAAU,CAAC;CAC3C;CAEA,AAAO,SAAS,MAAoB;EAClC,KAAK,SAAS,KAAK,IAAI;CACzB;CAEA,OAAc,SAAuC;EACnD,KAAK,MAAM,SAAS,KAAK,UACvB,MAAM;CAEV;CAEA,MAAa,QAAuB;EAClC,KAAK,SAAS;CAChB;AACF;;AAGA,IAAa,wBAAb,MAAgE;CAI9D,AAAO,YAAY,AAAiB,iBAAkC,CAAC,GAAG;EAAtC;wBAHsB,CAAC;CAGgB;CAE3E,MAAa,QAAQ,QAA4D;EAC/E,KAAK,eAAe,KAAK,MAAM;EAC/B,KAAK,iBAAiB,IAAI,uBAAuB,KAAK,cAAc;EACpE,OAAO,KAAK;CACd;AACF"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { RealtimeOptions, RealtimeSession } from "../contracts/realtime.contract.mjs";
|
|
2
|
+
|
|
3
|
+
//#region ../@warlock.js/ai-live/src/realtime/realtime.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* Open a live duplex voice session — the stateful primitive of
|
|
6
|
+
* `@warlock.js/ai-live`. Connects through the provided
|
|
7
|
+
* {@link RealtimeOptions.transport} (an OpenAI Realtime WebSocket
|
|
8
|
+
* adapter, a mock in tests), then hands back a {@link RealtimeSession}
|
|
9
|
+
* you drive: push audio/text in, consume the event stream out, and
|
|
10
|
+
* `close()` to end it and receive a `type: "realtime"` report for the
|
|
11
|
+
* cost/observability surfaces.
|
|
12
|
+
*
|
|
13
|
+
* Unlike the one-shot `ai.*` verbs, this returns a long-lived session —
|
|
14
|
+
* its closest sibling is `ai.orchestrator`. Keeping the transport
|
|
15
|
+
* pluggable is what lets the session surface ship without hard-wiring a
|
|
16
|
+
* WebSocket dependency.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* const session = await ai.realtime({ transport, model: "gpt-realtime", voice: "alloy" });
|
|
20
|
+
* session.sendAudio(micChunk, "audio/pcm");
|
|
21
|
+
* for await (const event of session.events()) {
|
|
22
|
+
* if (event.type === "audio") speaker.write(event.base64);
|
|
23
|
+
* }
|
|
24
|
+
* const report = await session.close();
|
|
25
|
+
*/
|
|
26
|
+
declare function realtime(options: RealtimeOptions): Promise<RealtimeSession>;
|
|
27
|
+
//#endregion
|
|
28
|
+
export { realtime };
|
|
29
|
+
//# sourceMappingURL=realtime.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtime.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/realtime/realtime.ts"],"mappings":";;;;;AA6BA;;;;;;;;;;;;;AAAiF;;;;;;;iBAA3D,QAAA,CAAS,OAAA,EAAS,eAAA,GAAkB,OAAA,CAAQ,eAAA"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { generateRunId } from "@warlock.js/ai";
|
|
2
|
+
|
|
3
|
+
//#region ../@warlock.js/ai-live/src/realtime/realtime.ts
|
|
4
|
+
/**
|
|
5
|
+
* Open a live duplex voice session — the stateful primitive of
|
|
6
|
+
* `@warlock.js/ai-live`. Connects through the provided
|
|
7
|
+
* {@link RealtimeOptions.transport} (an OpenAI Realtime WebSocket
|
|
8
|
+
* adapter, a mock in tests), then hands back a {@link RealtimeSession}
|
|
9
|
+
* you drive: push audio/text in, consume the event stream out, and
|
|
10
|
+
* `close()` to end it and receive a `type: "realtime"` report for the
|
|
11
|
+
* cost/observability surfaces.
|
|
12
|
+
*
|
|
13
|
+
* Unlike the one-shot `ai.*` verbs, this returns a long-lived session —
|
|
14
|
+
* its closest sibling is `ai.orchestrator`. Keeping the transport
|
|
15
|
+
* pluggable is what lets the session surface ship without hard-wiring a
|
|
16
|
+
* WebSocket dependency.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* const session = await ai.realtime({ transport, model: "gpt-realtime", voice: "alloy" });
|
|
20
|
+
* session.sendAudio(micChunk, "audio/pcm");
|
|
21
|
+
* for await (const event of session.events()) {
|
|
22
|
+
* if (event.type === "audio") speaker.write(event.base64);
|
|
23
|
+
* }
|
|
24
|
+
* const report = await session.close();
|
|
25
|
+
*/
|
|
26
|
+
async function realtime(options) {
|
|
27
|
+
const runId = generateRunId("realtime");
|
|
28
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
29
|
+
const startPerf = performance.now();
|
|
30
|
+
const connection = await options.transport.connect({
|
|
31
|
+
model: options.model,
|
|
32
|
+
voice: options.voice,
|
|
33
|
+
instructions: options.instructions
|
|
34
|
+
});
|
|
35
|
+
let report;
|
|
36
|
+
return {
|
|
37
|
+
sendAudio: (base64, mediaType) => connection.sendAudio(base64, mediaType),
|
|
38
|
+
sendText: (text) => connection.sendText(text),
|
|
39
|
+
events: () => connection.events(),
|
|
40
|
+
async close() {
|
|
41
|
+
if (report) return report;
|
|
42
|
+
await connection.close();
|
|
43
|
+
report = {
|
|
44
|
+
runId,
|
|
45
|
+
rootRunId: runId,
|
|
46
|
+
type: "realtime",
|
|
47
|
+
name: options.name ?? "realtime",
|
|
48
|
+
status: "completed",
|
|
49
|
+
startedAt,
|
|
50
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
51
|
+
duration: performance.now() - startPerf,
|
|
52
|
+
...options.sessionId ? { sessionId: options.sessionId } : {}
|
|
53
|
+
};
|
|
54
|
+
return report;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
//#endregion
|
|
60
|
+
export { realtime };
|
|
61
|
+
//# sourceMappingURL=realtime.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtime.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/realtime/realtime.ts"],"sourcesContent":["import { generateRunId } from \"@warlock.js/ai\";\nimport type {\n RealtimeOptions,\n RealtimeReport,\n RealtimeSession,\n} from \"../contracts/realtime.contract\";\n\n/**\n * Open a live duplex voice session — the stateful primitive of\n * `@warlock.js/ai-live`. Connects through the provided\n * {@link RealtimeOptions.transport} (an OpenAI Realtime WebSocket\n * adapter, a mock in tests), then hands back a {@link RealtimeSession}\n * you drive: push audio/text in, consume the event stream out, and\n * `close()` to end it and receive a `type: \"realtime\"` report for the\n * cost/observability surfaces.\n *\n * Unlike the one-shot `ai.*` verbs, this returns a long-lived session —\n * its closest sibling is `ai.orchestrator`. Keeping the transport\n * pluggable is what lets the session surface ship without hard-wiring a\n * WebSocket dependency.\n *\n * @example\n * const session = await ai.realtime({ transport, model: \"gpt-realtime\", voice: \"alloy\" });\n * session.sendAudio(micChunk, \"audio/pcm\");\n * for await (const event of session.events()) {\n * if (event.type === \"audio\") speaker.write(event.base64);\n * }\n * const report = await session.close();\n */\nexport async function realtime(options: RealtimeOptions): Promise<RealtimeSession> {\n const runId = generateRunId(\"realtime\");\n const startedAt = new Date().toISOString();\n const startPerf = performance.now();\n\n const connection = await options.transport.connect({\n model: options.model,\n voice: options.voice,\n instructions: options.instructions,\n });\n\n let report: RealtimeReport | undefined;\n\n return {\n sendAudio: (base64, mediaType) => connection.sendAudio(base64, mediaType),\n sendText: (text) => connection.sendText(text),\n events: () => connection.events(),\n async close(): Promise<RealtimeReport> {\n // Idempotent — closing twice returns the first report, never\n // re-tears-down the connection.\n if (report) {\n return report;\n }\n\n await connection.close();\n\n report = {\n runId,\n rootRunId: runId,\n type: \"realtime\",\n name: options.name ?? \"realtime\",\n status: \"completed\",\n startedAt,\n endedAt: new Date().toISOString(),\n duration: performance.now() - startPerf,\n ...(options.sessionId ? { sessionId: options.sessionId } : {}),\n };\n\n return report;\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,eAAsB,SAAS,SAAoD;CACjF,MAAM,QAAQ,cAAc,UAAU;CACtC,MAAM,6BAAY,IAAI,KAAK,EAAC,CAAC,YAAY;CACzC,MAAM,YAAY,YAAY,IAAI;CAElC,MAAM,aAAa,MAAM,QAAQ,UAAU,QAAQ;EACjD,OAAO,QAAQ;EACf,OAAO,QAAQ;EACf,cAAc,QAAQ;CACxB,CAAC;CAED,IAAI;CAEJ,OAAO;EACL,YAAY,QAAQ,cAAc,WAAW,UAAU,QAAQ,SAAS;EACxE,WAAW,SAAS,WAAW,SAAS,IAAI;EAC5C,cAAc,WAAW,OAAO;EAChC,MAAM,QAAiC;GAGrC,IAAI,QACF,OAAO;GAGT,MAAM,WAAW,MAAM;GAEvB,SAAS;IACP;IACA,WAAW;IACX,MAAM;IACN,MAAM,QAAQ,QAAQ;IACtB,QAAQ;IACR;IACA,0BAAS,IAAI,KAAK,EAAC,CAAC,YAAY;IAChC,UAAU,YAAY,IAAI,IAAI;IAC9B,GAAI,QAAQ,YAAY,EAAE,WAAW,QAAQ,UAAU,IAAI,CAAC;GAC9D;GAEA,OAAO;EACT;CACF;AACF"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { GeneratedVideo, VideoModelContract } from "../contracts/video.contract.mjs";
|
|
2
|
+
import { BaseReport, ExecuteResult, FlowObserveOption } from "@warlock.js/ai";
|
|
3
|
+
|
|
4
|
+
//#region ../@warlock.js/ai-live/src/video/video.d.ts
|
|
5
|
+
/** Parameters for {@link video}. `model` comes from an adapter's `video({ name })`. */
|
|
6
|
+
type VideoParams = {
|
|
7
|
+
model: VideoModelContract;
|
|
8
|
+
prompt: string;
|
|
9
|
+
durationSeconds?: number;
|
|
10
|
+
aspectRatio?: string;
|
|
11
|
+
resolution?: string;
|
|
12
|
+
negativePrompt?: string;
|
|
13
|
+
signal?: AbortSignal;
|
|
14
|
+
observe?: FlowObserveOption;
|
|
15
|
+
sessionId?: string;
|
|
16
|
+
name?: string;
|
|
17
|
+
options?: Record<string, unknown>;
|
|
18
|
+
};
|
|
19
|
+
/** Success payload of a {@link video} run. */
|
|
20
|
+
type VideoData = {
|
|
21
|
+
video: GeneratedVideo;
|
|
22
|
+
};
|
|
23
|
+
/** The report node a {@link video} run produces (`type: "video"`). */
|
|
24
|
+
type VideoReport = BaseReport & {
|
|
25
|
+
type: "video";
|
|
26
|
+
model: {
|
|
27
|
+
name: string;
|
|
28
|
+
provider: string;
|
|
29
|
+
}; /** Final clip duration in seconds, when reported. */
|
|
30
|
+
durationSeconds?: number;
|
|
31
|
+
};
|
|
32
|
+
/** Result envelope of {@link video} — the uniform `{ data, error, usage, report }`. */
|
|
33
|
+
type VideoResult = ExecuteResult<VideoData> & {
|
|
34
|
+
type: "video";
|
|
35
|
+
report: VideoReport;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Generate a video from a text prompt — the moving-image verb of the
|
|
39
|
+
* output-modality track. The adapter hides the provider's submit→poll
|
|
40
|
+
* job, so this returns the framework's uniform never-throws envelope:
|
|
41
|
+
*
|
|
42
|
+
* - **Never throws.** Provider failures surface as a typed `AIError`.
|
|
43
|
+
* - **Cost-truth.** `usage.cost` is filled per-second (Sora / Veo) or
|
|
44
|
+
* per-token, folding into the same `Usage.cost` rollup as everything else.
|
|
45
|
+
* - **Observable.** The completed {@link VideoReport} routes to any
|
|
46
|
+
* registered `Observer` via the shared `observe` seam.
|
|
47
|
+
*
|
|
48
|
+
* @example
|
|
49
|
+
* const { data, error } = await ai.video({
|
|
50
|
+
* model: sora.video({ name: "sora-2", pricing: { perSecond: 0.1 } }),
|
|
51
|
+
* prompt: "a timelapse of a city skyline at dusk, cinematic",
|
|
52
|
+
* durationSeconds: 8,
|
|
53
|
+
* });
|
|
54
|
+
* if (!error) download(data.video);
|
|
55
|
+
*/
|
|
56
|
+
declare function video(params: VideoParams): Promise<VideoResult>;
|
|
57
|
+
//#endregion
|
|
58
|
+
export { VideoData, VideoParams, VideoReport, VideoResult, video };
|
|
59
|
+
//# sourceMappingURL=video.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"video.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/video/video.ts"],"mappings":";;;;;KAqBY,WAAA;EACV,KAAA,EAAO,kBAAA;EACP,MAAA;EACA,eAAA;EACA,WAAA;EACA,UAAA;EACA,cAAA;EACA,MAAA,GAAS,WAAA;EACT,OAAA,GAAU,iBAAA;EACV,SAAA;EACA,IAAA;EACA,OAAA,GAAU,MAAA;AAAA;;KAIA,SAAA;EAAc,KAAA,EAAO,cAAc;AAAA;;KAGnC,WAAA,GAAc,UAAU;EAClC,IAAA;EACA,KAAA;IAAS,IAAA;IAAc,QAAA;EAAA,GATb;EAWV,eAAA;AAAA;AAPF;AAAA,KAWY,WAAA,GAAc,aAAA,CAAc,SAAA;EAAe,IAAA;EAAe,MAAA,EAAQ,WAAA;AAAA;AAR9E;;;;;;;;;;;AAIiB;AAIjB;;;;;;;AARA,iBA6BsB,KAAA,CAAM,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,WAAA"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { AIError, ProviderError, REPORT_SCHEMA_VERSION, computeCost, generateRunId, resolveObservers, stampReportLineage } from "@warlock.js/ai";
|
|
2
|
+
|
|
3
|
+
//#region ../@warlock.js/ai-live/src/video/video.ts
|
|
4
|
+
/**
|
|
5
|
+
* Generate a video from a text prompt — the moving-image verb of the
|
|
6
|
+
* output-modality track. The adapter hides the provider's submit→poll
|
|
7
|
+
* job, so this returns the framework's uniform never-throws envelope:
|
|
8
|
+
*
|
|
9
|
+
* - **Never throws.** Provider failures surface as a typed `AIError`.
|
|
10
|
+
* - **Cost-truth.** `usage.cost` is filled per-second (Sora / Veo) or
|
|
11
|
+
* per-token, folding into the same `Usage.cost` rollup as everything else.
|
|
12
|
+
* - **Observable.** The completed {@link VideoReport} routes to any
|
|
13
|
+
* registered `Observer` via the shared `observe` seam.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* const { data, error } = await ai.video({
|
|
17
|
+
* model: sora.video({ name: "sora-2", pricing: { perSecond: 0.1 } }),
|
|
18
|
+
* prompt: "a timelapse of a city skyline at dusk, cinematic",
|
|
19
|
+
* durationSeconds: 8,
|
|
20
|
+
* });
|
|
21
|
+
* if (!error) download(data.video);
|
|
22
|
+
*/
|
|
23
|
+
async function video(params) {
|
|
24
|
+
const { model, prompt } = params;
|
|
25
|
+
const runId = generateRunId("video");
|
|
26
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
27
|
+
const startPerf = performance.now();
|
|
28
|
+
const usage = {
|
|
29
|
+
input: 0,
|
|
30
|
+
output: 0,
|
|
31
|
+
total: 0
|
|
32
|
+
};
|
|
33
|
+
let data;
|
|
34
|
+
let error;
|
|
35
|
+
let status = "completed";
|
|
36
|
+
let durationSeconds;
|
|
37
|
+
try {
|
|
38
|
+
const response = await model.generate(prompt, {
|
|
39
|
+
durationSeconds: params.durationSeconds,
|
|
40
|
+
aspectRatio: params.aspectRatio,
|
|
41
|
+
resolution: params.resolution,
|
|
42
|
+
negativePrompt: params.negativePrompt,
|
|
43
|
+
signal: params.signal,
|
|
44
|
+
...params.options
|
|
45
|
+
});
|
|
46
|
+
Object.assign(usage, response.usage);
|
|
47
|
+
durationSeconds = response.durationSeconds;
|
|
48
|
+
if (usage.cost === void 0) {
|
|
49
|
+
const cost = computeVideoCost(usage, durationSeconds, model.pricing);
|
|
50
|
+
if (cost !== void 0) usage.cost = cost;
|
|
51
|
+
}
|
|
52
|
+
data = { video: response.video };
|
|
53
|
+
} catch (thrown) {
|
|
54
|
+
error = thrown instanceof AIError ? thrown : new ProviderError(toMessage(thrown), { cause: thrown });
|
|
55
|
+
status = params.signal?.aborted ? "cancelled" : "failed";
|
|
56
|
+
}
|
|
57
|
+
const report = {
|
|
58
|
+
runId,
|
|
59
|
+
rootRunId: runId,
|
|
60
|
+
name: params.name ?? "video",
|
|
61
|
+
type: "video",
|
|
62
|
+
status,
|
|
63
|
+
error,
|
|
64
|
+
startedAt,
|
|
65
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
66
|
+
duration: performance.now() - startPerf,
|
|
67
|
+
usage,
|
|
68
|
+
children: [],
|
|
69
|
+
model: {
|
|
70
|
+
name: model.name,
|
|
71
|
+
provider: model.provider
|
|
72
|
+
},
|
|
73
|
+
...durationSeconds !== void 0 ? { durationSeconds } : {},
|
|
74
|
+
reportSchemaVersion: REPORT_SCHEMA_VERSION
|
|
75
|
+
};
|
|
76
|
+
stampReportLineage(report, {
|
|
77
|
+
rootRunId: runId,
|
|
78
|
+
sessionId: params.sessionId
|
|
79
|
+
});
|
|
80
|
+
for (const observer of resolveObservers(params.observe)) try {
|
|
81
|
+
await observer.collect(report);
|
|
82
|
+
} catch {}
|
|
83
|
+
return {
|
|
84
|
+
type: "video",
|
|
85
|
+
data,
|
|
86
|
+
error,
|
|
87
|
+
usage,
|
|
88
|
+
report
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Price a video run: `perSecond × durationSeconds` (per-second metering,
|
|
93
|
+
* attributed to `cost.output`) wins when configured, otherwise the
|
|
94
|
+
* standard token math. Returns `undefined` when no usable pricing is
|
|
95
|
+
* present.
|
|
96
|
+
*/
|
|
97
|
+
function computeVideoCost(usage, durationSeconds, pricing) {
|
|
98
|
+
if (!pricing) return;
|
|
99
|
+
if (pricing.perSecond !== void 0) {
|
|
100
|
+
if (durationSeconds === void 0) return;
|
|
101
|
+
return {
|
|
102
|
+
input: 0,
|
|
103
|
+
output: durationSeconds * pricing.perSecond
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
if (pricing.input !== void 0 && pricing.output !== void 0) return computeCost(usage, {
|
|
107
|
+
input: pricing.input,
|
|
108
|
+
output: pricing.output
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
/** Best-effort message for a non-`AIError` thrown value. */
|
|
112
|
+
function toMessage(thrown) {
|
|
113
|
+
return thrown instanceof Error ? thrown.message : String(thrown);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
//#endregion
|
|
117
|
+
export { video };
|
|
118
|
+
//# sourceMappingURL=video.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"video.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai-live/src/video/video.ts"],"sourcesContent":["import {\n AIError,\n computeCost,\n generateRunId,\n ProviderError,\n REPORT_SCHEMA_VERSION,\n resolveObservers,\n stampReportLineage,\n type BaseReport,\n type ExecuteResult,\n type FlowObserveOption,\n type ModelPricing,\n type Usage,\n} from \"@warlock.js/ai\";\nimport type {\n GeneratedVideo,\n VideoModelContract,\n VideoModelPricing,\n} from \"../contracts/video.contract\";\n\n/** Parameters for {@link video}. `model` comes from an adapter's `video({ name })`. */\nexport type VideoParams = {\n model: VideoModelContract;\n prompt: string;\n durationSeconds?: number;\n aspectRatio?: string;\n resolution?: string;\n negativePrompt?: string;\n signal?: AbortSignal;\n observe?: FlowObserveOption;\n sessionId?: string;\n name?: string;\n options?: Record<string, unknown>;\n};\n\n/** Success payload of a {@link video} run. */\nexport type VideoData = { video: GeneratedVideo };\n\n/** The report node a {@link video} run produces (`type: \"video\"`). */\nexport type VideoReport = BaseReport & {\n type: \"video\";\n model: { name: string; provider: string };\n /** Final clip duration in seconds, when reported. */\n durationSeconds?: number;\n};\n\n/** Result envelope of {@link video} — the uniform `{ data, error, usage, report }`. */\nexport type VideoResult = ExecuteResult<VideoData> & { type: \"video\"; report: VideoReport };\n\n/**\n * Generate a video from a text prompt — the moving-image verb of the\n * output-modality track. The adapter hides the provider's submit→poll\n * job, so this returns the framework's uniform never-throws envelope:\n *\n * - **Never throws.** Provider failures surface as a typed `AIError`.\n * - **Cost-truth.** `usage.cost` is filled per-second (Sora / Veo) or\n * per-token, folding into the same `Usage.cost` rollup as everything else.\n * - **Observable.** The completed {@link VideoReport} routes to any\n * registered `Observer` via the shared `observe` seam.\n *\n * @example\n * const { data, error } = await ai.video({\n * model: sora.video({ name: \"sora-2\", pricing: { perSecond: 0.1 } }),\n * prompt: \"a timelapse of a city skyline at dusk, cinematic\",\n * durationSeconds: 8,\n * });\n * if (!error) download(data.video);\n */\nexport async function video(params: VideoParams): Promise<VideoResult> {\n const { model, prompt } = params;\n\n const runId = generateRunId(\"video\");\n const startedAt = new Date().toISOString();\n const startPerf = performance.now();\n\n const usage: Usage = { input: 0, output: 0, total: 0 };\n let data: VideoData | undefined;\n let error: AIError | undefined;\n let status: VideoReport[\"status\"] = \"completed\";\n let durationSeconds: number | undefined;\n\n try {\n const response = await model.generate(prompt, {\n durationSeconds: params.durationSeconds,\n aspectRatio: params.aspectRatio,\n resolution: params.resolution,\n negativePrompt: params.negativePrompt,\n signal: params.signal,\n ...params.options,\n });\n\n Object.assign(usage, response.usage);\n durationSeconds = response.durationSeconds;\n\n if (usage.cost === undefined) {\n const cost = computeVideoCost(usage, durationSeconds, model.pricing);\n if (cost !== undefined) {\n usage.cost = cost;\n }\n }\n\n data = { video: response.video };\n } catch (thrown) {\n error =\n thrown instanceof AIError ? thrown : new ProviderError(toMessage(thrown), { cause: thrown });\n status = params.signal?.aborted ? \"cancelled\" : \"failed\";\n }\n\n const report: VideoReport = {\n runId,\n rootRunId: runId,\n name: params.name ?? \"video\",\n type: \"video\",\n status,\n error,\n startedAt,\n endedAt: new Date().toISOString(),\n duration: performance.now() - startPerf,\n usage,\n children: [],\n model: { name: model.name, provider: model.provider },\n ...(durationSeconds !== undefined ? { durationSeconds } : {}),\n reportSchemaVersion: REPORT_SCHEMA_VERSION,\n };\n\n stampReportLineage(report, { rootRunId: runId, sessionId: params.sessionId });\n\n for (const observer of resolveObservers(params.observe)) {\n try {\n await observer.collect(report);\n } catch {\n // Isolate observer failures — never break the run.\n }\n }\n\n return { type: \"video\", data, error, usage, report };\n}\n\n/**\n * Price a video run: `perSecond × durationSeconds` (per-second metering,\n * attributed to `cost.output`) wins when configured, otherwise the\n * standard token math. Returns `undefined` when no usable pricing is\n * present.\n */\nfunction computeVideoCost(\n usage: Usage,\n durationSeconds: number | undefined,\n pricing: VideoModelPricing | undefined,\n): ModelPricing | undefined {\n if (!pricing) {\n return undefined;\n }\n\n if (pricing.perSecond !== undefined) {\n if (durationSeconds === undefined) {\n return undefined;\n }\n return { input: 0, output: durationSeconds * pricing.perSecond };\n }\n\n if (pricing.input !== undefined && pricing.output !== undefined) {\n return computeCost(usage, { input: pricing.input, output: pricing.output });\n }\n\n return undefined;\n}\n\n/** Best-effort message for a non-`AIError` thrown value. */\nfunction toMessage(thrown: unknown): string {\n return thrown instanceof Error ? thrown.message : String(thrown);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAoEA,eAAsB,MAAM,QAA2C;CACrE,MAAM,EAAE,OAAO,WAAW;CAE1B,MAAM,QAAQ,cAAc,OAAO;CACnC,MAAM,6BAAY,IAAI,KAAK,EAAC,CAAC,YAAY;CACzC,MAAM,YAAY,YAAY,IAAI;CAElC,MAAM,QAAe;EAAE,OAAO;EAAG,QAAQ;EAAG,OAAO;CAAE;CACrD,IAAI;CACJ,IAAI;CACJ,IAAI,SAAgC;CACpC,IAAI;CAEJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,SAAS,QAAQ;GAC5C,iBAAiB,OAAO;GACxB,aAAa,OAAO;GACpB,YAAY,OAAO;GACnB,gBAAgB,OAAO;GACvB,QAAQ,OAAO;GACf,GAAG,OAAO;EACZ,CAAC;EAED,OAAO,OAAO,OAAO,SAAS,KAAK;EACnC,kBAAkB,SAAS;EAE3B,IAAI,MAAM,SAAS,QAAW;GAC5B,MAAM,OAAO,iBAAiB,OAAO,iBAAiB,MAAM,OAAO;GACnE,IAAI,SAAS,QACX,MAAM,OAAO;EAEjB;EAEA,OAAO,EAAE,OAAO,SAAS,MAAM;CACjC,SAAS,QAAQ;EACf,QACE,kBAAkB,UAAU,SAAS,IAAI,cAAc,UAAU,MAAM,GAAG,EAAE,OAAO,OAAO,CAAC;EAC7F,SAAS,OAAO,QAAQ,UAAU,cAAc;CAClD;CAEA,MAAM,SAAsB;EAC1B;EACA,WAAW;EACX,MAAM,OAAO,QAAQ;EACrB,MAAM;EACN;EACA;EACA;EACA,0BAAS,IAAI,KAAK,EAAC,CAAC,YAAY;EAChC,UAAU,YAAY,IAAI,IAAI;EAC9B;EACA,UAAU,CAAC;EACX,OAAO;GAAE,MAAM,MAAM;GAAM,UAAU,MAAM;EAAS;EACpD,GAAI,oBAAoB,SAAY,EAAE,gBAAgB,IAAI,CAAC;EAC3D,qBAAqB;CACvB;CAEA,mBAAmB,QAAQ;EAAE,WAAW;EAAO,WAAW,OAAO;CAAU,CAAC;CAE5E,KAAK,MAAM,YAAY,iBAAiB,OAAO,OAAO,GACpD,IAAI;EACF,MAAM,SAAS,QAAQ,MAAM;CAC/B,QAAQ,CAER;CAGF,OAAO;EAAE,MAAM;EAAS;EAAM;EAAO;EAAO;CAAO;AACrD;;;;;;;AAQA,SAAS,iBACP,OACA,iBACA,SAC0B;CAC1B,IAAI,CAAC,SACH;CAGF,IAAI,QAAQ,cAAc,QAAW;EACnC,IAAI,oBAAoB,QACtB;EAEF,OAAO;GAAE,OAAO;GAAG,QAAQ,kBAAkB,QAAQ;EAAU;CACjE;CAEA,IAAI,QAAQ,UAAU,UAAa,QAAQ,WAAW,QACpD,OAAO,YAAY,OAAO;EAAE,OAAO,QAAQ;EAAO,QAAQ,QAAQ;CAAO,CAAC;AAI9E;;AAGA,SAAS,UAAU,QAAyB;CAC1C,OAAO,kBAAkB,QAAQ,OAAO,UAAU,OAAO,MAAM;AACjE"}
|
package/llms-full.txt
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# Warlock AI Live — full skills
|
|
2
|
+
|
|
3
|
+
> Package: `@warlock.js/ai-live`
|
|
4
|
+
|
|
5
|
+
> Generated artifact. Concatenates every SKILL.md and reference file under `@warlock.js/ai-live/skills/`. Re-run `node scripts/generate-llms.mjs` after any change.
|
|
6
|
+
|
|
7
|
+
## use-ai-live `@warlock.js/ai-live/use-ai-live/SKILL.md`
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
name: use-ai-live
|
|
11
|
+
description: 'Live & generative rich-media add-on for @warlock.js/ai. A side-effect import `import "@warlock.js/ai-live"` mounts two heavy modalities onto the shared `ai.*` facade: `ai.video({ model, prompt })` — text-to-video (Sora / Veo / Kling-class), the async submit→poll job hidden behind the same uniform never-throws `{ data, error, usage, report }` envelope as `ai.image`, with per-second cost-truth folded into `Usage.cost`; and `ai.realtime({ transport, model })` — a stateful duplex VOICE SESSION over a pluggable `RealtimeTransport` (sendAudio / sendText / events() / close()→RealtimeReport). Videos are a discriminated `GeneratedVideo = { type: "url" } | { type: "base64" }`; session output is a `RealtimeEvent` union (audio | transcript | tool-call | error | done). Ships `MockVideoModel` + `MockRealtimeTransport` for HTTP-free tests. Triggers: `ai.video`, `ai.realtime`, `VideoModelContract`, `GeneratedVideo`, `VideoModelPricing`, `RealtimeSession`, `RealtimeTransport`, `RealtimeEvent`, `RealtimeReport`, `MockVideoModel`, `MockRealtimeTransport`; ''generate a video'', ''text to video'', ''sora'', ''veo'', ''realtime voice'', ''duplex voice session'', ''live voice agent'', ''barge-in'', ''per-second video cost''; typical import `import "@warlock.js/ai-live"` + `import { ai } from "@warlock.js/ai"`. Skip: still-image OUTPUT — [[generate-images]]; text-to-speech / transcription (one-shot audio, not a session) — [[generate-speech]]; chat agents / tools / workflows — `@warlock.js/ai/run-ai-agent/SKILL.md`.'
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# Live rich-media — `ai.video` + `ai.realtime` (`@warlock.js/ai-live`)
|
|
15
|
+
|
|
16
|
+
`@warlock.js/ai` ships the synchronous modality verbs (`ai.agent`, `ai.image`, `ai.speech`, `ai.transcribe`). `ai-live` adds the two that need heavyweight, demand-gated machinery — a **long-running job** (video) and a **persistent network session** (realtime voice) — kept in their own package so the core stays dependency-light.
|
|
17
|
+
|
|
18
|
+
Both mount onto the same `ai.*` facade by a **side-effect import**:
|
|
19
|
+
|
|
20
|
+
```ts
|
|
21
|
+
import "@warlock.js/ai-live"; // lights up ai.video + ai.realtime on the shared Ai facade
|
|
22
|
+
import { ai } from "@warlock.js/ai";
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The mount mirrors how `@warlock.js/ai-tools` mounts `ai.mcp` and `@warlock.js/ai-workspace` mounts `ai.workspace`: a `declare module "@warlock.js/ai"` interface-merge plus `ai.video = video; ai.realtime = realtime` at load. Forget the side-effect import and `ai.video` is a compile-time `undefined` — not a silent runtime miss.
|
|
26
|
+
|
|
27
|
+
## `ai.video` — text-to-video, uniform envelope
|
|
28
|
+
|
|
29
|
+
Prompt in / one video out. The adapter hides the provider's async **submit→poll** lifecycle and resolves only when the clip is ready, so the verb returns the framework's uniform **never-throws** `{ data, error, usage, report }` — the exact shape `ai.image` returns, so video spend and traces fold into the same dashboards.
|
|
30
|
+
|
|
31
|
+
```ts
|
|
32
|
+
import { ai } from "@warlock.js/ai";
|
|
33
|
+
|
|
34
|
+
const { data, error, usage, report } = await ai.video({
|
|
35
|
+
model: sora.video({ name: "sora-2", pricing: { perSecond: 0.1 } }), // VideoModelContract
|
|
36
|
+
prompt: "a timelapse of a city skyline at dusk, cinematic",
|
|
37
|
+
durationSeconds: 8,
|
|
38
|
+
aspectRatio: "16:9",
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
if (error) {
|
|
42
|
+
console.warn(error.code); // typed AIError — NEVER thrown
|
|
43
|
+
} else {
|
|
44
|
+
const clip = data.video; // GeneratedVideo (discriminated)
|
|
45
|
+
if (clip.type === "url") download(clip.url);
|
|
46
|
+
else save(Buffer.from(clip.base64, "base64"), clip.mediaType);
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### Shape
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
type VideoResult = ExecuteResult<VideoData> & { type: "video"; report: VideoReport };
|
|
54
|
+
type VideoData = { video: GeneratedVideo };
|
|
55
|
+
|
|
56
|
+
type GeneratedVideo =
|
|
57
|
+
| { type: "url"; url: string; mediaType?: string }
|
|
58
|
+
| { type: "base64"; base64: string; mediaType: string };
|
|
59
|
+
|
|
60
|
+
// VideoParams (provider-neutral) — the model is the only required field beyond prompt:
|
|
61
|
+
await ai.video({
|
|
62
|
+
model, // VideoModelContract from an adapter's video({ name })
|
|
63
|
+
prompt: "...",
|
|
64
|
+
durationSeconds: 8, // requested clip length
|
|
65
|
+
aspectRatio: "9:16",
|
|
66
|
+
resolution: "1080p", // hint
|
|
67
|
+
negativePrompt: "blurry, watermark",
|
|
68
|
+
signal, // AbortSignal → status "cancelled"
|
|
69
|
+
observe: collector, // route the report to an Observer (panoptic), like agents
|
|
70
|
+
sessionId: "campaign-42",
|
|
71
|
+
name: "hero-clip", // report node name (defaults to "video")
|
|
72
|
+
options: { seed: 7 }, // provider-specific escape hatch, forwarded verbatim
|
|
73
|
+
});
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`VideoModelContract` is the moving-image sibling of `ImageModelContract` — `{ name, provider, pricing?, generate(prompt, options) }` — produced by an adapter's `video()` factory. An adapter with no video API simply doesn't define `video()`.
|
|
77
|
+
|
|
78
|
+
## Video cost-truth — per-second first
|
|
79
|
+
|
|
80
|
+
`VideoModelPricing` carries `{ perSecond?, input?, output? }`. Per-second wins when set (video is metered by clip length): `usage.cost = { input: 0, output: durationSeconds × perSecond }`, attributed to `cost.output`. A token-metered model falls back to the standard `computeCost` against reported tokens. No usable pricing → `usage.cost` stays `undefined` (honest "cost unknown", never a false zero). A pre-priced adapter response is honored, not overwritten. The final `durationSeconds` from the provider (not just the requested one) drives the math and lands on `report.durationSeconds`.
|
|
81
|
+
|
|
82
|
+
## `ai.realtime` — a duplex voice SESSION (not a one-shot)
|
|
83
|
+
|
|
84
|
+
Unlike every other `ai.*` verb, `ai.realtime()` returns a **stateful, long-lived session** — its closest sibling is `ai.orchestrator`. You open it over a pluggable **`RealtimeTransport`** (the low-level connection to the provider's realtime endpoint), push microphone audio / text turns in, consume an async **event stream** out, and `close()` to end it and receive a `RealtimeReport` for the cost/observability surfaces.
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
import { ai } from "@warlock.js/ai";
|
|
88
|
+
|
|
89
|
+
const session = await ai.realtime({
|
|
90
|
+
transport: openAiRealtime({ apiKey }), // RealtimeTransport (own adapter)
|
|
91
|
+
model: "gpt-realtime",
|
|
92
|
+
voice: "alloy",
|
|
93
|
+
instructions: "You are a friendly phone receptionist.",
|
|
94
|
+
sessionId: "call-8891",
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
session.sendAudio(micChunkBase64, "audio/pcm"); // push mic audio upstream
|
|
98
|
+
session.sendText("Please hold for one moment."); // or a text turn
|
|
99
|
+
|
|
100
|
+
for await (const event of session.events()) { // duplex output
|
|
101
|
+
switch (event.type) {
|
|
102
|
+
case "audio": speaker.write(event.base64); break;
|
|
103
|
+
case "transcript": if (event.final) log(event.role, event.text); break;
|
|
104
|
+
case "tool-call": await handleTool(event.name, event.input); break;
|
|
105
|
+
case "error": console.warn(event.error.code); break;
|
|
106
|
+
case "done": break;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const report = await session.close(); // RealtimeReport — idempotent
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### Session contract & event union
|
|
114
|
+
|
|
115
|
+
```ts
|
|
116
|
+
interface RealtimeSession {
|
|
117
|
+
sendAudio(base64: string, mediaType: string): void;
|
|
118
|
+
sendText(text: string): void;
|
|
119
|
+
events(): AsyncIterable<RealtimeEvent>;
|
|
120
|
+
close(): Promise<RealtimeReport>; // idempotent — second close() returns the first report
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
type RealtimeEvent =
|
|
124
|
+
| { type: "audio"; base64: string; mediaType: string }
|
|
125
|
+
| { type: "transcript"; role: "user" | "assistant"; text: string; final: boolean }
|
|
126
|
+
| { type: "tool-call"; id: string; name: string; input: unknown }
|
|
127
|
+
| { type: "error"; error: AIError } // typed, non-fatal
|
|
128
|
+
| { type: "done" }; // server ended the session
|
|
129
|
+
|
|
130
|
+
type RealtimeReport = {
|
|
131
|
+
runId: string; rootRunId: string; type: "realtime";
|
|
132
|
+
name: string; status: "completed" | "failed" | "cancelled";
|
|
133
|
+
startedAt: string; endedAt: string; duration: number; sessionId?: string;
|
|
134
|
+
};
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The transport seam is what lets `ai-live` ship the session surface without hard-wiring `ws` (it's an optional peer). A concrete transport implements just two interfaces: `RealtimeTransport.connect(config) → RealtimeConnection` (with `sendAudio` / `sendText` / `events()` / `close()`).
|
|
138
|
+
|
|
139
|
+
## Pattern — a live phone receptionist over a channel
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
import "@warlock.js/ai-live";
|
|
143
|
+
import { ai } from "@warlock.js/ai";
|
|
144
|
+
|
|
145
|
+
async function handleCall(caller: PhoneChannel, transport: RealtimeTransport) {
|
|
146
|
+
const session = await ai.realtime({
|
|
147
|
+
transport,
|
|
148
|
+
model: "gpt-realtime",
|
|
149
|
+
voice: "alloy",
|
|
150
|
+
instructions: "Greet the caller and route them to the right department.",
|
|
151
|
+
sessionId: caller.id,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
// Pump caller audio → session (fire-and-forget).
|
|
155
|
+
caller.onAudio((chunk) => session.sendAudio(chunk, "audio/pcm"));
|
|
156
|
+
|
|
157
|
+
// Pump session output → caller, until the model signals done.
|
|
158
|
+
for await (const event of session.events()) {
|
|
159
|
+
if (event.type === "audio") caller.playAudio(event.base64);
|
|
160
|
+
if (event.type === "done") break;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const report = await session.close();
|
|
164
|
+
metrics.record({ runId: report.runId, seconds: report.duration / 1000 });
|
|
165
|
+
}
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
## Testing — mocks, no HTTP, no sockets
|
|
169
|
+
|
|
170
|
+
`@warlock.js/ai-live` exports deterministic doubles so both verbs test offline.
|
|
171
|
+
|
|
172
|
+
```ts
|
|
173
|
+
import { ai } from "@warlock.js/ai";
|
|
174
|
+
import { MockVideoModel, MockRealtimeTransport } from "@warlock.js/ai-live";
|
|
175
|
+
|
|
176
|
+
// Video — script the returned clip / usage / duration; assert cost math + recorded calls.
|
|
177
|
+
const model = new MockVideoModel("mock-video", [{ durationSeconds: 8 }], { perSecond: 0.1 });
|
|
178
|
+
const { data, usage } = await ai.video({ model, prompt: "x", durationSeconds: 8 });
|
|
179
|
+
// data.video → { type: "url", url: "https://mock/video.mp4", ... }
|
|
180
|
+
// usage.cost.output === 0.8 (8 × 0.1)
|
|
181
|
+
// model.calls[0] → { prompt: "x", options: { durationSeconds: 8, ... } }
|
|
182
|
+
|
|
183
|
+
// Realtime — script the outbound event stream; assert what the session sent.
|
|
184
|
+
const transport = new MockRealtimeTransport([
|
|
185
|
+
{ type: "transcript", role: "assistant", text: "Hi!", final: true },
|
|
186
|
+
{ type: "done" },
|
|
187
|
+
]);
|
|
188
|
+
const session = await ai.realtime({ transport, model: "mock-realtime" });
|
|
189
|
+
session.sendText("hello");
|
|
190
|
+
for await (const e of session.events()) { /* ... */ }
|
|
191
|
+
const report = await session.close();
|
|
192
|
+
// transport.lastConnection.sentText → ["hello"]
|
|
193
|
+
// transport.connectConfigs[0] → { model: "mock-realtime", ... }
|
|
194
|
+
// report.status === "completed"
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Feed `MockVideoModel` an `{ error }` response and `ai.video` lands it on `result.error` (never throws), setting `report.status` to `"failed"` (or `"cancelled"` when the signal aborted).
|
|
198
|
+
|
|
199
|
+
## Status — contracts now, transports next
|
|
200
|
+
|
|
201
|
+
`4.6.0` introduces the package: the `VideoModelContract` / `RealtimeSession` contracts, the `ai.video()` verb (uniform envelope + per-second cost-truth, tested against `MockVideoModel`), and the `ai.realtime()` session primitive over a pluggable transport (tested against `MockRealtimeTransport`). The first **concrete provider transports** — an OpenAI Realtime **WebSocket** for `ai.realtime`, and **Sora / Veo** video adapters for `ai.video` — are the next implementation step; the seams are defined so they drop in without changing the verb surface.
|
|
202
|
+
|
|
203
|
+
## See also
|
|
204
|
+
|
|
205
|
+
- [[generate-images]] — still-image OUTPUT (`ai.image`), the synchronous sibling `ai.video` mirrors.
|
|
206
|
+
- [[generate-speech]] — one-shot text-to-speech / transcription (`ai.speech` / `ai.transcribe`); use `ai.realtime` instead when you need a live duplex conversation, not a single audio render.
|
|
207
|
+
- `@warlock.js/ai/observe-ai-flows/SKILL.md` — the `observe` seam both verbs route their reports through.
|
|
208
|
+
- `@warlock.js/ai/run-orchestrator/SKILL.md` — the other session-shaped primitive `ai.realtime` resembles.
|
|
209
|
+
|
|
210
|
+
|