@xopcai/xopc 0.0.466 → 0.0.467
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/manifest.json +1 -1
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{automations-page-_apIsCPT.js → automations-page-3WGWLdiW.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-workspace-WmcQUi9O.js → automations-workspace-CH9gYqIK.js} +2 -2
- package/dist/gateway/static/root/assets/{binary-plugins-C7i5pqkO.js → binary-plugins-MFVtTdvy.js} +1 -1
- package/dist/gateway/static/root/assets/capabilities-settings-panel-BjaLTd6i.js +8 -0
- package/dist/gateway/static/root/assets/chat-page-BtMkdtxJ.js +71 -0
- package/dist/gateway/static/root/assets/{home-page-Dgopm60U.js → home-page-BBTtXpDD.js} +3 -3
- package/dist/gateway/static/root/assets/index-BJj-uZNl.js +11 -0
- package/dist/gateway/static/root/assets/index-DiWUK4sJ.css +1 -0
- package/dist/gateway/static/root/assets/{media-plugins-DlMEwcS8.js → media-plugins-CW5UoTx6.js} +1 -1
- package/dist/gateway/static/root/assets/{note-detail-panel-yAPC_cZU.js → note-detail-panel-D5BtgBxy.js} +2 -2
- package/dist/gateway/static/root/assets/{notes-page-CUMhjKKd.js → notes-page-DyM-3O_9.js} +1 -1
- package/dist/gateway/static/root/assets/personal-page-BB4Sv68K.js +1 -0
- package/dist/gateway/static/root/assets/{project-detail-page-BjOzJyw1.js → project-detail-page-BG2nDt98.js} +1 -1
- package/dist/gateway/static/root/assets/route-preload-BYqg1BYz.js +2 -0
- package/dist/gateway/static/root/assets/{routine-panel-DSNsdpy8.js → routine-panel-Coh2B0j7.js} +2 -2
- package/dist/gateway/static/root/assets/{settings-page-u-U9D9qT.js → settings-page-D9A2DdJ_.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-BYPdlPIb.js → share-preview-page-CyCv69rh.js} +1 -1
- package/dist/gateway/static/root/assets/{task-detail-page-BPHt6S1V.js → task-detail-page-DgsLOz6h.js} +1 -1
- package/dist/gateway/static/root/assets/{use-composer-attachments-UhypPQH_.js → use-composer-attachments-C9GpCJQR.js} +1 -1
- package/dist/gateway/static/root/assets/{use-file-preview-expanded-BvEF3yHL.js → use-file-preview-expanded-Daf-LmiF.js} +2 -2
- package/dist/gateway/static/root/assets/video-generation-D08uZbsu.js +1 -0
- package/dist/gateway/static/root/assets/{workspace-file-preview-dialog-BRQIf4sc.js → workspace-file-preview-dialog-JSGIA79T.js} +1 -1
- package/dist/gateway/static/root/index.html +9 -10
- package/dist/package.js +1 -1
- package/dist/packages/gateway-contract/src/index.js +2 -0
- package/dist/packages/gateway-contract/src/music-generation.js +6 -0
- package/dist/src/agent/inbound/attachment-pipeline.d.ts +1 -0
- package/dist/src/agent/inbound/attachment-pipeline.js +5 -1
- package/dist/src/agent/prompt/sections/tooling.js +4 -0
- package/dist/src/agent/tools/factory.js +14 -1
- package/dist/src/agent/tools/music-generation-tool.d.ts +11 -0
- package/dist/src/agent/tools/music-generation-tool.js +70 -0
- package/dist/src/agent/tools/toolExposure.js +2 -0
- package/dist/src/agent/tools/video-draft-tool.d.ts +12 -0
- package/dist/src/agent/tools/video-draft-tool.js +75 -0
- package/dist/src/cli/commands/doctor/checks/database-schema.js +1 -1
- package/dist/src/gateway/hono/routes/lazy-bundles.js +9 -1
- package/dist/src/gateway/hono/routes/media.js +1 -20
- package/dist/src/gateway/hono/routes/music-generation.d.ts +3 -0
- package/dist/src/gateway/hono/routes/music-generation.js +59 -0
- package/dist/src/gateway/hono/routes/video-generation.js +51 -26
- package/dist/src/gateway/service.d.ts +1 -0
- package/dist/src/gateway/service.js +4 -1
- package/dist/src/media-shared/http/byte-range.d.ts +6 -0
- package/dist/src/media-shared/http/byte-range.js +23 -0
- package/dist/src/music-generation/artifact.d.ts +5 -0
- package/dist/src/music-generation/artifact.js +105 -0
- package/dist/src/music-generation/mureka.d.ts +3 -0
- package/dist/src/music-generation/mureka.js +96 -0
- package/dist/src/music-generation/provider.d.ts +31 -0
- package/dist/src/music-generation/provider.js +4 -0
- package/dist/src/music-generation/providers.d.ts +26 -0
- package/dist/src/music-generation/providers.js +56 -0
- package/dist/src/music-generation/repository.d.ts +50 -0
- package/dist/src/music-generation/repository.js +104 -0
- package/dist/src/music-generation/settings.d.ts +7 -0
- package/dist/src/music-generation/settings.js +24 -0
- package/dist/src/music-generation/worker.d.ts +8 -0
- package/dist/src/music-generation/worker.js +115 -0
- package/dist/src/personal-agent/policy.d.ts +1 -1
- package/dist/src/personal-agent/policy.js +3 -1
- package/dist/src/personal-agent/prompts.js +2 -0
- package/dist/src/session/store.js +8 -2
- package/dist/src/storage/sqlite/schema.d.ts +2 -2
- package/dist/src/storage/sqlite/schema.js +7 -7
- package/dist/src/storage/sqlite/schema.sql +31 -0
- package/dist/src/storage/sqlite/transcript-repository.d.ts +1 -0
- package/dist/src/storage/sqlite/transcript-repository.js +8 -1
- package/dist/src/storage/sqlite/upgrades/277_music_generation.sql +30 -0
- package/dist/src/video-generation/drafts.d.ts +17 -0
- package/dist/src/video-generation/drafts.js +89 -0
- package/dist/src/video-generation/worker.js +1 -1
- package/package.json +1 -1
- package/dist/gateway/static/root/assets/api-CG-SjNfw.js +0 -1
- package/dist/gateway/static/root/assets/capabilities-settings-panel-DuQGhgAl.js +0 -8
- package/dist/gateway/static/root/assets/chat-page-BMZwIVXH.js +0 -71
- package/dist/gateway/static/root/assets/index-BE3XMuQI.js +0 -11
- package/dist/gateway/static/root/assets/index-CY5vdQcc.css +0 -1
- package/dist/gateway/static/root/assets/personal-page-POCG1_kK.js +0 -1
- package/dist/gateway/static/root/assets/route-preload-DjYsRhY1.js +0 -2
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AgentTool } from '@earendil-works/pi-agent-core';
|
|
2
|
+
declare const schema: import("@sinclair/typebox").TObject<{
|
|
3
|
+
action: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"create">, import("@sinclair/typebox").TLiteral<"update">, import("@sinclair/typebox").TLiteral<"revise">, import("@sinclair/typebox").TLiteral<"inspect">]>;
|
|
4
|
+
prompt: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
5
|
+
duration: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TInteger>;
|
|
6
|
+
ratio: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"16:9">, import("@sinclair/typebox").TLiteral<"9:16">, import("@sinclair/typebox").TLiteral<"adaptive">]>>;
|
|
7
|
+
resolution: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"480P">, import("@sinclair/typebox").TLiteral<"720P">, import("@sinclair/typebox").TLiteral<"1080P">]>>;
|
|
8
|
+
image: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"current">, import("@sinclair/typebox").TLiteral<"none">]>>;
|
|
9
|
+
imageUri: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
10
|
+
}>;
|
|
11
|
+
export declare function createVideoDraftTool(getConversationId: () => string | undefined): AgentTool<typeof schema>;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { getCurrentTranscriptId, init_session_repository } from "../../storage/sqlite/session-repository.js";
|
|
2
|
+
import { getVideoJob } from "../../video-generation/repository.js";
|
|
3
|
+
import { getVideoDraft, prepareVideoDraft, videoDraftImages } from "../../video-generation/drafts.js";
|
|
4
|
+
import { Type } from "@sinclair/typebox";
|
|
5
|
+
import { Value } from "@sinclair/typebox/value";
|
|
6
|
+
//#region src/agent/tools/video-draft-tool.ts
|
|
7
|
+
init_session_repository();
|
|
8
|
+
const schema = Type.Object({
|
|
9
|
+
action: Type.Union([
|
|
10
|
+
Type.Literal("create"),
|
|
11
|
+
Type.Literal("update"),
|
|
12
|
+
Type.Literal("revise"),
|
|
13
|
+
Type.Literal("inspect")
|
|
14
|
+
]),
|
|
15
|
+
prompt: Type.Optional(Type.String({ maxLength: 1e3 })),
|
|
16
|
+
duration: Type.Optional(Type.Integer()),
|
|
17
|
+
ratio: Type.Optional(Type.Union([
|
|
18
|
+
Type.Literal("16:9"),
|
|
19
|
+
Type.Literal("9:16"),
|
|
20
|
+
Type.Literal("adaptive")
|
|
21
|
+
])),
|
|
22
|
+
resolution: Type.Optional(Type.Union([
|
|
23
|
+
Type.Literal("480P"),
|
|
24
|
+
Type.Literal("720P"),
|
|
25
|
+
Type.Literal("1080P")
|
|
26
|
+
])),
|
|
27
|
+
image: Type.Optional(Type.Union([Type.Literal("current"), Type.Literal("none")])),
|
|
28
|
+
imageUri: Type.Optional(Type.String({ maxLength: 512 }))
|
|
29
|
+
});
|
|
30
|
+
function createVideoDraftTool(getConversationId) {
|
|
31
|
+
return {
|
|
32
|
+
name: "video_draft",
|
|
33
|
+
label: "Prepare video draft",
|
|
34
|
+
description: "Prepare a video when the user asks to generate one in Web chat. Create a draft, update the pending draft, or revise the last submitted video as a new version. For a new draft, a single image in the current user message is used by default. An update or revision keeps its reference image unless the user requests another one; use image=current to switch to a new attachment. Use inspect to list recent images and the previous draft. Ask which image to use when the reference is ambiguous. Use video settings for any unspecified options. The draft appears as a card. Tell the user to click Generate or reply “confirm video generation” / “就按这个生成” to submit the reviewed draft. This tool cannot start generation.",
|
|
35
|
+
parameters: schema,
|
|
36
|
+
async execute(_id, args) {
|
|
37
|
+
if (!Value.Check(schema, args)) throw new Error("Invalid video draft parameters.");
|
|
38
|
+
const conversationId = getConversationId();
|
|
39
|
+
if (!conversationId) throw new Error("Video drafts are available in Web chat.");
|
|
40
|
+
if (args.action === "inspect") {
|
|
41
|
+
const transcriptId = getCurrentTranscriptId(conversationId);
|
|
42
|
+
const draft = transcriptId ? getVideoDraft(conversationId, transcriptId) : void 0;
|
|
43
|
+
const { recent } = videoDraftImages(conversationId);
|
|
44
|
+
return {
|
|
45
|
+
content: [{
|
|
46
|
+
type: "text",
|
|
47
|
+
text: JSON.stringify({
|
|
48
|
+
draft,
|
|
49
|
+
submitted: Boolean(draft && getVideoJob(conversationId, draft.id)),
|
|
50
|
+
images: recent.slice(0, 12).map(({ uri, name }) => ({
|
|
51
|
+
uri,
|
|
52
|
+
name
|
|
53
|
+
}))
|
|
54
|
+
})
|
|
55
|
+
}],
|
|
56
|
+
details: {}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const { action, ...input } = args;
|
|
60
|
+
const draft = prepareVideoDraft(conversationId, {
|
|
61
|
+
action,
|
|
62
|
+
...input
|
|
63
|
+
});
|
|
64
|
+
return {
|
|
65
|
+
content: [{
|
|
66
|
+
type: "text",
|
|
67
|
+
text: JSON.stringify(draft)
|
|
68
|
+
}],
|
|
69
|
+
details: {}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
//#endregion
|
|
75
|
+
export { createVideoDraftTool };
|
|
@@ -31,7 +31,7 @@ async function checkDatabaseSchema(ctx) {
|
|
|
31
31
|
id: "database-schema",
|
|
32
32
|
label: "Database schema",
|
|
33
33
|
status: "pass",
|
|
34
|
-
message: `SQLite schema v${readSchemaVersion(getSqliteDatabase())} (current release
|
|
34
|
+
message: `SQLite schema v${readSchemaVersion(getSqliteDatabase())} (current release v277).`,
|
|
35
35
|
hints: []
|
|
36
36
|
};
|
|
37
37
|
}
|
|
@@ -184,9 +184,17 @@ const AUTHENTICATED_LAZY_ROUTE_BUNDLES = [
|
|
|
184
184
|
return { register: registerRuntimeToolsRoutes };
|
|
185
185
|
}
|
|
186
186
|
},
|
|
187
|
+
{
|
|
188
|
+
id: "music-generation",
|
|
189
|
+
match: (path) => /^\/api\/music-generation\/settings(?:\/credentials\/[^/]+)?$/.test(path) || /^\/api\/music-generation\/sessions\/[^/]+\/jobs(?:\/[0-9a-f]{64}\/audio)?$/.test(path),
|
|
190
|
+
load: async () => {
|
|
191
|
+
const { registerMusicGenerationRoutes } = await import("./music-generation.js");
|
|
192
|
+
return { register: registerMusicGenerationRoutes };
|
|
193
|
+
}
|
|
194
|
+
},
|
|
187
195
|
{
|
|
188
196
|
id: "video-generation",
|
|
189
|
-
match: (path) => path === "/api/video-generation/settings" || /^\/api\/video-generation\/sessions\/[^/]+\/jobs(?:\/[^/]+\/video)?$/.test(path),
|
|
197
|
+
match: (path) => path === "/api/video-generation/settings" || /^\/api\/video-generation\/sessions\/[^/]+\/jobs(?:\/confirm-draft|\/[^/]+\/video)?$/.test(path),
|
|
190
198
|
load: async () => {
|
|
191
199
|
const { registerVideoGenerationRoutes } = await import("./video-generation.js");
|
|
192
200
|
return { register: registerVideoGenerationRoutes };
|
|
@@ -1,30 +1,11 @@
|
|
|
1
1
|
import { MEDIA_ARTIFACT_MAX_BYTES, mimeTypeFromMediaPath, saveMediaBuffer } from "../../../media/store.js";
|
|
2
2
|
import { readMediaReference } from "../../../media/media-reference.js";
|
|
3
3
|
import { MAX_WEBCHAT_ATTACHMENT_FILE_BYTES } from "../../chat-limits.js";
|
|
4
|
+
import { parseByteRange } from "../../../media-shared/http/byte-range.js";
|
|
4
5
|
import { createGatewayRouteLogger } from "../lib/route-logger.js";
|
|
5
6
|
import { resolveScopedMediaReference } from "../../media-access.js";
|
|
6
7
|
//#region src/gateway/hono/routes/media.ts
|
|
7
8
|
const log = createGatewayRouteLogger("Media");
|
|
8
|
-
function parseByteRange(header, size) {
|
|
9
|
-
if (!header) return null;
|
|
10
|
-
const match = /^bytes=(\d*)-(\d*)$/i.exec(header.trim());
|
|
11
|
-
if (!match || !match[1] && !match[2] || size <= 0) return "invalid";
|
|
12
|
-
if (!match[1]) {
|
|
13
|
-
const suffixLength = Number(match[2]);
|
|
14
|
-
if (!Number.isSafeInteger(suffixLength) || suffixLength <= 0) return "invalid";
|
|
15
|
-
return {
|
|
16
|
-
start: Math.max(0, size - suffixLength),
|
|
17
|
-
end: size - 1
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
const start = Number(match[1]);
|
|
21
|
-
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
22
|
-
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return "invalid";
|
|
23
|
-
return {
|
|
24
|
-
start,
|
|
25
|
-
end: Math.min(requestedEnd, size - 1)
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
9
|
function registerMediaRoutes(authenticated, deps) {
|
|
29
10
|
authenticated.post("/api/media", deps.strictRateLimitMiddleware, async (c) => {
|
|
30
11
|
const file = (await c.req.formData().catch(() => null))?.get("file");
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { musicCredentials, saveMusicDefaults, saveMusicModel } from "../../../music-generation/settings.js";
|
|
2
|
+
import { configuredMusicModel, hasMusicProvider, readMusicModelSettings, resolveMusicModel } from "../../../music-generation/providers.js";
|
|
3
|
+
import { getMusicJob, listMusicJobs } from "../../../music-generation/repository.js";
|
|
4
|
+
import { readMusicArtifact } from "../../../music-generation/artifact.js";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
//#region src/gateway/hono/routes/music-generation.ts
|
|
7
|
+
function registerMusicGenerationRoutes(app, deps) {
|
|
8
|
+
const settings = "/api/music-generation/settings";
|
|
9
|
+
const path = "/api/music-generation/sessions/:conversationId/jobs";
|
|
10
|
+
app.get(settings, async (c) => c.json(await readMusicModelSettings()));
|
|
11
|
+
app.put(settings, deps.strictRateLimitMiddleware, async (c) => {
|
|
12
|
+
const input = z.object({ modelRef: z.string().max(200) }).safeParse(await c.req.json().catch(() => null));
|
|
13
|
+
if (!input.success || !resolveMusicModel(input.data.modelRef)) return c.json({ error: "Unknown music model" }, 400);
|
|
14
|
+
saveMusicModel(input.data.modelRef);
|
|
15
|
+
return c.json(await readMusicModelSettings());
|
|
16
|
+
});
|
|
17
|
+
app.patch(settings, deps.strictRateLimitMiddleware, async (c) => {
|
|
18
|
+
const input = z.object({
|
|
19
|
+
modelRef: z.string().max(200),
|
|
20
|
+
mode: z.enum(["song", "instrumental"])
|
|
21
|
+
}).safeParse(await c.req.json().catch(() => null));
|
|
22
|
+
if (!input.success) return c.json({ error: "Invalid music defaults" }, 400);
|
|
23
|
+
const { modelRef, mode } = input.data;
|
|
24
|
+
if (!resolveMusicModel(modelRef)?.capabilities.modes.includes(mode)) return c.json({ error: "The selected model does not support this music mode" }, 400);
|
|
25
|
+
saveMusicDefaults(modelRef, { mode });
|
|
26
|
+
return c.json(await readMusicModelSettings());
|
|
27
|
+
});
|
|
28
|
+
app.put(`${settings}/credentials/:providerId`, deps.strictRateLimitMiddleware, async (c) => {
|
|
29
|
+
const providerId = c.req.param("providerId");
|
|
30
|
+
const input = z.object({ apiKey: z.string().trim().min(1).max(4096).regex(/^[!-~]+$/) }).safeParse(await c.req.json().catch(() => null));
|
|
31
|
+
if (!hasMusicProvider(providerId) || !input.success) return c.json({ error: "Invalid music provider or API key" }, 400);
|
|
32
|
+
await musicCredentials().saveApiKey(providerId, input.data.apiKey);
|
|
33
|
+
return c.json(await readMusicModelSettings());
|
|
34
|
+
});
|
|
35
|
+
app.delete(`${settings}/credentials/:providerId`, deps.strictRateLimitMiddleware, async (c) => {
|
|
36
|
+
const providerId = c.req.param("providerId");
|
|
37
|
+
if (!hasMusicProvider(providerId)) return c.json({ error: "Unknown music provider" }, 400);
|
|
38
|
+
await musicCredentials().deleteProfile(`${providerId}:default`);
|
|
39
|
+
return c.json(await readMusicModelSettings());
|
|
40
|
+
});
|
|
41
|
+
app.get(path, async (c) => {
|
|
42
|
+
const conversationId = c.req.param("conversationId");
|
|
43
|
+
const session = await deps.service.sessionIndexInstance.getSessionMetadata(conversationId);
|
|
44
|
+
if (!session?.transcriptId || session.sourceChannel !== "webchat" || session.sessionType !== "chat") return c.json({ error: "Conversation not found" }, 404);
|
|
45
|
+
return c.json({
|
|
46
|
+
configured: Boolean(await configuredMusicModel()),
|
|
47
|
+
jobs: listMusicJobs(conversationId, session.transcriptId)
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
app.get(`${path}/:jobId/audio`, async (c) => {
|
|
51
|
+
const conversationId = c.req.param("conversationId");
|
|
52
|
+
const session = await deps.service.sessionIndexInstance.getSessionMetadata(conversationId);
|
|
53
|
+
const job = getMusicJob(conversationId, c.req.param("jobId"));
|
|
54
|
+
if (!session?.transcriptId || job?.transcript_id !== session.transcriptId || job.state !== "completed") return c.json({ error: "Audio not found" }, 404);
|
|
55
|
+
return readMusicArtifact(job.id, c.req.header("Range"), c.req.query("download") === "1");
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
//#endregion
|
|
59
|
+
export { registerMusicGenerationRoutes };
|
|
@@ -1,18 +1,25 @@
|
|
|
1
|
+
import { init_transaction, runSqliteWriteTransaction } from "../../../storage/sqlite/transaction.js";
|
|
1
2
|
import { VIDEO_IMAGE_MAX_BYTES, VIDEO_MODELS, VIDEO_MODEL_REFS, VIDEO_RESOLUTIONS, WAN_VIDEO_MODEL_REF, isVideoDurationSupported } from "../../../../packages/gateway-contract/src/video-generation.js";
|
|
2
3
|
import { init_src } from "../../../../packages/gateway-contract/src/index.js";
|
|
3
4
|
import { readMediaReference } from "../../../media/media-reference.js";
|
|
4
|
-
import { readVideoArtifact } from "../../../video-generation/artifact.js";
|
|
5
5
|
import { getVideoDefaults, getVideoModel, setVideoDefaults, setVideoModel } from "../../../video-generation/settings.js";
|
|
6
6
|
import { enqueueVideoJob, getVideoJob, listVideoJobs } from "../../../video-generation/repository.js";
|
|
7
|
+
import { getVideoDraft } from "../../../video-generation/drafts.js";
|
|
8
|
+
import { readVideoArtifact } from "../../../video-generation/artifact.js";
|
|
7
9
|
import { getConfiguredVideoProviders, getVideoProvider } from "../../../video-generation/providers.js";
|
|
8
10
|
import { resolveScopedMediaReference } from "../../media-access.js";
|
|
9
11
|
import { createHash } from "node:crypto";
|
|
10
12
|
import { z } from "zod";
|
|
11
13
|
//#region src/gateway/hono/routes/video-generation.ts
|
|
12
14
|
init_src();
|
|
15
|
+
init_transaction();
|
|
13
16
|
const settingsPath = "/api/video-generation/settings";
|
|
14
17
|
const modelSchema = z.enum(VIDEO_MODEL_REFS);
|
|
15
18
|
const path = "/api/video-generation/sessions/:conversationId/jobs";
|
|
19
|
+
const confirmationSchema = z.object({
|
|
20
|
+
id: z.string().uuid(),
|
|
21
|
+
revision: z.number().int().positive()
|
|
22
|
+
});
|
|
16
23
|
const requestSchema = z.object({
|
|
17
24
|
id: z.string().uuid(),
|
|
18
25
|
modelRef: modelSchema.default(WAN_VIDEO_MODEL_REF),
|
|
@@ -84,32 +91,42 @@ function registerVideoGenerationRoutes(app, deps) {
|
|
|
84
91
|
if (!session?.transcriptId) return c.json({ error: "Conversation not found" }, 404);
|
|
85
92
|
const modelRef = getVideoModel();
|
|
86
93
|
const configured = Boolean(await getVideoProvider(modelRef).resolveKey().catch(() => void 0));
|
|
94
|
+
const jobs = listVideoJobs(conversationId);
|
|
95
|
+
const draft = getVideoDraft(conversationId, session.transcriptId);
|
|
87
96
|
return c.json({
|
|
88
97
|
transcriptId: session.transcriptId,
|
|
89
98
|
modelRef,
|
|
90
99
|
configured,
|
|
91
100
|
defaults: getVideoDefaults(modelRef),
|
|
92
|
-
jobs
|
|
101
|
+
jobs,
|
|
102
|
+
...draft && !jobs.some((job) => job.id === draft.id) ? { draft } : {}
|
|
93
103
|
});
|
|
94
104
|
});
|
|
95
|
-
app.post(path, deps.strictRateLimitMiddleware,
|
|
105
|
+
app.post(path, deps.strictRateLimitMiddleware, (c) => submit(c, false));
|
|
106
|
+
app.post(`${path}/confirm-draft`, deps.strictRateLimitMiddleware, (c) => submit(c, true));
|
|
107
|
+
async function submit(c, confirming) {
|
|
96
108
|
const conversationId = c.req.param("conversationId");
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
const transcriptId = (await deps.service.sessionIndexInstance.getSessionMetadata(conversationId))?.transcriptId;
|
|
110
|
+
if (!transcriptId) return c.json({ error: "Conversation not found" }, 404);
|
|
111
|
+
const confirmation = confirming ? confirmationSchema.safeParse(await c.req.json().catch(() => null)) : void 0;
|
|
112
|
+
if (confirmation && !confirmation.success) return c.json({ error: "Invalid draft confirmation" }, 400);
|
|
113
|
+
const draft = confirmation?.success ? getVideoDraft(conversationId, transcriptId) : void 0;
|
|
114
|
+
if (confirmation?.success && (draft?.id !== confirmation.data.id || draft.revision !== confirmation.data.revision)) return c.json({ error: "The video draft changed. Review the current version before generating." }, 409);
|
|
115
|
+
const form = confirming ? null : await c.req.formData().catch(() => null);
|
|
116
|
+
if (!draft && !form) return c.json({ error: "Invalid video request" }, 400);
|
|
117
|
+
const parsed = requestSchema.safeParse(draft ?? {
|
|
118
|
+
id: form?.get("id"),
|
|
119
|
+
modelRef: form?.get("modelRef") ?? void 0,
|
|
120
|
+
transcriptId: form?.get("transcriptId"),
|
|
121
|
+
prompt: form?.get("prompt"),
|
|
122
|
+
duration: form?.get("duration"),
|
|
123
|
+
ratio: form?.get("ratio"),
|
|
124
|
+
resolution: form?.get("resolution"),
|
|
125
|
+
imageUri: form?.get("imageUri")
|
|
109
126
|
});
|
|
110
127
|
if (!parsed.success) return c.json({ error: "Invalid video settings" }, 400);
|
|
111
128
|
const { imageUri, modelRef, ...request } = parsed.data;
|
|
112
|
-
const upload = form
|
|
129
|
+
const upload = form?.get("image") ?? null;
|
|
113
130
|
if (upload !== null && (!(upload instanceof File) || imageUri || !upload.size || upload.size > 5242880)) return c.json({ error: "Provide one PNG, JPEG or WebP image up to 5 MiB" }, 400);
|
|
114
131
|
const model = VIDEO_MODELS[modelRef];
|
|
115
132
|
const hasImage = Boolean(upload || imageUri);
|
|
@@ -142,20 +159,28 @@ function registerVideoGenerationRoutes(app, deps) {
|
|
|
142
159
|
imageName
|
|
143
160
|
}));
|
|
144
161
|
if (bytes) hash.update(bytes);
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
162
|
+
const requestHash = hash.digest("hex");
|
|
163
|
+
const result = runSqliteWriteTransaction(() => {
|
|
164
|
+
if (draft) {
|
|
165
|
+
const latest = getVideoDraft(conversationId, transcriptId);
|
|
166
|
+
if (latest?.id !== draft.id || latest.revision !== draft.revision) return { error: "draft_changed" };
|
|
167
|
+
}
|
|
168
|
+
return enqueueVideoJob(conversationId, {
|
|
169
|
+
...request,
|
|
170
|
+
modelRef,
|
|
171
|
+
imageName
|
|
172
|
+
}, requestHash, bytes && mime ? {
|
|
173
|
+
bytes,
|
|
174
|
+
mime
|
|
175
|
+
} : void 0);
|
|
176
|
+
});
|
|
177
|
+
if (result.error === "draft_changed") return c.json({ error: "The video draft changed. Review the current version before generating." }, 409);
|
|
153
178
|
if (result.error === "budget_exhausted") return c.json({ error: "The Gateway video generation budget for today has been reached." }, 429);
|
|
154
179
|
if (result.error === "queue_full") return c.json({ error: "The video queue is full. Try again after a task completes." }, 429);
|
|
155
180
|
if (result.error === "model_changed") return c.json({ error: "The video model changed. Review the updated model before submitting." }, 409);
|
|
156
|
-
if (result.error) return c.json({ error: "The conversation or video request changed.
|
|
181
|
+
if (result.error) return c.json({ error: "The conversation or video request changed. Review the current request and try again." }, 409);
|
|
157
182
|
return c.json(result.job, 202);
|
|
158
|
-
}
|
|
183
|
+
}
|
|
159
184
|
app.get(`${path}/:jobId/video`, async (c) => {
|
|
160
185
|
const job = getVideoJob(c.req.param("conversationId"), c.req.param("jobId"));
|
|
161
186
|
if (job?.state !== "completed") return c.json({ error: "Video not found" }, 404);
|
|
@@ -222,6 +222,7 @@ export declare class GatewayService {
|
|
|
222
222
|
private loadDeferredExtensions;
|
|
223
223
|
private schedulePostReadySidecars;
|
|
224
224
|
private runPostReadySidecars;
|
|
225
|
+
private readonly musicGenerationWorker;
|
|
225
226
|
private readonly videoGenerationWorker;
|
|
226
227
|
start(): Promise<void>;
|
|
227
228
|
private startRuntime;
|
|
@@ -146,6 +146,7 @@ import { NotificationService } from "../notifications/service.js";
|
|
|
146
146
|
import "../local-apps/index.js";
|
|
147
147
|
import { TaskAllocationDispatcher } from "../tasks/task-allocation-service.js";
|
|
148
148
|
import { TaskResultDeliveryService } from "../tasks/task-result-delivery-service.js";
|
|
149
|
+
import { MusicGenerationWorker } from "../music-generation/worker.js";
|
|
149
150
|
import { VideoGenerationWorker } from "../video-generation/worker.js";
|
|
150
151
|
import { scheduleGatewayUpdateCheck } from "../infra/update-startup.js";
|
|
151
152
|
import { resolveChannelConnectDeferSet } from "./resolve-channel-connect-defer.js";
|
|
@@ -1347,6 +1348,7 @@ var GatewayService = class {
|
|
|
1347
1348
|
}, `Deferred extension load failed: ${em}`);
|
|
1348
1349
|
}
|
|
1349
1350
|
}
|
|
1351
|
+
musicGenerationWorker = new MusicGenerationWorker();
|
|
1350
1352
|
videoGenerationWorker = new VideoGenerationWorker();
|
|
1351
1353
|
async start() {
|
|
1352
1354
|
try {
|
|
@@ -1590,6 +1592,7 @@ var GatewayService = class {
|
|
|
1590
1592
|
this.discussionLiveWorker.start();
|
|
1591
1593
|
this.discussionSealer.start();
|
|
1592
1594
|
this.videoGenerationWorker.start();
|
|
1595
|
+
this.musicGenerationWorker.start();
|
|
1593
1596
|
this.discussionWorker.start();
|
|
1594
1597
|
this.connectorSupervisor = startConnectorSupervisor({
|
|
1595
1598
|
getConfig: () => this.config,
|
|
@@ -1755,7 +1758,7 @@ var GatewayService = class {
|
|
|
1755
1758
|
this.homeIntelligenceHost = null;
|
|
1756
1759
|
this.realtime.close();
|
|
1757
1760
|
this.voiceRealtime.close();
|
|
1758
|
-
await this.videoGenerationWorker.stop();
|
|
1761
|
+
await Promise.all([this.videoGenerationWorker.stop(), this.musicGenerationWorker.stop()]);
|
|
1759
1762
|
await this.discussionWorker.stop();
|
|
1760
1763
|
await this.discussionSealer.stop();
|
|
1761
1764
|
await this.discussionLiveWorker.stop();
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
//#region src/media-shared/http/byte-range.ts
|
|
2
|
+
function parseByteRange(header, size) {
|
|
3
|
+
if (!header) return null;
|
|
4
|
+
const match = /^bytes=(\d*)-(\d*)$/i.exec(header.trim());
|
|
5
|
+
if (!match || !match[1] && !match[2] || size <= 0) return "invalid";
|
|
6
|
+
if (!match[1]) {
|
|
7
|
+
const suffixLength = Number(match[2]);
|
|
8
|
+
if (!Number.isSafeInteger(suffixLength) || suffixLength <= 0) return "invalid";
|
|
9
|
+
return {
|
|
10
|
+
start: Math.max(0, size - suffixLength),
|
|
11
|
+
end: size - 1
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
const start = Number(match[1]);
|
|
15
|
+
const requestedEnd = match[2] ? Number(match[2]) : size - 1;
|
|
16
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return "invalid";
|
|
17
|
+
return {
|
|
18
|
+
start,
|
|
19
|
+
end: Math.min(requestedEnd, size - 1)
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
//#endregion
|
|
23
|
+
export { parseByteRange };
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { MusicProvider } from './provider.js';
|
|
2
|
+
export declare function musicArtifactExists(id: string): Promise<boolean>;
|
|
3
|
+
export declare function deleteMusicArtifact(id: string): Promise<void>;
|
|
4
|
+
export declare function saveMusicArtifact(id: string, provider: MusicProvider, source: string, canPublish: () => boolean): Promise<void>;
|
|
5
|
+
export declare function readMusicArtifact(id: string, rangeHeader?: string, download?: boolean): Promise<Response>;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { resolveMediaBufferPath } from "../media/store.js";
|
|
2
|
+
import { assertSafeUrl } from "../media-shared/http/ssrf-guard.js";
|
|
3
|
+
import { parseByteRange } from "../media-shared/http/byte-range.js";
|
|
4
|
+
import { dirname } from "node:path";
|
|
5
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
7
|
+
import { link, mkdir, open, rm, stat } from "node:fs/promises";
|
|
8
|
+
import { pipeline } from "node:stream/promises";
|
|
9
|
+
import { Readable, Transform } from "node:stream";
|
|
10
|
+
//#region src/music-generation/artifact.ts
|
|
11
|
+
const MAX_AUDIO_BYTES = 24 * 1024 * 1024;
|
|
12
|
+
function artifactPath(id) {
|
|
13
|
+
return resolveMediaBufferPath(`music-${id}.mp3`, "outbound");
|
|
14
|
+
}
|
|
15
|
+
async function musicArtifactExists(id) {
|
|
16
|
+
const file = await stat(artifactPath(id)).catch(() => null);
|
|
17
|
+
return Boolean(file?.isFile() && file.size > 0 && file.size <= MAX_AUDIO_BYTES);
|
|
18
|
+
}
|
|
19
|
+
async function deleteMusicArtifact(id) {
|
|
20
|
+
await rm(artifactPath(id), { force: true });
|
|
21
|
+
}
|
|
22
|
+
async function saveMusicArtifact(id, provider, source, canPublish) {
|
|
23
|
+
const url = new URL(source);
|
|
24
|
+
if (url.protocol !== "https:" || url.username || url.password || url.port || !provider.allowsAudioHost(url.hostname)) throw new Error("Music provider returned an untrusted audio URL.");
|
|
25
|
+
await assertSafeUrl(url);
|
|
26
|
+
const path = artifactPath(id);
|
|
27
|
+
const temporary = `${path}.${randomUUID()}.part`;
|
|
28
|
+
await mkdir(dirname(path), {
|
|
29
|
+
recursive: true,
|
|
30
|
+
mode: 448
|
|
31
|
+
});
|
|
32
|
+
const response = await fetch(url, {
|
|
33
|
+
signal: AbortSignal.timeout(45e3),
|
|
34
|
+
redirect: "error"
|
|
35
|
+
});
|
|
36
|
+
if (!response.ok || !response.body || Number(response.headers.get("content-length")) > MAX_AUDIO_BYTES) {
|
|
37
|
+
await response.body?.cancel().catch(() => void 0);
|
|
38
|
+
throw new Error("Music download failed or exceeded the size limit.");
|
|
39
|
+
}
|
|
40
|
+
let size = 0;
|
|
41
|
+
const limit = new Transform({ transform(chunk, _encoding, callback) {
|
|
42
|
+
size += chunk.length;
|
|
43
|
+
if (size > MAX_AUDIO_BYTES) return callback(/* @__PURE__ */ new Error("Generated music exceeds the size limit."));
|
|
44
|
+
callback(null, chunk);
|
|
45
|
+
} });
|
|
46
|
+
try {
|
|
47
|
+
await pipeline(Readable.fromWeb(response.body), limit, createWriteStream(temporary, {
|
|
48
|
+
flags: "wx",
|
|
49
|
+
mode: 384
|
|
50
|
+
}));
|
|
51
|
+
const file = await open(temporary, "r");
|
|
52
|
+
try {
|
|
53
|
+
const header = Buffer.alloc(10);
|
|
54
|
+
const { bytesRead } = await file.read(header);
|
|
55
|
+
if (bytesRead !== header.length) throw new Error("Music provider returned incomplete MP3 audio.");
|
|
56
|
+
let frameOffset = 0;
|
|
57
|
+
if (header.subarray(0, 3).toString("ascii") === "ID3") {
|
|
58
|
+
const tagSize = header.subarray(6, 10);
|
|
59
|
+
if (tagSize.some((byte) => byte > 127)) throw new Error("Music provider returned an invalid ID3 tag.");
|
|
60
|
+
const footerSize = header[3] === 4 && (header[5] & 16) !== 0 ? 10 : 0;
|
|
61
|
+
frameOffset = tagSize.reduce((length, byte) => length << 7 | byte, 0) + 10 + footerSize;
|
|
62
|
+
}
|
|
63
|
+
const frame = Buffer.alloc(4);
|
|
64
|
+
if ((await file.read(frame, 0, frame.length, frameOffset)).bytesRead !== frame.length || size <= frameOffset + frame.length || frame[0] !== 255 || (frame[1] & 230) !== 226 || (frame[1] & 24) === 8 || (frame[2] & 240) === 240 || (frame[2] & 12) === 12) throw new Error("Music provider returned invalid MP3 audio.");
|
|
65
|
+
await file.sync();
|
|
66
|
+
} finally {
|
|
67
|
+
await file.close();
|
|
68
|
+
}
|
|
69
|
+
if (!canPublish()) return;
|
|
70
|
+
await link(temporary, path).catch((error) => {
|
|
71
|
+
if (error.code !== "EEXIST") throw error;
|
|
72
|
+
});
|
|
73
|
+
} finally {
|
|
74
|
+
await rm(temporary, { force: true });
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async function readMusicArtifact(id, rangeHeader, download = false) {
|
|
78
|
+
const path = artifactPath(id);
|
|
79
|
+
const file = await stat(path).catch(() => null);
|
|
80
|
+
if (!file?.isFile() || !file.size || file.size > MAX_AUDIO_BYTES) return new Response(null, { status: 404 });
|
|
81
|
+
const headers = new Headers({
|
|
82
|
+
"Accept-Ranges": "bytes",
|
|
83
|
+
"Content-Type": "audio/mpeg",
|
|
84
|
+
"Cache-Control": "private, no-store",
|
|
85
|
+
"X-Content-Type-Options": "nosniff"
|
|
86
|
+
});
|
|
87
|
+
if (download) headers.set("Content-Disposition", "attachment; filename=\"music.mp3\"");
|
|
88
|
+
const range = parseByteRange(rangeHeader, file.size);
|
|
89
|
+
if (range === "invalid") {
|
|
90
|
+
headers.set("Content-Range", `bytes */${file.size}`);
|
|
91
|
+
return new Response(null, {
|
|
92
|
+
status: 416,
|
|
93
|
+
headers
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
if (range) headers.set("Content-Range", `bytes ${range.start}-${range.end}/${file.size}`);
|
|
97
|
+
headers.set("Content-Length", String(range ? range.end - range.start + 1 : file.size));
|
|
98
|
+
const stream = createReadStream(path, range ?? void 0);
|
|
99
|
+
return new Response(Readable.toWeb(stream), {
|
|
100
|
+
status: range ? 206 : 200,
|
|
101
|
+
headers
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
//#endregion
|
|
105
|
+
export { deleteMusicArtifact, musicArtifactExists, readMusicArtifact, saveMusicArtifact };
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { MusicRequestRejected } from "./provider.js";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
//#region src/music-generation/mureka.ts
|
|
4
|
+
const task = z.object({ id: z.string().min(1).max(200) });
|
|
5
|
+
const status = z.object({
|
|
6
|
+
status: z.string(),
|
|
7
|
+
choices: z.array(z.object({
|
|
8
|
+
url: z.string().optional(),
|
|
9
|
+
duration: z.unknown().optional()
|
|
10
|
+
})).optional()
|
|
11
|
+
});
|
|
12
|
+
async function request(apiOrigin, key, path, input) {
|
|
13
|
+
const response = await fetch(new URL(path, apiOrigin), {
|
|
14
|
+
method: input ? "POST" : "GET",
|
|
15
|
+
headers: {
|
|
16
|
+
Authorization: `Bearer ${key}`,
|
|
17
|
+
...input ? { "Content-Type": "application/json" } : {}
|
|
18
|
+
},
|
|
19
|
+
body: input ? JSON.stringify(input) : void 0,
|
|
20
|
+
redirect: "error",
|
|
21
|
+
signal: AbortSignal.timeout(25e3)
|
|
22
|
+
});
|
|
23
|
+
if (!response.ok) {
|
|
24
|
+
await response.body?.cancel().catch(() => void 0);
|
|
25
|
+
const message = `Mureka returned HTTP ${response.status}.`;
|
|
26
|
+
if (input && [
|
|
27
|
+
400,
|
|
28
|
+
401,
|
|
29
|
+
402,
|
|
30
|
+
403,
|
|
31
|
+
404,
|
|
32
|
+
422,
|
|
33
|
+
429
|
|
34
|
+
].includes(response.status)) throw new MusicRequestRejected(message);
|
|
35
|
+
throw new Error(message);
|
|
36
|
+
}
|
|
37
|
+
return response.json();
|
|
38
|
+
}
|
|
39
|
+
function createMurekaProvider(id, apiOrigin) {
|
|
40
|
+
return {
|
|
41
|
+
id,
|
|
42
|
+
name: "Mureka",
|
|
43
|
+
models: { "mureka-9.5": {
|
|
44
|
+
modes: ["song", "instrumental"],
|
|
45
|
+
maxPromptCharacters: 1024,
|
|
46
|
+
maxLyricsCharacters: 5e3
|
|
47
|
+
} },
|
|
48
|
+
environmentKey: () => {
|
|
49
|
+
return (id === "mureka-cn" ? process.env.MUREKA_CN_API_KEY : process.env.MUREKA_GLOBAL_API_KEY)?.trim() || process.env.MUREKA_API_KEY?.trim() || void 0;
|
|
50
|
+
},
|
|
51
|
+
allowsAudioHost: (host) => host === "cdn.mureka.cn" || host === "cdn.mureka.ai" || (process.env.XOPC_MUREKA_AUDIO_HOSTS ?? "").split(",").some((value) => value.trim().toLowerCase() === host),
|
|
52
|
+
async submit(key, model, { mode, prompt, lyrics }) {
|
|
53
|
+
const result = await request(apiOrigin, key, `/v1/${mode === "song" ? "song" : "instrumental"}/generate`, {
|
|
54
|
+
model,
|
|
55
|
+
n: 1,
|
|
56
|
+
prompt,
|
|
57
|
+
...mode === "song" ? { lyrics } : {}
|
|
58
|
+
});
|
|
59
|
+
return task.parse(result).id;
|
|
60
|
+
},
|
|
61
|
+
async query(key, mode, taskId) {
|
|
62
|
+
const kind = mode === "song" ? "song" : "instrumental";
|
|
63
|
+
const result = status.parse(await request(apiOrigin, key, `/v1/${kind}/query/${encodeURIComponent(taskId)}`));
|
|
64
|
+
switch (result.status) {
|
|
65
|
+
case "preparing":
|
|
66
|
+
case "queued": return { state: "submitted" };
|
|
67
|
+
case "running":
|
|
68
|
+
case "streaming": return { state: "running" };
|
|
69
|
+
case "failed":
|
|
70
|
+
case "timeouted":
|
|
71
|
+
case "cancelled": return {
|
|
72
|
+
state: "failed",
|
|
73
|
+
reason: `Mureka task ended with status ${result.status}.`
|
|
74
|
+
};
|
|
75
|
+
case "succeeded": {
|
|
76
|
+
const choice = result.choices?.[0];
|
|
77
|
+
if (choice?.url) {
|
|
78
|
+
const duration = choice.duration;
|
|
79
|
+
const durationMs = typeof duration === "number" && Number.isSafeInteger(duration) && duration > 0 ? duration : null;
|
|
80
|
+
return {
|
|
81
|
+
state: "completed",
|
|
82
|
+
url: choice.url,
|
|
83
|
+
durationMs
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
throw new Error("Mureka returned an incomplete or unrecognized music task status.");
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
const murekaProvider = createMurekaProvider("mureka-cn", "https://api.mureka.cn");
|
|
94
|
+
const murekaGlobalProvider = createMurekaProvider("mureka-global", "https://api.mureka.ai");
|
|
95
|
+
//#endregion
|
|
96
|
+
export { murekaGlobalProvider, murekaProvider };
|