@miraiclip/assistant 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +650 -40
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +52 -3
- package/dist/index.d.ts +52 -3
- package/dist/index.js +644 -34
- package/dist/index.js.map +1 -1
- package/package.json +4 -2
package/dist/index.cjs
CHANGED
|
@@ -20,8 +20,10 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var src_exports = {};
|
|
22
22
|
__export(src_exports, {
|
|
23
|
+
BACKGROUND_TEMPLATE: () => BACKGROUND_TEMPLATE,
|
|
23
24
|
ChatModelError: () => ChatModelError,
|
|
24
25
|
DEFAULT_INSTRUCTIONS: () => DEFAULT_INSTRUCTIONS,
|
|
26
|
+
applyChecked: () => applyChecked,
|
|
25
27
|
applyCommandsSchema: () => applyCommandsSchema,
|
|
26
28
|
commitFork: () => commitFork,
|
|
27
29
|
createAssistant: () => createAssistant,
|
|
@@ -29,14 +31,18 @@ __export(src_exports, {
|
|
|
29
31
|
defineChatModel: () => defineChatModel,
|
|
30
32
|
defineTool: () => defineTool,
|
|
31
33
|
describeCommands: () => describeCommands,
|
|
34
|
+
describeForAssistant: () => describeForAssistant,
|
|
32
35
|
editorTools: () => editorTools,
|
|
33
36
|
forkProject: () => forkProject,
|
|
34
37
|
openAIChatModel: () => openAIChatModel,
|
|
35
38
|
parseToolArguments: () => parseToolArguments,
|
|
39
|
+
rateLimitedChatModel: () => rateLimitedChatModel,
|
|
36
40
|
remoteChatModel: () => remoteChatModel,
|
|
41
|
+
retryDelayMs: () => retryDelayMs,
|
|
37
42
|
scriptedChatModel: () => scriptedChatModel,
|
|
38
43
|
toOpenAIMessages: () => toOpenAIMessages,
|
|
39
44
|
toolsFromDefinitions: () => toolsFromDefinitions,
|
|
45
|
+
unknownFields: () => unknownFields,
|
|
40
46
|
withExplicitIds: () => withExplicitIds
|
|
41
47
|
});
|
|
42
48
|
module.exports = __toCommonJS(src_exports);
|
|
@@ -87,6 +93,21 @@ function parseToolArguments(raw) {
|
|
|
87
93
|
return { __invalidArguments: raw };
|
|
88
94
|
}
|
|
89
95
|
}
|
|
96
|
+
var RETRYABLE = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
97
|
+
function retryDelayMs(res, message, attempt) {
|
|
98
|
+
const header = Number(res.headers.get("retry-after-ms") ?? NaN) || Number(res.headers.get("retry-after") ?? NaN) * 1e3;
|
|
99
|
+
const m = /try again in ([\d.]+)\s*(ms|s)/i.exec(message);
|
|
100
|
+
const asked = header || (m ? Number(m[1]) * (m[2].toLowerCase() === "ms" ? 1 : 1e3) : 0);
|
|
101
|
+
const backoff = Math.min(3e4, 1e3 * 2 ** attempt);
|
|
102
|
+
return Math.round(Math.min(6e4, Math.max(asked + 250, asked ? 0 : backoff)) * (1 + Math.random() * 0.25));
|
|
103
|
+
}
|
|
104
|
+
function sleep(ms, signal) {
|
|
105
|
+
return new Promise((resolve, reject) => {
|
|
106
|
+
if (signal?.aborted) return reject(signal.reason ?? new Error("aborted"));
|
|
107
|
+
const t = setTimeout(resolve, ms);
|
|
108
|
+
signal?.addEventListener("abort", () => (clearTimeout(t), reject(signal.reason ?? new Error("aborted"))), { once: true });
|
|
109
|
+
});
|
|
110
|
+
}
|
|
90
111
|
var FINISH = {
|
|
91
112
|
stop: "stop",
|
|
92
113
|
tool_calls: "tool_calls",
|
|
@@ -118,15 +139,21 @@ function openAIChatModel(options) {
|
|
|
118
139
|
if (options.apiKey) headers["Authorization"] = `Bearer ${options.apiKey}`;
|
|
119
140
|
if (options.organization) headers["OpenAI-Organization"] = options.organization;
|
|
120
141
|
if (options.project) headers["OpenAI-Project"] = options.project;
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
142
|
+
const maxRetries = options.maxRetries ?? 4;
|
|
143
|
+
let res;
|
|
144
|
+
for (let attempt = 0; ; attempt++) {
|
|
145
|
+
res = await doFetch(`${base}/chat/completions`, {
|
|
146
|
+
method: "POST",
|
|
147
|
+
headers,
|
|
148
|
+
body: JSON.stringify(body),
|
|
149
|
+
...opts.signal ? { signal: opts.signal } : {}
|
|
150
|
+
});
|
|
151
|
+
if (res.ok) break;
|
|
128
152
|
const err = await res.json().catch(() => null);
|
|
129
|
-
|
|
153
|
+
const message = err?.error?.message ?? `HTTP ${res.status}`;
|
|
154
|
+
const permanent = /quota|credits|billing/i.test(message);
|
|
155
|
+
if (attempt >= maxRetries || !RETRYABLE.has(res.status) || permanent) throw new ChatModelError(id, message, res.status);
|
|
156
|
+
await sleep(retryDelayMs(res, message, attempt), opts.signal);
|
|
130
157
|
}
|
|
131
158
|
if (!stream || !res.body || !(res.headers.get("content-type") ?? "").includes("event-stream")) {
|
|
132
159
|
return fromCompletion(await res.json(), opts.onText);
|
|
@@ -303,8 +330,163 @@ async function remoteChatModel(endpoint, options = {}) {
|
|
|
303
330
|
};
|
|
304
331
|
}
|
|
305
332
|
|
|
306
|
-
// src/
|
|
333
|
+
// src/limits.ts
|
|
334
|
+
function rateLimitedChatModel(model, options) {
|
|
335
|
+
const tpm = options.tokensPerMinute ?? Infinity;
|
|
336
|
+
const rpm = options.requestsPerMinute ?? Infinity;
|
|
337
|
+
const estimate = options.estimate ?? ((r) => Math.ceil(JSON.stringify(r).length / 4) + (r.maxOutputTokens ?? 1e3));
|
|
338
|
+
const window = [];
|
|
339
|
+
let gate = Promise.resolve();
|
|
340
|
+
const prune = (now) => {
|
|
341
|
+
while (window.length && now - window[0].at >= 6e4) window.shift();
|
|
342
|
+
};
|
|
343
|
+
const used = () => window.reduce((n2, e) => n2 + e.tokens, 0);
|
|
344
|
+
const reserve = (tokens, signal) => {
|
|
345
|
+
const want = Math.min(tokens, tpm);
|
|
346
|
+
const turn = gate.then(async () => {
|
|
347
|
+
for (; ; ) {
|
|
348
|
+
if (signal?.aborted) throw signal.reason ?? new Error("aborted");
|
|
349
|
+
const now = Date.now();
|
|
350
|
+
prune(now);
|
|
351
|
+
if (window.length < rpm && used() + want <= tpm) break;
|
|
352
|
+
const oldest = window[0];
|
|
353
|
+
await new Promise((r) => setTimeout(r, oldest ? Math.max(50, 6e4 - (now - oldest.at)) : 50));
|
|
354
|
+
}
|
|
355
|
+
const entry = { at: Date.now(), tokens: want };
|
|
356
|
+
window.push(entry);
|
|
357
|
+
return entry;
|
|
358
|
+
});
|
|
359
|
+
gate = turn.then(
|
|
360
|
+
() => void 0,
|
|
361
|
+
() => void 0
|
|
362
|
+
);
|
|
363
|
+
return turn;
|
|
364
|
+
};
|
|
365
|
+
return {
|
|
366
|
+
...model,
|
|
367
|
+
async complete(request, opts = {}) {
|
|
368
|
+
const entry = await reserve(estimate(request), opts.signal);
|
|
369
|
+
const response = await model.complete(request, opts);
|
|
370
|
+
const real = (response.usage?.inputTokens ?? 0) + (response.usage?.outputTokens ?? 0);
|
|
371
|
+
if (real > 0) entry.tokens = real;
|
|
372
|
+
return response;
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// src/state.ts
|
|
307
378
|
var import_core = require("@miraiclip/core");
|
|
379
|
+
function describeForAssistant(doc) {
|
|
380
|
+
const lines = [
|
|
381
|
+
(0, import_core.describeProject)(doc),
|
|
382
|
+
"",
|
|
383
|
+
"track names: " + doc.trackOrder.map((id) => {
|
|
384
|
+
const t = doc.tracks[id];
|
|
385
|
+
const flags = [t?.hidden && "hidden", t?.muted && "muted", t?.solo && "solo", t?.locked && "locked"].filter(Boolean);
|
|
386
|
+
return `${id} = "${t?.name ?? id}"${flags.length ? ` (${flags.join(", ")})` : ""}`;
|
|
387
|
+
}).join(", ") + " (track flags muted / solo / locked / hidden: track/set-property)"
|
|
388
|
+
];
|
|
389
|
+
const cuts = (0, import_core.findCuts)(doc);
|
|
390
|
+
lines.push(
|
|
391
|
+
cuts.length ? `cuts (where one clip ends and the next starts on the same track; transitions go here): ${cuts.map((c) => `${n(c.atUs / 1e6)}s ${c.fromClipId}\u2192${c.toClipId}${c.transition ? ` (${c.transition.kind})` : ""}`).join(", ")}` : "cuts: none"
|
|
392
|
+
);
|
|
393
|
+
const clips = Object.values(doc.clips).sort((a, b) => a.trackId.localeCompare(b.trackId) || a.startUs - b.startUs);
|
|
394
|
+
if (clips.length) {
|
|
395
|
+
lines.push("clip details (transform x/y: 0.5 = center; scale 1 = full size; times in seconds):");
|
|
396
|
+
for (const clip of clips) lines.push(`- ${clip.id}: ${details(clip)}`);
|
|
397
|
+
}
|
|
398
|
+
return lines.join("\n");
|
|
399
|
+
}
|
|
400
|
+
var n = (v) => Number(v.toFixed(3));
|
|
401
|
+
function details(clip) {
|
|
402
|
+
const parts = [`${clip.kind} on ${clip.trackId}`, `${n(clip.startUs / 1e6)}\u2013${n((clip.startUs + clip.durationUs) / 1e6)}s`];
|
|
403
|
+
if (clip.kind !== "audio") {
|
|
404
|
+
const t = clip.transform;
|
|
405
|
+
parts.push(`x ${n(t.x)} y ${n(t.y)} scale ${n(t.scale)}${t.rotation ? ` rotation ${n(t.rotation)}\xB0` : ""} opacity ${n(t.opacity)}`);
|
|
406
|
+
}
|
|
407
|
+
if (clip.kind === "text") {
|
|
408
|
+
const c = clip;
|
|
409
|
+
parts.push(`font ${c.fontFamily} ${c.fontSizePx}px${c.fontWeight ? ` weight ${c.fontWeight}` : ""} color ${c.color}${c.textAlign ? ` align ${c.textAlign}` : ""}`);
|
|
410
|
+
}
|
|
411
|
+
if (clip.kind === "caption") {
|
|
412
|
+
const c = clip;
|
|
413
|
+
parts.push(
|
|
414
|
+
`words "${c.words.map((w) => w.text).join(" ").slice(0, 80)}" \xB7 style preset ${c.style.preset} color ${c.style.color} highlight ${c.style.highlightColor} box ${c.style.backgroundColor ?? "none"} (style.backgroundColor)`
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
if (clip.kind === "html") {
|
|
418
|
+
const c = clip;
|
|
419
|
+
parts.push(`params ${JSON.stringify(c.params)} (template placeholders: ${htmlPlaceholders(c.template).join(", ") || "none"})`);
|
|
420
|
+
}
|
|
421
|
+
if ("volume" in clip) {
|
|
422
|
+
const c = clip;
|
|
423
|
+
parts.push(`volume ${n(c.volume)}${c.fadeInUs ? ` fade-in ${n(c.fadeInUs / 1e6)}s` : ""}${c.fadeOutUs ? ` fade-out ${n(c.fadeOutUs / 1e6)}s` : ""}`);
|
|
424
|
+
parts.push(`source from ${n(c.trimStartUs / 1e6)}s`);
|
|
425
|
+
}
|
|
426
|
+
if (clip.effects?.length) parts.push(`effects ${clip.effects.map((e) => `${e.kind}#${e.id}${e.enabled ? "" : " (off)"}`).join(", ")}`);
|
|
427
|
+
const volumeKeys = clip.animations?.volume ?? [];
|
|
428
|
+
if (volumeKeys.length) parts.push(`volume keyframes ${volumeKeys.map((k) => `${n((clip.startUs + (0, import_core.keyframeTimeUs)(k, clip.durationUs)) / 1e6)}s=${n(k.value)}`).join(", ")}`);
|
|
429
|
+
const visual = Object.entries(clip.animations ?? {}).some(([prop, list]) => prop !== "volume" && list && list.length);
|
|
430
|
+
if (visual) {
|
|
431
|
+
const read = readClipAnimation(clip);
|
|
432
|
+
parts.push(read.custom ? `custom keyframes (not presets): ${describeKeyframes(clip)}` : `animation ${(0, import_core.describeAnimation)(read.recipe)}`);
|
|
433
|
+
}
|
|
434
|
+
return parts.join(" \xB7 ");
|
|
435
|
+
}
|
|
436
|
+
function htmlPlaceholders(template) {
|
|
437
|
+
return [...new Set([...template.matchAll(/\{\{\s*([\w.-]+)\s*\}\}/g)].map((m) => m[1]))];
|
|
438
|
+
}
|
|
439
|
+
function readClipAnimation(clip) {
|
|
440
|
+
const read = (0, import_core.readAnimation)(clip);
|
|
441
|
+
if (read.custom) return { recipe: {}, custom: true };
|
|
442
|
+
if (read.stale && read.recipe.loop) {
|
|
443
|
+
const preset = (0, import_core.animationPreset)(read.recipe.loop.preset);
|
|
444
|
+
const props = Object.keys(preset?.build(clip.transform, 1e6, "linear") ?? {});
|
|
445
|
+
const count = props.reduce((sum, prop) => sum + (clip.animations?.[prop]?.length ?? 0), 0);
|
|
446
|
+
if (count < 4) return { recipe: {}, custom: true };
|
|
447
|
+
}
|
|
448
|
+
return { recipe: read.recipe, custom: false };
|
|
449
|
+
}
|
|
450
|
+
function describeKeyframes(clip) {
|
|
451
|
+
return Object.entries(clip.animations ?? {}).filter(([prop, list]) => prop !== "volume" && list?.length).map(([prop, list]) => `${prop} ${list.map((k) => `${n((clip.startUs + (0, import_core.keyframeTimeUs)(k, clip.durationUs)) / 1e6)}s ${n(k.value)}`).join(" \u2192 ")}`).join("; ");
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// src/strict.ts
|
|
455
|
+
function branches(schema) {
|
|
456
|
+
const alts = schema.anyOf ?? schema.oneOf;
|
|
457
|
+
return alts ? alts.flatMap(branches) : [schema];
|
|
458
|
+
}
|
|
459
|
+
function unknownIn(schema, value, prefix = "") {
|
|
460
|
+
const props = schema.properties;
|
|
461
|
+
if (!props || schema.additionalProperties !== void 0 && schema.additionalProperties !== false) return [];
|
|
462
|
+
const out = [];
|
|
463
|
+
for (const [key, v] of Object.entries(value)) {
|
|
464
|
+
const sub = props[key];
|
|
465
|
+
if (!sub) out.push(prefix + key);
|
|
466
|
+
else if (!prefix && v && typeof v === "object" && !Array.isArray(v)) {
|
|
467
|
+
const objectBranch = branches(sub).find((b) => b.properties);
|
|
468
|
+
if (objectBranch) out.push(...unknownIn(objectBranch, v, `${key}.`));
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
return out;
|
|
472
|
+
}
|
|
473
|
+
function unknownFields(schema, payload) {
|
|
474
|
+
if (!schema || !payload || typeof payload !== "object" || Array.isArray(payload)) return null;
|
|
475
|
+
const value = payload;
|
|
476
|
+
let candidates = branches(schema).filter((b) => b.properties);
|
|
477
|
+
if (candidates.length === 0) return null;
|
|
478
|
+
const byKind = candidates.filter((b) => b.properties.kind?.const !== void 0 && b.properties.kind.const === value.kind);
|
|
479
|
+
if (byKind.length) candidates = byKind;
|
|
480
|
+
let best = null;
|
|
481
|
+
for (const b of candidates) {
|
|
482
|
+
const unknown = unknownIn(b, value);
|
|
483
|
+
if (!best || unknown.length < best.unknown.length) best = { unknown, allowed: Object.keys(b.properties) };
|
|
484
|
+
}
|
|
485
|
+
return best && best.unknown.length ? best : null;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// src/tools.ts
|
|
489
|
+
var import_core2 = require("@miraiclip/core");
|
|
308
490
|
function defineTool(tool) {
|
|
309
491
|
if (!/^[a-zA-Z0-9_-]{1,64}$/.test(tool.name)) throw new Error(`Tool name must match ^[a-zA-Z0-9_-]{1,64}$ (got "${tool.name}")`);
|
|
310
492
|
return tool;
|
|
@@ -327,7 +509,17 @@ function toolsFromDefinitions(definitions, run, options = {}) {
|
|
|
327
509
|
});
|
|
328
510
|
}
|
|
329
511
|
var S = 1e6;
|
|
512
|
+
var SLIDE_IN = { "from-right": "left", "from-left": "right", "from-bottom": "up", "from-top": "down" };
|
|
513
|
+
var SLIDE_OUT = { "to-left": "left", "to-right": "right", "to-top": "up", "to-bottom": "down" };
|
|
514
|
+
function slideAliases(input) {
|
|
515
|
+
const out = {};
|
|
516
|
+
if (typeof input.in === "string" && SLIDE_IN[input.in]) out.in = SLIDE_IN[input.in];
|
|
517
|
+
if (typeof input.out === "string" && SLIDE_OUT[input.out]) out.out = SLIDE_OUT[input.out];
|
|
518
|
+
return out;
|
|
519
|
+
}
|
|
520
|
+
var BACKGROUND_TEMPLATE = '<div style="width:100%;height:100%;background:{{color}}"></div>';
|
|
330
521
|
var seconds = (us) => `${Math.round(us / 1e5) / 10}s`;
|
|
522
|
+
var n3 = (v) => Number(v.toFixed(3));
|
|
331
523
|
var toUs = (s) => typeof s === "number" && Number.isFinite(s) ? Math.round(s * S) : void 0;
|
|
332
524
|
var PHRASES = {
|
|
333
525
|
"clip/add": (p) => p.kind === "text" ? `Added text "${String(p.text ?? "").slice(0, 40)}" at ${seconds(Number(p.startUs))}` : `Added a ${p.kind} clip at ${seconds(Number(p.startUs))}`,
|
|
@@ -365,19 +557,19 @@ function describeCommands(commands) {
|
|
|
365
557
|
}
|
|
366
558
|
var US_NOTE = "Times in commands are integer MICROSECONDS (1 s = 1000000).";
|
|
367
559
|
function editorTools(options = {}) {
|
|
368
|
-
const transitionKinds = options.transitionKinds ?? Object.keys(
|
|
369
|
-
const presetIds = (slot) =>
|
|
560
|
+
const transitionKinds = options.transitionKinds ?? Object.keys(import_core2.builtinTransitionParamSchemas);
|
|
561
|
+
const presetIds = (slot) => import_core2.ANIMATION_PRESETS.filter((p) => p.slot === slot).map((p) => p.id.split(":")[1]);
|
|
370
562
|
return [
|
|
371
563
|
defineTool({
|
|
372
564
|
name: "get_state",
|
|
373
|
-
description: `The project as it is now (after your edits so far): settings, assets, tracks with clips (ids, kinds, time ranges), transitions. ${US_NOTE}`,
|
|
565
|
+
description: `The project as it is now (after your edits so far): settings, assets, tracks with clips (ids, kinds, time ranges), transitions, and each clip's position, size, opacity, text style, volume, effects and animation. ${US_NOTE}`,
|
|
374
566
|
inputSchema: {
|
|
375
567
|
type: "object",
|
|
376
568
|
properties: { includeJson: { type: "boolean", description: "Also return the full document JSON (verbose; only for exact field values)." } },
|
|
377
569
|
additionalProperties: false
|
|
378
570
|
},
|
|
379
571
|
run(input, { project }) {
|
|
380
|
-
const summary = (
|
|
572
|
+
const summary = describeForAssistant(project.toJSON());
|
|
381
573
|
return { ok: true, result: input.includeJson ? { summary, document: project.toJSON() } : summary };
|
|
382
574
|
}
|
|
383
575
|
}),
|
|
@@ -399,18 +591,16 @@ function editorTools(options = {}) {
|
|
|
399
591
|
defineTool({
|
|
400
592
|
name: "apply_commands",
|
|
401
593
|
description: `Apply editing commands as one all-or-nothing batch. On failure nothing changes and the error names the failing index and why (invalid payload issues, or the engine's rejection reason) \u2014 fix it and retry. Give new clips, tracks and assets your own readable ids. ${US_NOTE}`,
|
|
402
|
-
inputSchema: applyCommandsSchema(options.commandCatalog ?? (0,
|
|
594
|
+
inputSchema: applyCommandsSchema(options.commandCatalog ?? (0, import_core2.commandCatalog)()),
|
|
403
595
|
run(input, { project }) {
|
|
404
596
|
const commands = input.commands;
|
|
405
597
|
if (!Array.isArray(commands) || commands.length === 0) return { ok: false, error: "commands must be a non-empty array of { type, payload }" };
|
|
406
|
-
|
|
407
|
-
if (!result.ok) return { ok: false, error: result };
|
|
408
|
-
return { ok: true, result: { applied: result.applied }, changes: describeCommands(commands) };
|
|
598
|
+
return applyChecked(project, commands);
|
|
409
599
|
}
|
|
410
600
|
}),
|
|
411
601
|
defineTool({
|
|
412
602
|
name: "add_transition",
|
|
413
|
-
description:
|
|
603
|
+
description: 'Add (or replace, or remove) a transition on cuts \u2014 where one clip ends exactly where the next starts on the same video track. Pick the cut by its two clip ids, by a time near it, or all cuts. A transition overlaps both clips around the cut, so its length is capped by the spare footage there; cuts with none are skipped and reported. When a video clip directly follows another on its track, "fade into / dissolve to / slide in / wipe to" that clip means a transition on the cut before it, not an animation. Slide / wipe direction is the direction of motion: "from the left" = right.',
|
|
414
604
|
inputSchema: {
|
|
415
605
|
type: "object",
|
|
416
606
|
properties: {
|
|
@@ -426,15 +616,23 @@ function editorTools(options = {}) {
|
|
|
426
616
|
additionalProperties: false
|
|
427
617
|
},
|
|
428
618
|
run(input, { project }) {
|
|
429
|
-
const cuts = (0,
|
|
619
|
+
const cuts = (0, import_core2.findCuts)(project.getState().doc);
|
|
430
620
|
if (cuts.length === 0) return { ok: false, error: "There are no cuts: transitions need two clips touching on the same video track." };
|
|
431
621
|
let targets;
|
|
432
622
|
if (input.allCuts === true) targets = cuts;
|
|
433
623
|
else if (typeof input.fromClipId === "string" || typeof input.toClipId === "string") {
|
|
434
624
|
targets = cuts.filter((c) => (!input.fromClipId || c.fromClipId === input.fromClipId) && (!input.toClipId || c.toClipId === input.toClipId));
|
|
625
|
+
if (targets.length === 0) return { ok: false, error: { error: noCutReason(project, input.fromClipId, input.toClipId), cuts: cuts.map(cutInfo) } };
|
|
435
626
|
} else {
|
|
436
627
|
const at = toUs(input.nearSeconds) ?? project.getState().playheadUs;
|
|
437
|
-
|
|
628
|
+
const nearest = cuts.reduce((a, b) => Math.abs(b.atUs - at) < Math.abs(a.atUs - at) ? b : a);
|
|
629
|
+
if (input.nearSeconds !== void 0 && Math.abs(nearest.atUs - at) > 1.5 * S) {
|
|
630
|
+
return {
|
|
631
|
+
ok: false,
|
|
632
|
+
error: { error: `There is no cut near ${seconds(at)} (the nearest is at ${seconds(nearest.atUs)}). Transitions only go on cuts; nothing was added. Use one of these cuts, or tell the user there isn't one there.`, cuts: cuts.map(cutInfo) }
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
targets = [nearest];
|
|
438
636
|
}
|
|
439
637
|
if (targets.length === 0) return { ok: false, error: { error: "no cut matches", cuts: cuts.map(cutInfo) } };
|
|
440
638
|
const kind = String(input.kind);
|
|
@@ -459,7 +657,7 @@ function editorTools(options = {}) {
|
|
|
459
657
|
changes.push(`${cut.transition ? "Replaced the transition with" : "Added"} ${kind} at ${seconds(cut.atUs)} (${seconds(durationUs)})`);
|
|
460
658
|
}
|
|
461
659
|
if (commands.length) {
|
|
462
|
-
const result = (0,
|
|
660
|
+
const result = (0, import_core2.applyCommands)(project, commands);
|
|
463
661
|
if (!result.ok) return { ok: false, error: result };
|
|
464
662
|
}
|
|
465
663
|
return { ok: true, result: { changed: changes.length, skipped }, changes };
|
|
@@ -467,27 +665,35 @@ function editorTools(options = {}) {
|
|
|
467
665
|
}),
|
|
468
666
|
defineTool({
|
|
469
667
|
name: "animate_clip",
|
|
470
|
-
description: `Give a visual clip (video, image, text) an entrance (in), a loop while on screen, and/or an exit (out), as keyframes. Slots you leave out keep their current animation; "none" removes one; reset:true removes all. In/out presets: ${presetIds("in").join(", ")}. Loops: ${presetIds("loop").join(", ")}. Exits stay on the clip's end through trims.`,
|
|
668
|
+
description: `Give a visual clip (video, image, text) an entrance (in), a loop while on screen, and/or an exit (out), as keyframes. Slots you leave out keep their current animation; "none" removes one; reset:true removes all. In/out presets: ${presetIds("in").join(", ")}. Loops: ${presetIds("loop").join(", ")}. Exits stay on the clip's end through trims. Slides: give the edge, in: "from-right" (enters from the right edge, moving left), "from-left", "from-top", "from-bottom"; out: "to-left", "to-right", "to-top", "to-bottom". For text, logos and overlays; between two video clips prefer add_transition. The clip's current animation is in get_state; pass only the slots to change. Fading the whole VIDEO in at the start and out at the end is two calls: in on the FIRST clip of the main video track, out on the LAST one (an out on the first clip would fade to black at its cut).`,
|
|
471
669
|
inputSchema: {
|
|
472
670
|
type: "object",
|
|
473
671
|
properties: {
|
|
474
672
|
clipId: { type: "string" },
|
|
475
|
-
in: { type: "string", enum: [...presetIds("in"), "none"] },
|
|
673
|
+
in: { type: "string", enum: [...presetIds("in"), ...Object.keys(SLIDE_IN), "none"], description: 'For slides, prefer "from-left" / "from-right" / "from-top" / "from-bottom": the edge the clip enters from.' },
|
|
476
674
|
loop: { type: "string", enum: [...presetIds("loop"), "none"] },
|
|
477
|
-
out: { type: "string", enum: [...presetIds("out"), "none"] },
|
|
675
|
+
out: { type: "string", enum: [...presetIds("out"), ...Object.keys(SLIDE_OUT), "none"], description: 'For slides, prefer "to-left" / "to-right" / "to-top" / "to-bottom": the edge the clip leaves by.' },
|
|
478
676
|
inSeconds: { type: "number", minimum: 0.2, maximum: 3, description: "Default 0.6." },
|
|
479
677
|
outSeconds: { type: "number", minimum: 0.2, maximum: 3, description: "Default 0.6." },
|
|
480
|
-
easing: { type: "string", enum:
|
|
678
|
+
easing: { type: "string", enum: import_core2.ANIMATION_EASINGS.map((e) => e.id), description: "Default smooth." },
|
|
481
679
|
reset: { type: "boolean" }
|
|
482
680
|
},
|
|
483
681
|
required: ["clipId"],
|
|
484
682
|
additionalProperties: false
|
|
485
683
|
},
|
|
486
684
|
run(input, { project }) {
|
|
685
|
+
input = { ...input, ...slideAliases(input) };
|
|
487
686
|
const clip = project.getState().doc.clips[String(input.clipId)];
|
|
488
687
|
if (!clip) return { ok: false, error: `no clip "${input.clipId}"` };
|
|
489
688
|
if (clip.kind === "audio") return { ok: false, error: "audio clips have no picture to animate (use volume or fades)" };
|
|
490
|
-
const
|
|
689
|
+
const reading = readClipAnimation(clip);
|
|
690
|
+
if (reading.custom && input.reset !== true) {
|
|
691
|
+
return {
|
|
692
|
+
ok: false,
|
|
693
|
+
error: `${clip.id} has its own keyframes (${describeKeyframes(clip)}), not presets; presets would overwrite them. To remove one property's animation, apply_commands keyframe/clear { clipId, property }. To edit them, use set_keyframes. To replace them all with presets, pass reset: true. Nothing was changed.`
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
const current = input.reset === true ? {} : reading.recipe;
|
|
491
697
|
const next = { ...current };
|
|
492
698
|
const easing = typeof input.easing === "string" ? input.easing : "smooth";
|
|
493
699
|
for (const slot of ["in", "loop", "out"]) {
|
|
@@ -500,20 +706,405 @@ function editorTools(options = {}) {
|
|
|
500
706
|
const durationUs = slot === "loop" ? 0 : toUs(input[slot === "in" ? "inSeconds" : "outSeconds"]) ?? current[slot]?.durationUs ?? 6e5;
|
|
501
707
|
next[slot] = { preset: `${slot}:${name}`, durationUs, easing: slot === "loop" ? "linear" : easing };
|
|
502
708
|
}
|
|
503
|
-
const recipe = (0,
|
|
504
|
-
const result = (0,
|
|
709
|
+
const recipe = (0, import_core2.fitAnimation)(next, clip.durationUs);
|
|
710
|
+
const result = (0, import_core2.applyCommands)(project, (0, import_core2.animationCommands)(clip, recipe));
|
|
505
711
|
if (!result.ok) return { ok: false, error: result };
|
|
506
|
-
const summary = (0,
|
|
507
|
-
|
|
712
|
+
const summary = (0, import_core2.describeAnimation)(recipe) || "no animation";
|
|
713
|
+
const atCut = input.in && input.in !== "none" && clip.kind === "video" ? (0, import_core2.findCuts)(project.getState().doc).find((c) => c.toClipId === clip.id) : void 0;
|
|
714
|
+
return {
|
|
715
|
+
ok: true,
|
|
716
|
+
result: {
|
|
717
|
+
clipId: clip.id,
|
|
718
|
+
animation: summary,
|
|
719
|
+
...atCut ? { note: `${clip.id} starts at a cut right after ${atCut.fromClipId}. If the user meant going from ${atCut.fromClipId} into ${clip.id} ("fade into", "slide in", "dissolve to"), that is add_transition on this cut: undo this entrance with in:"none" and add the transition.` } : {}
|
|
720
|
+
},
|
|
721
|
+
changes: [`${clip.id}: ${summary}`]
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
}),
|
|
725
|
+
defineTool({
|
|
726
|
+
name: "add_effect",
|
|
727
|
+
description: "Add a visual effect to one or more clips (video, image, text). Kinds, with their params (min\u2013max, default): " + import_core2.EFFECT_CATALOG.map((e) => `${e.kind} (${e.label}${e.params.length ? `: ${e.params.map((p) => p.type === "number" ? `${p.key} ${p.min}\u2013${p.max}, ${p.default}` : `${p.key} color`).join("; ")}` : ""})`).join(", ") + ". Omitted params take their defaults. Black & white is mono; brightness, contrast, saturation and hue are colorAdjust params (-1\u20131, 0 = unchanged).",
|
|
728
|
+
inputSchema: {
|
|
729
|
+
type: "object",
|
|
730
|
+
properties: {
|
|
731
|
+
clipIds: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
732
|
+
kind: { type: "string", enum: import_core2.EFFECT_CATALOG.map((e) => e.kind) },
|
|
733
|
+
params: { type: "object", description: "Effect params by key; see the list above." }
|
|
734
|
+
},
|
|
735
|
+
required: ["clipIds", "kind"],
|
|
736
|
+
additionalProperties: false
|
|
737
|
+
},
|
|
738
|
+
run(input, { project }) {
|
|
739
|
+
const ids = input.clipIds ?? [];
|
|
740
|
+
const doc = project.getState().doc;
|
|
741
|
+
const missing = ids.filter((id) => !doc.clips[id]);
|
|
742
|
+
if (missing.length) return { ok: false, error: `no clip ${missing.map((m) => `"${m}"`).join(", ")}` };
|
|
743
|
+
const audio = ids.filter((id) => doc.clips[id].kind === "audio");
|
|
744
|
+
if (audio.length) return { ok: false, error: `audio clips have no picture: ${audio.join(", ")}` };
|
|
745
|
+
const params = input.params && typeof input.params === "object" ? input.params : void 0;
|
|
746
|
+
const out = applyChecked(project, ids.map((clipId) => ({ type: "effect/add", payload: { clipId, kind: input.kind, ...params ? { params } : {} } })));
|
|
747
|
+
if (!out.ok) return out;
|
|
748
|
+
const label = import_core2.EFFECT_CATALOG.find((e) => e.kind === input.kind)?.label ?? String(input.kind);
|
|
749
|
+
return { ...out, changes: [`Added ${label} to ${ids.join(", ")}`] };
|
|
750
|
+
}
|
|
751
|
+
}),
|
|
752
|
+
defineTool({
|
|
753
|
+
name: "remove_effects",
|
|
754
|
+
description: "Delete effects from clips: one kind, or every effect when kind is omitted. To turn effects off but keep them, use set_effects_enabled.",
|
|
755
|
+
inputSchema: {
|
|
756
|
+
type: "object",
|
|
757
|
+
properties: {
|
|
758
|
+
clipIds: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
759
|
+
kind: { type: "string", description: "Only effects of this kind (default: all)." }
|
|
760
|
+
},
|
|
761
|
+
required: ["clipIds"],
|
|
762
|
+
additionalProperties: false
|
|
763
|
+
},
|
|
764
|
+
run(input, { project }) {
|
|
765
|
+
const doc = project.getState().doc;
|
|
766
|
+
const commands = [];
|
|
767
|
+
for (const id of input.clipIds ?? []) {
|
|
768
|
+
const clip = doc.clips[id];
|
|
769
|
+
if (!clip) return { ok: false, error: `no clip "${id}"` };
|
|
770
|
+
for (const e of clip.effects ?? []) if (!input.kind || e.kind === input.kind) commands.push({ type: "effect/remove", payload: { clipId: id, effectId: e.id } });
|
|
771
|
+
}
|
|
772
|
+
if (!commands.length) return { ok: true, result: { removed: 0, note: "those clips had no matching effects" } };
|
|
773
|
+
const out = applyChecked(project, commands);
|
|
774
|
+
return out.ok ? { ...out, result: { removed: commands.length }, changes: [`Removed ${commands.length} effect${commands.length > 1 ? "s" : ""}`] } : out;
|
|
775
|
+
}
|
|
776
|
+
}),
|
|
777
|
+
defineTool({
|
|
778
|
+
name: "trim_clip",
|
|
779
|
+
description: "Trim a clip like an editor's edge drag, in seconds. removeFromStart cuts that much off the beginning (the clip's end stays put; for video/audio the source moves on too). removeFromEnd cuts off the end. setDuration sets the length, keeping the start. startSeconds / endSeconds put the clip's edges at those timeline times (e.g. 0 and the composition length to cover the whole video; a video/audio clip's source shifts with its start edge). ripple:true shifts later clips on the same track to close (or open) the gap.",
|
|
780
|
+
inputSchema: {
|
|
781
|
+
type: "object",
|
|
782
|
+
properties: {
|
|
783
|
+
clipId: { type: "string" },
|
|
784
|
+
removeFromStart: { type: "number", exclusiveMinimum: 0 },
|
|
785
|
+
removeFromEnd: { type: "number", exclusiveMinimum: 0 },
|
|
786
|
+
setDuration: { type: "number", exclusiveMinimum: 0 },
|
|
787
|
+
startSeconds: { type: "number", minimum: 0, description: "Timeline time for the clip's start edge." },
|
|
788
|
+
endSeconds: { type: "number", exclusiveMinimum: 0, description: "Timeline time for the clip's end edge." },
|
|
789
|
+
ripple: { type: "boolean" }
|
|
790
|
+
},
|
|
791
|
+
required: ["clipId"],
|
|
792
|
+
additionalProperties: false
|
|
793
|
+
},
|
|
794
|
+
run(input, { project }) {
|
|
795
|
+
const doc = project.getState().doc;
|
|
796
|
+
const clip = doc.clips[String(input.clipId)];
|
|
797
|
+
if (!clip) return { ok: false, error: `no clip "${input.clipId}"` };
|
|
798
|
+
const head = toUs(input.removeFromStart) ?? 0;
|
|
799
|
+
const tail = toUs(input.removeFromEnd) ?? 0;
|
|
800
|
+
const ripple = input.ripple === true;
|
|
801
|
+
const startAt = toUs(input.startSeconds);
|
|
802
|
+
const endAt = toUs(input.endSeconds);
|
|
803
|
+
if ((startAt !== void 0 || endAt !== void 0) && (head || tail)) return { ok: false, error: "use either startSeconds / endSeconds or removeFromStart / removeFromEnd, not both" };
|
|
804
|
+
let duration = clip.durationUs - head - tail;
|
|
805
|
+
const set = toUs(input.setDuration);
|
|
806
|
+
if (set) duration = set;
|
|
807
|
+
let start = ripple ? clip.startUs : clip.startUs + head;
|
|
808
|
+
let sourceShift = head;
|
|
809
|
+
if (startAt !== void 0) {
|
|
810
|
+
start = startAt;
|
|
811
|
+
sourceShift = startAt - clip.startUs;
|
|
812
|
+
if (endAt === void 0 && !set) duration = clip.startUs + clip.durationUs - startAt;
|
|
813
|
+
}
|
|
814
|
+
if (endAt !== void 0) duration = endAt - start;
|
|
815
|
+
if (duration <= 0) return { ok: false, error: `that leaves the clip ${seconds(duration)} long; it is ${seconds(clip.durationUs)} (${seconds(clip.startUs)}\u2013${seconds(clip.startUs + clip.durationUs)})` };
|
|
816
|
+
const trimmable = "trimStartUs" in clip;
|
|
817
|
+
if (trimmable && clip.trimStartUs + sourceShift < 0) {
|
|
818
|
+
return { ok: false, error: `${clip.id}'s media has only ${seconds(clip.trimStartUs)} before its current start, so its start edge can move back at most that far` };
|
|
819
|
+
}
|
|
820
|
+
const commands = [
|
|
821
|
+
{
|
|
822
|
+
type: "clip/trim",
|
|
823
|
+
payload: {
|
|
824
|
+
clipId: clip.id,
|
|
825
|
+
startUs: start,
|
|
826
|
+
durationUs: duration,
|
|
827
|
+
...trimmable && sourceShift ? { trimStartUs: clip.trimStartUs + sourceShift } : {}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
];
|
|
831
|
+
if (ripple) {
|
|
832
|
+
const oldEnd = clip.startUs + clip.durationUs;
|
|
833
|
+
const delta = start + duration - oldEnd;
|
|
834
|
+
const later = Object.values(doc.clips).filter((c) => c.trackId === clip.trackId && c.id !== clip.id && c.startUs >= oldEnd).sort((x, y) => delta < 0 ? x.startUs - y.startUs : y.startUs - x.startUs);
|
|
835
|
+
for (const c of later) commands.push({ type: "clip/move", payload: { clipId: c.id, startUs: Math.max(0, c.startUs + delta) } });
|
|
836
|
+
}
|
|
837
|
+
const out = applyChecked(project, commands);
|
|
838
|
+
if (!out.ok) return out;
|
|
839
|
+
const after = project.getState().doc.clips[clip.id];
|
|
840
|
+
return { ...out, changes: [`Trimmed ${clip.id} to ${seconds(after.startUs)}\u2013${seconds(after.startUs + after.durationUs)}`] };
|
|
841
|
+
}
|
|
842
|
+
}),
|
|
843
|
+
defineTool({
|
|
844
|
+
name: "set_effects_enabled",
|
|
845
|
+
description: "Turn effects off (or back on) without deleting them, keeping their settings: one kind, or every effect when kind is omitted. To delete effects use remove_effects.",
|
|
846
|
+
inputSchema: {
|
|
847
|
+
type: "object",
|
|
848
|
+
properties: {
|
|
849
|
+
clipIds: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
850
|
+
kind: { type: "string", description: "Only effects of this kind (default: all)." },
|
|
851
|
+
enabled: { type: "boolean" }
|
|
852
|
+
},
|
|
853
|
+
required: ["clipIds", "enabled"],
|
|
854
|
+
additionalProperties: false
|
|
855
|
+
},
|
|
856
|
+
run(input, { project }) {
|
|
857
|
+
const doc = project.getState().doc;
|
|
858
|
+
const enabled = input.enabled === true;
|
|
859
|
+
const commands = [];
|
|
860
|
+
for (const id of input.clipIds ?? []) {
|
|
861
|
+
const clip = doc.clips[id];
|
|
862
|
+
if (!clip) return { ok: false, error: `no clip "${id}"` };
|
|
863
|
+
for (const e of clip.effects ?? []) {
|
|
864
|
+
if ((!input.kind || e.kind === input.kind) && e.enabled !== enabled) commands.push({ type: "effect/update", payload: { clipId: id, effectId: e.id, enabled } });
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
if (!commands.length) return { ok: true, result: { changed: 0, note: `no matching effects that are ${enabled ? "off" : "on"}` } };
|
|
868
|
+
const out = applyChecked(project, commands);
|
|
869
|
+
return out.ok ? { ...out, result: { changed: commands.length }, changes: [`Turned ${enabled ? "on" : "off"} ${commands.length} effect${commands.length > 1 ? "s" : ""}`] } : out;
|
|
870
|
+
}
|
|
871
|
+
}),
|
|
872
|
+
defineTool({
|
|
873
|
+
name: "set_keyframes",
|
|
874
|
+
description: `Animate one property of a clip so it CHANGES over time, with keyframes at TIMELINE seconds (the same times as get_state). Replaces that property's existing keyframes (including a preset's) unless keep:true. Between points the value moves smoothly; before the first and after the last it holds. Properties: x, y (0\u20131, 0.5 = center), scale (1 = full size), rotation (degrees; 360 = one full turn, so a static 360 looks like 0), opacity (0\u20131), volume (0\u20131, audio/video). Not for constant values ("rotate 45\xB0", "music at 30%", "louder": set those with clip/set-property) or for removing an animation (keyframe/clear). Use it for motion and timed changes the presets don't cover: a spin (rotation 0 \u2192 360 over the clip), a pan, ducking music (volume: normal \u2192 low just before a range, low \u2192 normal just after), or fading audio between two times.`,
|
|
875
|
+
inputSchema: {
|
|
876
|
+
type: "object",
|
|
877
|
+
properties: {
|
|
878
|
+
clipId: { type: "string" },
|
|
879
|
+
property: { type: "string", enum: ["x", "y", "scale", "rotation", "opacity", "volume"] },
|
|
880
|
+
points: {
|
|
881
|
+
type: "array",
|
|
882
|
+
minItems: 1,
|
|
883
|
+
items: {
|
|
884
|
+
type: "object",
|
|
885
|
+
properties: { atSeconds: { type: "number", minimum: 0 }, value: { type: "number" } },
|
|
886
|
+
required: ["atSeconds", "value"],
|
|
887
|
+
additionalProperties: false
|
|
888
|
+
}
|
|
889
|
+
},
|
|
890
|
+
easing: { type: "string", enum: ["linear", "easeIn", "easeOut", "easeInOut", "hold"], description: "Curve between points (default linear)." },
|
|
891
|
+
keep: { type: "boolean", description: "Keep the property's other keyframes (default false: replace them)." }
|
|
892
|
+
},
|
|
893
|
+
required: ["clipId", "property", "points"],
|
|
894
|
+
additionalProperties: false
|
|
895
|
+
},
|
|
896
|
+
run(input, { project }) {
|
|
897
|
+
const clip = project.getState().doc.clips[String(input.clipId)];
|
|
898
|
+
if (!clip) return { ok: false, error: `no clip "${input.clipId}"` };
|
|
899
|
+
const property = String(input.property);
|
|
900
|
+
if (property === "volume" && !("volume" in clip)) return { ok: false, error: `${clip.id} is a ${clip.kind} clip and has no volume` };
|
|
901
|
+
if (property !== "volume" && clip.kind === "audio") return { ok: false, error: "audio clips only animate volume" };
|
|
902
|
+
const end = clip.startUs + clip.durationUs;
|
|
903
|
+
const points = input.points.map((p) => ({ atUs: toUs(p.atSeconds), value: p.value }));
|
|
904
|
+
const outside = points.filter((p) => p.atUs < clip.startUs - 1e3 || p.atUs > end + 1e3);
|
|
905
|
+
if (outside.length) return { ok: false, error: `${clip.id} runs ${seconds(clip.startUs)}\u2013${seconds(end)}; points must be inside that range (got ${outside.map((p) => seconds(p.atUs)).join(", ")})` };
|
|
906
|
+
if (input.keep !== true && new Set(points.map((p) => p.value)).size < 2) {
|
|
907
|
+
const how = property === "volume" ? "clip/set-property { volume }" : `clip/set-property { transform: { ${property} } }`;
|
|
908
|
+
return {
|
|
909
|
+
ok: false,
|
|
910
|
+
error: `Keyframes are for a value that CHANGES over time; ${points.length === 1 ? "one point" : "these points"} hold${points.length === 1 ? "s" : ""} a single value. For a constant ${property}, set it directly with apply_commands ${how}. To remove an animation, apply keyframe/clear { clipId, property }. Nothing was changed.`
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
const commands = [];
|
|
914
|
+
if (input.keep !== true && (clip.animations?.[property]?.length ?? 0) > 0) commands.push({ type: "keyframe/clear", payload: { clipId: clip.id, property } });
|
|
915
|
+
const easing = typeof input.easing === "string" ? input.easing : "linear";
|
|
916
|
+
for (const p of points) {
|
|
917
|
+
commands.push({ type: "keyframe/set", payload: { clipId: clip.id, property, timeUs: Math.min(clip.durationUs, Math.max(0, p.atUs - clip.startUs)), value: p.value, easing } });
|
|
918
|
+
}
|
|
919
|
+
const out = applyChecked(project, commands);
|
|
920
|
+
if (!out.ok) return out;
|
|
921
|
+
const list = points.map((p) => `${seconds(p.atUs)} ${n3(p.value)}`).join(" \u2192 ");
|
|
922
|
+
return { ...out, changes: [`${clip.id}: ${property} ${list}`] };
|
|
923
|
+
}
|
|
924
|
+
}),
|
|
925
|
+
defineTool({
|
|
926
|
+
name: "set_background",
|
|
927
|
+
description: `A solid color background behind everything (an html clip on a Background track at the very bottom). Adds one, recolors or re-times the existing one, or removes it with color "none". Covers the whole video unless fromSeconds / toSeconds say otherwise. Only visible where nothing opaque is on top (behind scaled-down, faded or transparent clips, or where there's no footage).`,
|
|
928
|
+
inputSchema: {
|
|
929
|
+
type: "object",
|
|
930
|
+
properties: {
|
|
931
|
+
color: { type: "string", description: 'CSS color, e.g. "#1E3A8A", or "none" to remove the background.' },
|
|
932
|
+
fromSeconds: { type: "number", minimum: 0 },
|
|
933
|
+
toSeconds: { type: "number", exclusiveMinimum: 0 }
|
|
934
|
+
},
|
|
935
|
+
required: ["color"],
|
|
936
|
+
additionalProperties: false
|
|
937
|
+
},
|
|
938
|
+
run(input, { project }) {
|
|
939
|
+
const doc = project.getState().doc;
|
|
940
|
+
const color = String(input.color).trim();
|
|
941
|
+
const existing = Object.values(doc.clips).find(
|
|
942
|
+
(c) => c.kind === "html" && (c.template === BACKGROUND_TEMPLATE || doc.tracks[c.trackId]?.name === "Background")
|
|
943
|
+
);
|
|
944
|
+
if (color.toLowerCase() === "none") {
|
|
945
|
+
if (!existing) return { ok: true, result: { note: "there is no background to remove" } };
|
|
946
|
+
const commands2 = [{ type: "clip/remove", payload: { clipId: existing.id } }];
|
|
947
|
+
if (!Object.values(doc.clips).some((c) => c.trackId === existing.trackId && c.id !== existing.id)) commands2.push({ type: "track/remove", payload: { id: existing.trackId } });
|
|
948
|
+
const out2 = applyChecked(project, commands2);
|
|
949
|
+
return out2.ok ? { ...out2, changes: ["Removed the background"] } : out2;
|
|
950
|
+
}
|
|
951
|
+
let endUs = 0;
|
|
952
|
+
for (const c of Object.values(doc.clips)) if (c.id !== existing?.id) endUs = Math.max(endUs, c.startUs + c.durationUs);
|
|
953
|
+
const startUs = toUs(input.fromSeconds) ?? (existing && input.toSeconds === void 0 ? existing.startUs : 0);
|
|
954
|
+
const stopUs = toUs(input.toSeconds) ?? (existing && input.fromSeconds === void 0 ? existing.startUs + existing.durationUs : endUs || 10 * S);
|
|
955
|
+
if (stopUs <= startUs) return { ok: false, error: `toSeconds must be after fromSeconds (got ${seconds(startUs)}\u2013${seconds(stopUs)})` };
|
|
956
|
+
const commands = [];
|
|
957
|
+
if (existing) {
|
|
958
|
+
const params = existing;
|
|
959
|
+
const key = htmlPlaceholders(params.template).find((k) => /colou?r|background|bg/i.test(k)) ?? "color";
|
|
960
|
+
if (params.params[key] !== color) commands.push({ type: "clip/set-property", payload: { clipId: existing.id, params: { [key]: color } } });
|
|
961
|
+
if (existing.startUs !== startUs || existing.durationUs !== stopUs - startUs) commands.push({ type: "clip/trim", payload: { clipId: existing.id, startUs, durationUs: stopUs - startUs } });
|
|
962
|
+
if (!commands.length) return { ok: true, result: { note: "the background already looks like that" } };
|
|
963
|
+
} else {
|
|
964
|
+
const trackId = doc.tracks.background ? `background-${Object.keys(doc.tracks).length}` : "background";
|
|
965
|
+
commands.push({ type: "track/add", payload: { id: trackId, kind: "video", name: "Background", index: 0 } });
|
|
966
|
+
commands.push({ type: "clip/add", payload: { kind: "html", id: `${trackId}-clip`, trackId, startUs, durationUs: stopUs - startUs, template: BACKGROUND_TEMPLATE, params: { color } } });
|
|
967
|
+
}
|
|
968
|
+
const out = applyChecked(project, commands);
|
|
969
|
+
if (!out.ok) return out;
|
|
970
|
+
return { ...out, changes: [`${existing ? "Changed the background to" : "Added a background:"} ${color}, ${seconds(startUs)}\u2013${seconds(stopUs)}`] };
|
|
971
|
+
}
|
|
972
|
+
}),
|
|
973
|
+
defineTool({
|
|
974
|
+
name: "close_gaps",
|
|
975
|
+
description: "Close the empty gaps on a track so its clips run back to back, keeping their order (the first clip stays where it is, or starts at 0 with fromZero).",
|
|
976
|
+
inputSchema: {
|
|
977
|
+
type: "object",
|
|
978
|
+
properties: { trackId: { type: "string" }, fromZero: { type: "boolean" } },
|
|
979
|
+
required: ["trackId"],
|
|
980
|
+
additionalProperties: false
|
|
981
|
+
},
|
|
982
|
+
run(input, { project }) {
|
|
983
|
+
const doc = project.getState().doc;
|
|
984
|
+
if (!doc.tracks[String(input.trackId)]) return { ok: false, error: `no track "${input.trackId}"` };
|
|
985
|
+
const clips = Object.values(doc.clips).filter((c) => c.trackId === input.trackId).sort((a, b) => a.startUs - b.startUs);
|
|
986
|
+
let cursor = input.fromZero === true ? 0 : clips[0]?.startUs ?? 0;
|
|
987
|
+
const commands = [];
|
|
988
|
+
for (const c of clips) {
|
|
989
|
+
if (c.startUs !== cursor) commands.push({ type: "clip/move", payload: { clipId: c.id, startUs: cursor } });
|
|
990
|
+
cursor += c.durationUs;
|
|
991
|
+
}
|
|
992
|
+
if (!commands.length) return { ok: true, result: { moved: 0, note: "no gaps on that track" } };
|
|
993
|
+
const out = applyChecked(project, commands);
|
|
994
|
+
return out.ok ? { ...out, changes: [`Closed ${commands.length} gap${commands.length > 1 ? "s" : ""} on ${input.trackId}`] } : out;
|
|
508
995
|
}
|
|
509
996
|
})
|
|
510
997
|
];
|
|
511
998
|
}
|
|
999
|
+
var Abort = class extends Error {
|
|
1000
|
+
};
|
|
1001
|
+
function applyChecked(project, commands) {
|
|
1002
|
+
const catalog = project.commandCatalog();
|
|
1003
|
+
const nothing = commands.length > 1 ? "Nothing in this batch was applied (not even the commands before it): resend the whole corrected batch." : "Nothing was applied.";
|
|
1004
|
+
const templates = /* @__PURE__ */ new Map();
|
|
1005
|
+
for (const [index, command] of commands.entries()) {
|
|
1006
|
+
if (!catalog[command?.type]) {
|
|
1007
|
+
const looksLikeTool = /^[a-z]+(_[a-z]+)+$/.test(String(command?.type));
|
|
1008
|
+
return {
|
|
1009
|
+
ok: false,
|
|
1010
|
+
error: {
|
|
1011
|
+
failedIndex: index,
|
|
1012
|
+
kind: "unknown-command",
|
|
1013
|
+
commandType: command?.type,
|
|
1014
|
+
error: `"${command?.type}" is not a command${looksLikeTool ? ": it is a tool. Call it as its own tool call, not inside apply_commands" : ""}. ${nothing}`,
|
|
1015
|
+
validTypes: Object.keys(catalog).sort()
|
|
1016
|
+
}
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
const htmlError = checkHtmlParams(project, command, templates);
|
|
1020
|
+
if (htmlError) return { ok: false, error: { failedIndex: index, commandType: command.type, error: `${htmlError} ${nothing}` } };
|
|
1021
|
+
const extra = unknownFields(catalog[command?.type], command?.payload);
|
|
1022
|
+
if (extra) {
|
|
1023
|
+
return {
|
|
1024
|
+
ok: false,
|
|
1025
|
+
error: {
|
|
1026
|
+
failedIndex: index,
|
|
1027
|
+
commandType: command.type,
|
|
1028
|
+
error: `unknown field${extra.unknown.length > 1 ? "s" : ""} ${extra.unknown.map((f) => `"${f}"`).join(", ")}: ${command.type} would ignore ${extra.unknown.length > 1 ? "them" : "it"}. Its fields are: ${extra.allowed.join(", ")}. ${nothing}`
|
|
1029
|
+
}
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
const unchanged = [];
|
|
1034
|
+
let failure;
|
|
1035
|
+
try {
|
|
1036
|
+
project.transaction(() => {
|
|
1037
|
+
commands.forEach((command, index) => {
|
|
1038
|
+
const before = project.getState().doc;
|
|
1039
|
+
const result = (0, import_core2.tryDispatch)(project, command);
|
|
1040
|
+
if (!result.ok) {
|
|
1041
|
+
const hint = rejectionHint(result.error, command);
|
|
1042
|
+
const skipped = commands.length > 1 ? { notApplied: describeCommands(commands.filter((_, i) => i !== index)) } : {};
|
|
1043
|
+
failure = { failedIndex: index, ...result.error, ...hint ? { hint } : {}, note: nothing, ...skipped };
|
|
1044
|
+
throw new Abort();
|
|
1045
|
+
}
|
|
1046
|
+
if (project.getState().doc === before) unchanged.push(index);
|
|
1047
|
+
});
|
|
1048
|
+
});
|
|
1049
|
+
} catch (err) {
|
|
1050
|
+
if (err instanceof Abort) return { ok: false, error: failure };
|
|
1051
|
+
throw err;
|
|
1052
|
+
}
|
|
1053
|
+
const changed = commands.filter((_, i) => !unchanged.includes(i));
|
|
1054
|
+
return {
|
|
1055
|
+
ok: true,
|
|
1056
|
+
result: {
|
|
1057
|
+
applied: changed.length,
|
|
1058
|
+
...unchanged.length ? { unchanged: unchanged.map((i) => ({ index: i, type: commands[i].type, note: "changed nothing: the project already had these values. Check the field names and values." })) } : {}
|
|
1059
|
+
},
|
|
1060
|
+
...changed.length ? { changes: describeCommands(changed) } : {}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
function checkHtmlParams(project, command, templates) {
|
|
1064
|
+
const p = command.payload ?? {};
|
|
1065
|
+
let template;
|
|
1066
|
+
if (command.type === "clip/add" && p.kind === "html" && typeof p.template === "string") {
|
|
1067
|
+
template = p.template;
|
|
1068
|
+
if (typeof p.id === "string") templates.set(p.id, template);
|
|
1069
|
+
} else if (command.type === "clip/set-property" && p.params && typeof p.params === "object") {
|
|
1070
|
+
const clip = project.getState().doc.clips[String(p.clipId)];
|
|
1071
|
+
template = templates.get(String(p.clipId)) ?? (clip?.kind === "html" ? clip.template : void 0);
|
|
1072
|
+
}
|
|
1073
|
+
if (template === void 0 || !p.params || typeof p.params !== "object") return void 0;
|
|
1074
|
+
const names = htmlPlaceholders(template);
|
|
1075
|
+
const unknown = Object.keys(p.params).filter((k) => !names.includes(k));
|
|
1076
|
+
if (!unknown.length) return void 0;
|
|
1077
|
+
return `html params ${unknown.map((k) => `"${k}"`).join(", ")} match no {{placeholder}} in the template, so they would change nothing. Its placeholders: ${names.join(", ") || "none"}.`;
|
|
1078
|
+
}
|
|
1079
|
+
function rejectionHint(error, _command) {
|
|
1080
|
+
switch (error.code) {
|
|
1081
|
+
case "no-audio":
|
|
1082
|
+
return "fadeInUs / fadeOutUs fade audio only. To fade a text, image or video picture in or out, call animate_clip (in / out: fade).";
|
|
1083
|
+
case "asset-not-found":
|
|
1084
|
+
return "Clips can only use assets already in the project (see get_state). Audio found with search_audio is added with add_audio, which imports it.";
|
|
1085
|
+
case "not-caption":
|
|
1086
|
+
case "not-html":
|
|
1087
|
+
case "not-text":
|
|
1088
|
+
return "That field belongs to another clip kind: text clips take color / font fields, caption clips take style, html clips take params (named by the template's {{placeholders}}, shown in get_state).";
|
|
1089
|
+
default:
|
|
1090
|
+
return void 0;
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
function noCutReason(project, fromId, toId) {
|
|
1094
|
+
const doc = project.getState().doc;
|
|
1095
|
+
const a = typeof fromId === "string" ? doc.clips[fromId] : void 0;
|
|
1096
|
+
const b = typeof toId === "string" ? doc.clips[toId] : void 0;
|
|
1097
|
+
const tail = "Nothing was added. Don't substitute a different cut: tell the user, and offer an animation (animate_clip) instead if that fits.";
|
|
1098
|
+
for (const [given, clip] of [[fromId, a], [toId, b]]) if (typeof given === "string" && !clip) return `There is no clip "${given}". ${tail}`;
|
|
1099
|
+
if (a && b && a.trackId !== b.trackId) return `${a.id} is on track ${a.trackId} and ${b.id} on track ${b.trackId}: a transition joins two clips that touch on the SAME track, so these two have no cut between them. ${tail}`;
|
|
1100
|
+
if (a && b) return `${a.id} and ${b.id} don't touch (${a.id} ends at ${seconds(a.startUs + a.durationUs)}, ${b.id} starts at ${seconds(b.startUs)}). ${tail}`;
|
|
1101
|
+
return `No cut ${a ? `starts after ${a.id}` : `leads into ${b.id}`} (it isn't directly after/before another clip on its track). ${tail}`;
|
|
1102
|
+
}
|
|
512
1103
|
function cutInfo(c) {
|
|
513
1104
|
return { fromClipId: c.fromClipId, toClipId: c.toClipId, atSeconds: c.atUs / S, maxSeconds: c.maxTransitionUs / S, ...c.transition ? { transition: c.transition.kind } : {} };
|
|
514
1105
|
}
|
|
515
1106
|
function applyCommandsSchema(catalog) {
|
|
516
|
-
const [dispatch] = (0,
|
|
1107
|
+
const [dispatch] = (0, import_core2.toToolDefinitions)(catalog, { mode: "dispatch" });
|
|
517
1108
|
return {
|
|
518
1109
|
type: "object",
|
|
519
1110
|
properties: {
|
|
@@ -525,7 +1116,7 @@ function applyCommandsSchema(catalog) {
|
|
|
525
1116
|
}
|
|
526
1117
|
|
|
527
1118
|
// src/fork.ts
|
|
528
|
-
var
|
|
1119
|
+
var import_core3 = require("@miraiclip/core");
|
|
529
1120
|
var seq = 0;
|
|
530
1121
|
var genId = (prefix) => `${prefix}-${Date.now().toString(36)}${(++seq).toString(36)}${Math.random().toString(36).slice(2, 6)}`;
|
|
531
1122
|
function withExplicitIds(command) {
|
|
@@ -544,7 +1135,7 @@ function withExplicitIds(command) {
|
|
|
544
1135
|
}
|
|
545
1136
|
}
|
|
546
1137
|
function forkProject(project, options = {}) {
|
|
547
|
-
const fork = (options.createFork ?? ((doc) => (0,
|
|
1138
|
+
const fork = (options.createFork ?? ((doc) => (0, import_core3.createProject)(doc)))(project.toJSON());
|
|
548
1139
|
const state = project.getState();
|
|
549
1140
|
fork.setPlayhead(state.playheadUs);
|
|
550
1141
|
fork.setSelection([...state.selection]);
|
|
@@ -598,21 +1189,33 @@ function commitFork(project, fork, label) {
|
|
|
598
1189
|
}
|
|
599
1190
|
|
|
600
1191
|
// src/run.ts
|
|
601
|
-
var import_core3 = require("@miraiclip/core");
|
|
602
1192
|
var MAX_TOOL_RESULT = 2e4;
|
|
603
1193
|
var MAX_OLD_TOOL_RESULT = 2e3;
|
|
604
1194
|
var DEFAULT_INSTRUCTIONS = `You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.
|
|
605
1195
|
|
|
606
1196
|
How to work:
|
|
607
1197
|
- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.
|
|
608
|
-
- Prefer the high-level tools when they fit (add_transition, animate_clip, and any audio or media tools). Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it.
|
|
1198
|
+
- Prefer the high-level tools when they fit (add_transition, animate_clip, add_effect, set_effects_enabled, set_background, trim_clip, set_keyframes, close_gaps, and any audio or media tools). Call tools directly; apply_commands takes only editing commands like "clip/add", never tool names. Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it, and use only the fields it lists.
|
|
1199
|
+
- Fading a picture (text, image, logo, video) in or out is animate_clip (in / out: fade). fadeInUs / fadeOutUs are audio fades. "Fade the video in at the start / out at the end" means the first clip's entrance and the last clip's exit.
|
|
1200
|
+
- Constant values are set directly with clip/set-property: rotation ("rotate 45\xB0", "tilt"), volume ("lower to 30%", "louder"), position, size, opacity. Only values that CHANGE over a time range ("duck the music between 10 and 15 s", "fade to silence between 18 and 20 s", "spin one full turn", "pan across") are keyframes: set_keyframes, with times in timeline seconds. Remove an animation with keyframe/clear.
|
|
1201
|
+
- "Cuts" are the ones listed in the project summary; use their times (e.g. one sound per cut).
|
|
1202
|
+
- "Add N more seconds of footage" adds a new clip (clip/add with trimStartUs for the source point) after the last one; it does not stretch an existing clip.
|
|
1203
|
+
- Caption looks (box behind the text, outline, shadow, colors, word preset) are the caption clip's style (clip/set-property { style }).
|
|
1204
|
+
- "The whole video" means from 0 to the composition length (trim_clip startSeconds / endSeconds).
|
|
1205
|
+
- A solid color background behind everything is set_background (add, recolor, remove). Other html clips are recolored through the params their template uses (shown in the clip details). A new track goes on top unless track/add gets index 0 (the bottom).
|
|
1206
|
+
- "Replace X with Y" removes X and adds Y in the same turn, whichever tool adds Y (audio tools take replaceClipId, which does both).
|
|
1207
|
+
- Solo / mute / lock / hide a track with track/set-property (solo, muted, locked, hidden). "Solo X" is solo: true on X, not muting the others.
|
|
1208
|
+
- Relative requests ("bigger", "louder", "a bit brighter", "twice as big") start from the current values in the clip details below: make a visible change in the asked direction (about 25% for "bigger"/"smaller", \xD72 for "twice").
|
|
1209
|
+
- When something is selected, "this", "it", "the clip" and any request that names no target mean ONLY the selection, even with words like "at the end" (the end of that clip). Edit nothing else. "Here" / "now" / "this point" mean the playhead. "Clips" without a kind mean the video clips on the main video track.
|
|
1210
|
+
- "Again" / "also" / "another" mean keep the original and add a copy (clip/duplicate, or clip/add with the same asset); never move the original. "Instead" means move or replace.
|
|
1211
|
+
- Removing a clip leaves a gap; when asked to close it, also run close_gaps (or trim_clip with ripple).
|
|
609
1212
|
- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.
|
|
610
1213
|
- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.
|
|
611
1214
|
- Only reference media that exists in the project or that a tool returned. Never invent file URLs.
|
|
612
|
-
- If a tool fails, read the error, fix the input and retry. If something can't be done, say why.
|
|
1215
|
+
- If a tool fails, read the error, fix the input and retry. A failed apply_commands batch applied NONE of its commands: resend all of them, corrected. If something can't be done, say why.
|
|
613
1216
|
- If the request is ambiguous in a way that matters, ask one short question instead of guessing.
|
|
614
1217
|
|
|
615
|
-
When you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). No markdown headings.`;
|
|
1218
|
+
When you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). Describe only edits your tools reported as applied; if a tool says something changed nothing or failed, fix it or say so. No markdown headings.`;
|
|
616
1219
|
function createAssistant(options) {
|
|
617
1220
|
const tools = options.tools ?? editorTools();
|
|
618
1221
|
const byName = new Map(tools.map((t) => [t.name, t]));
|
|
@@ -632,8 +1235,9 @@ function createAssistant(options) {
|
|
|
632
1235
|
DEFAULT_INSTRUCTIONS,
|
|
633
1236
|
options.instructions?.trim(),
|
|
634
1237
|
`Current project:
|
|
635
|
-
${(
|
|
636
|
-
turn.context?.trim() ? `What the user is looking at: ${turn.context.trim()}` : void 0
|
|
1238
|
+
${describeForAssistant(fork.toJSON())}`,
|
|
1239
|
+
turn.context?.trim() ? `What the user is looking at: ${turn.context.trim()}` : void 0,
|
|
1240
|
+
selectionNote(fork.toJSON(), fork.getState().selection)
|
|
637
1241
|
].filter(Boolean).join("\n\n");
|
|
638
1242
|
let reply = "";
|
|
639
1243
|
let status = "no-changes";
|
|
@@ -760,6 +1364,12 @@ function abortError() {
|
|
|
760
1364
|
err.name = "AbortError";
|
|
761
1365
|
return err;
|
|
762
1366
|
}
|
|
1367
|
+
function selectionNote(doc, selection) {
|
|
1368
|
+
const clips = selection.map((id) => doc.clips[id]).filter((c) => !!c);
|
|
1369
|
+
if (!clips.length) return void 0;
|
|
1370
|
+
const list = clips.map((c) => `${c.id} (${c.kind}, ${Number((c.startUs / 1e6).toFixed(2))}\u2013${Number(((c.startUs + c.durationUs) / 1e6).toFixed(2))}s)`).join(", ");
|
|
1371
|
+
return `Selected: ${list}. In this request, "this", "it", "the clip" and anything that names no target mean ONLY ${clips.map((c) => c.id).join(", ")}. Leave every other clip alone.`;
|
|
1372
|
+
}
|
|
763
1373
|
|
|
764
1374
|
// src/testing.ts
|
|
765
1375
|
function scriptedChatModel(steps, options = {}) {
|
|
@@ -775,7 +1385,7 @@ function scriptedChatModel(steps, options = {}) {
|
|
|
775
1385
|
i++;
|
|
776
1386
|
const s = typeof step === "string" ? { text: step } : step;
|
|
777
1387
|
if (s.text) opts.onText?.(s.text);
|
|
778
|
-
const toolCalls = (s.toolCalls ?? []).map((c,
|
|
1388
|
+
const toolCalls = (s.toolCalls ?? []).map((c, n2) => ({ id: c.id ?? `call_${i}_${n2}`, name: c.name, arguments: c.arguments }));
|
|
779
1389
|
return { content: s.text ?? "", toolCalls, finish: toolCalls.length ? "tool_calls" : "stop", usage: { inputTokens: 10, outputTokens: 5 } };
|
|
780
1390
|
}
|
|
781
1391
|
});
|