@pixerate/editor 0.234.0 → 0.236.0

package/dist/video-editor/index.js

@@ -153,6 +153,13 @@ import {
 import {
   namedWithSuggestion
 } from "../chunk-EUKY4KFX.js";
+import {
+  effectLayerRole,
+  effectLayerSpan,
+  effectSequenceTiming,
+  effectTemplateTools,
+  fillEffectTemplate
+} from "../chunk-G3RUKJ5C.js";
 import "../chunk-EHLOCVWB.js";
 import {
   CAMERA_MOVE_MAX_MS,
@@ -3454,6 +3461,106 @@ var VideoPreviewEngine = class {
   }
 };
 
+// src/video-editor/sequence.ts
+var svgUrl = (svg) => svg.trimStart().startsWith("<") ? `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}` : svg;
+var GROUPS = ["base", "background", "subject", "overlay", "text"];
+function videoEditsFromSequence(project, sequence, options = {}) {
+  const at = options.at ?? videoProjectDuration(project);
+  const W = project.width;
+  const H = project.height;
+  const sx = W / sequence.stage.width;
+  const sy = H / sequence.stage.height;
+  const timing = effectSequenceTiming(sequence);
+  const provider = options.provider ? { provider: options.provider } : {};
+  const notes = [];
+  const usedClips = new Set(allVideoClipIds(project));
+  const usedTracks = new Set(project.tracks.map((t) => t.id));
+  const usedAssets = new Set(project.assets.map((a) => a.id));
+  const fresh = (used, base) => {
+    for (let i = 1; ; i++) if (!used.has(`${base}${i}`)) return used.add(`${base}${i}`), `${base}${i}`;
+  };
+  const edits = [];
+  const assets = [];
+  const placeholders = {};
+  const items = [];
+  const placement = (r) => {
+    if (!r) return {};
+    const x = (r.x + r.width / 2) * sx - W / 2;
+    const y = (r.y + r.height / 2) * sy - H / 2;
+    const scale = Math.min(r.width * sx / W, r.height * sy / H);
+    return { transform: { x: Math.round(x * 100) / 100, y: Math.round(y * 100) / 100, scale: Math.round(scale * 1e3) / 1e3 } };
+  };
+  const clipOf = (layer, start, duration, shot) => {
+    const id = fresh(usedClips, "tpl-c");
+    const common = { id, start, duration, ...layer.opacity !== void 0 && layer.opacity < 1 ? { opacity: layer.opacity } : {}, ...layer.fade ? { fadeIn: layer.fade, fadeOut: layer.fade } : {}, ...placement(layer.rect) };
+    const effects = layer.effect ? { effects: [{ id: "fx1", effect: layer.effect, ...provider, ...layer.values ? { values: layer.values } : {} }] } : {};
+    const s = layer.subject;
+    if (!s) return { ...common, type: "effect", effect: layer.effect, ...provider, ...layer.values ? { values: layer.values } : {} };
+    if ("slot" in s) {
+      (placeholders[s.slot] ??= []).push(id);
+      if (layer.effect) notes.push(`Shot ${shot + 1}: the "${s.slot}" slot is a grey placeholder; when you swap in the picture, apply ${layer.effect} to it again (apply_effect).`);
+      return { ...common, type: "color", color: "#475569", name: `Slot "${s.slot}"` };
+    }
+    if ("text" in s) return { ...common, type: "text", text: s.text, ...s.font ? { fontFamily: s.font } : {}, ...typeof s.weight === "number" ? { fontWeight: s.weight } : {}, ...effects };
+    const asset = fresh(usedAssets, "tpl-a");
+    assets.push(asset);
+    if ("video" in s) {
+      notes.push(`Shot ${shot + 1}: a video's length is not known here, so its asset is given the clip's length (${Math.round(duration)} ms); correct it with update_video_asset if it is longer.`);
+      edits.push({ op: "add_asset", asset: { id: asset, kind: "video", src: s.video, duration } });
+    } else edits.push({ op: "add_asset", asset: { id: asset, kind: "image", src: "image" in s ? s.image : svgUrl(s.svg) } });
+    return { ...common, type: "media", asset, fit: "contain", ...effects };
+  };
+  sequence.shots.forEach((shot, i) => {
+    const shotStart = timing.starts[i];
+    const shotEnd = timing.starts[i + 1] ?? timing.total;
+    let base;
+    shot.layers.forEach((layer, j) => {
+      const span = effectLayerSpan(layer, i, timing.starts, timing.total);
+      if (span.end - span.start <= 0) return void notes.push(`Shot ${i + 1}, layer ${j + 1} lasts no time, so it was left out.`);
+      const clip = clipOf(layer, at + span.start, span.end - span.start, i);
+      const isBase = j === 0 && !base && span.start === shotStart && span.end === shotEnd && effectLayerRole(layer) === "background" && !layer.rect;
+      const item = { group: isBase ? "base" : effectLayerRole(layer), clip };
+      if (isBase) base = item;
+      items.push(item);
+    });
+    if (!base) {
+      base = { group: "base", clip: { id: fresh(usedClips, "tpl-c"), type: "color", color: "#000000", name: `Shot ${i + 1}`, start: at + shotStart, duration: shotEnd - shotStart } };
+      items.push(base);
+    }
+    if (i > 0 && shot.transition) {
+      base.clip.transition = { type: "effect", duration: shot.transition.ms ?? 800, effect: { effect: shot.transition.effect, ...provider, ...shot.transition.values ? { values: shot.transition.values } : {} } };
+      if (shot.layers.length > 1) notes.push(`Shot ${i + 1}: its transition blends the clips on the base track; the layers above it cut.`);
+    }
+    const words = (text, y, size, name) => items.push({ group: "text", clip: { id: fresh(usedClips, "tpl-c"), type: "text", text, name, fontSize: Math.round(size), start: at + shotStart, duration: shotEnd - shotStart, transform: { x: 0, y: Math.round(y) } } });
+    if (shot.title) words(shot.title, 0, H * 0.09, "Title");
+    if (shot.caption) words(shot.caption, H * 0.35, H * 0.045, "Caption");
+    if (shot.camera?.in !== void 0 || shot.camera?.during !== void 0 || shot.camera?.framing) notes.push(`Shot ${i + 1}: its camera framing and moves were not carried over; set them with set_clip_camera_move on its clips.`);
+  });
+  const lanes = [];
+  for (const group of GROUPS) {
+    const mine = [];
+    for (const item of items.filter((x) => x.group === group).sort((a, b) => a.clip.start - b.clip.start)) {
+      let lane = group === "base" ? mine[0] : mine.find((l) => l.ends.every((e) => e <= item.clip.start + 1e-6));
+      if (!lane) mine.push(lane = { group, id: fresh(usedTracks, "tpl-v"), ends: [], clips: [] });
+      lane.ends.push(item.clip.start + item.clip.duration);
+      lane.clips.push(item);
+    }
+    lanes.push(...mine);
+  }
+  for (const lane of lanes) edits.push({ op: "add_track", track: { id: lane.id, kind: "video", name: `Template ${lane.group}` }, at: 0 });
+  for (const lane of lanes) for (const item of lane.clips) edits.push({ op: "add_clip", track: lane.id, clip: item.clip });
+  return {
+    edits,
+    clips: lanes.flatMap((l) => l.clips.map((c) => c.clip.id)),
+    tracks: lanes.map((l) => l.id).reverse(),
+    assets,
+    start: at,
+    end: at + timing.total,
+    placeholders,
+    notes
+  };
+}
+
 // src/video-editor/controller.ts
 var VideoEditorController = class _VideoEditorController {
   history;
@@ -3818,6 +3925,29 @@ var VideoEditorController = class _VideoEditorController {
     if (place) checkVideoEffectPlace(d, place);
     return d;
   }
+  /**
+   * Fills a provider sequence template and puts it on the timeline at `at` (default the end), on
+   * new tracks above the others (see `videoEditsFromSequence`). Values are checked first and every
+   * problem is refused at once. One undo step. Returns the new tracks, clips and assets, where it
+   * starts and ends, the placeholder clips standing in for open slots, and notes.
+   */
+  async fillTemplate(template, values = {}, options = {}) {
+    const reason = this.readOnlyReason();
+    if (reason) return this.refused(reason);
+    if (options.at !== void 0 && !(Number.isFinite(options.at) && options.at >= 0)) return this.refused(`"at" is a time in ms, 0 or more (got ${options.at}).`);
+    let filled;
+    try {
+      filled = await fillEffectTemplate(this.effects.provider, template, values, options.name ? { name: options.name } : {});
+    } catch (error) {
+      if (isEffectError(error) || error instanceof Error) return this.refused(error.message);
+      throw error;
+    }
+    const made = videoEditsFromSequence(this.getState().project, filled.sequence, { at: options.at, provider: this.effects.provider.id });
+    const r = this.apply(made.edits, `Template ${filled.template.name}`);
+    if (!r.ok) return r;
+    const { edits: _edits, notes, ...rest } = made;
+    return { ok: true, template, ...rest, openSlots: filled.openSlots, notes: [...filled.notes, ...notes] };
+  }
   refused(message) {
     return this.refuse(message);
   }
@@ -5923,6 +6053,20 @@ function createVideoEffectsToolProvider(controller, options = {}) {
     subscribe: (notify) => controller.subscribe(() => notify()),
     extraTools: (ctx) => [
       ...videoCameraMoveTools(controller, ctx),
+      ...effectTemplateTools(
+        {
+          provider: () => controller.effectsProvider,
+          fillDetails: `It goes on the timeline at "at" (default the end of the project), on new tracks above the others: each shot's bottom layer on a base track (with the shot's transition), other layers as effect, picture and text clips on tracks by role, titles and captions as text clips; an open subject slot becomes a grey placeholder clip named after it. Camera moves are not carried over. Returns { template, tracks, clips, assets, start, end (ms), placeholders (clip ids by open slot), openSlots, notes }. To fill an open slot, add the picture with add_video_asset and put it on the placeholder's track at its start with add_video_clip in overwrite mode.`,
+          fillOptions: { at: { type: "number", minimum: 0, description: "Where on the timeline it starts, ms (default the end of the project)." } },
+          fill: async (template, values, options2) => {
+            const r = await controller.fillTemplate(template, values, { name: options2.name, at: options2.at });
+            if (!r.ok) throw new Error(r.error);
+            const { ok: _ok, ...rest } = r;
+            return rest;
+          }
+        },
+        ctx
+      ),
       {
         name: "add_video_effect_clip",
         title: "Add an effect clip",
@@ -6684,6 +6828,7 @@ export {
   videoEaseBezier,
   videoEaseProgress,
   videoEaseToAnimationEasing,
+  videoEditsFromSequence,
   videoEffectFrameTime,
   videoEffectLayerKey,
   videoEffectValuesAt,

package/dist/video-editor/sequence.d.cts

@@ -0,0 +1,38 @@
+/**
+ * An effects provider's `EffectSequence` (what a sequence template fills to) as video edits, so a
+ * template lands on the timeline as one undo step:
+ *
+ * - Each shot's bottom layer, when it lasts exactly the shot, goes on a base track; otherwise the
+ *   shot gets a black colour clip there. A shot's transition goes on its base clip (a provider
+ *   `effect` transition from the base clip before), so it blends the base pictures.
+ * - Other layers go on tracks above by role (background, subject, overlay), packed so clips of a
+ *   track never overlap: an effect with no subject is an effect clip; a picture or SVG is a media
+ *   clip on a new image asset, carrying the layer's effect as a clip effect; words are a text clip;
+ *   an open slot is a grey placeholder colour clip named after it.
+ * - A layer's `rect` becomes the clip's placement; `opacity` and `fade` carry over.
+ * - Titles and captions are text clips on the top track.
+ *
+ * Camera moves are not carried over (notes say so); set them with `set_clip_camera_move`.
+ */
+import type { EffectSequence } from '../effects/types.cjs';
+import type { VideoEdit } from './edit.cjs';
+import type { VideoProject } from './types.cjs';
+export interface VideoSequenceEdits {
+    edits: VideoEdit[];
+    /** The new clips' ids, base clips first. */
+    clips: string[];
+    /** The new tracks' ids, top to bottom. */
+    tracks: string[];
+    assets: string[];
+    /** Where the sequence starts and ends on the timeline (ms). */
+    start: number;
+    end: number;
+    /** Placeholder clips standing in for open subject slots, by slot. */
+    placeholders: Record<string, string[]>;
+    notes: string[];
+}
+/** The edits that put a sequence on a project's timeline at `at` (default its end), on new tracks above the others. */
+export declare function videoEditsFromSequence(project: VideoProject, sequence: EffectSequence, options?: {
+    at?: number;
+    provider?: string;
+}): VideoSequenceEdits;

package/dist/video-editor/sequence.d.ts

@@ -0,0 +1,38 @@
+/**
+ * An effects provider's `EffectSequence` (what a sequence template fills to) as video edits, so a
+ * template lands on the timeline as one undo step:
+ *
+ * - Each shot's bottom layer, when it lasts exactly the shot, goes on a base track; otherwise the
+ *   shot gets a black colour clip there. A shot's transition goes on its base clip (a provider
+ *   `effect` transition from the base clip before), so it blends the base pictures.
+ * - Other layers go on tracks above by role (background, subject, overlay), packed so clips of a
+ *   track never overlap: an effect with no subject is an effect clip; a picture or SVG is a media
+ *   clip on a new image asset, carrying the layer's effect as a clip effect; words are a text clip;
+ *   an open slot is a grey placeholder colour clip named after it.
+ * - A layer's `rect` becomes the clip's placement; `opacity` and `fade` carry over.
+ * - Titles and captions are text clips on the top track.
+ *
+ * Camera moves are not carried over (notes say so); set them with `set_clip_camera_move`.
+ */
+import type { EffectSequence } from '../effects/types.js';
+import type { VideoEdit } from './edit.js';
+import type { VideoProject } from './types.js';
+export interface VideoSequenceEdits {
+    edits: VideoEdit[];
+    /** The new clips' ids, base clips first. */
+    clips: string[];
+    /** The new tracks' ids, top to bottom. */
+    tracks: string[];
+    assets: string[];
+    /** Where the sequence starts and ends on the timeline (ms). */
+    start: number;
+    end: number;
+    /** Placeholder clips standing in for open subject slots, by slot. */
+    placeholders: Record<string, string[]>;
+    notes: string[];
+}
+/** The edits that put a sequence on a project's timeline at `at` (default its end), on new tracks above the others. */
+export declare function videoEditsFromSequence(project: VideoProject, sequence: EffectSequence, options?: {
+    at?: number;
+    provider?: string;
+}): VideoSequenceEdits;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixerate/editor",
-  "version": "0.234.0",
+  "version": "0.236.0",
   "description": "UI-agnostic rich text and media editing core library",
   "author": "Pixerate",
   "license": "MIT",

package/skills/pixerate-rich-text/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: pixerate-rich-text
-description: How to read and edit Pixerate rich text documents and prompt editors through agent tools - reading text with document positions (get_text, find_text), minimal-diff edits (replace_range, replace_text, insert_text), formatting (format_text for bold, italic, links and colour; set_block_type for headings, lists and quotes; set_text_align), selections, tables (edit_table and table.* commands), charts and sparklines embedded in a document, find and replace commands, and prompt editing with templates, variables and instructions (prompt_insert_text, prompt_replace_range, prompt_set_variable, get_editor_state, view_editor). Use when asked to write, rewrite, proofread, restructure or format a document, report, note, heading, paragraph or table, to insert a chart or sparkline into text, or to edit an image-generation or LLM prompt with {{template}}, {variable} and __instruction__ tokens.
+description: How to read and edit Pixerate rich text documents and prompt editors through agent tools - reading text with document positions (get_text, find_text), minimal-diff edits (replace_range, replace_text, insert_text), formatting (format_text for bold, italic, links and colour; set_block_type for headings, lists and quotes; set_text_align), selections, tables (edit_table and table.* commands), charts and sparklines embedded in a document, comments and tracked changes (add_comment, list_changes, accept_change), find and replace commands, and prompt editing with templates, variables and instructions (prompt_insert_text, prompt_replace_range, prompt_set_variable, get_editor_state, view_editor). Use when asked to write, rewrite, proofread, restructure or format a document, report, note, heading, paragraph or table, to insert a chart or sparkline into text, to comment, suggest edits or accept changes, or to edit an image-generation or LLM prompt with {{template}}, {variable} and __instruction__ tokens.
 ---
 
 # Editing rich text documents and prompts
@@ -72,6 +72,13 @@ the pitfalls, and uses the tools below.
 - **Charts in text.** Introduce a chart in the sentence before it and state the takeaway in
   its title ("Revenue doubled in Q3", not "Revenue"). Use a sparkline for a trend inside a
   sentence or table cell and give it a `label` that says what it shows.
+- **Review.** When the user asks you to suggest edits rather than make them, call
+  `set_track_changes {enabled: true}` first so every write shows as a suggestion they can
+  accept or reject, and turn it back off only if it was off before. Comment (`add_comment`)
+  when a point needs the author's decision instead of a fix; keep each comment to one point and
+  say what you suggest. Resolve threads you have addressed rather than deleting them. Read
+  `list_changes` before accepting or rejecting, and act on ids one at a time unless the user
+  asked for all.
 - **Prompt hygiene.** Put everything that changes between runs in a `{variable}` and set it
   with `prompt_set_variable` rather than editing the text. Give variables descriptive
   snake_case names and, where useful, a default and a description:
@@ -114,6 +121,26 @@ the pitfalls, and uses the tools below.
 | `prompt_set_variable` | Set the value of a template variable (`name` without braces, `value` any JSON value). Prefer this over editing the prompt text when only a slot's value changes. |
 | `prompt_select` | Select prompt text between two positions (`from === to` places the cursor), e.g. to show the user what you changed. |
 
+### Review
+
+Present when the editor has the `Comments` and / or `TrackChanges` extensions (the host may give
+them their own namespace, e.g. `review__list_comments`). Positions are document positions, as
+above. The write tools are hidden while the document is read-only.
+
+| Tool | Use it to |
+| --- | --- |
+| `list_comments` | List comment threads in document order: `id`, the commented `from` / `to` and `quote`, `author`, `body`, `replies` (`id`, `author`, `body`), `resolved`, `orphaned` (its text was deleted; `from` / `to` are null). `includeResolved` default true. Call before replying, resolving or deleting. |
+| `add_comment` | Comment on `from`..`to` (from `find_text`) with a `body`. One undo step; the selection moves to the commented text. Returns the new thread with its `id`. Fails on an empty or out-of-range span. |
+| `reply_to_comment` | Reply to a thread by `commentId` with a `body`. Re-opens a resolved thread. Returns `replyId` and the thread. |
+| `resolve_comment` | Resolve a thread by `commentId` (`resolved: false` re-opens it). Keeps the thread and its text. Returns the thread. Prefer this over deleting once a point is addressed. |
+| `delete_comment` | Delete a thread and its replies and remove its highlight (one undo step). Returns `deleted` and how many remain. Only when the user asks, or for a comment you added by mistake. |
+| `list_changes` | List tracked changes in document order: `id`, `type` (`insertion`, `deletion`, `format`, `block`, `move`), `author`, `from` / `to`, `text`, a plain-language `description` (moves add `movedFrom` / `movedTo`), and `trackingEnabled`. `type` filters. Call before accepting or rejecting. |
+| `accept_change` | Accept one change by `changeId` (one undo step): inserted text stays, deleted text goes, a move keeps its destination. Returns the change and how many `remaining`. Ids of other changes stay valid. |
+| `reject_change` | Reject one change by `changeId` (one undo step): inserted text goes, deleted text and old formatting come back, a move returns to its source. Returns the change and `remaining`. |
+| `accept_all_changes` | Accept every tracked change in one undo step. Returns how many. Only when the user asks for all; otherwise go one by one. |
+| `reject_all_changes` | Reject every tracked change in one undo step. Returns how many. |
+| `set_track_changes` | Turn tracking on (`enabled: true`: every later edit, yours included, becomes a suggestion) or off. Returns `trackingEnabled`. Turn it on before suggesting edits. |
+
 ### Save and open
 
 The host may add the shared document tools (see `pixerate-agent-tools`). A rich text document is the editor's ProseMirror JSON (`{ "type": "doc", "content": [...] }`), with marks such as tracked changes and comment anchors; comment threads stay in their own store.

package/skills/pixerate-rich-text-revise/SKILL.md

@@ -29,6 +29,9 @@ This skill is the method. `pixerate-rich-text` has the full tool reference.
 
 - **Scope:** spelling only, grammar and style, or a rewrite of named passages.
 - **Exceptions:** text that must stay as written (quotes, legal text, product names, titles).
+- **Edit or suggest:** when the user wants to review your changes, turn on `set_track_changes`
+  first so each fix is a suggestion, and flag doubtful points with `add_comment` instead of
+  changing them.
 - **House style:** spelling variant (UK or US), how numbers and dates are written.
 
 ## Working method
@@ -47,7 +50,8 @@ This skill is the method. `pixerate-rich-text` has the full tool reference.
    **Check:** `get_text` `blocks`: the other blocks' text is unchanged.
 5. **Consistency.** Terms, numbers and dates one way: `find_text` with a `regex` for the
    variants, then `replace_text`.
-6. **Report** the fixes made, the matches you left and why.
+6. **Report** the fixes made, the matches you left and why. When tracking is on, `list_changes`
+   shows the suggestions you made; leave accepting them to the user.
 
 ## Decision table
 
@@ -58,6 +62,7 @@ This skill is the method. `pixerate-rich-text` has the full tool reference.
 | A family of variants (e-mail, email, E-mail) | `find_text` / `replace_text` with `regex: true` |
 | Rewrite a sentence or paragraph | `replace_range` over its exact range |
 | Insert a missing word | `insert_text` at the position from `find_text` |
+| A point only the author can decide | `add_comment` on the passage, saying what you suggest |
 | Formatting only | `format_text` on the range; leave the words |
 
 ## Recipes
@@ -90,3 +95,5 @@ blocks[0].to, text }` with one sentence.
 - `replace_range`, `insert_text`: rewrite or insert at exact positions
 - `format_text`: formatting fixes without touching words
 - `undo_rich_text`: take back a step
+- `set_track_changes`, `list_changes`: suggest fixes as tracked changes and list them
+- `add_comment`: flag what the author should decide