@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/timeline-preview/frames.d.ts

@@ -0,0 +1,193 @@
+/**
+ * Composite a timeline at chosen timecodes, headlessly.
+ *
+ * The question this answers is the one an agent editing a timeline could not
+ * ask before: *what does the cut actually look like at 2.4s?* Structural
+ * validation says the clips are wired correctly and `get_clip_frames` shows one
+ * clip's source media, but neither shows the frame — the picture under the
+ * title, the title's animation mid-flight, the transition halfway through, the
+ * track order that decides what covers what.
+ *
+ * It resolves the scene the same way every other render surface does
+ * (`computeActiveLayers` → `resolveAnimatedLayerProps`), rasterizes the
+ * text/shape/caption layers through the shared drawing rules, decodes the media
+ * layers with Mediabunny, and composites on `@napi-rs/canvas` through the
+ * shared Canvas 2D rules. So it needs no GPU, no ffmpeg and no browser, and it
+ * runs anywhere the agent runs.
+ *
+ * Effects run through CPU pixel passes matching the GPU chain. Unknown effect
+ * types are reported in `effects_not_applied`, and missing render resources
+ * are reported on each frame.
+ */
+import { type ClipModel3DCamera, type TimelineSequence } from "@nodetool-ai/timeline";
+import type { ActiveLayer, Canvas2DDegradationReason, DroppedLayerReason, MotionBlurOptions } from "@nodetool-ai/timeline/scene";
+export interface RenderTimelineFramesOptions {
+    sequence: TimelineSequence;
+    /** Absolute timeline positions to composite, in milliseconds. */
+    timesMs: readonly number[];
+    /** Output frame width in pixels; height follows the sequence aspect. */
+    width?: number;
+    /** Resolve a clip's asset id to its encoded bytes, or null when unavailable. */
+    loadAsset: (assetId: string) => Promise<Uint8Array | null>;
+    /** Draw these clips only, while resolving their layout and animations against the full sequence. */
+    onlyClipIds?: readonly string[];
+    /**
+     * Average N sub-frame instants into every frame instead of sampling one (D10).
+     * Absent or one sample is blur off, which is what every caller that does not
+     * ask about it gets. The frame interval the shutter opens inside comes from
+     * the sequence's own fps.
+     */
+    motionBlur?: MotionBlurOptions;
+}
+/**
+ * The camera a `model3d` layer was drawn with: the clip's authored pose folded
+ * with the four animated camera channels. Reported because the pixels alone do
+ * not say where the camera was — a turntable mid-sweep and a static pose are
+ * the same picture at one instant.
+ */
+export interface PreviewCameraReport {
+    mode: ClipModel3DCamera["mode"];
+    azimuth_deg: number;
+    elevation_deg: number;
+    fov_deg: number;
+    zoom: number;
+    /** `scene` mode: the glTF camera the layer was drawn through. */
+    scene_camera_name?: string;
+    /** Look-at offset from the bounding-sphere center, world units. */
+    target_offset?: [number, number, number];
+}
+/** What one layer contributed to a frame, for the report beside the pixels. */
+export interface PreviewLayerReport {
+    clip_id: string;
+    clip_name: string;
+    kind: ActiveLayer["kind"];
+    track_index: number;
+    /** Composite order: higher covers lower. */
+    z_index: number;
+    /** Final opacity — clip opacity × transition ramp × animation. */
+    opacity: number;
+    blend_mode: string;
+    /** Present when an animation or a wipe transition is masking the layer. */
+    wipe?: {
+        direction: string;
+        progress: number;
+    };
+    /**
+     * Present while a cut is in flight over this layer. Which side of it the
+     * layer is on is what an agent otherwise cannot see in the pixels: a frame
+     * mid-push is two half-frames, and the report says which is arriving.
+     */
+    transition?: {
+        type: string;
+        role: string;
+        progress: number;
+    };
+    /** Present when a shape mask is cutting the layer. */
+    mask?: {
+        kind: string;
+    };
+    /**
+     * Present when a track matte is driving the layer's alpha. The source clip
+     * draws nothing of its own, so naming it here is the only way to see in the
+     * report that it is on the timeline at all.
+     */
+    matte?: {
+        source_clip_id: string;
+        mode: string;
+        invert: boolean;
+    };
+    /** The text a text or caption layer drew. */
+    text?: string;
+    /** The camera a `model3d` layer was drawn with. */
+    camera?: PreviewCameraReport;
+    /**
+     * Present on a `model3d` layer: the glTF animation's own clock in seconds —
+     * the clip's source time times the style's `animation.speed` (D3). It is not
+     * the timeline time, so a trimmed or retimed clip's animation can be read.
+     */
+    animation_time_sec?: number;
+    /** Why the layer contributed no pixels, when it didn't. */
+    skipped?: string;
+}
+/**
+ * One way the Canvas 2D compositor drew this frame differently from the GPU
+ * render, where the difference is not an effect type `effects_not_applied`
+ * could name — a feathered mask drawn hard, a matte skipped, a group's blend
+ * lost, a second drop shadow not cast, a brightness applied as a multiply.
+ */
+export interface PreviewDegradation {
+    /** The clip it happened to, when the layer carried one. */
+    clip_id?: string;
+    clip_name?: string;
+    reason: PreviewDegradationReason;
+}
+/**
+ * Every way this pass can draw a frame differently from the GPU render: the
+ * Canvas 2D compositor's own list, plus the one only this host can hit — a
+ * `model3d` layer whose headless renderer would not launch (D5, host 2).
+ */
+export type PreviewDegradationReason = Canvas2DDegradationReason
+/** A 3D layer left out because headless Chromium is missing or would not run. */
+ | "model3d_unavailable";
+/** A clip that was active at the frame's time and still did not draw. */
+export interface PreviewDroppedLayer {
+    clip_id: string;
+    clip_name: string;
+    /** Why the scene model refused it. */
+    reason: DroppedLayerReason;
+}
+/** A failure across the shutter window, separate from representative geometry. */
+export interface PreviewSampleFailure {
+    kind: "skipped" | "dropped" | "degraded";
+    clip_id?: string;
+    clip_name?: string;
+    reason: string;
+    /** Every contributing sample affected by this failure. Index is zero-based. */
+    samples: {
+        index: number;
+        time_ms: number;
+    }[];
+}
+export interface PreviewFrame {
+    /** False when any requested visual content was skipped or approximated. */
+    complete: boolean;
+    time_ms: number;
+    /** PNG bytes of the composited frame. */
+    png: Uint8Array;
+    width: number;
+    height: number;
+    /** Geometry from the first contributing sample. See failures for the full window. */
+    layers: PreviewLayerReport[];
+    /**
+     * Clips the scene model left out of this frame. Empty on almost every
+     * frame; non-empty means the picture is missing a layer the document asks
+     * for, which is otherwise invisible in the pixels.
+     */
+    dropped: PreviewDroppedLayer[];
+    /**
+     * How this frame differs from the GPU render beyond the effects it could not
+     * draw. Empty on almost every frame; an entry means the picture is a
+     * degraded version of the one an export would produce.
+     */
+    degraded: PreviewDegradation[];
+    /** Failures from every contributing sample, deduplicated with affected taps. */
+    failures: PreviewSampleFailure[];
+}
+export interface RenderTimelineFramesResult {
+    /** True only when every requested frame rendered without missing content. */
+    complete: boolean;
+    frames: PreviewFrame[];
+    /** Effect types present on the timeline that Canvas 2D cannot draw. */
+    effectsNotApplied: string[];
+    /**
+     * Families a text or caption layer names that are neither bundled nor
+     * resolvable from Google Fonts — drawn in the fallback face (D8).
+     */
+    fontsUnavailable: string[];
+}
+/**
+ * Composite `timesMs` and return the PNG of each, plus what each layer
+ * contributed. Asset bytes are loaded once per asset across all timecodes.
+ */
+export declare function renderTimelineFrames(options: RenderTimelineFramesOptions): Promise<RenderTimelineFramesResult>;
+//# sourceMappingURL=frames.d.ts.map

package/dist/timeline-preview/frames.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frames.d.ts","sourceRoot":"","sources":["../../src/timeline-preview/frames.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,EAGL,KAAK,iBAAiB,EAEtB,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EACV,WAAW,EAKX,yBAAyB,EAGzB,kBAAkB,EAClB,iBAAiB,EAGlB,MAAM,6BAA6B,CAAC;AA0ErC,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,iEAAiE;IACjE,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3B,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC3D,oGAAoG;IACpG,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED,+EAA+E;AAC/E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,IAAI,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C;;;;OAIG;IACH,UAAU,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9D,sDAAsD;IACtD,IAAI,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAClE,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,wBAAwB,CAAC;CAClC;AAED;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAChC,yBAAyB;AAC3B,iFAAiF;GAC/E,qBAAqB,CAAC;AAE1B,yEAAyE;AACzE,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/C;AAED,MAAM,WAAW,YAAY;IAC3B,2EAA2E;IAC3E,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,qFAAqF;IACrF,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAC7B;;;;OAIG;IACH,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,gFAAgF;IAChF,QAAQ,EAAE,oBAAoB,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,0BAA0B;IACzC,6EAA6E;IAC7E,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B;;;OAGG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAqED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,0BAA0B,CAAC,CA4uBrC"}