@pictor-sdk/video-android 0.1.5 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pictor-sdk/video-android",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "description": "Optional Android video playback and export adapter for Pictor. Requires Android API 29+.",
   "license": "MIT",
   "main": "./dist/index.js",
@@ -29,7 +29,7 @@
     "react": ">=18.0.0",
     "@expo/config-plugins": ">=54.0.0",
     "react-native-reanimated": ">=3.16.0",
-    "@pictor-sdk/video-core": "^0.1.1"
+    "@pictor-sdk/video-core": "^0.2.0"
   },
   "devDependencies": {
     "react-native": "^0.81.0",
@@ -40,7 +40,7 @@
     "@types/react": "^19.0.0",
     "@expo/config-plugins": "^54.0.5",
     "react-native-reanimated": "^4.1.0",
-    "@pictor-sdk/video-core": "0.1.1"
+    "@pictor-sdk/video-core": "0.2.0"
   },
   "codegenConfig": {
     "name": "PictorVideoSpec",

package/src/NativePictorVideo.ts

@@ -248,6 +248,24 @@ export interface Spec extends TurboModule {
    *
    * Resolves with the same JSON outcome the blocking call returns. @experimental
    */
+  /**
+   * Export one still photograph with animated overlays.
+   *
+   * `pngPath` is a local PNG the host rendered; native never draws the base
+   * twice. No audio track is produced, so this is the photo-story path rather
+   * than a shortened video export that happened to have no sound.
+   *
+   * @experimental
+   */
+  exportRunPhotoProjectAsync(
+    jobId: string,
+    pngPath: string,
+    programJson: string,
+    overlaysJson: string,
+    durationUs: number,
+    timeoutSeconds: number,
+    destinationJson: string,
+  ): Promise<string>
   exportRunProjectAsync(
     jobId: string,
     shardsJson: string,

package/src/compositionSpike.ts

@@ -106,6 +106,16 @@ export interface CompositionSpikeState {
   releases?: number
   readinessTimeouts?: number
   readinessFailure?: string | null
+  /**
+   * How many resumes had to be re-primed because no frame could satisfy them.
+   *
+   * A resume whose target sits past the newest frame cannot be released on
+   * anything that exists, so playback seeks to the frame on screen to make the
+   * decoder produce one. Counted because it is a recovery, not a normal path:
+   * a number that climbs steadily is the renderer falling behind the transport
+   * more often than it should.
+   */
+  primedResumes?: number
   /** How long the last hold lasted before playback was released. */
   holdMs?: number
   /**

package/src/exporter.ts

@@ -16,6 +16,7 @@ import {
   validateAudioMixProgram,
   validateProject,
   VIDEO_SCENE_PROGRAM_VERSION,
+  type AnimatedFrameSchedule,
   type ExportSourceContext,
   type MediaSource,
   type ResampleRequest,
@@ -190,6 +191,18 @@ export interface Media3ExporterOptions {
    * @experimental
    */
   videoSpeedVia?: 'cadence' | 'media3'
+  /**
+   * Render one photo clip to the PNG a photo-story export encodes.
+   *
+   * A photo project has no video source for the base frame, so the host renders
+   * it through the same document renderer the preview uses and returns a local
+   * PNG path. Without this a photo project is refused exactly as before.
+   *
+   * Only a project consisting of exactly one photo is accepted: a sequence of
+   * stills is a slideshow with its own timeline rules, and silently treating it
+   * as one long still would be a different composition.
+   */
+  renderPhotoBase?(source: MediaSource, clipId: string): Promise<string>
 }
 
 /** Why an export will not start. @experimental */
@@ -205,6 +218,24 @@ export class VideoExportRejected extends Error {
 /** The frame rate an export runs at when the spec does not say. */
 const DEFAULT_FPS = 30
 
+/**
+ * Timings carried by animated stickers in this project.
+ *
+ * Without them the compiled scene names the overlay once, so native is handed
+ * the overlay's base id instead of a per-frame resource and refuses the first
+ * frame it cannot find. The schedule lives on the overlay's own data, which is
+ * where the app that built the story put it.
+ */
+function animatedSchedulesFor(project: VideoProject): Record<string, AnimatedFrameSchedule> {
+  const schedules: Record<string, AnimatedFrameSchedule> = {}
+  for (const overlay of project.overlays) {
+    if (overlay.type !== 'sticker') continue
+    const data = overlay.data as { animation?: AnimatedFrameSchedule } | null
+    if (data && data.animation) schedules[overlay.id] = data.animation
+  }
+  return schedules
+}
+
 
 /**
  * What this slice cannot encode, stated rather than skipped.
@@ -212,12 +243,22 @@ const DEFAULT_FPS = 30
  * Each of these has a milestone. Until then the export fails naming the clip or
  * the field, because the alternative is a file that looks finished.
  */
-function describeUnsupported(project: VideoProject, spec: VideoExportSpec): string[] {
+function describeUnsupported(
+  project: VideoProject,
+  spec: VideoExportSpec,
+  options: Media3ExporterOptions,
+): string[] {
   const reasons: string[] = []
 
   for (const clip of project.clips) {
     if (clip.kind === 'photo') {
-      reasons.push(`clip '${clip.id}' is a photo, which this exporter does not encode yet`)
+      if (!options.renderPhotoBase) {
+        reasons.push(`clip '${clip.id}' is a photo, and no photo renderer was supplied`)
+      } else if (project.clips.length !== 1) {
+        reasons.push(
+          `clip '${clip.id}' is a photo in a project of ${project.clips.length} clips; only a single-photo story is supported`,
+        )
+      }
       continue
     }
   }
@@ -273,7 +314,7 @@ export function createMedia3VideoExporter(options: Media3ExporterOptions): Video
       if (issues.length > 0) {
         throw new VideoExportRejected(issues.map((issue) => issue.message))
       }
-      const unsupported = describeUnsupported(project, spec)
+      const unsupported = describeUnsupported(project, spec, options)
       if (unsupported.length > 0) throw new VideoExportRejected(unsupported)
 
       /*
@@ -287,6 +328,116 @@ export function createMedia3VideoExporter(options: Media3ExporterOptions): Video
       if (decision.destination === null) throw new VideoExportRejected(decision.reasons)
       const destination = decision.destination
 
+      /*
+        A photo story is encoded as one still, not as a video of the same
+        length. The compiled program still carries the animated overlays, and
+        native draws them over the image through the same scene path.
+      */
+      const photoOnly = project.clips.length === 1 && project.clips[0]!.kind === 'photo'
+      if (photoOnly) {
+        const clip = project.clips[0]!
+        if (!options.renderPhotoBase) {
+          throw new VideoExportRejected([`clip '${clip.id}' is a photo, and no photo renderer was supplied`])
+        }
+        const pngPath = await options.renderPhotoBase(clip.source, clip.id)
+        job.checkpoint('resolving')
+        const photoProgram = compileVideoSceneProgram(project, {
+          animatedSchedules: animatedSchedulesFor(project),
+        })
+        if (photoProgram.segments.length === 0) {
+          throw new VideoExportRejected(['the compiled program has no segments'])
+        }
+        const named = new Set<string>()
+        for (const segment of photoProgram.segments) {
+          for (const command of segment.commands) {
+            if (command.op === 'overlay') named.add(command.overlayId)
+          }
+        }
+        const overlayPlan: OverlayPreparationResult = options.prepareOverlays
+          ? options.prepareOverlays(photoProgram, project.size, 1)
+          : {
+              resources: [],
+              pending: [],
+              invalid: [...named].map((overlayId) => ({
+                overlayId,
+                kind: 'unknown',
+                reason: 'no prepareOverlays was supplied, so this overlay has no portable form',
+              })),
+            }
+        const stagedNow = overlayPlan.resources.map((resource) => resource.file)
+        if (overlayPlan.pending.length > 0) {
+          options.releaseResources?.(stagedNow)
+          throw new VideoExportRejected(
+            overlayPlan.pending.map(
+              (id: string) => `overlay '${id}' is still preparing; export would omit it`,
+            ),
+          )
+        }
+        if (overlayPlan.invalid.length > 0) {
+          options.releaseResources?.(stagedNow)
+          throw new VideoExportRejected(
+            overlayPlan.invalid.map(
+              (bad: { overlayId: string; kind: string; reason: string }) =>
+                `overlay '${bad.overlayId}' (${bad.kind}): ${bad.reason}`,
+            ),
+          )
+        }
+        job.checkpoint('overlays')
+        let outcome: { ok: boolean; uri: string; error: string }
+        try {
+          outcome = JSON.parse(
+            await nativeModule().exportRunPhotoProjectAsync(
+              job.id,
+              pngPath,
+              JSON.stringify({
+                version: VIDEO_SCENE_PROGRAM_VERSION,
+                projectSize: { width: project.size.width, height: project.size.height },
+                durationUs: photoProgram.durationUs,
+                segments: photoProgram.segments,
+              }),
+              JSON.stringify(overlayPlan.resources),
+              projectDuration(project),
+              180,
+              destination.kind === 'file'
+                ? JSON.stringify({
+                    kind: 'file',
+                    path: destination.path,
+                    overwrite: destination.overwrite,
+                  })
+                : '',
+            ),
+          ) as { ok: boolean; uri: string; error: string }
+        } finally {
+          options.releaseResources?.(stagedNow)
+        }
+        if (!outcome.ok) {
+          const error = outcome.error || 'the export failed'
+          if (isRefusal(error)) throw new VideoExportRejected([error])
+          throw new VideoExportFailed(job.id, 'encoding', error)
+        }
+        const probe = JSON.parse(nativeModule().exportProbeOutput(outcome.uri)) as {
+          readable: boolean
+          error?: string
+        }
+        if (!probe.readable) {
+          throw new VideoExportFailed(
+            job.id,
+            'publishing',
+            `${describeTarget(outcome.uri)} could not be read back: ${probe.error ?? 'no video track'}`,
+          )
+        }
+        if (destination.kind === 'file') return { kind: 'file', uri: outcome.uri }
+        const temporaryUri = outcome.uri
+        return {
+          kind: 'temporary',
+          uri: temporaryUri,
+          release: async () => {
+            await Promise.resolve()
+            nativeModule().exportReleaseTemporaryOutput(temporaryUri)
+          },
+        }
+      }
+
       const playable = project.clips.filter((clip) => isPlayableWindow(clip))
       if (playable.length === 0) throw new VideoExportRejected(['the project has no playable clips'])
       if (!(projectDuration(project) > 0)) {
@@ -317,7 +468,9 @@ export function createMedia3VideoExporter(options: Media3ExporterOptions): Video
         targetFps: fps,
       }))
       const plan = planClipShards(requests, fps)
-      const program = compileVideoSceneProgram(project)
+      const program = compileVideoSceneProgram(project, {
+        animatedSchedules: animatedSchedulesFor(project),
+      })
       if (program.segments.length === 0) {
         throw new VideoExportRejected(['the compiled program has no segments'])
       }

package/src/useCompositionPlayback.ts

@@ -105,6 +105,19 @@ export interface CompositionPlaybackState {
   width: number
   height: number
   error: string | null
+  /**
+   * Why the transport refused to start, or null.
+   *
+   * Playback is released on a frame that shows the position asked for, and if
+   * no such frame arrives the hold times out and the transport stays stopped
+   * rather than starting on an unverified picture. That used to be invisible:
+   * the control stayed enabled, the press was accepted, and nothing happened,
+   * with no way for a host to tell that apart from a person who had not pressed
+   * it yet. Pressing play again clears it and tries afresh.
+   *
+   * @experimental
+   */
+  readinessFailure: string | null
 }
 
 function composedState(snapshot: CompositionSpikeState): CompositionPlaybackState {
@@ -130,6 +143,7 @@ function composedState(snapshot: CompositionSpikeState): CompositionPlaybackStat
     width: 0,
     height: 0,
     error: snapshot.error ?? null,
+    readinessFailure: snapshot.readinessFailure ?? null,
   }
 }