@airisu-dev/react-native-live2d-skia 0.0.0-stage → 0.1.1

package/src/rendering/ticker/index.ts

@@ -0,0 +1,5 @@
+// One animation loop for every <Live2D> on the live2d worklet runtime: each
+// frame, every shown model is updated and drawn once, in one pass. Everything
+// here runs on that runtime, and its state lives in the runtime's globalThis.
+export { addToTicker } from './addToTicker'
+export type { Live2DFrameStats, PublishedFrame, TickerModel, TickerView } from './types'

package/src/rendering/ticker/loop.ts

@@ -0,0 +1,115 @@
+import { scheduleOnRN } from 'react-native-worklets'
+import { now } from '../../utils/clock'
+import type { SkPicture } from '../skia'
+import {
+  CAP_TOLERANCE_MS,
+  DEFAULT_FRAME_MS,
+  MAX_DELTA_SECONDS,
+  MAX_FRAME_MS,
+  MAX_FRAMES_AHEAD,
+  MIN_FRAME_MS,
+  STATS_INTERVAL_MS,
+} from './constants'
+import { freeFrames, oldestShown, releaseView, stopEntry } from './frames'
+import { type Entry, reportError, tickerState } from './state'
+import { reportEntry } from './stats'
+
+function drawEntry(entry: Entry, frameStart: number, wantStats: boolean) {
+  'worklet'
+  const dt = entry.last === null ? 0 : Math.min((frameStart - entry.last) / 1000, MAX_DELTA_SECONDS)
+  entry.last = frameStart
+  // Timed from here: frameStart is when the whole loop frame began.
+  const start = wantStats ? now() : 0
+  entry.native.update(dt)
+  const updated = wantStats ? now() : 0
+  const stats = wantStats ? entry.drawStats : undefined
+  let copied = updated
+  let picture: SkPicture
+  if (entry.nativeDraw) {
+    picture = entry.nativeDraw(stats)
+  } else {
+    const drawables = entry.native.getDrawables()
+    copied = wantStats ? now() : 0
+    picture = entry.jsDraw!(drawables, stats)
+  }
+  entry.frame++
+  for (const { view } of entry.views) {
+    view.target.setBlocking({ frame: entry.frame, picture })
+  }
+  entry.frames.push({ frame: entry.frame, picture })
+  if (wantStats) {
+    const end = now()
+    entry.statsFrames++
+    entry.updateMs += updated - start
+    entry.drawablesMs += copied - updated
+    entry.drawMs += end - copied
+  }
+}
+
+/** Updates and draws every model once. Returns the delay until the next frame, or null to stop. */
+export function runFrame(): number | null {
+  'worklet'
+  const ticker = tickerState()
+  const frameStart = now()
+  let displayMs = 0
+  for (const [id, entry] of ticker.entries) {
+    const stopped = entry.views.filter((s) => !s.view.running.getDirty())
+    if (stopped.length > 0) {
+      for (const slot of stopped) releaseView(entry, slot)
+      entry.views = entry.views.filter((s) => !stopped.includes(s))
+    }
+    if (entry.views.length === 0) {
+      stopEntry(ticker, id, entry)
+      continue
+    }
+    freeFrames(entry)
+    for (const { view } of entry.views) {
+      const ms = view.displayMs.getDirty()
+      if (ms > 0 && (displayMs === 0 || ms < displayMs)) displayMs = ms
+    }
+    // The fastest view sets the model's frame rate.
+    const capMs = Math.min(...entry.views.map((s) => s.view.capMs))
+    if (entry.last !== null && frameStart - entry.last < capMs - CAP_TOLERANCE_MS) {
+      continue
+    }
+    // Wait for a UI thread that's behind, rather than draw frames it won't show.
+    if (entry.frame - Math.min(...entry.views.map(oldestShown)) >= MAX_FRAMES_AHEAD) {
+      continue
+    }
+    const wantStats = entry.views.some((s) => s.view.report !== null)
+    try {
+      drawEntry(entry, frameStart, wantStats)
+    } catch (e) {
+      // An uncaught error on this runtime aborts the app: drop the model instead.
+      for (const slot of entry.views) {
+        slot.view.running.setBlocking(false)
+        releaseView(entry, slot)
+      }
+      stopEntry(ticker, id, entry)
+      scheduleOnRN(reportError, String((e as Error)?.stack ?? e))
+    }
+  }
+
+  const end = now()
+  ticker.loopWorkMs += end - frameStart
+  ticker.loopFrames++
+  const interval = Math.min(MAX_FRAME_MS, Math.max(MIN_FRAME_MS, displayMs || DEFAULT_FRAME_MS))
+  for (const entry of ticker.entries.values()) {
+    const capMs = Math.min(...entry.views.map((s) => s.view.capMs))
+    reportEntry(ticker, entry, end, Math.max(interval, capMs))
+  }
+  if (end - ticker.loopStatsStart >= STATS_INTERVAL_MS) {
+    ticker.loopStatsStart = end
+    ticker.loopWorkMs = 0
+    ticker.loopFrames = 0
+  }
+
+  if (ticker.entries.size === 0) {
+    ticker.running = false
+    return null
+  }
+  ticker.nextFrameAt += interval
+  // More than a frame behind (a stall, or the first frame): start over from now.
+  if (ticker.nextFrameAt < end - interval) ticker.nextFrameAt = end + interval
+  return Math.max(0, ticker.nextFrameAt - end)
+}

package/src/rendering/ticker/state.ts

@@ -0,0 +1,65 @@
+import type { Live2DModel as NativeModel } from '../../specs/react-native-live2d-skia.nitro'
+import type { createDrawer, DrawStats } from '../drawer'
+import type { SkPicture } from '../skia'
+import type { TickerView } from './types'
+
+export interface Slot {
+  view: TickerView
+  /** The entry's newest frame when the view joined: the oldest it can show before acking one. */
+  since: number
+}
+
+export interface Frame {
+  frame: number
+  picture: SkPicture
+}
+
+export interface Entry {
+  native: NativeModel
+  /** Draws the model's current state: in C++ when built with Skia, else in JS. */
+  nativeDraw: ((stats?: DrawStats) => SkPicture) | null
+  jsDraw: ReturnType<typeof createDrawer> | null
+  views: Slot[]
+  /**
+   * Frames a view may still show, oldest first. Freed explicitly once no view
+   * can show them: the GC doesn't see their native memory.
+   */
+  frames: Frame[]
+  frame: number
+  last: number | null
+  statsStart: number
+  statsFrames: number
+  updateMs: number
+  drawablesMs: number
+  drawMs: number
+  drawStats: DrawStats
+}
+
+export interface Ticker {
+  entries: Map<number, Entry>
+  running: boolean
+  /** When the next frame is due (absolute, so late timer wake-ups don't add up). */
+  nextFrameAt: number
+  loopStatsStart: number
+  loopWorkMs: number
+  loopFrames: number
+}
+
+export const reportError = (message: string) =>
+  console.error('[react-native-live2d-skia] animation stopped:', message)
+
+export function tickerState(): Ticker {
+  'worklet'
+  const global = globalThis as unknown as { __live2dSkiaTicker?: Ticker }
+  if (!global.__live2dSkiaTicker) {
+    global.__live2dSkiaTicker = {
+      entries: new Map(),
+      running: false,
+      nextFrameAt: 0,
+      loopStatsStart: 0,
+      loopWorkMs: 0,
+      loopFrames: 0,
+    }
+  }
+  return global.__live2dSkiaTicker
+}

package/src/rendering/ticker/stats.ts

@@ -0,0 +1,36 @@
+import { scheduleOnRN } from 'react-native-worklets'
+import { STATS_INTERVAL_MS } from './constants'
+import type { Entry, Ticker } from './state'
+import type { Live2DFrameStats } from './types'
+
+export function reportEntry(ticker: Ticker, entry: Entry, at: number, frameMs: number) {
+  'worklet'
+  const elapsed = at - entry.statsStart
+  if (elapsed < STATS_INTERVAL_MS || entry.statsFrames === 0) return
+  const n = entry.statsFrames
+  const s = entry.drawStats
+  const stats: Live2DFrameStats = {
+    fps: (n * 1000) / elapsed,
+    targetFps: 1000 / frameMs,
+    workMs: (entry.updateMs + entry.drawablesMs + entry.drawMs) / n,
+    updateMs: entry.updateMs / n,
+    drawablesMs: entry.drawablesMs / n,
+    drawMs: entry.drawMs / n,
+    meshMs: s.meshMs / n,
+    meshesBuilt: s.meshesBuilt / n,
+    layers: s.layers / n,
+    models: ticker.entries.size,
+    loopWorkMs: ticker.loopFrames > 0 ? ticker.loopWorkMs / ticker.loopFrames : 0,
+  }
+  for (const { view } of entry.views) {
+    if (view.report) scheduleOnRN(view.report, stats)
+  }
+  entry.statsStart = at
+  entry.statsFrames = 0
+  entry.updateMs = 0
+  entry.drawablesMs = 0
+  entry.drawMs = 0
+  s.meshMs = 0
+  s.meshesBuilt = 0
+  s.layers = 0
+}

package/src/rendering/ticker/types.ts

@@ -0,0 +1,57 @@
+import type { Synchronizable } from 'react-native-worklets'
+import type { Live2DModel as NativeModel } from '../../specs/react-native-live2d-skia.nitro'
+import type { CanvasInfo } from '../drawer'
+import type { SkImage, SkPicture, SkShader } from '../skia'
+
+/** Averages per frame over the last second, in milliseconds unless noted. */
+export interface Live2DFrameStats {
+  fps: number
+  /** The frame rate the loop aims for: the display's refresh rate, or `maxFps`. */
+  targetFps: number
+  /** All work for this model's frame on the animation thread. */
+  workMs: number
+  /** Motions, physics and so on in C++ (`update`). */
+  updateMs: number
+  /** Copying vertex data out of C++ (`getDrawables`). 0 when drawn in C++. */
+  drawablesMs: number
+  /** Recording the picture, including `meshMs`. */
+  drawMs: number
+  /** Building Skia vertices for drawables that moved. */
+  meshMs: number
+  /** Meshes rebuilt per frame (a count). */
+  meshesBuilt: number
+  /**
+   * Mask layers per frame (a count). Their cost is in the UI thread's Skia
+   * draw, which these stats don't cover.
+   */
+  layers: number
+  /** Models animated by the loop, this one included (a count). */
+  models: number
+  /** All work for a frame of the loop, every model included. */
+  loopWorkMs: number
+}
+
+/** A newly drawn frame, for the UI thread to pick up. */
+export type PublishedFrame = { frame: number; picture: SkPicture } | null
+
+export interface TickerView {
+  target: Synchronizable<PublishedFrame>
+  /** The frame the UI thread shows (-1 before the first). Frames are freed once every view is past them. */
+  shown: Synchronizable<number>
+  /** Set to false to stop showing frames here. */
+  running: Synchronizable<boolean>
+  /** The display's frame interval, measured on the UI thread; 0 until known. */
+  displayMs: Synchronizable<number>
+  /** Shortest frame time for this view (`maxFps`); 0 for no cap. */
+  capMs: number
+  /** Gets frame stats about once a second; null if not wanted. */
+  report: ((stats: Live2DFrameStats) => void) | null
+}
+
+export interface TickerModel {
+  id: number
+  native: NativeModel
+  textures: SkImage[]
+  shaders: SkShader[]
+  canvas: CanvasInfo
+}

package/src/specs/react-native-live2d-skia.nitro.ts

@@ -0,0 +1,88 @@
+import type { HybridObject } from 'react-native-nitro-modules'
+
+export interface DrawableData {
+  /** Index into the model's drawables; `masks` refers to these. */
+  index: number
+  /** Cubism's visibility flag. Hidden drawables can still be used as masks. */
+  visible: boolean
+  textureIndex: number
+  opacity: number
+  /** Cubism constant flags (`csmBlendAdditive`, `csmIsInvertedMask`, ...). */
+  constantFlags: number
+  /** Indices of the drawables that clip this one. Empty if unmasked. */
+  masks: number[]
+  /** float32 x/y pairs in model units, y-up. */
+  positions: ArrayBuffer
+  /** float32 u/v pairs in 0..1, v-up. */
+  uvs: ArrayBuffer
+  /** uint16 triangle indices. */
+  indices: ArrayBuffer
+}
+
+/** Like the Cubism SDK samples: a higher priority interrupts a lower one. */
+export type MotionPriority = 'idle' | 'normal' | 'force'
+
+/**
+ * Thread-safe: the model can be updated on one thread while another starts
+ * motions or moves the focus.
+ */
+export interface Live2DModel extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  readonly canvasWidth: number
+  readonly canvasHeight: number
+  readonly originX: number
+  readonly originY: number
+  readonly pixelsPerUnit: number
+  readonly drawableCount: number
+  /** All drawables sorted by render order (back to front), including hidden ones. */
+  getDrawables(): DrawableData[]
+  loadPhysics(json: string): void
+  loadPose(json: string): void
+  /**
+   * Loads one `motion3.json` as entry `index` of motion `group`.
+   * Fade times come from `model3.json` when it defines them.
+   */
+  loadMotion(group: string, index: number, json: string): void
+  /**
+   * Starts a loaded motion, or a random one of `group` without `index`.
+   * Returns false if it isn't loaded or a higher priority motion is playing.
+   */
+  startMotion(group: string, index?: number, priority?: MotionPriority): boolean
+  loadExpression(name: string, json: string): void
+  /**
+   * Fades to a loaded expression, or a random one without `name`.
+   * Returns false if it isn't loaded.
+   */
+  setExpression(name?: string): boolean
+  clearExpression(): void
+  /**
+   * Holds a parameter at `value` until `clearParameter`. Applied after motions,
+   * expressions, eye blink, look and breath, before physics and pose.
+   */
+  setParameter(id: string, value: number): void
+  clearParameter(id: string): void
+  /**
+   * Makes the model look at a point, easing towards it: -1..1 on each axis,
+   * y-up, (0, 0) is straight ahead.
+   */
+  focus(x: number, y: number): void
+  /**
+   * Names of the `model3.json` hit areas containing a point in model units
+   * (y-up, see `originX`/`pixelsPerUnit`), in the order they're listed.
+   */
+  hitTest(x: number, y: number): string[]
+  /**
+   * Advances the model by `deltaSeconds`: motions (a random `Idle` motion
+   * whenever nothing plays), expression, eye blink, look, breath, held
+   * parameters, physics and pose.
+   * Pass 0 to just apply the current state.
+   */
+  update(deltaSeconds: number): void
+}
+
+export interface Live2dSkia extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  /**
+   * Loads a `.moc3` file with its `model3.json` (used for eye blink and lip
+   * sync parameters and motion fade times). Throws if the moc is invalid.
+   */
+  loadModel(moc: ArrayBuffer, settingsJson: string): Live2DModel
+}

package/src/utils/clock.ts

@@ -0,0 +1,10 @@
+// performance.now() exists on the JS thread and on worklet runtimes, but this
+// package's TypeScript lib (esnext, no DOM) doesn't declare it.
+type WithPerformance = { performance: { now(): number } }
+
+export function now(): number {
+  'worklet'
+  // Read globalThis here, not from a module constant: a worklet copies what it
+  // captures to its runtime, and the global object can't be copied (it's cyclic).
+  return (globalThis as unknown as WithPerformance).performance.now()
+}