@airisu-dev/react-native-live2d-skia 0.0.0-stage → 0.1.0

package/src/hooks/useLive2DModel.ts

@@ -0,0 +1,47 @@
+import { useCallback, useEffect, useState } from 'react'
+import { type Live2DModel, type Live2DModelSource, loadLive2DModel } from '../loader/loadModel'
+import type { Live2DModelContext } from '../loader/modelContext'
+import type { Live2DLoadOptions } from '../loader/modelUrl'
+
+export interface Live2DModelState {
+  /** The loaded model, `null` while loading or after an error. */
+  model: Live2DModel | null
+  /** Why the last load failed, e.g. a network error for a URL source. */
+  error: Error | null
+  loading: boolean
+  /** Loads the model again, e.g. after an error. */
+  retry: () => void
+}
+
+/** Loads a model on mount. `options` apply to remote files (see `Live2DLoadOptions`). */
+export function useLive2DModel(
+  source: Live2DModelSource | Live2DModelContext | string,
+  options?: Live2DLoadOptions,
+): Live2DModelState {
+  const [model, setModel] = useState<Live2DModel | null>(null)
+  const [error, setError] = useState<Error | null>(null)
+  const [attempt, setAttempt] = useState(0)
+  // Read once: remount to load another model. Wrapped because a require.context
+  // is a function, which useState would call as an initializer.
+  const [initialSource] = useState(() => source)
+  const [initialOptions] = useState(options)
+
+  useEffect(() => {
+    let cancelled = false
+    loadLive2DModel(initialSource, initialOptions).then(
+      (m) => !cancelled && setModel(m),
+      (e: unknown) => !cancelled && setError(e instanceof Error ? e : new Error(String(e))),
+    )
+    return () => {
+      cancelled = true
+    }
+  }, [initialSource, initialOptions, attempt])
+
+  const retry = useCallback(() => {
+    setError(null)
+    setModel(null)
+    setAttempt((n) => n + 1)
+  }, [])
+
+  return { model, error, loading: !model && !error, retry }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './components'
+export * from './hooks'
+export * from './loader'
+export { Live2dSkia } from './native'
+export type { MotionPriority } from './specs/react-native-live2d-skia.nitro'

package/src/loader/cubismJson.ts

@@ -0,0 +1,23 @@
+// Serializes a parsed Cubism JSON file (model3, physics3, motion3, ...) for the
+// Cubism Framework's own JSON parser, which is stricter than JSON:
+// - a number must be followed by a newline or a comma, never `}`, `]` or a space
+//   (CubismJson::ParseNumeric), so every value goes on its own line;
+// - exponents aren't supported, so numbers like 1e-7 are written out in full.
+// Files from Cubism Editor already look like this; `JSON.stringify(x)` does not.
+
+const PLACEHOLDER = '\u0000num:'
+
+function plainNumber(n: number): string {
+  const s = String(n)
+  if (!/e/i.test(s)) return s
+  // toFixed(20) keeps far more precision than the Framework's float32 parser reads.
+  return n.toFixed(20).replace(/\.?0+$/, '')
+}
+
+export function toCubismJson(value: unknown): string {
+  return JSON.stringify(
+    value,
+    (_key, v) => (typeof v === 'number' && /e/i.test(String(v)) ? PLACEHOLDER + plainNumber(v) : v),
+    '\t',
+  ).replace(/"\\u0000num:([^"]+)"/g, '$1')
+}

package/src/loader/index.ts

@@ -0,0 +1,9 @@
+export type {
+  AssetSource,
+  CubismJson,
+  Live2DModel,
+  Live2DModelSource,
+} from './loadModel'
+export { loadLive2DModel } from './loadModel'
+export type { Live2DModelContext } from './modelContext'
+export type { Live2DLoadOptions } from './modelUrl'

package/src/loader/loadModel.ts

@@ -0,0 +1,194 @@
+import { Asset } from 'expo-asset'
+import { File } from 'expo-file-system'
+import { Live2dSkia } from '../native'
+import {
+  type CanvasInfo,
+  createDrawer,
+  createNativeDrawer,
+  setNativeTextures,
+} from '../rendering/drawer'
+import {
+  FilterMode,
+  MipmapMode,
+  type SkImage,
+  Skia,
+  type SkPicture,
+  type SkShader,
+  TileMode,
+} from '../rendering/skia'
+import type {
+  MotionPriority,
+  Live2DModel as NativeModel,
+} from '../specs/react-native-live2d-skia.nitro'
+import { toCubismJson } from './cubismJson'
+import { isModelContext, type Live2DModelContext, sourceFromContext } from './modelContext'
+import { fetchBytes, isRemote, type Live2DLoadOptions, redactUrl, sourceFromUrl } from './modelUrl'
+
+export type AssetSource = number | string
+
+/** A parsed Cubism JSON file, e.g. `require('../model.physics3.json')`. */
+export type CubismJson = object
+
+export interface Live2DModelSource {
+  /** Parsed `model3.json`. */
+  settings: CubismJson
+  moc: AssetSource
+  /** In the order of `FileReferences.Textures`. */
+  textures: AssetSource[]
+  physics?: CubismJson
+  pose?: CubismJson
+  /** By group, in the order of `FileReferences.Motions`. A random `Idle` motion plays when nothing else does. */
+  motions?: Record<string, CubismJson[]>
+  expressions?: Record<string, CubismJson>
+}
+
+/** A loaded model. Points are in model pixels: native canvas size, origin top-left, like `picture`. */
+export interface Live2DModel {
+  /**
+   * Plays a motion of `group`, a random one without `index`. Returns false
+   * if there's no such motion or one with a higher priority is playing.
+   */
+  motion(group: string, index?: number, priority?: MotionPriority): boolean
+  /** Fades to an expression, a random one without `name`. Returns false if there's none. */
+  expression(name?: string): boolean
+  resetExpression(): void
+  /** Holds a parameter at `value`, over motions, until `resetParameter`. */
+  setParameter(id: string, value: number): void
+  resetParameter(id: string): void
+  focus(x: number, y: number): void
+  resetFocus(): void
+  /** Names of the `model3.json` hit areas at a point, e.g. `['Head', 'Body']`. */
+  hitTest(x: number, y: number): string[]
+  motionGroups: string[]
+  expressionNames: string[]
+
+  /** Unique per loaded model: <Live2D>s showing the same model share its frames. */
+  id: number
+  /** Native model. Thread-safe. */
+  native: NativeModel
+  /** The model drawn at load, at its native canvas size, origin top-left. */
+  picture: SkPicture
+  width: number
+  height: number
+  /** Draws the model's current state (after `native.update`) into a new picture. */
+  draw(): SkPicture
+  /** For drawing on another worklet runtime with `createDrawer`. */
+  textures: SkImage[]
+  shaders: SkShader[]
+  canvas: CanvasInfo
+}
+
+// For error messages: URLs lose their query, which can hold tokens.
+const describe = (source: AssetSource) =>
+  typeof source === 'string' ? redactUrl(source) : String(source)
+
+async function readAsset(source: AssetSource, options: Live2DLoadOptions): Promise<ArrayBuffer> {
+  // Remote files are fetched directly: expo-asset can't send headers.
+  if (isRemote(source)) return fetchBytes(source, options)
+  const asset = Asset.fromModule(source)
+  // Android release builds embed images as drawables, and expo-asset marks them
+  // downloaded with a bare resource name (enough for <Image>), which isn't a file.
+  // Download for real: that copies the resource into the cache.
+  if (asset.localUri && !asset.localUri.includes(':')) {
+    asset.downloaded = false
+    asset.localUri = null
+  }
+  await asset.downloadAsync()
+  if (!asset.localUri) {
+    throw new Error(`Live2D: failed to download asset ${describe(source)}`)
+  }
+  return new File(asset.localUri).arrayBuffer()
+}
+
+async function loadTexture(source: AssetSource, options: Live2DLoadOptions): Promise<SkImage> {
+  const bytes = new Uint8Array(await readAsset(source, options))
+  const image = Skia.Image.MakeImageFromEncoded(Skia.Data.fromBytes(bytes))
+  if (!image) {
+    throw new Error(`Live2D: failed to decode texture ${describe(source)}`)
+  }
+  return image
+}
+
+let nextModelId = 1
+
+/**
+ * Loads a model from its files, from a `require.context` of its folder, or
+ * from the URL of its `model3.json` (the files it lists are fetched next to it).
+ * `options` apply to every remote file.
+ */
+export async function loadLive2DModel(
+  input: Live2DModelSource | Live2DModelContext | string,
+  options: Live2DLoadOptions = {},
+): Promise<Live2DModel> {
+  const source =
+    typeof input === 'string'
+      ? await sourceFromUrl(input, options)
+      : isModelContext(input)
+        ? sourceFromContext(input)
+        : input
+  const [moc, textures] = await Promise.all([
+    readAsset(source.moc, options),
+    Promise.all(source.textures.map((t) => loadTexture(t, options))),
+  ])
+  const native = Live2dSkia.loadModel(moc, toCubismJson(source.settings))
+  if (source.physics) native.loadPhysics(toCubismJson(source.physics))
+  if (source.pose) native.loadPose(toCubismJson(source.pose))
+  for (const [group, motions] of Object.entries(source.motions ?? {})) {
+    motions.forEach((motion, i) => {
+      native.loadMotion(group, i, toCubismJson(motion))
+    })
+  }
+  for (const [name, expression] of Object.entries(source.expressions ?? {})) {
+    native.loadExpression(name, toCubismJson(expression))
+  }
+  // Apply the pose and the first idle motion's starting state.
+  native.update(0)
+
+  // Trilinear: models are usually drawn far smaller than their 2048px textures,
+  // and without mipmaps thin lines (hair, outlines) break up into jaggies.
+  const shaders = textures.map((t) =>
+    t.makeShaderOptions(TileMode.Clamp, TileMode.Clamp, FilterMode.Linear, MipmapMode.Linear),
+  )
+  const width = native.canvasWidth
+  const height = native.canvasHeight
+  const canvas: CanvasInfo = {
+    width,
+    height,
+    originX: native.originX,
+    originY: native.originY,
+    pixelsPerUnit: native.pixelsPerUnit,
+  }
+  setNativeTextures(native, textures, shaders)
+  const nativeDraw = createNativeDrawer(native, canvas)
+  const drawer = createDrawer(textures, shaders, canvas)
+  const draw = nativeDraw ?? (() => drawer(native.getDrawables()))
+  // Model pixels (y-down) to model units (y-up).
+  const toUnits = (x: number, y: number) =>
+    [
+      (x - canvas.originX) / canvas.pixelsPerUnit,
+      (canvas.originY - y) / canvas.pixelsPerUnit,
+    ] as const
+
+  return {
+    id: nextModelId++,
+    motion: (group, index, priority) => native.startMotion(group, index, priority),
+    expression: (name) => native.setExpression(name),
+    resetExpression: () => native.clearExpression(),
+    setParameter: (id, value) => native.setParameter(id, value),
+    resetParameter: (id) => native.clearParameter(id),
+    // Relative to the canvas center, -1..1, y-up.
+    focus: (x, y) => native.focus((x / width) * 2 - 1, 1 - (y / height) * 2),
+    resetFocus: () => native.focus(0, 0),
+    hitTest: (x, y) => native.hitTest(...toUnits(x, y)),
+    motionGroups: Object.keys(source.motions ?? {}),
+    expressionNames: Object.keys(source.expressions ?? {}),
+    native,
+    picture: draw(),
+    width,
+    height,
+    draw,
+    textures,
+    shaders,
+    canvas,
+  }
+}

package/src/loader/modelContext.ts

@@ -0,0 +1,80 @@
+import type { CubismJson, Live2DModelSource } from './loadModel'
+
+/** A Metro `require.context` of a model's folder; the path must be a string literal for Metro to bundle it. */
+export interface Live2DModelContext {
+  keys(): string[]
+  (id: string): unknown
+}
+
+export interface Model3Json {
+  FileReferences?: {
+    Moc?: string
+    Textures?: string[]
+    Physics?: string
+    Pose?: string
+    Motions?: Record<string, { File: string }[]>
+    Expressions?: { Name: string; File: string }[]
+  }
+}
+
+export function isModelContext(
+  source: Live2DModelSource | Live2DModelContext,
+): source is Live2DModelContext {
+  return typeof source === 'function'
+}
+
+/** Resolves `file` relative to the folder of `from`, a context key or a URL. */
+export function resolvePath(from: string, file: string): string {
+  const parts = from
+    .replace(/[?#].*$/, '')
+    .split('/')
+    .slice(0, -1)
+  for (const part of file.split('/')) {
+    if (part === '..') parts.pop()
+    else if (part !== '.' && part !== '') parts.push(part)
+  }
+  return parts.join('/')
+}
+
+export function sourceFromContext(context: Live2DModelContext): Live2DModelSource {
+  const keys = context.keys()
+  const model3Keys = keys.filter((k) => k.endsWith('.model3.json'))
+  if (model3Keys.length !== 1) {
+    throw new Error(
+      `Live2D: expected one .model3.json in the require.context, found ${
+        model3Keys.length === 0 ? 'none' : model3Keys.join(', ')
+      }`,
+    )
+  }
+  const model3Key = model3Keys[0]!
+  const settings = context(model3Key) as Model3Json
+  const refs = settings.FileReferences ?? {}
+
+  const load = <T>(file: string): T => {
+    const key = resolvePath(model3Key, file)
+    if (!keys.includes(key)) {
+      throw new Error(
+        `Live2D: ${file} (listed in ${model3Key}) isn't in the require.context. Check its filter.`,
+      )
+    }
+    return context(key) as T
+  }
+
+  if (!refs.Moc) throw new Error(`Live2D: ${model3Key} has no FileReferences.Moc`)
+  return {
+    settings: settings as CubismJson,
+    moc: load(refs.Moc),
+    textures: (refs.Textures ?? []).map((t) => load(t)),
+    physics: refs.Physics ? load(refs.Physics) : undefined,
+    pose: refs.Pose ? load(refs.Pose) : undefined,
+    motions: Object.fromEntries(
+      Object.entries(refs.Motions ?? {}).map(([group, motions]) => [
+        group,
+        motions.map((m) => load<CubismJson>(m.File)),
+      ]),
+    ),
+    expressions: Object.fromEntries(
+      (refs.Expressions ?? []).map((e) => [e.Name, load<CubismJson>(e.File)]),
+    ),
+  }
+}

package/src/loader/modelUrl.ts

@@ -0,0 +1,112 @@
+import { fetch } from 'expo/fetch'
+import type { CubismJson, Live2DModelSource } from './loadModel'
+import { type Model3Json, resolvePath } from './modelContext'
+
+/** How remote model files are requested. */
+export interface Live2DLoadOptions {
+  /** Sent with every request for a remote file, e.g. `{ Authorization: 'Bearer …' }`. */
+  headers?: Record<string, string>
+  /**
+   * Gives the URL to request for a file the `model3.json` lists, e.g. to sign
+   * each one through your backend. `url` is resolved next to the `model3.json`,
+   * without a query; `file` is the path as listed. Replaces the default, which
+   * reuses the `model3.json` URL's query (e.g. a token) for files on its host.
+   */
+  resolveUrl?: (url: string, file: string) => string | Promise<string>
+}
+
+const ABSOLUTE = /^[a-z][a-z0-9+.-]*:\/\//i
+
+export function isRemote(source: unknown): source is string {
+  return typeof source === 'string' && /^https?:\/\//i.test(source)
+}
+
+/** A URL without its query and hash, which can hold tokens: for error messages. */
+export function redactUrl(url: string): string {
+  return url.replace(/[?#].*$/, '')
+}
+
+const origin = (url: string) => /^[a-z][a-z0-9+.-]*:\/\/[^/?#]*/i.exec(url)?.[0]
+const queryOf = (url: string) => /\?[^#]*/.exec(url)?.[0] ?? ''
+
+async function request(url: string, options: Live2DLoadOptions) {
+  let res: Awaited<ReturnType<typeof fetch>>
+  try {
+    res = await fetch(url, { headers: options.headers })
+  } catch (e) {
+    const reason = e instanceof Error ? e.message : String(e)
+    throw new Error(`Live2D: failed to fetch ${redactUrl(url)} (${reason})`)
+  }
+  if (!res.ok) {
+    throw new Error(`Live2D: failed to fetch ${redactUrl(url)} (${res.status})`)
+  }
+  return res
+}
+
+export async function fetchBytes(url: string, options: Live2DLoadOptions): Promise<ArrayBuffer> {
+  return (await request(url, options)).arrayBuffer()
+}
+
+async function fetchJson<T>(url: string, options: Live2DLoadOptions): Promise<T> {
+  return (await (await request(url, options)).json()) as T
+}
+
+// Escapes a model3.json file path for a URL: names can have spaces. Already
+// escaped segments are decoded first, so they aren't escaped twice.
+function encodePath(file: string): string {
+  return file
+    .split('/')
+    .map((part) => {
+      if (part === '.' || part === '..') return part
+      try {
+        return encodeURIComponent(decodeURIComponent(part))
+      } catch {
+        return encodeURIComponent(part)
+      }
+    })
+    .join('/')
+}
+
+/** Fetches a `model3.json` and the JSON files it lists; the moc and textures stay URLs. */
+export async function sourceFromUrl(
+  url: string,
+  options: Live2DLoadOptions = {},
+): Promise<Live2DModelSource> {
+  const settings = await fetchJson<Model3Json>(url, options)
+  const refs = settings.FileReferences ?? {}
+  if (!refs.Moc) {
+    throw new Error(`Live2D: ${redactUrl(url)} has no FileReferences.Moc`)
+  }
+
+  const query = queryOf(url)
+  const resolve = async (file: string): Promise<string> => {
+    const plain = ABSOLUTE.test(file) ? file : resolvePath(url, encodePath(file))
+    if (options.resolveUrl) return options.resolveUrl(plain, file)
+    // The model3.json's query (e.g. a signing token) goes to its own host only.
+    if (!query || origin(plain) !== origin(url)) return plain
+    return plain.includes('?') ? `${plain}&${query.slice(1)}` : plain + query
+  }
+  const json = async (file: string) => fetchJson<CubismJson>(await resolve(file), options)
+
+  const motionGroups = Object.entries(refs.Motions ?? {})
+  const [moc, textures, physics, pose, motions, expressions] = await Promise.all([
+    resolve(refs.Moc),
+    Promise.all((refs.Textures ?? []).map(resolve)),
+    refs.Physics ? json(refs.Physics) : undefined,
+    refs.Pose ? json(refs.Pose) : undefined,
+    Promise.all(motionGroups.map(([, list]) => Promise.all(list.map((m) => json(m.File))))),
+    Promise.all((refs.Expressions ?? []).map((e) => json(e.File))),
+  ])
+
+  return {
+    settings: settings as CubismJson,
+    moc,
+    textures,
+    physics,
+    pose,
+    motions: Object.fromEntries(motionGroups.map(([group], i) => [group, motions[i]!])),
+    expressions: Object.fromEntries(
+      (refs.Expressions ?? []).map((e, i) => [e.Name, expressions[i]!]),
+    ),
+  }
+}

package/src/native.ts

@@ -0,0 +1,4 @@
+import { NitroModules } from 'react-native-nitro-modules'
+import type { Live2dSkia as Live2dSkiaSpec } from './specs/react-native-live2d-skia.nitro'
+
+export const Live2dSkia = NitroModules.createHybridObject<Live2dSkiaSpec>('Live2dSkia')