@hypit/hypit 0.0.1 → 0.1.0

package/packages/media/src/surface.ts

@@ -0,0 +1,150 @@
+import { artifactTypes } from "@hypit/artifact";
+import type {
+  StructuredElement,
+  StructuredSurfaceHandler,
+} from "@hypit/markup";
+
+import { assertFontArtifactRef } from "./render.js";
+import { mediaTypes } from "./manifest.js";
+
+const IMAGE_MEDIA_TYPES = new Map([
+  [".avif", "image/avif"],
+  [".gif", "image/gif"],
+  [".jpeg", "image/jpeg"],
+  [".jpg", "image/jpeg"],
+  [".png", "image/png"],
+  [".webp", "image/webp"],
+]);
+
+const AUDIO_MEDIA_TYPES = new Map([
+  [".aac", "audio/aac"],
+  [".flac", "audio/flac"],
+  [".m4a", "audio/mp4"],
+  [".mp3", "audio/mpeg"],
+  [".oga", "audio/ogg"],
+  [".ogg", "audio/ogg"],
+  [".opus", "audio/opus"],
+  [".wav", "audio/wav"],
+]);
+
+const VIDEO_MEDIA_TYPES = new Map([
+  [".m4v", "video/x-m4v"],
+  [".mov", "video/quicktime"],
+  [".mp4", "video/mp4"],
+  [".webm", "video/webm"],
+]);
+
+const FONT_MEDIA_TYPES = new Map([
+  [".otf", "font/otf"],
+  [".ttf", "font/ttf"],
+  [".woff", "font/woff"],
+  [".woff2", "font/woff2"],
+]);
+
+function stringAttribute(element: StructuredElement, name: string): string {
+  const value = element.attributes[name];
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`${element.name}.${name} must be a non-empty string`);
+  }
+  return value.trim();
+}
+
+function assertChildrenEmpty(element: StructuredElement): void {
+  if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
+    throw new Error(`${element.name} does not accept children`);
+  }
+}
+
+function mediaTypeFor(
+  element: StructuredElement,
+  source: string,
+  kind: "image" | "audio" | "video" | "font",
+  known: ReadonlyMap<string, string>,
+): string {
+  const explicit = element.attributes["media-type"];
+  if (explicit !== undefined) {
+    if (typeof explicit !== "string" || !explicit.startsWith(`${kind}/`)) {
+      throw new Error(`${element.name}.media-type must be an ${kind} media type`);
+    }
+    return explicit;
+  }
+  const clean = source.split(/[?#]/u, 1)[0]!.toLocaleLowerCase("en");
+  const dot = clean.lastIndexOf(".");
+  const inferred = dot < 0 ? undefined : known.get(clean.slice(dot));
+  if (inferred === undefined) {
+    throw new Error(`${element.name}.src needs a known ${kind} extension or an explicit media-type`);
+  }
+  return inferred;
+}
+
+/** Host resolves and stages the bytes; this Surface only declares their authored media meaning. */
+async function decodeMediaAssetSurface(
+  element: StructuredElement,
+  resolveAsset: Parameters<StructuredSurfaceHandler>[0]["resolveAsset"],
+  kind: "image" | "audio" | "video",
+  known: ReadonlyMap<string, string>,
+) {
+  const names = Object.keys(element.attributes).sort();
+  if (names.join(",") !== "id,src" && names.join(",") !== "id,media-type,src") {
+    throw new Error(`${element.name} requires id and src, with optional media-type`);
+  }
+  assertChildrenEmpty(element);
+  const id = stringAttribute(element, "id");
+  const source = stringAttribute(element, "src");
+  const mediaType = mediaTypeFor(element, source, kind, known);
+  const resolved = await resolveAsset({ from: source, mediaType, range: element.range });
+  if (!resolved.artifact.mediaType.startsWith(`${kind}/`)) {
+    throw new Error(`${element.name}.src did not resolve to an ${kind} artifact`);
+  }
+  return {
+    records: [{
+      id,
+      type: artifactTypes.blob,
+      value: resolved.artifact,
+      range: element.range,
+    }],
+    components: [],
+    fragments: [],
+  };
+}
+
+export const decodeMediaImageSurface: StructuredSurfaceHandler = async ({ element, resolveAsset }) =>
+  await decodeMediaAssetSurface(element, resolveAsset, "image", IMAGE_MEDIA_TYPES);
+
+export const decodeMediaAudioSurface: StructuredSurfaceHandler = async ({ element, resolveAsset }) =>
+  await decodeMediaAssetSurface(element, resolveAsset, "audio", AUDIO_MEDIA_TYPES);
+
+export const decodeMediaVideoSurface: StructuredSurfaceHandler = async ({ element, resolveAsset }) =>
+  await decodeMediaAssetSurface(element, resolveAsset, "video", VIDEO_MEDIA_TYPES);
+
+export const decodeMediaFontSurface: StructuredSurfaceHandler = async ({ element, resolveAsset }) => {
+  const names = Object.keys(element.attributes).sort();
+  if (names.join(",") !== "id,src,style,weight" && names.join(",") !== "id,media-type,src,style,weight") {
+    throw new Error(`${element.name} requires id, src, weight and style, with optional media-type`);
+  }
+  assertChildrenEmpty(element);
+  const id = stringAttribute(element, "id");
+  const source = stringAttribute(element, "src");
+  const weight = Number(stringAttribute(element, "weight"));
+  if (!Number.isSafeInteger(weight) || weight < 1 || weight > 1_000) {
+    throw new Error(`${element.name}.weight must be an integer from 1 to 1000`);
+  }
+  const style = stringAttribute(element, "style");
+  if (style !== "normal" && style !== "italic" && style !== "oblique") {
+    throw new Error(`${element.name}.style must be normal, italic or oblique`);
+  }
+  const fontStyle = style as "normal" | "italic" | "oblique";
+  const mediaType = mediaTypeFor(element, source, "font", FONT_MEDIA_TYPES);
+  const resolved = await resolveAsset({ from: source, mediaType, range: element.range });
+  const font = {
+    sources: [{ artifact: resolved.artifact }],
+    weight,
+    style: fontStyle,
+  };
+  assertFontArtifactRef(font, `${element.name}.${id}`);
+  return {
+    records: [{ id, type: mediaTypes.fontArtifact, value: { kind: "inline" as const, value: font }, range: element.range }],
+    components: [],
+    fragments: [],
+  };
+};

package/packages/media/src/types.ts

@@ -0,0 +1,121 @@
+import type { BlobRef } from "@hypit/protocol";
+
+export type MediaRational = {
+  readonly numerator: number;
+  readonly denominator: number;
+};
+
+/** JSON-safe presentation timestamp: ticks * timeBase.numerator / timeBase.denominator. */
+export type MediaTimestamp = {
+  readonly ticks: string;
+  readonly timeBase: MediaRational;
+};
+
+export type MediaDisposition = {
+  readonly default: boolean;
+  readonly attachedPicture: boolean;
+};
+
+type MediaStreamBase = {
+  readonly index: number;
+  readonly codecType: string;
+  readonly codecName: string;
+  readonly disposition: MediaDisposition;
+  readonly timingStatus: "admissible" | "missing" | "non-monotonic" | "discontinuous";
+  readonly timeBase?: MediaRational;
+  readonly startPts?: MediaTimestamp;
+  readonly endPts?: MediaTimestamp;
+  readonly decodedUnitCount: number;
+};
+
+export type MediaVideoStream = MediaStreamBase & {
+  readonly kind: "video";
+  readonly codecType: "video";
+  readonly role: "moving" | "attached-picture" | "still";
+  readonly width: number;
+  readonly height: number;
+  /** Coded-pixel width/height become display geometry only after this ratio and rotation are normalized. */
+  readonly sampleAspectRatio: MediaRational;
+  readonly rotationDegrees: 0 | 90 | 180 | 270;
+  readonly averageFrameRate?: MediaRational;
+  readonly nominalFrameRate?: MediaRational;
+};
+
+export type MediaAudioStream = MediaStreamBase & {
+  readonly kind: "audio";
+  readonly codecType: "audio";
+  readonly sampleRate: number;
+  readonly channels: number;
+  readonly channelLayout?: string;
+  readonly decodedSampleFrames: number;
+};
+
+/** Subtitle, data, attachment and unknown streams remain visible instead of being discarded. */
+export type MediaOtherStream = MediaStreamBase & {
+  readonly kind: "other";
+};
+
+export type MediaStream = MediaVideoStream | MediaAudioStream | MediaOtherStream;
+
+/** Immutable observed facts about one exact media container. */
+export type MediaInspection = {
+  readonly container: {
+    readonly formatNames: readonly string[];
+  };
+  readonly streams: readonly MediaStream[];
+};
+
+export type MediaStreamSelection = {
+  readonly videoStreamIndex?: number;
+  readonly audioStreamIndex?: number;
+  readonly spanAuthority: "video" | "audio";
+  readonly policy:
+    | "primary-moving@1"
+    | "default-audio@1"
+    | "primary-moving-default-audio@1"
+    | "explicit-streams@1";
+};
+
+export type SynchronizedMedia = {
+  readonly timeline: {
+    readonly frameRate: MediaRational;
+    readonly frameCount: number;
+  };
+  readonly visual?: {
+    readonly artifact: BlobRef;
+    readonly width: number;
+    readonly height: number;
+  };
+  readonly audio?: {
+    readonly artifact: BlobRef;
+  };
+};
+
+/** Silent, frame-exact visual output from a renderer such as HyperFrames. */
+export type RenderedVisual = {
+  readonly frameRate: MediaRational;
+  readonly frameCount: number;
+  readonly canvas: {
+    readonly width: number;
+    readonly height: number;
+  };
+  readonly artifact: BlobRef;
+};
+
+/** Exact 48 kHz stereo PCM result of one explicit timeline-audio plan. */
+export type TimelineAudio = {
+  readonly artifact: BlobRef;
+  readonly sampleFrames: number;
+};
+
+/** Final mux result; visual rendering and program-audio preparation remain separate facts. */
+export type MuxedMedia = {
+  readonly frameRate: MediaRational;
+  readonly frameCount: number;
+  readonly canvas: {
+    readonly width: number;
+    readonly height: number;
+  };
+  readonly presentationSampleFrames: number;
+  readonly artifact: BlobRef;
+};

package/packages/media-execution/README.md

@@ -0,0 +1,28 @@
+# `@hypit/media-execution`
+
+Shared FFmpeg execution body for Hypit media Providers.
+
+Timeline-audio requests accept an optional `range: { startFrame, endFrameExclusive }` on the
+original programme clock. Only overlapping clips are read. Loop phase, tempo, gain and fades
+are evaluated in their original clip coordinates before trimming and rebasing the selected mix.
+The output remains 48 kHz stereo PCM. At rational frame rates, its sample count is rounded from
+the selected frame count; reconciling that zero-based clock can trim or pad one boundary sample.
+
+`@hypit/media-pipeline` owns provider-neutral Needs, plans and result contracts. This package
+owns the byte-level implementations of the eight current operations:
+
+- inspect every media stream;
+- normalize selected audio/video streams;
+- transform synchronized media with ordered trim and pitch-preserving retime operations;
+- extract one selected audio stream as an ordinary 48 kHz stereo WAV Artifact;
+- extract a first, last, indexed or time-selected frame as PNG;
+- project canonical 16 kHz mono speech-evidence audio;
+- render an exact sample-domain audio program;
+- mux rendered visual and audio products.
+
+Local and AWS Lambda Providers call these same functions with different Artifact I/O and FFmpeg
+launch environments. They therefore share stream selection, timing, codec and validation behavior
+instead of reimplementing media semantics per deployment.
+
+This is not an author package, Provider, queue or Core extension. It performs no endpoint selection,
+credential lookup or SVML parsing.

package/packages/media-execution/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@hypit/media-execution",
+  "version": "0.0.0-dev",
+  "license": "SEE LICENSE IN LICENSE",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "dependencies": {
+    "@hypit/media": "workspace:*",
+    "@hypit/media-pipeline": "workspace:*",
+    "@hypit/program-space": "workspace:*",
+    "@hypit/protocol": "workspace:*",
+    "@hypit/speech": "workspace:*"
+  }
+}