@pipedog/core 0.0.0 → 0.2.0

package/dist/bus/ui/messages.js

@@ -0,0 +1,171 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/bus/ui/messages.py
+/**
+ * Bus carriers for the UI Worker protocol.
+ *
+ * These classes are the on-the-bus shape that a UI worker (see `workers/ui`)
+ * and the client-facing worker exchange. They are NOT the on-the-wire format
+ * the client sees: the client-facing worker translates between the two, in
+ * whichever format its own client speaks.
+ *
+ * For example, `PipelineWorker` with RTVI enabled republishes inbound client
+ * messages onto the bus in `onUiMessage` and turns outbound carriers into
+ * RTVI frames in `onBusMessage`; the wire types it produces live in
+ * `processors/frameworks/rtvi/models` (`UIEventMessage`, `UICommandMessage`,
+ * `UIJobGroupMessage`, ...).
+ *
+ * All carriers subclass `BusUIDataMessage`, which a client-facing worker
+ * dispatches on to pick the outbound ones out of its bus traffic.
+ *
+ * - `BusUIEventMessage` and `BusUICommandMessage` carry client events and
+ *   server commands respectively.
+ * - `BusUIJobGroupStartedMessage`, `BusUIJobUpdateMessage`,
+ *   `BusUIJobCompletedMessage`, and `BusUIJobGroupCompletedMessage` carry the
+ *   four phases of a user-facing job group's lifecycle (see `UIWorker`).
+ *
+ * The carriers live in the `bus` layer (rather than alongside `UIWorker`)
+ * because both `PipelineWorker` (in `pipeline`) and `UIWorker` (in
+ * `workers`) reference them, and `pipeline` must not import from `workers`.
+ */
+import { BusDataMessage } from "../messages.js";
+/**
+ * The `eventName` of the client's accessibility snapshot on the bus. A
+ * client-facing worker republishes a `ui-snapshot` wire message as a
+ * `BusUIEventMessage` with this name, and `UIWorker` keeps the payload as its
+ * latest snapshot. The leading double underscore keeps app-defined
+ * `@ui_event` names from colliding with it.
+ */
+export const UI_SNAPSHOT_EVENT_NAME = "__ui_snapshot";
+/**
+ * The `eventName` of the client's request to cancel a job group. A
+ * client-facing worker republishes a `ui-cancel-job-group` wire message as a
+ * `BusUIEventMessage` with this name, and the worker that dispatched the
+ * group turns it into a cancel request.
+ */
+export const UI_CANCEL_JOB_GROUP_EVENT_NAME = "__cancel_job_group";
+/**
+ * Base for all UI Worker protocol bus carriers.
+ *
+ * A client-facing worker dispatches on this type to translate a worker's
+ * outbound UI carriers into its client's wire format, so every UI bus message
+ * below subclasses it.
+ */
+export class BusUIDataMessage extends BusDataMessage {
+}
+/**
+ * A UI event sent from the client to a server-side worker.
+ *
+ * Emitted by the client-facing worker when the client dispatches an event via
+ * `PipecatClient.sendUIEvent(event, payload)`. `UIWorker` subclasses dispatch
+ * these to `@ui_event(name)` handlers.
+ */
+export class BusUIEventMessage extends BusUIDataMessage {
+    eventName;
+    payload;
+    constructor(options) {
+        super(options);
+        this.eventName = options.eventName ?? "";
+        this.payload = options.payload ?? null;
+    }
+}
+/**
+ * A UI command sent from a server-side worker to the client.
+ *
+ * Published by `UIWorker.sendCommand(name, payload)`. The client-facing
+ * worker translates it into a command on its client's wire format; over RTVI
+ * that is an `RTVIUICommandFrame({ command: commandName, payload })` pushed
+ * through the pipeline.
+ */
+export class BusUICommandMessage extends BusUIDataMessage {
+    commandName;
+    payload;
+    constructor(options) {
+        super(options);
+        this.commandName = options.commandName ?? "";
+        this.payload = options.payload ?? null;
+    }
+}
+/**
+ * A user-facing job group has been dispatched.
+ *
+ * Published by a `UIWorker` as it dispatches the group. The client-facing
+ * worker forwards it as a `ui-job-group` envelope with
+ * `kind = "group_started"`.
+ */
+export class BusUIJobGroupStartedMessage extends BusUIDataMessage {
+    jobId;
+    workers;
+    label;
+    cancellable;
+    at;
+    constructor(options) {
+        super(options);
+        this.jobId = options.jobId ?? "";
+        this.workers = options.workers ?? null;
+        this.label = options.label ?? null;
+        this.cancellable = options.cancellable ?? true;
+        this.at = options.at ?? 0;
+    }
+}
+/**
+ * Per-worker progress for a user-facing job group.
+ *
+ * Forwarded by a `UIWorker` whenever a worker of one of its job groups emits
+ * a `BusJobUpdateMessage`. The client-facing worker forwards it as a
+ * `ui-job-group` envelope with `kind = "job_update"`.
+ */
+export class BusUIJobUpdateMessage extends BusUIDataMessage {
+    jobId;
+    workerName;
+    data;
+    at;
+    constructor(options) {
+        super(options);
+        this.jobId = options.jobId ?? "";
+        this.workerName = options.workerName ?? "";
+        this.data = options.data ?? null;
+        this.at = options.at ?? 0;
+    }
+}
+/**
+ * A worker in a user-facing job group has completed.
+ *
+ * Forwarded by a `UIWorker` when a worker of one of its job groups reaches a
+ * terminal state. The client-facing worker forwards it as a `ui-job-group`
+ * envelope with `kind = "job_completed"`.
+ */
+export class BusUIJobCompletedMessage extends BusUIDataMessage {
+    jobId;
+    workerName;
+    status;
+    response;
+    at;
+    constructor(options) {
+        super(options);
+        this.jobId = options.jobId ?? "";
+        this.workerName = options.workerName ?? "";
+        this.status = options.status ?? "";
+        this.response = options.response ?? null;
+        this.at = options.at ?? 0;
+    }
+}
+/**
+ * A user-facing job group has completed.
+ *
+ * Published by a `UIWorker` once every worker in the group has finished, or
+ * the group was cancelled. The client-facing worker forwards it as a
+ * `ui-job-group` envelope with `kind = "group_completed"`.
+ */
+export class BusUIJobGroupCompletedMessage extends BusUIDataMessage {
+    jobId;
+    at;
+    constructor(options) {
+        super(options);
+        this.jobId = options.jobId ?? "";
+        this.at = options.at ?? 0;
+    }
+}
+//# sourceMappingURL=messages.js.map

package/dist/classifiers/base_classifier.d.ts

@@ -0,0 +1,263 @@
+import type { ImageRawFrame } from "../frames/frames.ts";
+import { type LLMTokenUsage, type MetricsData } from "../metrics/metrics.ts";
+import { BaseObject, type BaseObjectOptions } from "../utils/base_object.ts";
+/** A classifier could not answer a question. */
+export declare class ClassifierError extends Error {
+    name: string;
+}
+/** What a question, or the state it is about, can be: text or structured data. */
+export type ClassifierValue = string | Record<string, unknown> | unknown[];
+export interface YesNoQuestionOptions {
+    /**
+     * What is being checked for, as a yes or no question. Text, or structured
+     * data holding the question in one field and what it refers to in others.
+     */
+    instructions: ClassifierValue;
+    /** What counts as a yes, when the question alone leaves it open. */
+    yes?: ClassifierValue | null;
+    /** What counts as a no. */
+    no?: ClassifierValue | null;
+}
+/** Whether the state meets a condition. */
+export declare class YesNoQuestion {
+    instructions: ClassifierValue;
+    yes: ClassifierValue | null;
+    no: ClassifierValue | null;
+    constructor(options: YesNoQuestionOptions);
+}
+export interface ChoiceQuestionOptions {
+    /** What is being decided. */
+    instructions: ClassifierValue;
+    /**
+     * The options to choose from, each mapped to a description of when it
+     * applies, or `null` when the option itself says enough.
+     */
+    options: Record<string, ClassifierValue | null>;
+}
+/** Which of several options fits the state. */
+export declare class ChoiceQuestion {
+    instructions: ClassifierValue;
+    options: Record<string, ClassifierValue | null>;
+    constructor(options: ChoiceQuestionOptions);
+}
+export interface ScoreQuestionOptions {
+    /** What is being rated. */
+    instructions: ClassifierValue;
+    /**
+     * The levels of the scale in order, lowest first, each described in a few
+     * words or as structured data. At least two.
+     */
+    levels: ClassifierValue[];
+}
+/** Where the state falls on an ordered scale. */
+export declare class ScoreQuestion {
+    instructions: ClassifierValue;
+    levels: ClassifierValue[];
+    constructor(options: ScoreQuestionOptions);
+}
+export type ClassifierQuestion = YesNoQuestion | ChoiceQuestion | ScoreQuestion;
+/** The image types every classifier that sees images accepts. */
+export type ClassifierImageType = "image/png" | "image/jpeg" | "image/webp";
+export interface ClassifierImageOptions {
+    /** The encoded image: PNG, JPEG or WebP. */
+    data: Uint8Array;
+    /** The image's MIME type, read from `data` when not given. */
+    contentType?: ClassifierImageType;
+}
+/** An encoded image the questions can look at, alongside the state. */
+export declare class ClassifierImage {
+    data: Uint8Array;
+    contentType: ClassifierImageType;
+    constructor(options: ClassifierImageOptions);
+    /**
+     * Build an image from an image frame, such as a frame from the user's camera.
+     *
+     * Raw pixels are encoded as a JPEG. A frame whose format is already a MIME
+     * type keeps its bytes.
+     *
+     * @param frame The image frame.
+     * @returns The image.
+     */
+    static fromFrame(frame: ImageRawFrame): Promise<ClassifierImage>;
+    /**
+     * The MIME type of the encoded image, or an error if it is not one
+     * classifiers take.
+     *
+     * PORTING NOTE: upstream opens the image with Pillow; this reads the
+     * format's signature bytes instead.
+     */
+    protected static _contentType(data: Uint8Array): ClassifierImageType;
+    toString(): string;
+}
+export interface YesNoResultOptions {
+    /** How likely the answer is yes, from 0 to 1. */
+    probability: number;
+}
+/** Answer to a `YesNoQuestion`. */
+export declare class YesNoResult {
+    probability: number;
+    constructor(options: YesNoResultOptions);
+    /**
+     * Whether yes is the likelier answer.
+     *
+     * Callers that need more certainty than that compare `probability` with a
+     * threshold of their own.
+     */
+    get isYes(): boolean;
+}
+export interface ChoiceResultOptions {
+    /** The option that fits best. */
+    choice: string;
+    /** How likely each option is, keyed by option. */
+    probabilities: Record<string, number>;
+    /** How sure the classifier is of `choice`, from 0 to 1. */
+    confidence: number;
+}
+/** Answer to a `ChoiceQuestion`. */
+export declare class ChoiceResult {
+    choice: string;
+    probabilities: Record<string, number>;
+    confidence: number;
+    constructor(options: ChoiceResultOptions);
+}
+export interface ScoreLevelOptions {
+    /** The level as the question gave it. */
+    level: ClassifierValue;
+    /** How likely the state is at this level, from 0 to 1. */
+    probability: number;
+}
+/** One level of a `ScoreQuestion`'s scale and how likely it is. */
+export declare class ScoreLevel {
+    level: ClassifierValue;
+    probability: number;
+    constructor(options: ScoreLevelOptions);
+}
+export interface ScoreResultOptions {
+    /**
+     * Where the state falls on the scale, as a position from 0 (the first
+     * level) to one less than the number of levels. It is the
+     * probability-weighted position, so it may fall between two levels.
+     */
+    score: number;
+    /** How likely each level is, in the question's order. */
+    levels: ScoreLevel[];
+    /** How sure the classifier is of `score`, from 0 to 1. */
+    confidence: number;
+}
+/** Answer to a `ScoreQuestion`. */
+export declare class ScoreResult {
+    score: number;
+    levels: ScoreLevel[];
+    confidence: number;
+    constructor(options: ScoreResultOptions);
+    /**
+     * How likely one level is.
+     *
+     * @param level The level as the question gave it.
+     * @returns The level's probability.
+     * @throws Error If the scale has no such level (upstream's KeyError).
+     */
+    probability(level: ClassifierValue): number;
+}
+export type ClassifierResult = YesNoResult | ChoiceResult | ScoreResult;
+/** Options for the questions-asking methods. */
+export interface ClassifierAskOptions {
+    /** Images the questions can look at, for a classifier that supports them. */
+    images?: readonly ClassifierImage[] | null;
+}
+export type BaseClassifierOptions = BaseObjectOptions;
+/**
+ * Answers typed questions about a state.
+ *
+ * Every question is about a `state`: plain text, or structured data such as
+ * a transcript with speaker labels or a trimmed screen snapshot. `ask()`
+ * answers any number of questions about one state, by name; the three typed
+ * methods are built on it and take questions of one kind. Subclasses
+ * implement `_ask()`.
+ *
+ * Classifiers that can see images say so in `supportsImages`, and then take
+ * `images` with the state. The others refuse them.
+ *
+ * An owner that runs inside a worker calls `setup()` with its task manager
+ * before the first question, and `cleanup()` when it is done.
+ *
+ * Event handlers available:
+ *
+ * - on_metrics: Called after every call with its metrics, the time it took
+ *   and, when the classifier knows it, the tokens it used. A classifier
+ *   cannot push frames, so the owner is the one to put them in a
+ *   `MetricsFrame`.
+ *
+ * @example
+ * await this._classifier.setup(this.taskManager);
+ *
+ * this._classifier.eventHandler("on_metrics", async (classifier, data: MetricsData[]) => {
+ *   await this.pushFrame(new MetricsFrame({ data }));
+ * });
+ */
+export declare abstract class BaseClassifier extends BaseObject {
+    constructor(options?: BaseClassifierOptions);
+    /** The model that answers, named in the metrics. */
+    get model(): string | null;
+    /** Whether questions can come with images. */
+    get supportsImages(): boolean;
+    /**
+     * Answer several questions about one state.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns One result per question, by the same names, each of the type
+     *   its question calls for.
+     * @throws ClassifierError If the answers could not be produced, or not in
+     *   time: every classifier answers or raises within a bound of its own, so
+     *   a caller waiting on it is never left hanging. Also if images come to a
+     *   classifier that cannot see them.
+     */
+    ask(state: ClassifierValue, questions: Readonly<Record<string, ClassifierQuestion>>, options?: ClassifierAskOptions): Promise<Record<string, ClassifierResult>>;
+    /**
+     * Ask whether the state meets each condition.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns How likely each answer is yes, by the same names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    yesNo(state: ClassifierValue, questions: Readonly<Record<string, YesNoQuestion>>, options?: ClassifierAskOptions): Promise<Record<string, YesNoResult>>;
+    /**
+     * Ask which option fits the state, for each question.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns The option that fits and how likely each one is, by the same
+     *   names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    choice(state: ClassifierValue, questions: Readonly<Record<string, ChoiceQuestion>>, options?: ClassifierAskOptions): Promise<Record<string, ChoiceResult>>;
+    /**
+     * Ask where the state falls on each scale.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns The position on each scale and how likely each level is, by
+     *   the same names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    score(state: ClassifierValue, questions: Readonly<Record<string, ScoreQuestion>>, options?: ClassifierAskOptions): Promise<Record<string, ScoreResult>>;
+    /**
+     * Answer the questions, and say what tokens the call used if that is known.
+     *
+     * `images` is always empty for a classifier that does not support images.
+     */
+    protected abstract _ask(state: ClassifierValue, questions: Readonly<Record<string, ClassifierQuestion>>, images?: readonly ClassifierImage[]): Promise<[Record<string, ClassifierResult>, LLMTokenUsage | null]>;
+    protected _metrics(seconds: number, usage: LLMTokenUsage | null): MetricsData[];
+    protected _typed<R extends ClassifierResult>(results: Record<string, ClassifierResult>, resultType: new (...args: never[]) => R): Record<string, R>;
+}
+//# sourceMappingURL=base_classifier.d.ts.map