@pipedog/core 0.0.0 → 0.1.0

package/dist/classifiers/base_classifier.js

@@ -0,0 +1,346 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/classifiers/base_classifier.py
+/**
+ * Classifiers answer typed questions about some state.
+ *
+ * A classifier is a plain object. Whoever needs answers creates one, keeps
+ * it, and calls it. It does not sit in a pipeline and no frames flow into it.
+ * A question is one of `YesNoQuestion`, `ChoiceQuestion` or `ScoreQuestion`.
+ * Questions are asked by name, several about one state at once:
+ * `BaseClassifier.ask()` takes any mix of kinds, and
+ * `BaseClassifier.yesNo()`, `BaseClassifier.choice()` and
+ * `BaseClassifier.score()` take questions of one kind and return typed
+ * results. Classifiers that can see images take `ClassifierImage` alongside
+ * the state.
+ */
+import { monotonic } from "@pipedog/asyncio";
+import { LLMUsageMetricsData, ProcessingMetricsData, } from "../metrics/metrics.js";
+import { BaseObject } from "../utils/base_object.js";
+import { encodeImage } from "../utils/image.js";
+/** A classifier could not answer a question. */
+export class ClassifierError extends Error {
+    name = "ClassifierError";
+}
+/** Whether the state meets a condition. */
+export class YesNoQuestion {
+    instructions;
+    yes;
+    no;
+    constructor(options) {
+        this.instructions = options.instructions;
+        this.yes = options.yes ?? null;
+        this.no = options.no ?? null;
+    }
+}
+/** Which of several options fits the state. */
+export class ChoiceQuestion {
+    instructions;
+    options;
+    constructor(options) {
+        this.instructions = options.instructions;
+        this.options = options.options;
+    }
+}
+/** Where the state falls on an ordered scale. */
+export class ScoreQuestion {
+    instructions;
+    levels;
+    constructor(options) {
+        if (options.levels.length < 2) {
+            throw new Error("ScoreQuestion levels should have at least 2 items");
+        }
+        this.instructions = options.instructions;
+        this.levels = options.levels;
+    }
+}
+/** An encoded image the questions can look at, alongside the state. */
+export class ClassifierImage {
+    data;
+    contentType;
+    constructor(options) {
+        const contentType = options.contentType;
+        // Upstream's pydantic Literal rejects any other given type.
+        if (contentType != null &&
+            contentType !== "image/png" &&
+            contentType !== "image/jpeg" &&
+            contentType !== "image/webp") {
+            throw new Error(`ClassifierImage takes PNG, JPEG or WebP, got ${contentType}`);
+        }
+        this.data = options.data;
+        this.contentType = contentType ?? ClassifierImage._contentType(options.data);
+    }
+    /**
+     * Build an image from an image frame, such as a frame from the user's camera.
+     *
+     * Raw pixels are encoded as a JPEG. A frame whose format is already a MIME
+     * type keeps its bytes.
+     *
+     * @param frame The image frame.
+     * @returns The image.
+     */
+    static async fromFrame(frame) {
+        const [data, contentType] = await encodeImage(frame.image, frame.size, frame.format);
+        return new ClassifierImage({ data, contentType: contentType });
+    }
+    /**
+     * The MIME type of the encoded image, or an error if it is not one
+     * classifiers take.
+     *
+     * PORTING NOTE: upstream opens the image with Pillow; this reads the
+     * format's signature bytes instead.
+     */
+    static _contentType(data) {
+        const contentType = sniffImageType(data);
+        if (contentType === null) {
+            throw new Error("ClassifierImage data is not an image");
+        }
+        if (contentType !== "image/png" &&
+            contentType !== "image/jpeg" &&
+            contentType !== "image/webp") {
+            throw new Error(`ClassifierImage takes PNG, JPEG or WebP, got ${contentType}`);
+        }
+        return contentType;
+    }
+    toString() {
+        return `ClassifierImage(content_type='${this.contentType}')`;
+    }
+}
+function startsWith(data, signature, offset = 0) {
+    if (data.length < offset + signature.length) {
+        return false;
+    }
+    return signature.every((b, i) => data[offset + i] === b);
+}
+const ascii = (s) => [...s].map((c) => c.charCodeAt(0));
+/** The MIME type of an encoded image read from its signature, or null if unknown. */
+function sniffImageType(data) {
+    if (startsWith(data, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))
+        return "image/png";
+    if (startsWith(data, [0xff, 0xd8, 0xff]))
+        return "image/jpeg";
+    if (startsWith(data, ascii("RIFF")) && startsWith(data, ascii("WEBP"), 8))
+        return "image/webp";
+    if (startsWith(data, ascii("GIF87a")) || startsWith(data, ascii("GIF89a")))
+        return "image/gif";
+    if (startsWith(data, ascii("BM")))
+        return "image/bmp";
+    if (startsWith(data, [0x49, 0x49, 0x2a, 0x00]) || startsWith(data, [0x4d, 0x4d, 0x00, 0x2a])) {
+        return "image/tiff";
+    }
+    if (startsWith(data, [0x00, 0x00, 0x01, 0x00]))
+        return "image/x-icon";
+    return null;
+}
+/** Answer to a `YesNoQuestion`. */
+export class YesNoResult {
+    probability;
+    constructor(options) {
+        this.probability = options.probability;
+    }
+    /**
+     * Whether yes is the likelier answer.
+     *
+     * Callers that need more certainty than that compare `probability` with a
+     * threshold of their own.
+     */
+    get isYes() {
+        return this.probability >= 0.5;
+    }
+}
+/** Answer to a `ChoiceQuestion`. */
+export class ChoiceResult {
+    choice;
+    probabilities;
+    confidence;
+    constructor(options) {
+        this.choice = options.choice;
+        this.probabilities = options.probabilities;
+        this.confidence = options.confidence;
+    }
+}
+/** One level of a `ScoreQuestion`'s scale and how likely it is. */
+export class ScoreLevel {
+    level;
+    probability;
+    constructor(options) {
+        this.level = options.level;
+        this.probability = options.probability;
+    }
+}
+/** Answer to a `ScoreQuestion`. */
+export class ScoreResult {
+    score;
+    levels;
+    confidence;
+    constructor(options) {
+        this.score = options.score;
+        this.levels = options.levels;
+        this.confidence = options.confidence;
+    }
+    /**
+     * How likely one level is.
+     *
+     * @param level The level as the question gave it.
+     * @returns The level's probability.
+     * @throws Error If the scale has no such level (upstream's KeyError).
+     */
+    probability(level) {
+        for (const item of this.levels) {
+            if (deepEqual(item.level, level)) {
+                return item.probability;
+            }
+        }
+        throw new Error(`not a level of this scale: ${JSON.stringify(level)}`);
+    }
+}
+/** Python `==` on JSON-like values: dicts compare regardless of key order. */
+function deepEqual(a, b) {
+    if (a === b) {
+        return true;
+    }
+    if (Array.isArray(a) || Array.isArray(b)) {
+        return (Array.isArray(a) &&
+            Array.isArray(b) &&
+            a.length === b.length &&
+            a.every((v, i) => deepEqual(v, b[i])));
+    }
+    if (typeof a === "object" && typeof b === "object" && a !== null && b !== null) {
+        const ka = Object.keys(a);
+        const kb = Object.keys(b);
+        return (ka.length === kb.length &&
+            ka.every((k) => Object.hasOwn(b, k) &&
+                deepEqual(a[k], b[k])));
+    }
+    return false;
+}
+/**
+ * Answers typed questions about a state.
+ *
+ * Every question is about a `state`: plain text, or structured data such as
+ * a transcript with speaker labels or a trimmed screen snapshot. `ask()`
+ * answers any number of questions about one state, by name; the three typed
+ * methods are built on it and take questions of one kind. Subclasses
+ * implement `_ask()`.
+ *
+ * Classifiers that can see images say so in `supportsImages`, and then take
+ * `images` with the state. The others refuse them.
+ *
+ * An owner that runs inside a worker calls `setup()` with its task manager
+ * before the first question, and `cleanup()` when it is done.
+ *
+ * Event handlers available:
+ *
+ * - on_metrics: Called after every call with its metrics, the time it took
+ *   and, when the classifier knows it, the tokens it used. A classifier
+ *   cannot push frames, so the owner is the one to put them in a
+ *   `MetricsFrame`.
+ *
+ * @example
+ * await this._classifier.setup(this.taskManager);
+ *
+ * this._classifier.eventHandler("on_metrics", async (classifier, data: MetricsData[]) => {
+ *   await this.pushFrame(new MetricsFrame({ data }));
+ * });
+ */
+export class BaseClassifier extends BaseObject {
+    constructor(options = {}) {
+        super(options);
+        this._registerEventHandler("on_metrics");
+    }
+    /** The model that answers, named in the metrics. */
+    get model() {
+        return null;
+    }
+    /** Whether questions can come with images. */
+    get supportsImages() {
+        return false;
+    }
+    /**
+     * Answer several questions about one state.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns One result per question, by the same names, each of the type
+     *   its question calls for.
+     * @throws ClassifierError If the answers could not be produced, or not in
+     *   time: every classifier answers or raises within a bound of its own, so
+     *   a caller waiting on it is never left hanging. Also if images come to a
+     *   classifier that cannot see them.
+     */
+    async ask(state, questions, options = {}) {
+        const images = options.images;
+        if (images && images.length > 0 && !this.supportsImages) {
+            throw new ClassifierError(`${this} cannot see images`);
+        }
+        const started = monotonic();
+        const [results, usage] = await this._ask(state, questions, [...(images ?? [])]);
+        await this._callEventHandler("on_metrics", this._metrics(monotonic() - started, usage));
+        return results;
+    }
+    /**
+     * Ask whether the state meets each condition.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns How likely each answer is yes, by the same names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    async yesNo(state, questions, options = {}) {
+        return this._typed(await this.ask(state, questions, options), YesNoResult);
+    }
+    /**
+     * Ask which option fits the state, for each question.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns The option that fits and how likely each one is, by the same
+     *   names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    async choice(state, questions, options = {}) {
+        return this._typed(await this.ask(state, questions, options), ChoiceResult);
+    }
+    /**
+     * Ask where the state falls on each scale.
+     *
+     * @param state What the questions are about.
+     * @param questions The questions, by name.
+     * @param options Images the questions can look at, for a classifier that
+     *   supports them.
+     * @returns The position on each scale and how likely each level is, by
+     *   the same names.
+     * @throws ClassifierError If the answers could not be produced.
+     */
+    async score(state, questions, options = {}) {
+        return this._typed(await this.ask(state, questions, options), ScoreResult);
+    }
+    _metrics(seconds, usage) {
+        const data = [
+            new ProcessingMetricsData({ processor: this.name, model: this.model, value: seconds }),
+        ];
+        if (usage) {
+            data.push(new LLMUsageMetricsData({ processor: this.name, model: this.model, value: usage }));
+        }
+        return data;
+    }
+    _typed(results, resultType) {
+        const typed = {};
+        for (const [name, result] of Object.entries(results)) {
+            if (!(result instanceof resultType)) {
+                throw new ClassifierError(`expected a ${resultType.name} for '${name}', got ${result.constructor.name}`);
+            }
+            typed[name] = result;
+        }
+        return typed;
+    }
+}
+//# sourceMappingURL=base_classifier.js.map

package/dist/classifiers/cloudflare/clef/classifier.d.ts

@@ -0,0 +1,70 @@
+import { LLMTokenUsage } from "../../../metrics/metrics.ts";
+import type { BaseTaskManager } from "../../../utils/asyncio/task_manager.ts";
+import { BaseClassifier, type BaseClassifierOptions, type ClassifierImage, type ClassifierQuestion, type ClassifierResult, type ClassifierValue } from "../../base_classifier.ts";
+import { ClefClient } from "./client.ts";
+/** The most options Clef takes in one choice question. */
+export declare const CLEF_MAX_CHOICE_OPTIONS = 255;
+/** The most images Clef takes in one request. */
+export declare const CLEF_MAX_IMAGES = 4;
+export interface ClefClassifierOptions extends BaseClassifierOptions {
+    /** Cloudflare account ID, when the classifier should have a client of its own. */
+    accountId?: string | null;
+    /** Cloudflare API token, when the classifier should have a client of its own. */
+    apiKey?: string | null;
+    /** A client to share. Either `client`, or both `accountId` and `apiKey`, are required. */
+    client?: ClefClient | null;
+    /** The Clef model a client of its own asks: `clef`, or `clef-flash` for faster answers. */
+    model?: string;
+    /** Seconds a client of its own waits for an answer. */
+    timeout?: number;
+}
+/**
+ * Answers questions by asking Clef.
+ *
+ * Build one with a Cloudflare account ID and API token to get a client of
+ * its own, or pass a `ClefClient` to share one between several classifiers.
+ * A client the classifier created is closed in `cleanup()`; a shared one is
+ * left to whoever made it.
+ *
+ * Clef can see images: up to `CLEF_MAX_IMAGES` per call, 4 MiB each and
+ * 8 MiB in all.
+ *
+ * @example
+ * const classifier = new ClefClassifier({
+ *   accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
+ *   apiKey: process.env.CLOUDFLARE_API_KEY,
+ * });
+ */
+export declare class ClefClassifier extends BaseClassifier {
+    protected _ownsClient: boolean;
+    protected _client: ClefClient;
+    constructor(options?: ClefClassifierOptions);
+    /** The client this classifier asks through. */
+    get client(): ClefClient;
+    /**
+     * Open the connection to Workers AI ahead of the first question.
+     *
+     * A connection that cannot be opened now is only a warning: the first
+     * question opens it itself.
+     *
+     * @param taskManager The task manager of the owner.
+     */
+    setup(taskManager: BaseTaskManager): Promise<void>;
+    /** Close the client if this classifier created it. */
+    cleanup(): Promise<void>;
+    /** The Clef model the questions go to. */
+    get model(): string;
+    /** Clef can see images. */
+    get supportsImages(): boolean;
+    /** Answer the questions in one request. */
+    protected _ask(state: ClassifierValue, questions: Readonly<Record<string, ClassifierQuestion>>, images?: readonly ClassifierImage[]): Promise<[Record<string, ClassifierResult>, LLMTokenUsage]>;
+    /** A question in Clef's own format. */
+    protected _toClef(question: ClassifierQuestion): Record<string, unknown>;
+    /** A result built from Clef's answer to the question. */
+    protected _fromClef(question: ClassifierQuestion, answer: Record<string, unknown>): ClassifierResult;
+    /** The number Clef wrote under `key`, or a ClassifierError if it did not. */
+    protected _number(answer: Record<string, unknown>, key: string): number;
+    /** The probability Clef wrote for each key, 0 for the ones it left out. */
+    protected _probabilities(answer: Record<string, unknown>, keys: string[]): Record<string, number>;
+}
+//# sourceMappingURL=classifier.d.ts.map

package/dist/classifiers/cloudflare/clef/classifier.js

@@ -0,0 +1,204 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/classifiers/cloudflare/clef/classifier.py
+/**
+ * Classifier backed by Clef, Cloudflare's decision model on Workers AI.
+ *
+ * `ClefClassifier` turns each question into a request and each reply into a
+ * result, through a `ClefClient`.
+ */
+import { rethrowIfCancelled } from "@pipedog/asyncio";
+import { LLMTokenUsage } from "../../../metrics/metrics.js";
+import { logger } from "../../../utils/logger.js";
+import { BaseClassifier, ChoiceQuestion, ChoiceResult, ClassifierError, ScoreLevel, ScoreResult, YesNoQuestion, YesNoResult, } from "../../base_classifier.js";
+import { bytesToBase64, isDict, orEmpty, toFloat } from "../../http_client.js";
+import { ClefClient, DEFAULT_MODEL, DEFAULT_TIMEOUT } from "./client.js";
+/** The most options Clef takes in one choice question. */
+export const CLEF_MAX_CHOICE_OPTIONS = 255;
+/** The most images Clef takes in one request. */
+export const CLEF_MAX_IMAGES = 4;
+/**
+ * Answers questions by asking Clef.
+ *
+ * Build one with a Cloudflare account ID and API token to get a client of
+ * its own, or pass a `ClefClient` to share one between several classifiers.
+ * A client the classifier created is closed in `cleanup()`; a shared one is
+ * left to whoever made it.
+ *
+ * Clef can see images: up to `CLEF_MAX_IMAGES` per call, 4 MiB each and
+ * 8 MiB in all.
+ *
+ * @example
+ * const classifier = new ClefClassifier({
+ *   accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
+ *   apiKey: process.env.CLOUDFLARE_API_KEY,
+ * });
+ */
+export class ClefClassifier extends BaseClassifier {
+    _ownsClient;
+    _client;
+    constructor(options = {}) {
+        super(options);
+        this._ownsClient = options.client == null;
+        let client = options.client ?? null;
+        if (!client) {
+            if (!options.accountId || !options.apiKey) {
+                throw new Error("ClefClassifier needs an account ID and API key, or a ClefClient");
+            }
+            client = new ClefClient({
+                accountId: options.accountId,
+                apiKey: options.apiKey,
+                model: options.model ?? DEFAULT_MODEL,
+                timeout: options.timeout ?? DEFAULT_TIMEOUT,
+            });
+        }
+        this._client = client;
+    }
+    /** The client this classifier asks through. */
+    get client() {
+        return this._client;
+    }
+    /**
+     * Open the connection to Workers AI ahead of the first question.
+     *
+     * A connection that cannot be opened now is only a warning: the first
+     * question opens it itself.
+     *
+     * @param taskManager The task manager of the owner.
+     */
+    async setup(taskManager) {
+        await super.setup(taskManager);
+        try {
+            await this._client.connect();
+        }
+        catch (e) {
+            if (!(e instanceof ClassifierError)) {
+                throw e;
+            }
+            logger.warning(`${this}: ${e.message}`);
+        }
+    }
+    /** Close the client if this classifier created it. */
+    async cleanup() {
+        await super.cleanup();
+        if (this._ownsClient) {
+            await this._client.close();
+        }
+    }
+    /** The Clef model the questions go to. */
+    get model() {
+        return this._client.model;
+    }
+    /** Clef can see images. */
+    get supportsImages() {
+        return true;
+    }
+    /** Answer the questions in one request. */
+    async _ask(state, questions, images = []) {
+        if (images.length > CLEF_MAX_IMAGES) {
+            throw new ClassifierError(`Clef takes at most ${CLEF_MAX_IMAGES} images, got ${images.length}`);
+        }
+        const clefQuestions = {};
+        for (const [name, q] of Object.entries(questions)) {
+            clefQuestions[name] = this._toClef(q);
+        }
+        const [answers, usage] = await this._client.ask(state, clefQuestions, images.map((image) => ({
+            content_type: image.contentType,
+            base64: bytesToBase64(image.data),
+        })));
+        const results = {};
+        for (const [name, question] of Object.entries(questions)) {
+            results[name] = this._fromClef(question, answers[name]);
+        }
+        return [
+            results,
+            new LLMTokenUsage({
+                promptTokens: usage.inputTokens,
+                completionTokens: usage.outputTokens,
+                totalTokens: usage.inputTokens + usage.outputTokens,
+            }),
+        ];
+    }
+    /** A question in Clef's own format. */
+    _toClef(question) {
+        if (question instanceof YesNoQuestion) {
+            const clef = { type: "noul", instructions: question.instructions };
+            if (question.yes !== null || question.no !== null) {
+                clef.criteria = { true: orEmpty(question.yes), false: orEmpty(question.no) };
+            }
+            return clef;
+        }
+        if (question instanceof ChoiceQuestion) {
+            const count = Object.keys(question.options).length;
+            if (count > CLEF_MAX_CHOICE_OPTIONS) {
+                throw new ClassifierError(`Clef takes at most ${CLEF_MAX_CHOICE_OPTIONS} options, got ${count}`);
+            }
+            return {
+                type: "choice",
+                instructions: question.instructions,
+                criteria: question.options,
+            };
+        }
+        return { type: "score", instructions: question.instructions, criteria: question.levels };
+    }
+    /** A result built from Clef's answer to the question. */
+    _fromClef(question, answer) {
+        if (question instanceof YesNoQuestion) {
+            return new YesNoResult({ probability: this._number(answer, "noul") });
+        }
+        if (question instanceof ChoiceQuestion) {
+            const choice = String(answer.choice ?? "");
+            if (!Object.hasOwn(question.options, choice)) {
+                throw new ClassifierError(`Clef chose '${choice}', which is not an option`);
+            }
+            const probabilities = this._probabilities(answer, Object.keys(question.options));
+            return new ChoiceResult({
+                choice,
+                probabilities,
+                confidence: this._number(answer, "confidence"),
+            });
+        }
+        // Clef keys level probabilities by position.
+        const positions = question.levels.map((_level, index) => String(index));
+        const probabilities = this._probabilities(answer, positions);
+        return new ScoreResult({
+            score: this._number(answer, "score"),
+            levels: question.levels.map((level, index) => new ScoreLevel({ level, probability: probabilities[positions[index]] ?? 0 })),
+            confidence: this._number(answer, "confidence"),
+        });
+    }
+    /** The number Clef wrote under `key`, or a ClassifierError if it did not. */
+    _number(answer, key) {
+        try {
+            if (!Object.hasOwn(answer, key)) {
+                throw new RangeError(key);
+            }
+            return toFloat(answer[key]);
+        }
+        catch (e) {
+            rethrowIfCancelled(e);
+            throw new ClassifierError(`Clef reply has no usable '${key}'`, { cause: e });
+        }
+    }
+    /** The probability Clef wrote for each key, 0 for the ones it left out. */
+    _probabilities(answer, keys) {
+        // Python's `answer.get("probabilities") or {}`: an empty list is falsy too.
+        const raw = answer.probabilities;
+        const given = !raw || (Array.isArray(raw) && raw.length === 0) ? {} : raw;
+        try {
+            if (!isDict(given)) {
+                throw new TypeError("probabilities is not an object");
+            }
+            return Object.fromEntries(keys.map((key) => [key, toFloat(Object.hasOwn(given, key) ? given[key] : 0.0)]));
+        }
+        catch (e) {
+            rethrowIfCancelled(e);
+            throw new ClassifierError(`Clef reply has an unusable probability: ${e.message}`, {
+                cause: e,
+            });
+        }
+    }
+}
+//# sourceMappingURL=classifier.js.map