@poolzin/pool-bot 2026.2.11 → 2026.2.18

package/dist/media-understanding/runner.entries.js

@@ -0,0 +1,459 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
+import { applyTemplate } from "../auto-reply/templating.js";
+import { logVerbose, shouldLogVerbose } from "../globals.js";
+import { runExec } from "../process/exec.js";
+import { CLI_OUTPUT_MAX_BUFFER, DEFAULT_AUDIO_MODELS, DEFAULT_TIMEOUT_SECONDS, } from "./defaults.js";
+import { MediaUnderstandingSkipError } from "./errors.js";
+import { fileExists } from "./fs.js";
+import { extractGeminiResponse } from "./output-extract.js";
+import { describeImageWithModel } from "./providers/image.js";
+import { getMediaUnderstandingProvider, normalizeMediaProviderId } from "./providers/index.js";
+import { resolveMaxBytes, resolveMaxChars, resolvePrompt, resolveTimeoutMs } from "./resolve.js";
+import { estimateBase64Size, resolveVideoMaxBase64Bytes } from "./video.js";
+function trimOutput(text, maxChars) {
+    const trimmed = text.trim();
+    if (!maxChars || trimmed.length <= maxChars) {
+        return trimmed;
+    }
+    return trimmed.slice(0, maxChars).trim();
+}
+function extractSherpaOnnxText(raw) {
+    const tryParse = (value) => {
+        const trimmed = value.trim();
+        if (!trimmed) {
+            return null;
+        }
+        const head = trimmed[0];
+        if (head !== "{" && head !== '"') {
+            return null;
+        }
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (typeof parsed === "string") {
+                return tryParse(parsed);
+            }
+            if (parsed && typeof parsed === "object") {
+                const text = parsed.text;
+                if (typeof text === "string" && text.trim()) {
+                    return text.trim();
+                }
+            }
+        }
+        catch { }
+        return null;
+    };
+    const direct = tryParse(raw);
+    if (direct) {
+        return direct;
+    }
+    const lines = raw
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean);
+    for (let i = lines.length - 1; i >= 0; i -= 1) {
+        const parsed = tryParse(lines[i] ?? "");
+        if (parsed) {
+            return parsed;
+        }
+    }
+    return null;
+}
+function commandBase(command) {
+    return path.parse(command).name;
+}
+function findArgValue(args, keys) {
+    for (let i = 0; i < args.length; i += 1) {
+        if (keys.includes(args[i] ?? "")) {
+            const value = args[i + 1];
+            if (value) {
+                return value;
+            }
+        }
+    }
+    return undefined;
+}
+function hasArg(args, keys) {
+    return args.some((arg) => keys.includes(arg));
+}
+function resolveWhisperOutputPath(args, mediaPath) {
+    const outputDir = findArgValue(args, ["--output_dir", "-o"]);
+    const outputFormat = findArgValue(args, ["--output_format"]);
+    if (!outputDir || !outputFormat) {
+        return null;
+    }
+    const formats = outputFormat.split(",").map((value) => value.trim());
+    if (!formats.includes("txt")) {
+        return null;
+    }
+    const base = path.parse(mediaPath).name;
+    return path.join(outputDir, `${base}.txt`);
+}
+function resolveWhisperCppOutputPath(args) {
+    if (!hasArg(args, ["-otxt", "--output-txt"])) {
+        return null;
+    }
+    const outputBase = findArgValue(args, ["-of", "--output-file"]);
+    if (!outputBase) {
+        return null;
+    }
+    return `${outputBase}.txt`;
+}
+async function resolveCliOutput(params) {
+    const commandId = commandBase(params.command);
+    const fileOutput = commandId === "whisper-cli"
+        ? resolveWhisperCppOutputPath(params.args)
+        : commandId === "whisper"
+            ? resolveWhisperOutputPath(params.args, params.mediaPath)
+            : null;
+    if (fileOutput && (await fileExists(fileOutput))) {
+        try {
+            const content = await fs.readFile(fileOutput, "utf8");
+            if (content.trim()) {
+                return content.trim();
+            }
+        }
+        catch { }
+    }
+    if (commandId === "gemini") {
+        const response = extractGeminiResponse(params.stdout);
+        if (response) {
+            return response;
+        }
+    }
+    if (commandId === "sherpa-onnx-offline") {
+        const response = extractSherpaOnnxText(params.stdout);
+        if (response) {
+            return response;
+        }
+    }
+    return params.stdout.trim();
+}
+function normalizeProviderQuery(options) {
+    if (!options) {
+        return undefined;
+    }
+    const query = {};
+    for (const [key, value] of Object.entries(options)) {
+        if (value === undefined) {
+            continue;
+        }
+        query[key] = value;
+    }
+    return Object.keys(query).length > 0 ? query : undefined;
+}
+function buildDeepgramCompatQuery(options) {
+    if (!options) {
+        return undefined;
+    }
+    const query = {};
+    if (typeof options.detectLanguage === "boolean") {
+        query.detect_language = options.detectLanguage;
+    }
+    if (typeof options.punctuate === "boolean") {
+        query.punctuate = options.punctuate;
+    }
+    if (typeof options.smartFormat === "boolean") {
+        query.smart_format = options.smartFormat;
+    }
+    return Object.keys(query).length > 0 ? query : undefined;
+}
+function normalizeDeepgramQueryKeys(query) {
+    const normalized = { ...query };
+    if ("detectLanguage" in normalized) {
+        normalized.detect_language = normalized.detectLanguage;
+        delete normalized.detectLanguage;
+    }
+    if ("smartFormat" in normalized) {
+        normalized.smart_format = normalized.smartFormat;
+        delete normalized.smartFormat;
+    }
+    return normalized;
+}
+function resolveProviderQuery(params) {
+    const { providerId, config, entry } = params;
+    const mergedOptions = normalizeProviderQuery({
+        ...config?.providerOptions?.[providerId],
+        ...entry.providerOptions?.[providerId],
+    });
+    if (providerId !== "deepgram") {
+        return mergedOptions;
+    }
+    const query = normalizeDeepgramQueryKeys(mergedOptions ?? {});
+    const compat = buildDeepgramCompatQuery({ ...config?.deepgram, ...entry.deepgram });
+    for (const [key, value] of Object.entries(compat ?? {})) {
+        if (query[key] === undefined) {
+            query[key] = value;
+        }
+    }
+    return Object.keys(query).length > 0 ? query : undefined;
+}
+export function buildModelDecision(params) {
+    if (params.entryType === "cli") {
+        const command = params.entry.command?.trim();
+        return {
+            type: "cli",
+            provider: command ?? "cli",
+            model: params.entry.model ?? command,
+            outcome: params.outcome,
+            reason: params.reason,
+        };
+    }
+    const providerIdRaw = params.entry.provider?.trim();
+    const providerId = providerIdRaw ? normalizeMediaProviderId(providerIdRaw) : undefined;
+    return {
+        type: "provider",
+        provider: providerId ?? providerIdRaw,
+        model: params.entry.model,
+        outcome: params.outcome,
+        reason: params.reason,
+    };
+}
+function resolveEntryRunOptions(params) {
+    const { capability, entry, cfg } = params;
+    const maxBytes = resolveMaxBytes({ capability, entry, cfg, config: params.config });
+    const maxChars = resolveMaxChars({ capability, entry, cfg, config: params.config });
+    const timeoutMs = resolveTimeoutMs(entry.timeoutSeconds ??
+        params.config?.timeoutSeconds ??
+        cfg.tools?.media?.[capability]?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS[capability]);
+    const prompt = resolvePrompt(capability, entry.prompt ?? params.config?.prompt ?? cfg.tools?.media?.[capability]?.prompt, maxChars);
+    return { maxBytes, maxChars, timeoutMs, prompt };
+}
+export function formatDecisionSummary(decision) {
+    const total = decision.attachments.length;
+    const success = decision.attachments.filter((entry) => entry.chosen?.outcome === "success").length;
+    const chosen = decision.attachments.find((entry) => entry.chosen)?.chosen;
+    const provider = chosen?.provider?.trim();
+    const model = chosen?.model?.trim();
+    const modelLabel = provider ? (model ? `${provider}/${model}` : provider) : undefined;
+    const reason = decision.attachments
+        .flatMap((entry) => entry.attempts.map((attempt) => attempt.reason).filter(Boolean))
+        .find(Boolean);
+    const shortReason = reason ? reason.split(":")[0]?.trim() : undefined;
+    const countLabel = total > 0 ? ` (${success}/${total})` : "";
+    const viaLabel = modelLabel ? ` via ${modelLabel}` : "";
+    const reasonLabel = shortReason ? ` reason=${shortReason}` : "";
+    return `${decision.capability}: ${decision.outcome}${countLabel}${viaLabel}${reasonLabel}`;
+}
+export async function runProviderEntry(params) {
+    const { entry, capability, cfg } = params;
+    const providerIdRaw = entry.provider?.trim();
+    if (!providerIdRaw) {
+        throw new Error(`Provider entry missing provider for ${capability}`);
+    }
+    const providerId = normalizeMediaProviderId(providerIdRaw);
+    const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions({
+        capability,
+        entry,
+        cfg,
+        config: params.config,
+    });
+    if (capability === "image") {
+        if (!params.agentDir) {
+            throw new Error("Image understanding requires agentDir");
+        }
+        const modelId = entry.model?.trim();
+        if (!modelId) {
+            throw new Error("Image understanding requires model id");
+        }
+        const media = await params.cache.getBuffer({
+            attachmentIndex: params.attachmentIndex,
+            maxBytes,
+            timeoutMs,
+        });
+        const provider = getMediaUnderstandingProvider(providerId, params.providerRegistry);
+        const result = provider?.describeImage
+            ? await provider.describeImage({
+                buffer: media.buffer,
+                fileName: media.fileName,
+                mime: media.mime,
+                model: modelId,
+                provider: providerId,
+                prompt,
+                timeoutMs,
+                profile: entry.profile,
+                preferredProfile: entry.preferredProfile,
+                agentDir: params.agentDir,
+                cfg: params.cfg,
+            })
+            : await describeImageWithModel({
+                buffer: media.buffer,
+                fileName: media.fileName,
+                mime: media.mime,
+                model: modelId,
+                provider: providerId,
+                prompt,
+                timeoutMs,
+                profile: entry.profile,
+                preferredProfile: entry.preferredProfile,
+                agentDir: params.agentDir,
+                cfg: params.cfg,
+            });
+        return {
+            kind: "image.description",
+            attachmentIndex: params.attachmentIndex,
+            text: trimOutput(result.text, maxChars),
+            provider: providerId,
+            model: result.model ?? modelId,
+        };
+    }
+    const provider = getMediaUnderstandingProvider(providerId, params.providerRegistry);
+    if (!provider) {
+        throw new Error(`Media provider not available: ${providerId}`);
+    }
+    if (capability === "audio") {
+        if (!provider.transcribeAudio) {
+            throw new Error(`Audio transcription provider "${providerId}" not available.`);
+        }
+        const media = await params.cache.getBuffer({
+            attachmentIndex: params.attachmentIndex,
+            maxBytes,
+            timeoutMs,
+        });
+        const auth = await resolveApiKeyForProvider({
+            provider: providerId,
+            cfg,
+            profileId: entry.profile,
+            preferredProfile: entry.preferredProfile,
+            agentDir: params.agentDir,
+        });
+        const apiKey = requireApiKey(auth, providerId);
+        const providerConfig = cfg.models?.providers?.[providerId];
+        const baseUrl = entry.baseUrl ?? params.config?.baseUrl ?? providerConfig?.baseUrl;
+        const mergedHeaders = {
+            ...providerConfig?.headers,
+            ...params.config?.headers,
+            ...entry.headers,
+        };
+        const headers = Object.keys(mergedHeaders).length > 0 ? mergedHeaders : undefined;
+        const providerQuery = resolveProviderQuery({
+            providerId,
+            config: params.config,
+            entry,
+        });
+        const model = entry.model?.trim() || DEFAULT_AUDIO_MODELS[providerId] || entry.model;
+        const result = await provider.transcribeAudio({
+            buffer: media.buffer,
+            fileName: media.fileName,
+            mime: media.mime,
+            apiKey,
+            baseUrl,
+            headers,
+            model,
+            language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language,
+            prompt,
+            query: providerQuery,
+            timeoutMs,
+        });
+        return {
+            kind: "audio.transcription",
+            attachmentIndex: params.attachmentIndex,
+            text: trimOutput(result.text, maxChars),
+            provider: providerId,
+            model: result.model ?? model,
+        };
+    }
+    if (!provider.describeVideo) {
+        throw new Error(`Video understanding provider "${providerId}" not available.`);
+    }
+    const media = await params.cache.getBuffer({
+        attachmentIndex: params.attachmentIndex,
+        maxBytes,
+        timeoutMs,
+    });
+    const estimatedBase64Bytes = estimateBase64Size(media.size);
+    const maxBase64Bytes = resolveVideoMaxBase64Bytes(maxBytes);
+    if (estimatedBase64Bytes > maxBase64Bytes) {
+        throw new MediaUnderstandingSkipError("maxBytes", `Video attachment ${params.attachmentIndex + 1} base64 payload ${estimatedBase64Bytes} exceeds ${maxBase64Bytes}`);
+    }
+    const auth = await resolveApiKeyForProvider({
+        provider: providerId,
+        cfg,
+        profileId: entry.profile,
+        preferredProfile: entry.preferredProfile,
+        agentDir: params.agentDir,
+    });
+    const apiKey = requireApiKey(auth, providerId);
+    const providerConfig = cfg.models?.providers?.[providerId];
+    const result = await provider.describeVideo({
+        buffer: media.buffer,
+        fileName: media.fileName,
+        mime: media.mime,
+        apiKey,
+        baseUrl: providerConfig?.baseUrl,
+        headers: providerConfig?.headers,
+        model: entry.model,
+        prompt,
+        timeoutMs,
+    });
+    return {
+        kind: "video.description",
+        attachmentIndex: params.attachmentIndex,
+        text: trimOutput(result.text, maxChars),
+        provider: providerId,
+        model: result.model ?? entry.model,
+    };
+}
+export async function runCliEntry(params) {
+    const { entry, capability, cfg, ctx } = params;
+    const command = entry.command?.trim();
+    const args = entry.args ?? [];
+    if (!command) {
+        throw new Error(`CLI entry missing command for ${capability}`);
+    }
+    const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions({
+        capability,
+        entry,
+        cfg,
+        config: params.config,
+    });
+    const pathResult = await params.cache.getPath({
+        attachmentIndex: params.attachmentIndex,
+        maxBytes,
+        timeoutMs,
+    });
+    const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-media-cli-"));
+    const mediaPath = pathResult.path;
+    const outputBase = path.join(outputDir, path.parse(mediaPath).name);
+    const templCtx = {
+        ...ctx,
+        MediaPath: mediaPath,
+        MediaDir: path.dirname(mediaPath),
+        OutputDir: outputDir,
+        OutputBase: outputBase,
+        Prompt: prompt,
+        MaxChars: maxChars,
+    };
+    const argv = [command, ...args].map((part, index) => index === 0 ? part : applyTemplate(part, templCtx));
+    try {
+        if (shouldLogVerbose()) {
+            logVerbose(`Media understanding via CLI: ${argv.join(" ")}`);
+        }
+        const { stdout } = await runExec(argv[0], argv.slice(1), {
+            timeoutMs,
+            maxBuffer: CLI_OUTPUT_MAX_BUFFER,
+        });
+        const resolved = await resolveCliOutput({
+            command,
+            args: argv.slice(1),
+            stdout,
+            mediaPath,
+        });
+        const text = trimOutput(resolved, maxChars);
+        if (!text) {
+            return null;
+        }
+        return {
+            kind: capability === "audio" ? "audio.transcription" : `${capability}.description`,
+            attachmentIndex: params.attachmentIndex,
+            text,
+            provider: "cli",
+            model: command,
+        };
+    }
+    finally {
+        await fs.rm(outputDir, { recursive: true, force: true }).catch(() => { });
+    }
+}

package/dist/memory/batch-error-utils.js

@@ -0,0 +1,11 @@
+export function extractBatchErrorMessage(lines) {
+    const first = lines.find((line) => line.error?.message || line.response?.body?.error);
+    return (first?.error?.message ??
+        (typeof first?.response?.body?.error?.message === "string"
+            ? first?.response?.body?.error?.message
+            : undefined));
+}
+export function formatUnavailableBatchError(err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return message ? `error file unavailable: ${message}` : undefined;
+}

package/dist/memory/batch-http.js

@@ -0,0 +1,27 @@
+import { retryAsync } from "../infra/retry.js";
+export async function postJsonWithRetry(params) {
+    const res = await retryAsync(async () => {
+        const res = await fetch(params.url, {
+            method: "POST",
+            headers: params.headers,
+            body: JSON.stringify(params.body),
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            const err = new Error(`${params.errorPrefix}: ${res.status} ${text}`);
+            err.status = res.status;
+            throw err;
+        }
+        return res;
+    }, {
+        attempts: 3,
+        minDelayMs: 300,
+        maxDelayMs: 2000,
+        jitter: 0.2,
+        shouldRetry: (err) => {
+            const status = err.status;
+            return status === 429 || (typeof status === "number" && status >= 500);
+        },
+    });
+    return (await res.json());
+}

package/dist/memory/batch-output.js

@@ -0,0 +1,29 @@
+export function applyEmbeddingBatchOutputLine(params) {
+    const customId = params.line.custom_id;
+    if (!customId) {
+        return;
+    }
+    params.remaining.delete(customId);
+    const errorMessage = params.line.error?.message;
+    if (errorMessage) {
+        params.errors.push(`${customId}: ${errorMessage}`);
+        return;
+    }
+    const response = params.line.response;
+    const statusCode = response?.status_code ?? 0;
+    if (statusCode >= 400) {
+        const messageFromObject = response?.body && typeof response.body === "object"
+            ? response.body.error?.message
+            : undefined;
+        const messageFromString = typeof response?.body === "string" ? response.body : undefined;
+        params.errors.push(`${customId}: ${messageFromObject ?? messageFromString ?? "unknown error"}`);
+        return;
+    }
+    const data = response?.body && typeof response.body === "object" ? (response.body.data ?? []) : [];
+    const embedding = data[0]?.embedding ?? [];
+    if (embedding.length === 0) {
+        params.errors.push(`${customId}: empty embedding`);
+        return;
+    }
+    params.byCustomId.set(customId, embedding);
+}

package/dist/memory/batch-runner.js

@@ -0,0 +1,22 @@
+import { splitBatchRequests } from "./batch-utils.js";
+import { runWithConcurrency } from "./internal.js";
+export async function runEmbeddingBatchGroups(params) {
+    if (params.requests.length === 0) {
+        return new Map();
+    }
+    const groups = splitBatchRequests(params.requests, params.maxRequests);
+    const byCustomId = new Map();
+    const tasks = groups.map((group, groupIndex) => async () => {
+        await params.runGroup({ group, groupIndex, groups: groups.length, byCustomId });
+    });
+    params.debug?.(params.debugLabel, {
+        requests: params.requests.length,
+        groups: groups.length,
+        wait: params.wait,
+        concurrency: params.concurrency,
+        pollIntervalMs: params.pollIntervalMs,
+        timeoutMs: params.timeoutMs,
+    });
+    await runWithConcurrency(tasks, params.concurrency);
+    return byCustomId;
+}

package/dist/memory/batch-upload.js

@@ -0,0 +1,23 @@
+import { buildBatchHeaders, normalizeBatchBaseUrl, } from "./batch-utils.js";
+import { hashText } from "./internal.js";
+export async function uploadBatchJsonlFile(params) {
+    const baseUrl = normalizeBatchBaseUrl(params.client);
+    const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
+    const form = new FormData();
+    form.append("purpose", "batch");
+    form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`);
+    const fileRes = await fetch(`${baseUrl}/files`, {
+        method: "POST",
+        headers: buildBatchHeaders(params.client, { json: false }),
+        body: form,
+    });
+    if (!fileRes.ok) {
+        const text = await fileRes.text();
+        throw new Error(`${params.errorPrefix}: ${fileRes.status} ${text}`);
+    }
+    const filePayload = (await fileRes.json());
+    if (!filePayload.id) {
+        throw new Error(`${params.errorPrefix}: missing file id`);
+    }
+    return filePayload.id;
+}

package/dist/memory/batch-utils.js

@@ -0,0 +1,26 @@
+export function normalizeBatchBaseUrl(client) {
+    return client.baseUrl?.replace(/\/$/, "") ?? "";
+}
+export function buildBatchHeaders(client, params) {
+    const headers = client.headers ? { ...client.headers } : {};
+    if (params.json) {
+        if (!headers["Content-Type"] && !headers["content-type"]) {
+            headers["Content-Type"] = "application/json";
+        }
+    }
+    else {
+        delete headers["Content-Type"];
+        delete headers["content-type"];
+    }
+    return headers;
+}
+export function splitBatchRequests(requests, maxRequests) {
+    if (requests.length <= maxRequests) {
+        return [requests];
+    }
+    const groups = [];
+    for (let i = 0; i < requests.length; i += maxRequests) {
+        groups.push(requests.slice(i, i + maxRequests));
+    }
+    return groups;
+}

package/dist/memory/embeddings-debug.js

@@ -0,0 +1,11 @@
+import { isTruthyEnvValue } from "../infra/env.js";
+import { createSubsystemLogger } from "../logging/subsystem.js";
+const debugEmbeddings = isTruthyEnvValue(process.env.POOLBOT_DEBUG_MEMORY_EMBEDDINGS);
+const log = createSubsystemLogger("memory/embeddings");
+export function debugEmbeddingsLog(message, meta) {
+    if (!debugEmbeddings) {
+        return;
+    }
+    const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
+    log.raw(`${message}${suffix}`);
+}

package/dist/memory/embeddings-remote-client.js

@@ -0,0 +1,22 @@
+import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
+export async function resolveRemoteEmbeddingBearerClient(params) {
+    const remote = params.options.remote;
+    const remoteApiKey = remote?.apiKey?.trim();
+    const remoteBaseUrl = remote?.baseUrl?.trim();
+    const providerConfig = params.options.config.models?.providers?.[params.provider];
+    const apiKey = remoteApiKey
+        ? remoteApiKey
+        : requireApiKey(await resolveApiKeyForProvider({
+            provider: params.provider,
+            cfg: params.options.config,
+            agentDir: params.options.agentDir,
+        }), params.provider);
+    const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || params.defaultBaseUrl;
+    const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
+    const headers = {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+        ...headerOverrides,
+    };
+    return { baseUrl, headers };
+}

package/dist/memory/embeddings-remote-fetch.js

@@ -0,0 +1,14 @@
+export async function fetchRemoteEmbeddingVectors(params) {
+    const res = await fetch(params.url, {
+        method: "POST",
+        headers: params.headers,
+        body: JSON.stringify(params.body),
+    });
+    if (!res.ok) {
+        const text = await res.text();
+        throw new Error(`${params.errorPrefix}: ${res.status} ${text}`);
+    }
+    const payload = (await res.json());
+    const data = payload.data ?? [];
+    return data.map((entry) => entry.embedding ?? []);
+}