@poolzin/pool-bot 2026.2.0 → 2026.2.2

package/dist/memory/batch-voyage.js

@@ -0,0 +1,277 @@
+import { createInterface } from "node:readline";
+import { Readable } from "node:stream";
+import { retryAsync } from "../infra/retry.js";
+import { hashText, runWithConcurrency } from "./internal.js";
+export const VOYAGE_BATCH_ENDPOINT = "/v1/embeddings";
+const VOYAGE_BATCH_COMPLETION_WINDOW = "12h";
+const VOYAGE_BATCH_MAX_REQUESTS = 50000;
+function getVoyageBaseUrl(client) {
+    return client.baseUrl?.replace(/\/$/, "") ?? "";
+}
+function getVoyageHeaders(client, params) {
+    const headers = client.headers ? { ...client.headers } : {};
+    if (params.json) {
+        if (!headers["Content-Type"] && !headers["content-type"]) {
+            headers["Content-Type"] = "application/json";
+        }
+    }
+    else {
+        delete headers["Content-Type"];
+        delete headers["content-type"];
+    }
+    return headers;
+}
+function splitVoyageBatchRequests(requests) {
+    if (requests.length <= VOYAGE_BATCH_MAX_REQUESTS) {
+        return [requests];
+    }
+    const groups = [];
+    for (let i = 0; i < requests.length; i += VOYAGE_BATCH_MAX_REQUESTS) {
+        groups.push(requests.slice(i, i + VOYAGE_BATCH_MAX_REQUESTS));
+    }
+    return groups;
+}
+async function submitVoyageBatch(params) {
+    const baseUrl = getVoyageBaseUrl(params.client);
+    const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
+    const form = new FormData();
+    form.append("purpose", "batch");
+    form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`);
+    // 1. Upload file using Voyage Files API
+    const fileRes = await fetch(`${baseUrl}/files`, {
+        method: "POST",
+        headers: getVoyageHeaders(params.client, { json: false }),
+        body: form,
+    });
+    if (!fileRes.ok) {
+        const text = await fileRes.text();
+        throw new Error(`voyage batch file upload failed: ${fileRes.status} ${text}`);
+    }
+    const filePayload = (await fileRes.json());
+    if (!filePayload.id) {
+        throw new Error("voyage batch file upload failed: missing file id");
+    }
+    // 2. Create batch job using Voyage Batches API
+    const batchRes = await retryAsync(async () => {
+        const res = await fetch(`${baseUrl}/batches`, {
+            method: "POST",
+            headers: getVoyageHeaders(params.client, { json: true }),
+            body: JSON.stringify({
+                input_file_id: filePayload.id,
+                endpoint: VOYAGE_BATCH_ENDPOINT,
+                completion_window: VOYAGE_BATCH_COMPLETION_WINDOW,
+                request_params: {
+                    model: params.client.model,
+                    input_type: "document",
+                },
+                metadata: {
+                    source: "clawdbot-memory",
+                    agent: params.agentId,
+                },
+            }),
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            const err = new Error(`voyage batch create failed: ${res.status} ${text}`);
+            err.status = res.status;
+            throw err;
+        }
+        return res;
+    }, {
+        attempts: 3,
+        minDelayMs: 300,
+        maxDelayMs: 2000,
+        jitter: 0.2,
+        shouldRetry: (err) => {
+            const status = err.status;
+            return status === 429 || (typeof status === "number" && status >= 500);
+        },
+    });
+    return (await batchRes.json());
+}
+async function fetchVoyageBatchStatus(params) {
+    const baseUrl = getVoyageBaseUrl(params.client);
+    const res = await fetch(`${baseUrl}/batches/${params.batchId}`, {
+        headers: getVoyageHeaders(params.client, { json: true }),
+    });
+    if (!res.ok) {
+        const text = await res.text();
+        throw new Error(`voyage batch status failed: ${res.status} ${text}`);
+    }
+    return (await res.json());
+}
+async function readVoyageBatchError(params) {
+    try {
+        const baseUrl = getVoyageBaseUrl(params.client);
+        const res = await fetch(`${baseUrl}/files/${params.errorFileId}/content`, {
+            headers: getVoyageHeaders(params.client, { json: true }),
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            throw new Error(`voyage batch error file content failed: ${res.status} ${text}`);
+        }
+        const text = await res.text();
+        if (!text.trim()) {
+            return undefined;
+        }
+        const lines = text
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+        const first = lines.find((line) => line.error?.message || line.response?.body?.error);
+        const message = first?.error?.message ??
+            (typeof first?.response?.body?.error?.message === "string"
+                ? first?.response?.body?.error?.message
+                : undefined);
+        return message;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return message ? `error file unavailable: ${message}` : undefined;
+    }
+}
+async function waitForVoyageBatch(params) {
+    const start = Date.now();
+    let current = params.initial;
+    while (true) {
+        const status = current ??
+            (await fetchVoyageBatchStatus({
+                client: params.client,
+                batchId: params.batchId,
+            }));
+        const state = status.status ?? "unknown";
+        if (state === "completed") {
+            if (!status.output_file_id) {
+                throw new Error(`voyage batch ${params.batchId} completed without output file`);
+            }
+            return {
+                outputFileId: status.output_file_id,
+                errorFileId: status.error_file_id ?? undefined,
+            };
+        }
+        if (["failed", "expired", "cancelled", "canceled"].includes(state)) {
+            const detail = status.error_file_id
+                ? await readVoyageBatchError({ client: params.client, errorFileId: status.error_file_id })
+                : undefined;
+            const suffix = detail ? `: ${detail}` : "";
+            throw new Error(`voyage batch ${params.batchId} ${state}${suffix}`);
+        }
+        if (!params.wait) {
+            throw new Error(`voyage batch ${params.batchId} still ${state}; wait disabled`);
+        }
+        if (Date.now() - start > params.timeoutMs) {
+            throw new Error(`voyage batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
+        }
+        params.debug?.(`voyage batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
+        await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
+        current = undefined;
+    }
+}
+export async function runVoyageEmbeddingBatches(params) {
+    if (params.requests.length === 0) {
+        return new Map();
+    }
+    const groups = splitVoyageBatchRequests(params.requests);
+    const byCustomId = new Map();
+    const tasks = groups.map((group, groupIndex) => async () => {
+        const batchInfo = await submitVoyageBatch({
+            client: params.client,
+            requests: group,
+            agentId: params.agentId,
+        });
+        if (!batchInfo.id) {
+            throw new Error("voyage batch create failed: missing batch id");
+        }
+        params.debug?.("memory embeddings: voyage batch created", {
+            batchId: batchInfo.id,
+            status: batchInfo.status,
+            group: groupIndex + 1,
+            groups: groups.length,
+            requests: group.length,
+        });
+        if (!params.wait && batchInfo.status !== "completed") {
+            throw new Error(`voyage batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`);
+        }
+        const completed = batchInfo.status === "completed"
+            ? {
+                outputFileId: batchInfo.output_file_id ?? "",
+                errorFileId: batchInfo.error_file_id ?? undefined,
+            }
+            : await waitForVoyageBatch({
+                client: params.client,
+                batchId: batchInfo.id,
+                wait: params.wait,
+                pollIntervalMs: params.pollIntervalMs,
+                timeoutMs: params.timeoutMs,
+                debug: params.debug,
+                initial: batchInfo,
+            });
+        if (!completed.outputFileId) {
+            throw new Error(`voyage batch ${batchInfo.id} completed without output file`);
+        }
+        const baseUrl = getVoyageBaseUrl(params.client);
+        const contentRes = await fetch(`${baseUrl}/files/${completed.outputFileId}/content`, {
+            headers: getVoyageHeaders(params.client, { json: true }),
+        });
+        if (!contentRes.ok) {
+            const text = await contentRes.text();
+            throw new Error(`voyage batch file content failed: ${contentRes.status} ${text}`);
+        }
+        const errors = [];
+        const remaining = new Set(group.map((request) => request.custom_id));
+        if (contentRes.body) {
+            const reader = createInterface({
+                input: Readable.fromWeb(contentRes.body),
+                terminal: false,
+            });
+            for await (const rawLine of reader) {
+                if (!rawLine.trim()) {
+                    continue;
+                }
+                const line = JSON.parse(rawLine);
+                const customId = line.custom_id;
+                if (!customId) {
+                    continue;
+                }
+                remaining.delete(customId);
+                if (line.error?.message) {
+                    errors.push(`${customId}: ${line.error.message}`);
+                    continue;
+                }
+                const response = line.response;
+                const statusCode = response?.status_code ?? 0;
+                if (statusCode >= 400) {
+                    const message = response?.body?.error?.message ??
+                        (typeof response?.body === "string" ? response.body : undefined) ??
+                        "unknown error";
+                    errors.push(`${customId}: ${message}`);
+                    continue;
+                }
+                const data = response?.body?.data ?? [];
+                const embedding = data[0]?.embedding ?? [];
+                if (embedding.length === 0) {
+                    errors.push(`${customId}: empty embedding`);
+                    continue;
+                }
+                byCustomId.set(customId, embedding);
+            }
+        }
+        if (errors.length > 0) {
+            throw new Error(`voyage batch ${batchInfo.id} failed: ${errors.join("; ")}`);
+        }
+        if (remaining.size > 0) {
+            throw new Error(`voyage batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
+        }
+    });
+    params.debug?.("memory embeddings: voyage batch submit", {
+        requests: params.requests.length,
+        groups: groups.length,
+        wait: params.wait,
+        concurrency: params.concurrency,
+        pollIntervalMs: params.pollIntervalMs,
+        timeoutMs: params.timeoutMs,
+    });
+    await runWithConcurrency(tasks, params.concurrency);
+    return byCustomId;
+}

package/dist/memory/embeddings-voyage.js

@@ -0,0 +1,75 @@
+import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
+export const DEFAULT_VOYAGE_EMBEDDING_MODEL = "voyage-4-large";
+const DEFAULT_VOYAGE_BASE_URL = "https://api.voyageai.com/v1";
+export function normalizeVoyageModel(model) {
+    const trimmed = model.trim();
+    if (!trimmed) {
+        return DEFAULT_VOYAGE_EMBEDDING_MODEL;
+    }
+    if (trimmed.startsWith("voyage/")) {
+        return trimmed.slice("voyage/".length);
+    }
+    return trimmed;
+}
+export async function createVoyageEmbeddingProvider(options) {
+    const client = await resolveVoyageEmbeddingClient(options);
+    const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
+    const embed = async (input, input_type) => {
+        if (input.length === 0) {
+            return [];
+        }
+        const body = {
+            model: client.model,
+            input,
+        };
+        if (input_type) {
+            body.input_type = input_type;
+        }
+        const res = await fetch(url, {
+            method: "POST",
+            headers: client.headers,
+            body: JSON.stringify(body),
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            throw new Error(`voyage embeddings failed: ${res.status} ${text}`);
+        }
+        const payload = (await res.json());
+        const data = payload.data ?? [];
+        return data.map((entry) => entry.embedding ?? []);
+    };
+    return {
+        provider: {
+            id: "voyage",
+            model: client.model,
+            embedQuery: async (text) => {
+                const [vec] = await embed([text], "query");
+                return vec ?? [];
+            },
+            embedBatch: async (texts) => embed(texts, "document"),
+        },
+        client,
+    };
+}
+export async function resolveVoyageEmbeddingClient(options) {
+    const remote = options.remote;
+    const remoteApiKey = remote?.apiKey?.trim();
+    const remoteBaseUrl = remote?.baseUrl?.trim();
+    const apiKey = remoteApiKey
+        ? remoteApiKey
+        : requireApiKey(await resolveApiKeyForProvider({
+            provider: "voyage",
+            cfg: options.config,
+            agentDir: options.agentDir,
+        }), "voyage");
+    const providerConfig = options.config.models?.providers?.voyage;
+    const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_VOYAGE_BASE_URL;
+    const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
+    const headers = {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+        ...headerOverrides,
+    };
+    const model = normalizeVoyageModel(options.model);
+    return { baseUrl, headers, model };
+}

package/dist/memory/embeddings.js

@@ -1,15 +1,27 @@
 import fsSync from "node:fs";
+import { formatErrorMessage } from "../infra/errors.js";
 import { resolveUserPath } from "../utils.js";
 import { createGeminiEmbeddingProvider } from "./embeddings-gemini.js";
 import { createOpenAiEmbeddingProvider } from "./embeddings-openai.js";
+import { createVoyageEmbeddingProvider } from "./embeddings-voyage.js";
 import { importNodeLlamaCpp } from "./node-llama.js";
+function sanitizeAndNormalizeEmbedding(vec) {
+    const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
+    const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
+    if (magnitude < 1e-10) {
+        return sanitized;
+    }
+    return sanitized.map((value) => value / magnitude);
+}
 const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
 function canAutoSelectLocal(options) {
     const modelPath = options.local?.modelPath?.trim();
-    if (!modelPath)
+    if (!modelPath) {
         return false;
-    if (/^(hf:|https?:)/i.test(modelPath))
+    }
+    if (/^(hf:|https?:)/i.test(modelPath)) {
         return false;
+    }
     const resolved = resolveUserPath(modelPath);
     try {
         return fsSync.statSync(resolved).isFile();
@@ -19,7 +31,7 @@ function canAutoSelectLocal(options) {
     }
 }
 function isMissingApiKeyError(err) {
-    const message = formatError(err);
+    const message = formatErrorMessage(err);
     return message.includes("No API key found for provider");
 }
 async function createLocalEmbeddingProvider(options) {
@@ -49,13 +61,13 @@ async function createLocalEmbeddingProvider(options) {
         embedQuery: async (text) => {
             const ctx = await ensureContext();
             const embedding = await ctx.getEmbeddingFor(text);
-            return Array.from(embedding.vector);
+            return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
         },
         embedBatch: async (texts) => {
             const ctx = await ensureContext();
             const embeddings = await Promise.all(texts.map(async (text) => {
                 const embedding = await ctx.getEmbeddingFor(text);
-                return Array.from(embedding.vector);
+                return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
             }));
             return embeddings;
         },
@@ -73,10 +85,14 @@ export async function createEmbeddingProvider(options) {
             const { provider, client } = await createGeminiEmbeddingProvider(options);
             return { provider, gemini: client };
         }
+        if (id === "voyage") {
+            const { provider, client } = await createVoyageEmbeddingProvider(options);
+            return { provider, voyage: client };
+        }
         const { provider, client } = await createOpenAiEmbeddingProvider(options);
         return { provider, openAi: client };
     };
-    const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err);
+    const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatErrorMessage(err);
     if (requestedProvider === "auto") {
         const missingKeyErrors = [];
         let localError = null;
@@ -89,7 +105,7 @@ export async function createEmbeddingProvider(options) {
                 localError = formatLocalSetupError(err);
             }
         }
-        for (const provider of ["openai", "gemini"]) {
+        for (const provider of ["openai", "gemini", "voyage"]) {
             try {
                 const result = await createProvider(provider);
                 return { ...result, requestedProvider };
@@ -100,7 +116,7 @@ export async function createEmbeddingProvider(options) {
                     missingKeyErrors.push(message);
                     continue;
                 }
-                throw new Error(message);
+                throw new Error(message, { cause: err });
             }
         }
         const details = [...missingKeyErrors, localError].filter(Boolean);
@@ -126,17 +142,12 @@ export async function createEmbeddingProvider(options) {
                 };
             }
             catch (fallbackErr) {
-                throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatError(fallbackErr)}`);
+                throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatErrorMessage(fallbackErr)}`, { cause: fallbackErr });
             }
         }
-        throw new Error(reason);
+        throw new Error(reason, { cause: primaryErr });
     }
 }
-function formatError(err) {
-    if (err instanceof Error)
-        return err.message;
-    return String(err);
-}
 function isNodeLlamaCppMissing(err) {
     if (!(err instanceof Error))
         return false;
@@ -147,7 +158,7 @@ function isNodeLlamaCppMissing(err) {
     return false;
 }
 function formatLocalSetupError(err) {
-    const detail = formatError(err);
+    const detail = formatErrorMessage(err);
     const missing = isNodeLlamaCppMissing(err);
     return [
         "Local embeddings unavailable.",
@@ -164,6 +175,7 @@ function formatLocalSetupError(err) {
             : null,
         "3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
         'Or set agents.defaults.memorySearch.provider = "openai" (remote).',
+        'Or set agents.defaults.memorySearch.provider = "voyage" (remote).',
     ]
         .filter(Boolean)
         .join("\n");

package/dist/memory/internal.js

@@ -13,6 +13,16 @@ export function normalizeRelPath(value) {
     const trimmed = value.trim().replace(/^[./]+/, "");
     return trimmed.replace(/\\/g, "/");
 }
+export function normalizeExtraMemoryPaths(workspaceDir, extraPaths) {
+    if (!extraPaths?.length) {
+        return [];
+    }
+    const resolved = extraPaths
+        .map((value) => value.trim())
+        .filter(Boolean)
+        .map((value) => path.isAbsolute(value) ? path.resolve(value) : path.resolve(workspaceDir, value));
+    return Array.from(new Set(resolved));
+}
 export function isMemoryPath(relPath) {
     const normalized = normalizeRelPath(relPath);
     if (!normalized)
@@ -21,15 +31,6 @@ export function isMemoryPath(relPath) {
         return true;
     return normalized.startsWith("memory/");
 }
-async function exists(filePath) {
-    try {
-        await fs.access(filePath);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
 async function walkDir(dir, files) {
     const entries = await fs.readdir(dir, { withFileTypes: true });
     for (const entry of entries) {
@@ -45,20 +46,55 @@ async function walkDir(dir, files) {
         files.push(full);
     }
 }
-export async function listMemoryFiles(workspaceDir) {
+export async function listMemoryFiles(workspaceDir, extraPaths) {
     const result = [];
     const memoryFile = path.join(workspaceDir, "MEMORY.md");
     const altMemoryFile = path.join(workspaceDir, "memory.md");
-    if (await exists(memoryFile))
-        result.push(memoryFile);
-    if (await exists(altMemoryFile))
-        result.push(altMemoryFile);
     const memoryDir = path.join(workspaceDir, "memory");
-    if (await exists(memoryDir)) {
-        await walkDir(memoryDir, result);
+    const addMarkdownFile = async (absPath) => {
+        try {
+            const stat = await fs.lstat(absPath);
+            if (stat.isSymbolicLink() || !stat.isFile()) {
+                return;
+            }
+            if (!absPath.endsWith(".md")) {
+                return;
+            }
+            result.push(absPath);
+        }
+        catch { }
+    };
+    await addMarkdownFile(memoryFile);
+    await addMarkdownFile(altMemoryFile);
+    try {
+        const dirStat = await fs.lstat(memoryDir);
+        if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
+            await walkDir(memoryDir, result);
+        }
     }
-    if (result.length <= 1)
+    catch { }
+    const normalizedExtraPaths = normalizeExtraMemoryPaths(workspaceDir, extraPaths);
+    if (normalizedExtraPaths.length > 0) {
+        for (const inputPath of normalizedExtraPaths) {
+            try {
+                const stat = await fs.lstat(inputPath);
+                if (stat.isSymbolicLink()) {
+                    continue;
+                }
+                if (stat.isDirectory()) {
+                    await walkDir(inputPath, result);
+                    continue;
+                }
+                if (stat.isFile() && inputPath.endsWith(".md")) {
+                    result.push(inputPath);
+                }
+            }
+            catch { }
+        }
+    }
+    if (result.length <= 1) {
         return result;
+    }
     const seen = new Set();
     const deduped = [];
     for (const entry of result) {
@@ -67,8 +103,9 @@ export async function listMemoryFiles(workspaceDir) {
             key = await fs.realpath(entry);
         }
         catch { }
-        if (seen.has(key))
+        if (seen.has(key)) {
             continue;
+        }
         seen.add(key);
         deduped.push(entry);
     }
@@ -187,3 +224,49 @@ export function cosineSimilarity(a, b) {
         return 0;
     return dot / (Math.sqrt(normA) * Math.sqrt(normB));
 }
+export async function runWithConcurrency(tasks, limit) {
+    if (tasks.length === 0) {
+        return [];
+    }
+    const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
+    const results = Array.from({ length: tasks.length });
+    let next = 0;
+    let firstError = null;
+    const workers = Array.from({ length: resolvedLimit }, async () => {
+        while (true) {
+            if (firstError) {
+                return;
+            }
+            const index = next;
+            next += 1;
+            if (index >= tasks.length) {
+                return;
+            }
+            try {
+                results[index] = await tasks[index]();
+            }
+            catch (err) {
+                firstError = err;
+                return;
+            }
+        }
+    });
+    await Promise.allSettled(workers);
+    if (firstError) {
+        throw firstError;
+    }
+    return results;
+}
+/**
+ * Remap chunk startLine/endLine using a lineMap from session file building.
+ */
+export function remapChunkLines(chunks, lineMap) {
+    if (!lineMap || lineMap.length === 0) {
+        return;
+    }
+    for (const chunk of chunks) {
+        // startLine/endLine are 1-indexed; lineMap is 0-indexed by content line
+        chunk.startLine = lineMap[chunk.startLine - 1] ?? chunk.startLine;
+        chunk.endLine = lineMap[chunk.endLine - 1] ?? chunk.endLine;
+    }
+}