@hope666/melu 1.0.0

package/dist/embedder.js

@@ -0,0 +1,231 @@
+/**
+ * Melu Embedding 封装。
+ *
+ * 使用 node-llama-cpp 加载 GGUF embedding 模型，
+ * 运行时使用预编译 llama.cpp binary，不要求用户自行安装系统编译依赖。
+ */
+import { getLlama, LlamaLogLevel, } from "node-llama-cpp";
+import { createConnection } from "node:net";
+import { loadConfig } from "./config.js";
+import { ensureEmbeddingModelReady } from "./model-bootstrap.js";
+import { splitTextIntoOverlappingWindows } from "./text-chunking.js";
+// 英文 instruction 前缀，跨语言效果最优
+const DEFAULT_INSTRUCTION = "Instruct: Retrieve semantically similar memories for the user\nQuery: ";
+const EMBEDDER_SOCKET_ENV = "MELU_EMBEDDER_SOCKET";
+const RUN_ID_ENV = "MELU_RUN_ID";
+const REMOTE_EMBEDDER_TIMEOUT_MS = 10000;
+let _embedder = null;
+let _loading = null;
+/**
+ * 获取或初始化 embedding backend（单例，懒加载）。
+ */
+export async function getEmbedder(modelId) {
+    if (_embedder)
+        return _embedder;
+    if (_loading)
+        return _loading;
+    _loading = (async () => {
+        const config = loadConfig();
+        const { modelPath } = await ensureEmbeddingModelReady({
+            config,
+            interactive: false,
+            showProgress: false,
+        });
+        console.log(`[Melu] 加载 GGUF embedding 模型: ${modelId}`);
+        console.log(`[Melu] 模型文件: ${modelPath}`);
+        const llama = await getLlama({
+            build: "never",
+            gpu: "auto",
+            logLevel: LlamaLogLevel.warn,
+            progressLogs: false,
+        });
+        const model = await llama.loadModel({
+            gpuLayers: "auto",
+            modelPath,
+            useMmap: true,
+        });
+        // 切块 2000 字符，纯中文最坏 ≈ 1400 tokens，3072 留足余量
+        const embeddingContext = await model.createEmbeddingContext({
+            contextSize: 3072,
+        });
+        _embedder = {
+            llama,
+            model,
+            embeddingContext,
+        };
+        console.log("[Melu] Embedding 模型就绪");
+        return _embedder;
+    })();
+    return _loading;
+}
+/**
+ * 对单条文本生成 embedding 向量。
+ */
+export async function embed(text, modelId, options = {}) {
+    const purpose = options.purpose ?? "query";
+    if (shouldUseRemoteEmbedder()) {
+        const vectors = await requestRemoteEmbeddings([text], purpose);
+        return vectors[0];
+    }
+    const { embeddingContext } = await getEmbedder(modelId);
+    const inputText = DEFAULT_INSTRUCTION + text;
+    const output = await embeddingContext.getEmbeddingFor(inputText);
+    return normalizeVector(new Float32Array(output.vector));
+}
+/**
+ * 对长文本分块后做 embedding，并对所有 chunk 向量取平均。
+ */
+export async function embedLongText(text, modelId, options = {}) {
+    const chunks = splitTextIntoOverlappingWindows(text);
+    if (chunks.length === 0) {
+        throw new Error("Cannot embed empty text");
+    }
+    if (chunks.length === 1) {
+        return embed(chunks[0], modelId, options);
+    }
+    const vectors = await embedBatch(chunks, modelId, options);
+    return averageVectors(vectors);
+}
+/**
+ * 批量 embedding。
+ */
+export async function embedBatch(texts, modelId, options = {}) {
+    const purpose = options.purpose ?? "query";
+    if (shouldUseRemoteEmbedder()) {
+        return requestRemoteEmbeddings(texts, purpose);
+    }
+    const results = [];
+    for (const text of texts) {
+        results.push(await embed(text, modelId, options));
+    }
+    return results;
+}
+/**
+ * 释放模型，回收内存。
+ */
+export async function disposeEmbedder() {
+    if (_embedder) {
+        await _embedder.embeddingContext.dispose();
+        await _embedder.model.dispose();
+        await _embedder.llama.dispose();
+        _embedder = null;
+    }
+    _loading = null;
+}
+function normalizeVector(vector) {
+    let norm = 0;
+    for (let i = 0; i < vector.length; i++) {
+        norm += vector[i] * vector[i];
+    }
+    if (norm <= 1e-12) {
+        return vector;
+    }
+    const scale = 1 / Math.sqrt(norm);
+    for (let i = 0; i < vector.length; i++) {
+        vector[i] *= scale;
+    }
+    return vector;
+}
+function averageVectors(vectors) {
+    if (vectors.length === 0) {
+        throw new Error("Cannot average empty vectors");
+    }
+    const result = new Float32Array(vectors[0].length);
+    for (const vector of vectors) {
+        if (vector.length !== result.length) {
+            throw new Error("Vectors have different lengths");
+        }
+        for (let i = 0; i < vector.length; i++) {
+            result[i] += vector[i];
+        }
+    }
+    const scale = 1 / vectors.length;
+    for (let i = 0; i < result.length; i++) {
+        result[i] *= scale;
+    }
+    return normalizeVector(result);
+}
+function shouldUseRemoteEmbedder() {
+    return Boolean(process.env[EMBEDDER_SOCKET_ENV]?.trim());
+}
+async function requestRemoteEmbeddings(texts, purpose) {
+    const socketPath = process.env[EMBEDDER_SOCKET_ENV]?.trim();
+    if (!socketPath) {
+        throw new Error("Remote embedder socket is not configured");
+    }
+    if (texts.length === 0) {
+        return [];
+    }
+    const response = await sendRemoteEmbeddingRequest({
+        id: `embed-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`,
+        run_id: process.env[RUN_ID_ENV]?.trim() || "legacy",
+        type: purpose,
+        texts,
+    }, socketPath);
+    const embeddings = response.embeddings;
+    if (!Array.isArray(embeddings)) {
+        throw new Error("Remote embedder returned no embeddings");
+    }
+    if (embeddings.length !== texts.length) {
+        throw new Error("Remote embedder returned mismatched embedding count");
+    }
+    return embeddings.map((vector) => normalizeVector(new Float32Array(vector)));
+}
+function sendRemoteEmbeddingRequest(request, socketPath) {
+    return new Promise((resolve, reject) => {
+        const socket = createConnection(socketPath);
+        let buffer = "";
+        let settled = false;
+        const finish = (error, response) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            socket.destroy();
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve(response);
+            }
+        };
+        const timer = setTimeout(() => {
+            finish(new Error(`Remote embedder timed out after ${REMOTE_EMBEDDER_TIMEOUT_MS}ms`));
+        }, REMOTE_EMBEDDER_TIMEOUT_MS);
+        socket.setEncoding("utf-8");
+        socket.on("connect", () => {
+            socket.write(`${JSON.stringify(request)}\n`);
+        });
+        socket.on("data", (chunk) => {
+            buffer += chunk;
+            const newlineIndex = buffer.indexOf("\n");
+            if (newlineIndex === -1)
+                return;
+            const line = buffer.slice(0, newlineIndex).trim();
+            if (!line) {
+                finish(new Error("Remote embedder returned an empty response"));
+                return;
+            }
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed.error) {
+                    finish(new Error(parsed.error));
+                    return;
+                }
+                finish(undefined, parsed);
+            }
+            catch (error) {
+                finish(error instanceof Error ? error : new Error(String(error)));
+            }
+        });
+        socket.on("error", (error) => {
+            finish(error);
+        });
+        socket.on("close", () => {
+            if (!settled) {
+                finish(new Error("Remote embedder closed before responding"));
+            }
+        });
+    });
+}
+//# sourceMappingURL=embedder.js.map

package/dist/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,QAAQ,EACR,aAAa,GAId,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,+BAA+B,EAAE,MAAM,oBAAoB,CAAC;AAErE,4BAA4B;AAC5B,MAAM,mBAAmB,GAAG,wEAAwE,CAAC;AACrG,MAAM,mBAAmB,GAAG,sBAAsB,CAAC;AACnD,MAAM,UAAU,GAAG,aAAa,CAAC;AACjC,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAUzC,IAAI,SAAS,GAA0B,IAAI,CAAC;AAC5C,IAAI,QAAQ,GAAmC,IAAI,CAAC;AAEpD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAe;IAC/C,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,QAAQ,GAAG,CAAC,KAAK,IAAI,EAAE;QACrB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,yBAAyB,CAAC;YACpD,MAAM;YACN,WAAW,EAAE,KAAK;YAClB,YAAY,EAAE,KAAK;SACpB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;QAEzC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,KAAK,EAAE,OAAO;YACd,GAAG,EAAE,MAAM;YACX,QAAQ,EAAE,aAAa,CAAC,IAAI;YAC5B,YAAY,EAAE,KAAK;SACpB,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;YAClC,SAAS,EAAE,MAAM;YACjB,SAAS;YACT,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,2CAA2C;QAC3C,MAAM,gBAAgB,GAAG,MAAM,KAAK,CAAC,sBAAsB,CAAC;YAC1D,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,SAAS,GAAG;YACV,KAAK;YACL,KAAK;YACL,gBAAgB;SACjB,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,IAAY,EACZ,OAAe,EACf,UAA0C,EAAE;IAE5C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC;IAE3C,IAAI,uBAAuB,EAAE,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC;IAC7C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IACjE,OAAO,eAAe,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAY,EACZ,OAAe,EACf,UAA0C,EAAE;IAE5C,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO,cAAc,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAe,EACf,UAA0C,EAAE;IAE5C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC;IAE3C,IAAI,uBAAuB,EAAE,EAAE,CAAC;QAC9B,OAAO,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAC3C,MAAM,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAChC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,QAAQ,GAAG,IAAI,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,MAAoB;IAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;QAClB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IACrB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,OAAuB;IAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IACrB,CAAC;IAED,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,uBAAuB;IAC9B,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,KAAe,EAAE,OAAyB;IAC/E,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;QAChD,EAAE,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACnE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,QAAQ;QACnD,IAAI,EAAE,OAAO;QACb,KAAK;KACN,EAAE,UAAU,CAAC,CAAC;IAEf,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;IACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AAkBD,SAAS,0BAA0B,CACjC,OAA+B,EAC/B,UAAkB;IAElB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,QAAkC,EAAE,EAAE;YACnE,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,QAAS,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,0BAA0B,IAAI,CAAC,CAAC,CAAC;QACvF,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAE/B,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC;YAChB,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,YAAY,KAAK,CAAC,CAAC;gBAAE,OAAO;YAEhC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;YAClD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC,CAAC;gBAChE,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;gBAC3D,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBACjB,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBAChC,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC3B,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/extraction-queue.d.ts

@@ -0,0 +1,53 @@
+export interface ExtractionQueuePaths {
+    runId: string;
+    runDir: string;
+    pendingDir: string;
+    workingDir: string;
+    failedDir: string;
+}
+export interface ExtractionQueueJob {
+    id: string;
+    runId: string;
+    text: string;
+    ts: number;
+    attempts: number;
+    memoryPath: string | null;
+    lastError: string | null;
+    nextAttemptAt: number | null;
+    updatedAt: number;
+}
+export interface ClaimedPendingExtractionJob extends ExtractionQueueJob {
+    claimedPath: string;
+}
+export interface ExtractionQueueSnapshot {
+    runId: string;
+    pending: number;
+    working: number;
+    retryableFailed: number;
+    terminalFailed: number;
+    remaining: number;
+    nextRetryAt: number | null;
+}
+export interface RequeueClaimedPendingExtractionJobPolicy {
+    maxAttempts?: number;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+}
+export interface RequeueClaimedPendingExtractionJobResult {
+    job: ExtractionQueueJob;
+    terminal: boolean;
+    path: string;
+}
+export declare function resolveExtractionRunId(value?: string | null): string;
+export declare function getExtractionQueuePaths(runId: string): ExtractionQueuePaths;
+export declare function preparePendingExtractionQueue(runId?: string | null): void;
+export declare function preparePendingExtractionQueueForRun(runId: string): void;
+export declare function enqueuePendingExtractionJob(text: string, memoryPath?: string | null, runId?: string | null): string | null;
+export declare function enqueuePendingExtractionJobForRun(runId: string, text: string, memoryPath?: string | null): string | null;
+export declare function claimNextPendingExtractionJob(runId?: string | null): ClaimedPendingExtractionJob | null;
+export declare function claimNextPendingExtractionJobForRun(runId: string): ClaimedPendingExtractionJob | null;
+export declare function completeClaimedPendingExtractionJob(job: ClaimedPendingExtractionJob): void;
+export declare function requeueClaimedPendingExtractionJob(job: ClaimedPendingExtractionJob, updates?: Partial<ExtractionQueueJob>, policy?: RequeueClaimedPendingExtractionJobPolicy | null): RequeueClaimedPendingExtractionJobResult;
+export declare function getExtractionQueueSnapshot(runId?: string | null): ExtractionQueueSnapshot;
+export declare function getExtractionQueueSnapshotForRun(runId: string): ExtractionQueueSnapshot;
+//# sourceMappingURL=extraction-queue.d.ts.map

package/dist/extraction-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-queue.d.ts","sourceRoot":"","sources":["../src/extraction-queue.ts"],"names":[],"mappings":"AAqBA,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,2BAA4B,SAAQ,kBAAkB;IACrE,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,wCAAwC;IACvD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,wCAAwC;IACvD,GAAG,EAAE,kBAAkB,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AASD,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAMpE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,oBAAoB,CAU3E;AAgPD,wBAAgB,6BAA6B,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAEzE;AAED,wBAAgB,mCAAmC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAMvE;AAED,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,EAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,MAAM,GAAG,IAAI,CAEf;AAED,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GACzB,MAAM,GAAG,IAAI,CAUf;AAED,wBAAgB,6BAA6B,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,2BAA2B,GAAG,IAAI,CAEvG;AAED,wBAAgB,mCAAmC,CAAC,KAAK,EAAE,MAAM,GAAG,2BAA2B,GAAG,IAAI,CA+BrG;AAED,wBAAgB,mCAAmC,CACjD,GAAG,EAAE,2BAA2B,GAC/B,IAAI,CAMN;AAaD,wBAAgB,kCAAkC,CAChD,GAAG,EAAE,2BAA2B,EAChC,OAAO,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,MAAM,CAAC,EAAE,wCAAwC,GAAG,IAAI,GACvD,wCAAwC,CA0C1C;AAED,wBAAgB,0BAA0B,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,uBAAuB,CAEzF;AAED,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,MAAM,GAAG,uBAAuB,CA4CvF"}