forge-openclaw-plugin 0.2.115 → 0.2.117

package/dist/server/server/src/managers/platform/background-job-manager.js

@@ -24,6 +24,7 @@ export class BackgroundJobManager extends AbstractManager {
                 },
                 functionName: "enqueue"
             });
+            this.scheduleDrain();
             return;
         }
         this.queue.push(input);

package/dist/server/server/src/managers/platform/openai-responses-provider.js

@@ -20,8 +20,12 @@ const MODEL_CONTEXT_WINDOWS = {
 };
 const DEFAULT_CONTEXT_WINDOW = 400_000;
 const RESERVED_RESPONSE_TOKENS = 140_000;
+const CODEX_WIKI_COMPILE_CONTEXT_WINDOW = 120_000;
+const CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS = 60_000;
 const APPROX_CHARS_PER_TOKEN = 4;
 const REQUEST_TIMEOUT_MS = 90_000;
+const CODEX_FOREGROUND_COMPILE_TIMEOUT_MS = 10 * 60_000;
+const CODEX_STREAM_READ_TIMEOUT_MS = 10 * 60_000;
 const BACKGROUND_POLL_INTERVAL_MS = 2_000;
 const DEFAULT_CODEX_BASE_URL = "https://chatgpt.com/backend-api";
 const CODEX_JWT_CLAIM_PATH = "https://api.openai.com/auth";
@@ -261,9 +265,52 @@ function parseCodexEventStreamPayload(streamText) {
 }
 async function readProviderPayload(response, profile) {
     return isCodexProfile(profile)
-        ? parseCodexEventStreamPayload(await response.text())
+        ? parseCodexEventStreamPayload(await readResponseTextWithTimeout(response, CODEX_STREAM_READ_TIMEOUT_MS))
         : readJsonPayload(response);
 }
+async function readResponseTextWithTimeout(response, timeoutMs) {
+    if (!response.body) {
+        return response.text();
+    }
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    const chunks = [];
+    const deadline = Date.now() + timeoutMs;
+    try {
+        while (true) {
+            const remainingMs = Math.max(1, deadline - Date.now());
+            const result = await readStreamChunkWithTimeout(reader, remainingMs, timeoutMs);
+            if (result.done) {
+                break;
+            }
+            chunks.push(decoder.decode(result.value, { stream: true }));
+        }
+        chunks.push(decoder.decode());
+        return chunks.join("");
+    }
+    catch (error) {
+        await reader.cancel().catch(() => undefined);
+        throw error;
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+function readStreamChunkWithTimeout(reader, remainingMs, totalTimeoutMs) {
+    let timeout = null;
+    return Promise.race([
+        reader.read(),
+        new Promise((_, reject) => {
+            timeout = setTimeout(() => {
+                reject(new Error(`Codex stream read timed out after ${Math.round(totalTimeoutMs / 1000)}s.`));
+            }, remainingMs);
+        })
+    ]).finally(() => {
+        if (timeout) {
+            clearTimeout(timeout);
+        }
+    });
+}
 function readReasoningEffort(profile) {
     return typeof profile.metadata.reasoningEffort === "string"
         ? profile.metadata.reasoningEffort
@@ -358,8 +405,14 @@ function estimateTokens(text) {
     return Math.ceil(text.length / APPROX_CHARS_PER_TOKEN);
 }
 function computeSourceExcerpt(profile, sourceText) {
-    const contextWindow = MODEL_CONTEXT_WINDOWS[profile.model] ?? DEFAULT_CONTEXT_WINDOW;
-    const inputBudget = Math.max(16_000, contextWindow - RESERVED_RESPONSE_TOKENS);
+    const configuredContextWindow = MODEL_CONTEXT_WINDOWS[profile.model] ?? DEFAULT_CONTEXT_WINDOW;
+    const contextWindow = isCodexProfile(profile)
+        ? Math.min(configuredContextWindow, CODEX_WIKI_COMPILE_CONTEXT_WINDOW)
+        : configuredContextWindow;
+    const reservedResponseTokens = isCodexProfile(profile)
+        ? CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS
+        : RESERVED_RESPONSE_TOKENS;
+    const inputBudget = Math.max(16_000, contextWindow - reservedResponseTokens);
     const estimatedTokens = estimateTokens(sourceText);
     if (estimatedTokens <= inputBudget) {
         return {
@@ -617,6 +670,10 @@ export class OpenAiResponsesProvider {
             "- For chats and transcripts, extract the durable parts: people, relationships, ongoing projects, commitments, habits, values, decisions, questions, sources, and evidence.",
             "- Merge repetitive back-and-forth into concise summaries.",
             "- Use short quotes only when the exact phrase matters.",
+            "Sensitive information rules:",
+            "- Never store or reproduce secrets, passwords, passphrases, recovery codes, API keys, access tokens, refresh tokens, session cookies, private keys, seed phrases, one-time codes, full payment card numbers, or equivalent credentials.",
+            "- If the source contains a secret or credential, replace the value with [REDACTED SECRET] and keep only the minimum non-secret context needed to explain why it appeared.",
+            "- Do not create wiki pages, notes, entity proposals, aliases, tags, titles, or page update suggestions that preserve secret values.",
             "How to split pages:",
             "- Keep markdown as the overview page for this source.",
             "- If one topic deserves its own page, put it in articleCandidates with title, slug, summary, rationale, markdown, tags, aliases, and parentSlug.",
@@ -820,7 +877,9 @@ export class OpenAiResponsesProvider {
                             }
                         })
                     }),
-                    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+                    signal: AbortSignal.timeout(isCodexProfile(profile)
+                        ? CODEX_FOREGROUND_COMPILE_TIMEOUT_MS
+                        : REQUEST_TIMEOUT_MS)
                 });
             }
             catch (error) {

package/dist/server/server/src/repositories/wiki-memory.js

@@ -10,6 +10,8 @@ import { createNoteLinkSchema, crudEntityTypeSchema, noteKindSchema, noteSchema
 import { deleteEncryptedSecret, readEncryptedSecret, storeEncryptedSecret } from "./calendar.js";
 import { isEntityDeleted } from "./deleted-entities.js";
 import { recordDiagnosticLog } from "./diagnostic-logs.js";
+const MAX_WIKI_INGEST_TEXT_CHUNK_CHARS = 220_000;
+const WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS = 2_000;
 const wikiSpaceSchema = z.object({
     id: z.string(),
     slug: z.string(),
@@ -2017,6 +2019,115 @@ export async function reindexWikiEmbeddings(input, secrets) {
         chunkCount
     };
 }
+function isChunkableWikiIngestTextAsset(asset) {
+    const mimeType = asset.mime_type.toLowerCase();
+    const fileName = asset.file_name.toLowerCase();
+    if (!existsSync(asset.file_path)) {
+        return false;
+    }
+    if (asset.size_bytes <= MAX_WIKI_INGEST_TEXT_CHUNK_CHARS) {
+        return false;
+    }
+    const metadata = parseJsonRecord(asset.metadata_json);
+    if (metadata?.chunkParentAssetId || metadata?.textChunked) {
+        return false;
+    }
+    return (mimeType.startsWith("text/") ||
+        fileName.endsWith(".txt") ||
+        fileName.endsWith(".md") ||
+        fileName.endsWith(".markdown") ||
+        fileName.endsWith(".csv") ||
+        fileName.endsWith(".json"));
+}
+function splitWikiIngestTextIntoChunks(sourceText, maxChars) {
+    const text = sourceText.trim();
+    if (text.length <= maxChars) {
+        return [text];
+    }
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        const hardEnd = Math.min(text.length, start + maxChars);
+        let end = hardEnd;
+        if (hardEnd < text.length) {
+            const newline = text.lastIndexOf("\n", hardEnd);
+            if (newline > start + Math.floor(maxChars * 0.65)) {
+                end = newline + 1;
+            }
+        }
+        const chunk = text.slice(start, end).trim();
+        if (chunk.length > 0) {
+            chunks.push(chunk);
+        }
+        if (end >= text.length) {
+            break;
+        }
+        start = Math.max(0, end - WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS);
+    }
+    return chunks;
+}
+async function splitLargeWikiIngestTextAsset(options) {
+    if (!isChunkableWikiIngestTextAsset(options.asset)) {
+        return 0;
+    }
+    const sourceText = await readFile(options.asset.file_path, "utf8");
+    const chunks = splitWikiIngestTextIntoChunks(sourceText, MAX_WIKI_INGEST_TEXT_CHUNK_CHARS);
+    if (chunks.length <= 1) {
+        return 0;
+    }
+    const extension = path.extname(options.asset.file_name) || ".txt";
+    const baseName = path.basename(options.asset.file_name, extension) ||
+        options.asset.file_name ||
+        "source";
+    const width = String(chunks.length).length;
+    for (const [index, chunk] of chunks.entries()) {
+        const chunkNumber = index + 1;
+        const chunkFileName = `${baseName}-part-${String(chunkNumber).padStart(width, "0")}-of-${String(chunks.length).padStart(width, "0")}${extension}`;
+        const chunkHeader = [
+            `Source file: ${options.asset.file_name}`,
+            `Source locator: ${options.asset.source_locator || options.asset.file_name}`,
+            `Chunk: ${chunkNumber}/${chunks.length}`,
+            `Parent checksum: ${options.asset.checksum}`,
+            "",
+            chunk
+        ].join("\n");
+        const persisted = await persistIngestUpload({
+            jobId: options.jobId,
+            fileName: chunkFileName,
+            mimeType: options.asset.mime_type || "text/plain",
+            payload: Buffer.from(chunkHeader, "utf8")
+        });
+        createWikiIngestAssetRecord({
+            jobId: options.jobId,
+            sourceKind: "upload",
+            sourceLocator: `${options.asset.source_locator || options.asset.file_name}#chunk-${chunkNumber}`,
+            fileName: chunkFileName,
+            mimeType: options.asset.mime_type || "text/plain",
+            filePath: persisted.filePath,
+            sizeBytes: persisted.sizeBytes,
+            checksum: persisted.checksum,
+            metadata: {
+                chunkParentAssetId: options.asset.id,
+                chunkParentChecksum: options.asset.checksum,
+                chunkIndex: chunkNumber,
+                chunkCount: chunks.length,
+                chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS,
+                chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS
+            }
+        });
+    }
+    updateWikiIngestAsset(options.asset.id, {
+        status: "completed",
+        metadata: {
+            ...parseJsonRecord(options.asset.metadata_json),
+            textChunked: true,
+            textChunkCount: chunks.length,
+            textChunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
+            textChunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
+        }
+    });
+    return chunks.length;
+}
 function getWikiIngestJobDir(jobId) {
     return path.join(resolveDataDir(), "wiki-ingest", jobId);
 }
@@ -2615,7 +2726,10 @@ export async function processWikiIngestJob(jobId, options) {
     const initialAssets = listWikiIngestJobAssetsInternal(jobId);
     let processedFiles = initialAssets.filter((asset) => asset.status === "completed").length;
     let totalFiles = Math.max(job.total_files, initialAssets.length);
-    let hadSuccess = false;
+    let hadSuccess = (() => {
+        const counts = refreshCounts();
+        return counts.pageCount + counts.entityCount > 0;
+    })();
     while (assetQueue().length > 0) {
         const nextAsset = assetQueue().find((asset) => ["processing", "queued"].includes(asset.status));
         if (!nextAsset) {
@@ -2677,6 +2791,27 @@ export async function processWikiIngestJob(jobId, options) {
             }
             continue;
         }
+        const derivedChunkCount = await splitLargeWikiIngestTextAsset({
+            jobId,
+            asset: nextAsset
+        });
+        if (derivedChunkCount > 0) {
+            totalFiles = Math.max(derivedChunkCount, totalFiles - 1 + derivedChunkCount);
+            updateWikiIngestJob(jobId, {
+                totalFiles,
+                latestMessage: `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`
+            });
+            createWikiIngestLog(jobId, `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`, "info", {
+                sourceAssetId: nextAsset.id,
+                fileName: nextAsset.file_name,
+                sourceLocator: nextAsset.source_locator,
+                checksum: nextAsset.checksum,
+                chunkCount: derivedChunkCount,
+                chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
+                chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
+            });
+            continue;
+        }
         updateWikiIngestAsset(nextAsset.id, { status: "processing" });
         currentAssetContext = {
             assetId: nextAsset.id,

package/dist/server/server/src/services/companion-iroh.js

@@ -301,6 +301,10 @@ function candidateIrohBinaries() {
     return candidateIrohAssetRoots().flatMap((root) => [
         path.join(root, "companion-iroh", "target", "release", binaryName),
         path.join(root, "companion-iroh", "target", "debug", binaryName),
+        path.join(root, "companion-iroh-src", "target", "release", binaryName),
+        path.join(root, "companion-iroh-src", "target", "debug", binaryName),
+        path.join(root, "dist", "companion-iroh-src", "target", "release", binaryName),
+        path.join(root, "dist", "companion-iroh-src", "target", "debug", binaryName),
         path.join(root, "openclaw-plugin", "dist", "companion-iroh", platformKey, binaryName),
         path.join(root, "companion-iroh", platformKey, binaryName),
         path.join(root, "companion-iroh", binaryName)
@@ -309,7 +313,8 @@ function candidateIrohBinaries() {
 function resolveCompanionIrohManifestPath() {
     const candidates = candidateIrohAssetRoots().flatMap((root) => [
         path.join(root, "companion-iroh", "Cargo.toml"),
-        path.join(root, "companion-iroh-src", "Cargo.toml")
+        path.join(root, "companion-iroh-src", "Cargo.toml"),
+        path.join(root, "dist", "companion-iroh-src", "Cargo.toml")
     ]);
     return candidates.find((candidate) => existsSync(candidate)) ?? null;
 }

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "forge-openclaw-plugin",
   "name": "Forge",
   "description": "Curated OpenClaw adapter for the Forge collaboration API, UI entrypoint, and localhost auto-start runtime.",
-  "version": "0.2.115",
+  "version": "0.2.117",
   "activation": {
     "onStartup": true,
     "onCapabilities": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "forge-openclaw-plugin",
-  "version": "0.2.115",
+  "version": "0.2.117",
   "description": "Curated OpenClaw adapter for the Forge collaboration API, UI entrypoint, and localhost auto-start runtime.",
   "type": "module",
   "license": "Apache-2.0",