@martian-engineering/lossless-claw 0.4.0 → 0.5.1

package/src/large-files.ts

@@ -512,6 +512,25 @@ export function formatFileReference(input: {
   ].join("\n");
 }
 
+export function formatToolOutputReference(input: {
+  fileId: string;
+  toolName?: string;
+  byteSize: number;
+  summary: string;
+}): string {
+  const toolName = input.toolName?.trim() || "unknown";
+  const byteSize = Math.max(0, input.byteSize);
+
+  return [
+    `[LCM Tool Output: ${input.fileId} | tool=${toolName} | ${byteSize.toLocaleString("en-US")} bytes]`,
+    "",
+    "Exploration Summary:",
+    input.summary.trim() || "(no summary available)",
+    "",
+    "Use lcm_describe with the file id to inspect the full output.",
+  ].join("\n");
+}
+
 export async function generateExplorationSummary(input: ExplorationSummaryInput): Promise<string> {
   const extension = extensionFromNameOrMime(input.fileName, input.mimeType);
 

package/src/plugin/index.ts

@@ -100,6 +100,17 @@ type RuntimeModelAuth = {
 const MODEL_AUTH_PR_URL = "https://github.com/openclaw/openclaw/pull/41090";
 const MODEL_AUTH_MERGE_COMMIT = "4790e40";
 const MODEL_AUTH_REQUIRED_RELEASE = "the first OpenClaw release after 2026.3.8";
+const AUTH_ERROR_TEXT_PATTERN =
+  /\b401\b|unauthorized|unauthorised|invalid[_ -]?token|invalid[_ -]?api[_ -]?key|authentication failed|authorization failed|missing scope|insufficient scope|model\.request\b/i;
+const AUTH_ERROR_STATUS_KEYS = ["status", "statusCode", "status_code"] as const;
+const AUTH_ERROR_NESTED_KEYS = ["error", "response", "cause", "details", "data", "body"] as const;
+
+type CompletionBridgeErrorInfo = {
+  kind: "provider_auth";
+  statusCode?: number;
+  code?: string;
+  message?: string;
+};
 
 /** Capture plugin env values once during initialization. */
 function snapshotPluginEnv(env: NodeJS.ProcessEnv = process.env): PluginEnvSnapshot {
@@ -115,6 +126,93 @@ function snapshotPluginEnv(env: NodeJS.ProcessEnv = process.env): PluginEnvSnaps
   };
 }
 
+function truncateErrorMessage(message: string, maxChars = 240): string {
+  return message.length <= maxChars ? message : `${message.slice(0, maxChars)}...`;
+}
+
+function collectErrorText(value: unknown, out: string[], depth = 0): void {
+  if (depth >= 4) {
+    return;
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (trimmed) {
+      out.push(trimmed);
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value.slice(0, 8)) {
+      collectErrorText(entry, out, depth + 1);
+    }
+    return;
+  }
+  if (!isRecord(value)) {
+    return;
+  }
+
+  for (const entry of Object.values(value).slice(0, 12)) {
+    collectErrorText(entry, out, depth + 1);
+  }
+}
+
+function extractErrorStatusCode(value: unknown, depth = 0): number | undefined {
+  if (depth >= 4 || !isRecord(value)) {
+    return undefined;
+  }
+
+  for (const key of AUTH_ERROR_STATUS_KEYS) {
+    const candidate = value[key];
+    if (typeof candidate === "number" && Number.isFinite(candidate)) {
+      return Math.trunc(candidate);
+    }
+    if (typeof candidate === "string") {
+      const parsed = Number.parseInt(candidate, 10);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  }
+
+  for (const key of AUTH_ERROR_NESTED_KEYS) {
+    const nested = value[key];
+    const statusCode = extractErrorStatusCode(nested, depth + 1);
+    if (statusCode !== undefined) {
+      return statusCode;
+    }
+  }
+
+  return undefined;
+}
+
+function detectProviderAuthError(error: unknown): CompletionBridgeErrorInfo | undefined {
+  const statusCode = extractErrorStatusCode(error);
+  const textParts: string[] = [];
+  collectErrorText(error, textParts);
+  const normalizedMessage = textParts.join(" ").replace(/\s+/g, " ").trim();
+
+  if (statusCode !== 401 && !AUTH_ERROR_TEXT_PATTERN.test(normalizedMessage)) {
+    return undefined;
+  }
+
+  const directCode =
+    isRecord(error) && typeof error.code === "string" && error.code.trim()
+      ? error.code.trim()
+      : isRecord(error) &&
+          isRecord(error.error) &&
+          typeof error.error.code === "string" &&
+          error.error.code.trim()
+        ? error.error.code.trim()
+        : undefined;
+
+  return {
+    kind: "provider_auth",
+    ...(statusCode !== undefined ? { statusCode } : {}),
+    ...(directCode ? { code: directCode } : {}),
+    ...(normalizedMessage ? { message: truncateErrorMessage(normalizedMessage) } : {}),
+  };
+}
+
 /** Read OpenClaw's configured default model from the validated runtime config. */
 function readDefaultModelFromConfig(config: unknown): string {
   if (!config || typeof config !== "object") {
@@ -740,6 +838,53 @@ async function resolveApiKeyFromAuthProfiles(params: {
     const expires = credential.expires;
     const isExpired =
       typeof expires === "number" && Number.isFinite(expires) && expires > 0 && Date.now() >= expires;
+    const shouldPreferOAuthHelper =
+      typeof params.piAiModule.getOAuthApiKey === "function" &&
+      normalizeProviderId(params.provider) === "openai-codex";
+
+    if (shouldPreferOAuthHelper) {
+      try {
+        const oauthCredential = {
+          access: credential.access ?? "",
+          refresh: credential.refresh ?? "",
+          expires: typeof credential.expires === "number" ? credential.expires : 0,
+          ...(typeof credential.projectId === "string" ? { projectId: credential.projectId } : {}),
+          ...(typeof credential.accountId === "string" ? { accountId: credential.accountId } : {}),
+        };
+        const refreshed = await params.piAiModule.getOAuthApiKey(params.provider, {
+          [params.provider]: oauthCredential,
+        });
+        if (refreshed?.apiKey) {
+          mergedStore.profiles[profileId] = {
+            ...credential,
+            ...refreshed.newCredentials,
+            type: "oauth",
+          };
+          if (persistPath) {
+            try {
+              writeFileSync(
+                persistPath,
+                JSON.stringify(
+                  {
+                    version: 1,
+                    profiles: mergedStore.profiles,
+                    ...(mergedStore.order ? { order: mergedStore.order } : {}),
+                  },
+                  null,
+                  2,
+                ),
+                "utf8",
+              );
+            } catch {
+              // Ignore persistence errors: refreshed credentials remain usable in-memory for this run.
+            }
+          }
+          return refreshed.apiKey;
+        }
+      } catch {
+        // Fall back to the cached access token below when helper resolution fails.
+      }
+    }
 
     if (!isExpired && access) {
       if (
@@ -1016,6 +1161,26 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
               };
 
         let resolvedApiKey = apiKey?.trim();
+        if (!resolvedApiKey && modelAuth) {
+          try {
+            resolvedApiKey = resolveApiKeyFromAuthResult(
+              await modelAuth.getApiKeyForModel({
+                model: buildModelAuthLookupModel({
+                  provider: providerId,
+                  model: modelId,
+                  api: resolvedModel.api,
+                }),
+                cfg: api.config,
+                ...(authProfileId ? { profileId: authProfileId } : {}),
+              }),
+            );
+          } catch (err) {
+            console.error(
+              `[lcm] modelAuth.getApiKeyForModel FAILED:`,
+              err instanceof Error ? err.message : err,
+            );
+          }
+        }
         if (!resolvedApiKey && modelAuth) {
           try {
             resolvedApiKey = resolveApiKeyFromAuthResult(
@@ -1032,13 +1197,13 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
             );
           }
         }
-        if (!resolvedApiKey && !modelAuth) {
+        if (!resolvedApiKey) {
           resolvedApiKey = resolveApiKey(providerId, readEnv);
         }
-        if (!resolvedApiKey && !modelAuth && typeof mod.getEnvApiKey === "function") {
+        if (!resolvedApiKey && typeof mod.getEnvApiKey === "function") {
           resolvedApiKey = mod.getEnvApiKey(providerId)?.trim();
         }
-        if (!resolvedApiKey && !modelAuth) {
+        if (!resolvedApiKey) {
           resolvedApiKey = await resolveApiKeyFromAuthProfiles({
             provider: providerId,
             authProfileId,
@@ -1072,6 +1237,19 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
           temperature,
           reasoning,
         });
+        const requestMetadata = {
+          request_provider: providerId,
+          request_model: modelId,
+          request_api: resolvedModel.api,
+          request_reasoning:
+            typeof reasoning === "string" && reasoning.trim() ? reasoning.trim() : "(none)",
+          request_has_system: typeof system === "string" && system.trim().length > 0 ? "true" : "false",
+          request_temperature:
+            typeof completeOptions.temperature === "number"
+              ? String(completeOptions.temperature)
+              : "(omitted)",
+          request_temperature_sent: typeof completeOptions.temperature === "number" ? "true" : "false",
+        };
 
         const result = await mod.completeSimple(
           resolvedModel,
@@ -1091,40 +1269,22 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
         if (!isRecord(result)) {
           return {
             content: [],
-            request_provider: providerId,
-            request_model: modelId,
-            request_api: resolvedModel.api,
-            request_reasoning:
-              typeof reasoning === "string" && reasoning.trim() ? reasoning.trim() : "(none)",
-            request_has_system:
-              typeof system === "string" && system.trim().length > 0 ? "true" : "false",
-            request_temperature:
-              typeof completeOptions.temperature === "number"
-                ? String(completeOptions.temperature)
-                : "(omitted)",
-            request_temperature_sent:
-              typeof completeOptions.temperature === "number" ? "true" : "false",
+            ...requestMetadata,
           };
         }
 
         return {
           ...result,
           content: Array.isArray(result.content) ? result.content : [],
-          request_provider: providerId,
-          request_model: modelId,
-          request_api: resolvedModel.api,
-          request_reasoning:
-            typeof reasoning === "string" && reasoning.trim() ? reasoning.trim() : "(none)",
-          request_has_system: typeof system === "string" && system.trim().length > 0 ? "true" : "false",
-          request_temperature:
-            typeof completeOptions.temperature === "number"
-              ? String(completeOptions.temperature)
-              : "(omitted)",
-          request_temperature_sent: typeof completeOptions.temperature === "number" ? "true" : "false",
+          ...requestMetadata,
         };
       } catch (err) {
         console.error(`[lcm] completeSimple error:`, err instanceof Error ? err.message : err);
-        return { content: [] };
+        const authError = detectProviderAuthError(err);
+        return {
+          content: [],
+          ...(authError ? { error: authError } : {}),
+        };
       }
     },
     callGateway: async (params) => {

package/src/store/conversation-store.ts

@@ -1,7 +1,7 @@
 import type { DatabaseSync } from "node:sqlite";
 import { randomUUID } from "node:crypto";
 import { sanitizeFts5Query } from "./fts5-sanitize.js";
-import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
+import { buildLikeSearchPlan, containsCjk, createFallbackSnippet } from "./full-text-fallback.js";
 
 export type ConversationId = number;
 export type MessageId = number;
@@ -205,6 +205,47 @@ function toMessagePartRecord(row: MessagePartRow): MessagePartRecord {
   };
 }
 
+function normalizeMessageContentForFullTextIndex(content: string): string | null {
+  const trimmed = content.trim();
+  if (!trimmed) {
+    return null;
+  }
+
+  const isExternalizedReference =
+    trimmed.startsWith("[LCM File:") || trimmed.startsWith("[LCM Tool Output:");
+  if (!isExternalizedReference) {
+    return content;
+  }
+
+  const lines = trimmed
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+  if (lines.length === 0) {
+    return null;
+  }
+
+  const header = lines[0] ?? "";
+  const summaryLines: string[] = [];
+  let inSummary = false;
+  for (let index = 1; index < lines.length; index += 1) {
+    const line = lines[index]!;
+    if (line === "Exploration Summary:") {
+      inSummary = true;
+      continue;
+    }
+    if (line.startsWith("Use lcm_describe")) {
+      continue;
+    }
+    if (inSummary) {
+      summaryLines.push(line);
+    }
+  }
+
+  const normalized = [header, ...summaryLines].filter((line) => line.length > 0).join("\n");
+  return normalized || null;
+}
+
 // ── ConversationStore ─────────────────────────────────────────────────────────
 
 export class ConversationStore {
@@ -622,6 +663,17 @@ export class ConversationStore {
     const limit = input.limit ?? 50;
 
     if (input.mode === "full_text") {
+      // FTS5 unicode61 can return incomplete matches for CJK text, so route
+      // those queries through the existing LIKE fallback path immediately.
+      if (containsCjk(input.query)) {
+        return this.searchLike(
+          input.query,
+          limit,
+          input.conversationId,
+          input.since,
+          input.before,
+        );
+      }
       if (this.fts5Available) {
         try {
           return this.searchFullText(
@@ -650,10 +702,14 @@ export class ConversationStore {
     if (!this.fts5Available) {
       return;
     }
+    const normalizedContent = normalizeMessageContentForFullTextIndex(content);
+    if (!normalizedContent) {
+      return;
+    }
     try {
       this.db
         .prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`)
-        .run(messageId, content);
+        .run(messageId, normalizedContent);
     } catch {
       // Full-text indexing is optional. Message persistence must still succeed.
     }
@@ -748,14 +804,24 @@ export class ConversationStore {
       )
       .all(...args) as unknown as MessageRow[];
 
-    return rows.map((row) => ({
-      messageId: row.message_id,
-      conversationId: row.conversation_id,
-      role: row.role,
-      snippet: createFallbackSnippet(row.content, plan.terms),
-      createdAt: new Date(row.created_at),
-      rank: 0,
-    }));
+    return rows
+      .map((row) => {
+        const normalizedContent = normalizeMessageContentForFullTextIndex(row.content) ?? row.content;
+        const haystack = normalizedContent.toLowerCase();
+        const matchesAllTerms = plan.terms.every((term) => haystack.includes(term));
+        if (!matchesAllTerms) {
+          return null;
+        }
+        return {
+          messageId: row.message_id,
+          conversationId: row.conversation_id,
+          role: row.role,
+          snippet: createFallbackSnippet(normalizedContent, plan.terms),
+          createdAt: new Date(row.created_at),
+          rank: 0,
+        };
+      })
+      .filter((row): row is MessageSearchResult => row !== null);
   }
 
   private searchRegex(

package/src/store/full-text-fallback.ts

@@ -1,4 +1,13 @@
 const RAW_TERM_RE = /"([^"]+)"|(\S+)/g;
+
+/**
+ * Detect whether a query contains CJK characters that FTS5 unicode61
+ * tokenizer cannot index properly.
+ */
+const CJK_RE = /[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\uAC00-\uD7AF\u3040-\u309F\u30A0-\u30FF]/;
+export function containsCjk(text: string): boolean {
+  return CJK_RE.test(text);
+}
 const EDGE_PUNCTUATION_RE = /^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;
 
 export type LikeSearchPlan = {

package/src/store/index.ts

@@ -26,4 +26,6 @@ export type {
   SummarySearchResult,
   CreateLargeFileInput,
   LargeFileRecord,
+  UpsertConversationBootstrapStateInput,
+  ConversationBootstrapStateRecord,
 } from "./summary-store.js";