@martian-engineering/lossless-claw 0.4.0 → 0.5.1

package/src/store/summary-store.ts

@@ -1,6 +1,6 @@
 import type { DatabaseSync } from "node:sqlite";
 import { sanitizeFts5Query } from "./fts5-sanitize.js";
-import { buildLikeSearchPlan, createFallbackSnippet } from "./full-text-fallback.js";
+import { buildLikeSearchPlan, containsCjk, createFallbackSnippet } from "./full-text-fallback.js";
 
 export type SummaryKind = "leaf" | "condensed";
 export type ContextItemType = "message" | "summary";
@@ -18,6 +18,7 @@ export type CreateSummaryInput = {
   descendantCount?: number;
   descendantTokenCount?: number;
   sourceMessageTokenCount?: number;
+  model?: string;
 };
 
 export type SummaryRecord = {
@@ -33,6 +34,7 @@ export type SummaryRecord = {
   descendantCount: number;
   descendantTokenCount: number;
   sourceMessageTokenCount: number;
+  model: string;
   createdAt: Date;
 };
 
@@ -91,6 +93,25 @@ export type LargeFileRecord = {
   createdAt: Date;
 };
 
+export type UpsertConversationBootstrapStateInput = {
+  conversationId: number;
+  sessionFilePath: string;
+  lastSeenSize: number;
+  lastSeenMtimeMs: number;
+  lastProcessedOffset: number;
+  lastProcessedEntryHash?: string | null;
+};
+
+export type ConversationBootstrapStateRecord = {
+  conversationId: number;
+  sessionFilePath: string;
+  lastSeenSize: number;
+  lastSeenMtimeMs: number;
+  lastProcessedOffset: number;
+  lastProcessedEntryHash: string | null;
+  updatedAt: Date;
+};
+
 // ── DB row shapes (snake_case) ────────────────────────────────────────────────
 
 interface SummaryRow {
@@ -106,6 +127,7 @@ interface SummaryRow {
   descendant_count: number | null;
   descendant_token_count: number | null;
   source_message_token_count: number | null;
+  model: string | null;
   created_at: string;
 }
 
@@ -161,6 +183,16 @@ interface LargeFileRow {
   created_at: string;
 }
 
+interface ConversationBootstrapStateRow {
+  conversation_id: number;
+  session_file_path: string;
+  last_seen_size: number;
+  last_seen_mtime_ms: number;
+  last_processed_offset: number;
+  last_processed_entry_hash: string | null;
+  updated_at: string;
+}
+
 // ── Row mappers ───────────────────────────────────────────────────────────────
 
 function toSummaryRecord(row: SummaryRow): SummaryRecord {
@@ -198,6 +230,7 @@ function toSummaryRecord(row: SummaryRow): SummaryRecord {
       row.source_message_token_count >= 0
         ? Math.floor(row.source_message_token_count)
         : 0,
+    model: typeof row.model === "string" ? row.model : "unknown",
     createdAt: new Date(row.created_at),
   };
 }
@@ -237,6 +270,20 @@ function toLargeFileRecord(row: LargeFileRow): LargeFileRecord {
   };
 }
 
+function toConversationBootstrapStateRecord(
+  row: ConversationBootstrapStateRow,
+): ConversationBootstrapStateRecord {
+  return {
+    conversationId: row.conversation_id,
+    sessionFilePath: row.session_file_path,
+    lastSeenSize: row.last_seen_size,
+    lastSeenMtimeMs: row.last_seen_mtime_ms,
+    lastProcessedOffset: row.last_processed_offset,
+    lastProcessedEntryHash: row.last_processed_entry_hash,
+    updatedAt: new Date(row.updated_at),
+  };
+}
+
 // ── SummaryStore ──────────────────────────────────────────────────────────────
 
 export class SummaryStore {
@@ -294,9 +341,10 @@ export class SummaryStore {
           latest_at,
           descendant_count,
           descendant_token_count,
-          source_message_token_count
+          source_message_token_count,
+          model
         )
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       )
       .run(
         input.summaryId,
@@ -311,13 +359,14 @@ export class SummaryStore {
         descendantCount,
         descendantTokenCount,
         sourceMessageTokenCount,
+        input.model ?? "unknown",
       );
 
     const row = this.db
       .prepare(
         `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
                 earliest_at, latest_at, descendant_count, created_at
-                , descendant_token_count, source_message_token_count
+                , descendant_token_count, source_message_token_count, model
        FROM summaries WHERE summary_id = ?`,
       )
       .get(input.summaryId) as unknown as SummaryRow;
@@ -345,7 +394,7 @@ export class SummaryStore {
       .prepare(
         `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
                 earliest_at, latest_at, descendant_count, created_at
-                , descendant_token_count, source_message_token_count
+                , descendant_token_count, source_message_token_count, model
        FROM summaries WHERE summary_id = ?`,
       )
       .get(summaryId) as unknown as SummaryRow | undefined;
@@ -357,7 +406,7 @@ export class SummaryStore {
       .prepare(
         `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
                 earliest_at, latest_at, descendant_count, created_at
-                , descendant_token_count, source_message_token_count
+                , descendant_token_count, source_message_token_count, model
        FROM summaries
        WHERE conversation_id = ?
        ORDER BY created_at`,
@@ -416,7 +465,7 @@ export class SummaryStore {
       .prepare(
         `SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
                 s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
-                , s.descendant_token_count, s.source_message_token_count
+                , s.descendant_token_count, s.source_message_token_count, s.model
        FROM summaries s
        JOIN summary_parents sp ON sp.summary_id = s.summary_id
        WHERE sp.parent_summary_id = ?
@@ -434,7 +483,7 @@ export class SummaryStore {
       .prepare(
         `SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
                 s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
-                , s.descendant_token_count, s.source_message_token_count
+                , s.descendant_token_count, s.source_message_token_count, s.model
        FROM summaries s
        JOIN summary_parents sp ON sp.parent_summary_id = s.summary_id
        WHERE sp.summary_id = ?
@@ -474,6 +523,7 @@ export class SummaryStore {
            s.descendant_count,
            s.descendant_token_count,
            s.source_message_token_count,
+           s.model,
            s.created_at,
            subtree.depth_from_root,
            subtree.parent_summary_id,
@@ -700,6 +750,17 @@ export class SummaryStore {
     const limit = input.limit ?? 50;
 
     if (input.mode === "full_text") {
+      // FTS5 unicode61 can return incomplete matches for CJK text, so route
+      // those queries through the existing LIKE fallback path immediately.
+      if (containsCjk(input.query)) {
+        return this.searchLike(
+          input.query,
+          limit,
+          input.conversationId,
+          input.since,
+          input.before,
+        );
+      }
       if (this.fts5Available) {
         try {
           return this.searchFullText(
@@ -796,7 +857,7 @@ export class SummaryStore {
       .prepare(
         `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
                 earliest_at, latest_at, descendant_count, descendant_token_count,
-                source_message_token_count, created_at
+                source_message_token_count, model, created_at
          FROM summaries
          ${whereClause}
          ORDER BY created_at DESC
@@ -851,7 +912,7 @@ export class SummaryStore {
       .prepare(
         `SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
                 earliest_at, latest_at, descendant_count, descendant_token_count,
-                source_message_token_count, created_at
+                source_message_token_count, model, created_at
          FROM summaries
          ${whereClause}
          ORDER BY created_at DESC`,
@@ -930,4 +991,63 @@ export class SummaryStore {
       .all(conversationId) as unknown as LargeFileRow[];
     return rows.map(toLargeFileRecord);
   }
+
+  // ── Bootstrap state ──────────────────────────────────────────────────────
+
+  async getConversationBootstrapState(
+    conversationId: number,
+  ): Promise<ConversationBootstrapStateRecord | null> {
+    const row = this.db
+      .prepare(
+        `SELECT conversation_id, session_file_path, last_seen_size, last_seen_mtime_ms,
+                last_processed_offset, last_processed_entry_hash, updated_at
+         FROM conversation_bootstrap_state
+         WHERE conversation_id = ?`,
+      )
+      .get(conversationId) as unknown as ConversationBootstrapStateRow | undefined;
+    return row ? toConversationBootstrapStateRecord(row) : null;
+  }
+
+  async upsertConversationBootstrapState(
+    input: UpsertConversationBootstrapStateInput,
+  ): Promise<ConversationBootstrapStateRecord> {
+    this.db
+      .prepare(
+        `INSERT INTO conversation_bootstrap_state (
+           conversation_id,
+           session_file_path,
+           last_seen_size,
+           last_seen_mtime_ms,
+           last_processed_offset,
+           last_processed_entry_hash
+         )
+         VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT (conversation_id) DO UPDATE SET
+           session_file_path = excluded.session_file_path,
+           last_seen_size = excluded.last_seen_size,
+           last_seen_mtime_ms = excluded.last_seen_mtime_ms,
+           last_processed_offset = excluded.last_processed_offset,
+           last_processed_entry_hash = excluded.last_processed_entry_hash,
+           updated_at = datetime('now')`,
+      )
+      .run(
+        input.conversationId,
+        input.sessionFilePath,
+        Math.max(0, Math.floor(input.lastSeenSize)),
+        Math.max(0, Math.floor(input.lastSeenMtimeMs)),
+        Math.max(0, Math.floor(input.lastProcessedOffset)),
+        input.lastProcessedEntryHash ?? null,
+      );
+
+    const row = this.db
+      .prepare(
+        `SELECT conversation_id, session_file_path, last_seen_size, last_seen_mtime_ms,
+                last_processed_offset, last_processed_entry_hash, updated_at
+         FROM conversation_bootstrap_state
+         WHERE conversation_id = ?`,
+      )
+      .get(input.conversationId) as unknown as ConversationBootstrapStateRow;
+
+    return toConversationBootstrapStateRecord(row);
+  }
 }

package/src/summarize.ts

@@ -31,6 +31,41 @@ const DIAGNOSTIC_MAX_OBJECT_KEYS = 16;
 const DIAGNOSTIC_MAX_CHARS = 1200;
 const DIAGNOSTIC_SENSITIVE_KEY_PATTERN =
   /(api[-_]?key|authorization|token|secret|password|cookie|set-cookie|private[-_]?key|bearer)/i;
+const AUTH_ERROR_TEXT_PATTERN =
+  /\b401\b|unauthorized|unauthorised|invalid[_ -]?token|invalid[_ -]?api[_ -]?key|authentication failed|authorization failed|missing scope|insufficient scope|model\.request\b/i;
+const AUTH_ERROR_STATUS_KEYS = ["status", "statusCode", "status_code"] as const;
+const AUTH_ERROR_NESTED_KEYS = ["error", "response", "cause", "details", "data", "body"] as const;
+
+type ProviderAuthFailure = {
+  statusCode?: number;
+  message?: string;
+  missingModelRequestScope: boolean;
+};
+
+/**
+ * Default timeout for a single summarizer LLM call.  Long enough for large
+ * context windows on slower providers, short enough to prevent the gateway
+ * event loop from starving when a provider hangs.
+ */
+const SUMMARIZER_TIMEOUT_MS = 60_000;
+
+/** Error used to distinguish summarizer timeouts from provider failures. */
+class SummarizerTimeoutError extends Error {
+  constructor(ms: number, label: string) {
+    super(`[lcm] summarizer timeout after ${ms}ms (${label})`);
+    this.name = "SummarizerTimeoutError";
+  }
+}
+
+function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new SummarizerTimeoutError(ms, label)), ms);
+    promise.then(
+      (val) => { clearTimeout(timer); resolve(val); },
+      (err) => { clearTimeout(timer); reject(err); },
+    );
+  });
+}
 
 /** Normalize provider ids for stable config/profile lookup. */
 function normalizeProviderId(provider: string): string {
@@ -272,6 +307,133 @@ function formatDiagnosticPayload(value: unknown): string {
   }
 }
 
+function collectAuthFailureText(value: unknown, out: string[], depth = 0): void {
+  if (depth >= 4) {
+    return;
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (trimmed) {
+      out.push(trimmed);
+    }
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const entry of value.slice(0, DIAGNOSTIC_MAX_ARRAY_ITEMS)) {
+      collectAuthFailureText(entry, out, depth + 1);
+    }
+    return;
+  }
+  if (!isRecord(value)) {
+    return;
+  }
+
+  for (const entry of Object.values(value).slice(0, DIAGNOSTIC_MAX_OBJECT_KEYS)) {
+    collectAuthFailureText(entry, out, depth + 1);
+  }
+}
+
+function extractAuthFailureStatusCode(value: unknown, depth = 0): number | undefined {
+  if (depth >= 4 || !isRecord(value)) {
+    return undefined;
+  }
+
+  for (const key of AUTH_ERROR_STATUS_KEYS) {
+    const candidate = value[key];
+    if (typeof candidate === "number" && Number.isFinite(candidate)) {
+      return Math.trunc(candidate);
+    }
+    if (typeof candidate === "string") {
+      const parsed = Number.parseInt(candidate, 10);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  }
+
+  for (const key of AUTH_ERROR_NESTED_KEYS) {
+    const nested = value[key];
+    const statusCode = extractAuthFailureStatusCode(nested, depth + 1);
+    if (statusCode !== undefined) {
+      return statusCode;
+    }
+  }
+
+  return undefined;
+}
+
+function pickAuthInspectionValue(value: unknown): unknown {
+  if (!isRecord(value)) {
+    return value;
+  }
+  if (isRecord(value.error) && value.error.kind === "provider_auth") {
+    return value.error;
+  }
+
+  const subset: Record<string, unknown> = {};
+  for (const key of [
+    "error",
+    "errorMessage",
+    "message",
+    "status",
+    "statusCode",
+    "status_code",
+    "code",
+    "details",
+    "response",
+    "cause",
+  ]) {
+    if (key in value) {
+      subset[key] = value[key];
+    }
+  }
+  return Object.keys(subset).length > 0 ? subset : value;
+}
+
+function extractProviderAuthFailure(value: unknown): ProviderAuthFailure | undefined {
+  const inspectValue = pickAuthInspectionValue(value);
+  const statusCode = extractAuthFailureStatusCode(inspectValue);
+  const textParts: string[] = [];
+  collectAuthFailureText(inspectValue, textParts);
+  const normalizedMessage = textParts.join(" ").replace(/\s+/g, " ").trim();
+  const missingModelRequestScope = /\bmodel\.request\b/i.test(normalizedMessage);
+  const hasScopeSignal =
+    missingModelRequestScope || /\b(missing|insufficient)\s+scope\b/i.test(normalizedMessage);
+
+  if (statusCode !== 401 && !hasScopeSignal && !AUTH_ERROR_TEXT_PATTERN.test(normalizedMessage)) {
+    return undefined;
+  }
+
+  return {
+    ...(statusCode !== undefined ? { statusCode } : {}),
+    ...(normalizedMessage ? { message: truncateDiagnosticText(normalizedMessage, 240) } : {}),
+    missingModelRequestScope,
+  };
+}
+
+function buildProviderAuthWarning(params: {
+  provider: string;
+  model: string;
+  failure: ProviderAuthFailure;
+}): string {
+  const detailParts: string[] = [];
+  if (params.failure.statusCode === 401) {
+    detailParts.push("401");
+  }
+  if (params.failure.missingModelRequestScope) {
+    detailParts.push("missing model.request scope");
+  }
+  const detail =
+    detailParts.length > 0
+      ? `provider auth error (${detailParts.join(" / ")})`
+      : "provider auth error";
+  const messageSuffix =
+    params.failure.message && !params.failure.missingModelRequestScope
+      ? ` Detail: ${params.failure.message}`
+      : "";
+  return `[lcm] compaction failed: ${detail}. Check that the configured summaryProvider has valid API credentials. Current: ${params.provider}/${params.model}${messageSuffix}`;
+}
+
 /**
  * Extract safe diagnostic metadata from a provider response envelope.
  *
@@ -641,7 +803,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
   deps: LcmDependencies;
   legacyParams: LcmSummarizerLegacyParams;
   customInstructions?: string;
-}): Promise<LcmSummarizeFn | undefined> {
+}): Promise<{ fn: LcmSummarizeFn; model: string } | undefined> {
   const readModelRef = (value: unknown): string => {
     if (typeof value === "string") {
       return value.trim();
@@ -735,11 +897,14 @@ export async function createLcmSummarizeFromLegacyParams(params: {
     console.error(`[lcm] createLcmSummarize: empty provider="${provider}" or model="${model}"`);
     return undefined;
   }
-  const authProfileId =
+  const legacyAuthProfileId =
     typeof params.legacyParams.authProfileId === "string" &&
     params.legacyParams.authProfileId.trim()
       ? params.legacyParams.authProfileId.trim()
       : undefined;
+  // When LCM selects a dedicated summarizer model/provider, do not leak the
+  // active session's auth profile into that separate credential lookup.
+  const authProfileId = resolvedSummary === undefined ? legacyAuthProfileId : undefined;
   const agentDir =
     typeof params.legacyParams.agentDir === "string" && params.legacyParams.agentDir.trim()
       ? params.legacyParams.agentDir.trim()
@@ -752,7 +917,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
       ? params.deps.config.condensedTargetTokens
       : DEFAULT_CONDENSED_TARGET_TOKENS;
 
-  return async (
+  const fn: LcmSummarizeFn = async (
     text: string,
     aggressive?: boolean,
     options?: LcmSummarizeOptions,
@@ -795,7 +960,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
 
     let result: Awaited<ReturnType<typeof params.deps.complete>>;
     try {
-      result = await params.deps.complete({
+      result = await withTimeout(params.deps.complete({
         provider,
         model,
         apiKey,
@@ -812,11 +977,30 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         ],
         maxTokens: targetTokens,
         temperature: aggressive ? 0.1 : 0.2,
-      });
+      }), SUMMARIZER_TIMEOUT_MS, "initial");
     } catch (err) {
-      console.error(
-        `[lcm] summarizer call failed; provider=${provider}; model=${model}; error=${err instanceof Error ? err.message : String(err)}`,
+      const authFailure = extractProviderAuthFailure(err);
+      if (authFailure) {
+        console.warn(buildProviderAuthWarning({ provider, model, failure: authFailure }));
+        return "";
+      }
+      const errMsg = err instanceof Error ? err.message : String(err);
+      const isTimeout = errMsg.includes("summarizer timeout");
+      console.warn(
+        `[lcm] summarizer ${isTimeout ? "timed out" : "failed"}; provider=${provider}; model=${model}; timeout=${SUMMARIZER_TIMEOUT_MS}ms; error=${errMsg}`,
       );
+      if (err instanceof SummarizerTimeoutError) {
+        console.error(
+          `[lcm] summarizer timed out; provider=${provider}; model=${model}; source=fallback`,
+        );
+        return buildDeterministicFallbackSummary(text, targetTokens);
+      }
+      return "";
+    }
+
+    const authFailure = extractProviderAuthFailure(result);
+    if (authFailure) {
+      console.warn(buildProviderAuthWarning({ provider, model, failure: authFailure }));
       return "";
     }
 
@@ -859,7 +1043,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
       // reasoning budget to coax a textual response from providers that
       // sometimes return reasoning-only or empty blocks on the first pass.
       try {
-        const retryResult = await params.deps.complete({
+        const retryResult = await withTimeout(params.deps.complete({
           provider,
           model,
           apiKey,
@@ -877,7 +1061,12 @@ export async function createLcmSummarizeFromLegacyParams(params: {
           maxTokens: targetTokens,
           temperature: 0.05,
           reasoning: "low",
-        });
+        }), SUMMARIZER_TIMEOUT_MS, "retry");
+        const retryAuthFailure = extractProviderAuthFailure(retryResult);
+        if (retryAuthFailure) {
+          console.warn(buildProviderAuthWarning({ provider, model, failure: retryAuthFailure }));
+          return "";
+        }
 
         const retryNormalized = normalizeCompletionSummary(retryResult.content);
         summary = retryNormalized.summary;
@@ -903,11 +1092,16 @@ export async function createLcmSummarizeFromLegacyParams(params: {
           console.error(`${retryParts.join("; ")}; falling back to truncation`);
         }
       } catch (retryErr) {
+        const retryAuthFailure = extractProviderAuthFailure(retryErr);
+        if (retryAuthFailure) {
+          console.warn(buildProviderAuthWarning({ provider, model, failure: retryAuthFailure }));
+          return "";
+        }
         // Retry is best-effort; log and proceed to deterministic fallback.
-        console.error(
-          `[lcm] retry failed; provider=${provider} model=${model}; error=${
-            retryErr instanceof Error ? retryErr.message : String(retryErr)
-          }; falling back to truncation`,
+        const retryErrMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
+        const isRetryTimeout = retryErrMsg.includes("summarizer timeout");
+        console.warn(
+          `[lcm] retry ${isRetryTimeout ? "timed out" : "failed"}; provider=${provider}; model=${model}; timeout=${SUMMARIZER_TIMEOUT_MS}ms; error=${retryErrMsg}; falling back to truncation`,
         );
       }
     }
@@ -928,4 +1122,6 @@ export async function createLcmSummarizeFromLegacyParams(params: {
 
     return summary;
   };
+
+  return { fn, model };
 }

package/src/types.ts

@@ -17,8 +17,17 @@ export type CompletionContentBlock = {
   [key: string]: unknown;
 };
 
+export type CompletionErrorInfo = {
+  kind?: string;
+  message?: string;
+  code?: string;
+  statusCode?: number;
+  [key: string]: unknown;
+};
+
 export type CompletionResult = {
   content: CompletionContentBlock[];
+  error?: CompletionErrorInfo;
   [key: string]: unknown;
 };