@martian-engineering/lossless-claw 0.6.3 → 0.8.0

package/src/store/full-text-sort.ts

@@ -0,0 +1,21 @@
+export type SearchSort = "recency" | "relevance" | "hybrid";
+
+export const AGE_DECAY_RATE = 0.001;
+
+/**
+ * Build the ORDER BY clause for FTS5-backed searches.
+ *
+ * `rank` is FTS5's BM25 score where lower (more negative) is better.
+ * `hybrid` keeps that relevance signal but applies a mild age penalty before
+ * LIMIT is enforced so older strong matches can still surface.
+ */
+export function buildFtsOrderBy(sort: SearchSort | undefined, createdAtExpr: string): string {
+  switch (sort ?? "recency") {
+    case "relevance":
+      return `rank ASC, ${createdAtExpr} DESC`;
+    case "hybrid":
+      return `(rank / (1 + ((julianday('now') - julianday(${createdAtExpr})) * 24 * ${AGE_DECAY_RATE}))) ASC, ${createdAtExpr} DESC`;
+    default:
+      return `${createdAtExpr} DESC`;
+  }
+}

package/src/store/index.ts

@@ -29,3 +29,11 @@ export type {
   UpsertConversationBootstrapStateInput,
   ConversationBootstrapStateRecord,
 } from "./summary-store.js";
+
+export { CompactionTelemetryStore } from "./compaction-telemetry-store.js";
+export type {
+  CacheState,
+  ActivityBand,
+  ConversationCompactionTelemetryRecord,
+  UpsertConversationCompactionTelemetryInput,
+} from "./compaction-telemetry-store.js";

package/src/store/summary-store.ts

@@ -3,6 +3,7 @@ import { withDatabaseTransaction } from "../transaction-mutex.js";
 import { sanitizeFts5Query } from "./fts5-sanitize.js";
 import { buildLikeSearchPlan, containsCjk, createFallbackSnippet } from "./full-text-fallback.js";
 import { parseUtcTimestamp, parseUtcTimestampOrNull } from "./parse-utc-timestamp.js";
+import { buildFtsOrderBy, type SearchSort } from "./full-text-sort.js";
 
 export type SummaryKind = "leaf" | "condensed";
 export type ContextItemType = "message" | "summary";
@@ -68,6 +69,7 @@ export type SummarySearchInput = {
   since?: Date;
   before?: Date;
   limit?: number;
+  sort?: SearchSort;
 };
 
 export type SummarySearchResult = {
@@ -958,7 +960,10 @@ export class SummaryStore {
       .run(conversationId, startOrdinal, summaryId);
 
     // 3. Resequence all ordinals to maintain contiguity (no gaps).
-    //    Fetch current items, then update ordinals in order.
+    //    Pre-compute ranks from a SELECT (safe snapshot), then apply
+    //    via 2-pass UPDATE loop using negative temps to avoid UNIQUE
+    //    constraint violations. The SELECT reads post-delete/insert
+    //    state and provides a consistent snapshot for resequencing.
     const items = this.db
       .prepare(
         `SELECT ordinal FROM context_items
@@ -967,18 +972,17 @@ export class SummaryStore {
       )
       .all(conversationId) as unknown as { ordinal: number }[];
 
-    const updateStmt = this.db.prepare(
-      `UPDATE context_items
-       SET ordinal = ?
-       WHERE conversation_id = ? AND ordinal = ?`,
-    );
-
-    // Use negative temp ordinals first to avoid unique constraint conflicts.
-    for (let i = 0; i < items.length; i++) {
-      updateStmt.run(-(i + 1), conversationId, items[i].ordinal);
-    }
-    for (let i = 0; i < items.length; i++) {
-      updateStmt.run(i, conversationId, -(i + 1));
+    if (items.length > 0 && items.some((item, i) => item.ordinal !== i)) {
+      const updateStmt = this.db.prepare(
+        `UPDATE context_items SET ordinal = ?
+         WHERE conversation_id = ? AND ordinal = ?`,
+      );
+      for (let i = 0; i < items.length; i++) {
+        updateStmt.run(-(i + 1), conversationId, items[i].ordinal);
+      }
+      for (let i = 0; i < items.length; i++) {
+        updateStmt.run(i, conversationId, -(i + 1));
+      }
     }
   }
 
@@ -1051,6 +1055,7 @@ export class SummaryStore {
             input.conversationId,
             input.since,
             input.before,
+            input.sort,
           );
         } catch {
           return this.searchLike(
@@ -1073,6 +1078,7 @@ export class SummaryStore {
     conversationId?: number,
     since?: Date,
     before?: Date,
+    sort?: SearchSort,
   ): SummarySearchResult[] {
     const where: string[] = ["summaries_fts MATCH ?"];
     const args: Array<string | number> = [sanitizeFts5Query(query)];
@@ -1089,6 +1095,7 @@ export class SummaryStore {
       args.push(before.toISOString());
     }
     args.push(limit);
+    const orderBy = buildFtsOrderBy(sort, "s.created_at");
 
     const sql = `SELECT
          summaries_fts.summary_id,
@@ -1100,7 +1107,7 @@ export class SummaryStore {
        FROM summaries_fts
        JOIN summaries s ON s.summary_id = summaries_fts.summary_id
        WHERE ${where.join(" AND ")}
-       ORDER BY s.created_at DESC
+       ORDER BY ${orderBy}
        LIMIT ?`;
     const rows = this.db.prepare(sql).all(...args) as unknown as SummarySearchRow[];
     return rows.map(toSearchResult);

package/src/summarize.ts

@@ -1,4 +1,6 @@
+import { describeLogError } from "./lcm-log.js";
 import type { LcmDependencies } from "./types.js";
+import { estimateTokens } from "./estimate-tokens.js";
 
 export type LcmSummarizeOptions = {
   previousSummary?: string;
@@ -173,10 +175,6 @@ function resolveProviderApiFromLegacyConfig(
   return undefined;
 }
 
-/** Approximate token estimate used for target-sizing prompts. */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
 
 /** Narrow unknown values to plain object records. */
 function isRecord(value: unknown): value is Record<string, unknown> {
@@ -1068,6 +1066,17 @@ function resolveSummaryCandidates(params: {
     },
   ];
 
+  // Append explicit fallback providers from config.
+  for (const fb of params.deps.config.fallbackProviders ?? []) {
+    resolutionCandidates.push({
+      levelName: `explicit fallback (${fb.provider}/${fb.model})`,
+      modelRef: `${fb.provider}/${fb.model}`,
+      providerHint: fb.provider,
+      hasExplicitProvider: true,
+      useLegacyAuthProfile: false,
+    });
+  }
+
   const resolvedCandidates: ResolvedSummaryCandidate[] = [];
   for (const candidate of resolutionCandidates) {
     if (!candidate.modelRef) {
@@ -1088,9 +1097,8 @@ function resolveSummaryCandidates(params: {
         });
       }
     } catch (err) {
-      console.error(
-        `[lcm] createLcmSummarize: resolveModel FAILED at ${candidate.levelName}:`,
-        err instanceof Error ? err.message : err,
+      params.deps.log.error(
+        `[lcm] createLcmSummarize: resolveModel FAILED at ${candidate.levelName}: ${describeLogError(err)}`,
       );
     }
   }
@@ -1111,7 +1119,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
 }): Promise<{ fn: LcmSummarizeFn; model: string; breakerKey: string } | undefined> {
   const resolvedCandidates = resolveSummaryCandidates(params);
   if (resolvedCandidates.length === 0) {
-    console.error("[lcm] createLcmSummarize: no summary model candidates resolved");
+    params.deps.log.error("[lcm] createLcmSummarize: no summary model candidates resolved");
     return undefined;
   }
 
@@ -1197,6 +1205,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         requestApiKey: string | undefined,
         label: string,
         reasoning?: string,
+        options?: { skipModelAuth?: boolean },
       ) =>
         withTimeout(params.deps.complete({
           provider,
@@ -1215,6 +1224,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
           ],
           maxTokens: targetTokens,
           ...(reasoning ? { reasoning } : {}),
+          ...(options?.skipModelAuth === true ? { skipModelAuth: true } : {}),
         }), summarizerTimeoutMs, label);
 
       const retryWithoutModelAuth = async (
@@ -1226,8 +1236,8 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         if (runtimeManagedAuth) {
           throw initialAuthError;
         }
-        console.warn(initialAuthError.message);
-        console.warn(
+        params.deps.log.warn(initialAuthError.message);
+        params.deps.log.warn(
           `[lcm] summarizer auth retry: retrying ${provider}/${model} without runtime.modelAuth credentials.`,
         );
 
@@ -1236,14 +1246,16 @@ export async function createLcmSummarizeFromLegacyParams(params: {
           skipModelAuth: true,
         });
         if (!directApiKey) {
-          console.warn(
+          params.deps.log.warn(
             `[lcm] summarizer auth retry unavailable: no direct credentials found for ${provider}/${model}.`,
           );
           throw initialAuthError;
         }
 
         try {
-          const directResult = await runSummarizerCall(directApiKey, "auth-retry", reasoning);
+          const directResult = await runSummarizerCall(directApiKey, "auth-retry", reasoning, {
+            skipModelAuth: true,
+          });
           // Use requireStructuralSignal on the retry success path too — the
           // summary text may legitimately contain auth-error phrases.
           const directFailure = extractProviderAuthFailure(directResult, {
@@ -1255,10 +1267,10 @@ export async function createLcmSummarizeFromLegacyParams(params: {
               model,
               failure: directFailure,
             });
-            console.warn(retryAuthError.message);
+            params.deps.log.warn(retryAuthError.message);
             throw retryAuthError;
           }
-          console.warn(
+          params.deps.log.info(
             `[lcm] summarizer auth retry succeeded; provider=${provider}; model=${model}; source=direct-credentials`,
           );
           return directResult;
@@ -1277,7 +1289,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
               model,
               failure: directFailure,
             });
-            console.warn(retryAuthError.message);
+            params.deps.log.warn(retryAuthError.message);
             throw retryAuthError;
           }
           throw directErr;
@@ -1317,31 +1329,35 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         if (err instanceof LcmProviderAuthError) {
           lastAuthError = err;
           if (nextCandidate) {
-            console.warn(
-              `[lcm] summarizer auth fallback: retrying with ${nextCandidate.provider}/${nextCandidate.model} after ${provider}/${model} failed auth.`,
+            params.deps.log.warn(
+              `[lcm] PROVIDER FALLBACK: ${provider}/${model} auth failed → trying ${nextCandidate.provider}/${nextCandidate.model}`,
             );
+            const backoffMs = Math.min(500 * Math.pow(2, index), 8000);
+            await new Promise((r) => setTimeout(r, backoffMs));
             continue;
           }
           throw lastAuthError;
         }
         const errMsg = err instanceof Error ? err.message : String(err);
         const isTimeout = errMsg.includes("summarizer timeout");
-        console.warn(
+        params.deps.log.warn(
           `[lcm] summarizer ${isTimeout ? "timed out" : "failed"}; provider=${provider}; model=${model}; timeout=${summarizerTimeoutMs}ms; error=${errMsg}`,
         );
         if (nextCandidate) {
-          console.warn(
-            `[lcm] summarizer candidate fallback: retrying with ${nextCandidate.provider}/${nextCandidate.model} after ${provider}/${model} ${isTimeout ? "timed out" : "failed"}.`,
+          params.deps.log.warn(
+            `[lcm] PROVIDER FALLBACK: ${provider}/${model} ${isTimeout ? "timed out" : "failed"} → trying ${nextCandidate.provider}/${nextCandidate.model}`,
           );
+          const backoffMs = Math.min(500 * Math.pow(2, index), 8000);
+          await new Promise((r) => setTimeout(r, backoffMs));
           continue;
         }
         if (err instanceof SummarizerTimeoutError) {
-          console.error(
+          params.deps.log.warn(
             `[lcm] summarizer timed out; provider=${provider}; model=${model}; source=fallback`,
           );
           return buildDeterministicFallbackSummary(text, targetTokens);
         }
-        return "";
+        break;
       }
 
       const normalized = normalizeCompletionSummary(result.content);
@@ -1358,7 +1374,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         if (envelopeNormalized.summary) {
           summary = envelopeNormalized.summary;
           summarySource = "envelope";
-          console.error(
+          params.deps.log.info(
             `[lcm] recovered summary from response envelope; provider=${provider}; model=${model}; ` +
               `block_types=${formatBlockTypes(envelopeNormalized.blockTypes)}; source=envelope`,
           );
@@ -1386,7 +1402,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
         if (responseDiag) {
           diagParts.push(responseDiag);
         }
-        console.error(`${diagParts.join("; ")}; retrying with conservative settings`);
+        params.deps.log.warn(`${diagParts.join("; ")}; retrying with conservative settings`);
 
         // Single retry with conservative parameters: low temperature and low
         // reasoning budget to coax a textual response from providers that
@@ -1401,7 +1417,7 @@ export async function createLcmSummarizeFromLegacyParams(params: {
 
           if (summary) {
             summarySource = "retry";
-            console.error(
+            params.deps.log.info(
               `[lcm] retry succeeded; provider=${provider}; model=${model}; ` +
                 `block_types=${formatBlockTypes(retryEnvelopeNormalized.blockTypes)}; source=retry`,
             );
@@ -1418,21 +1434,23 @@ export async function createLcmSummarizeFromLegacyParams(params: {
               retryParts.push(retryDiag);
             }
             if (nextCandidate) {
-              console.warn(
+              params.deps.log.warn(
                 `${retryParts.join("; ")}; retrying with ${nextCandidate.provider}/${nextCandidate.model}`,
               );
               continue;
             }
-            console.error(`${retryParts.join("; ")}; falling back to truncation`);
+            params.deps.log.warn(`${retryParts.join("; ")}; falling back to truncation`);
             summary = initialSummary;
           }
         } catch (retryErr) {
           if (retryErr instanceof LcmProviderAuthError) {
             lastAuthError = retryErr;
             if (nextCandidate) {
-              console.warn(
-                `[lcm] summarizer auth fallback: retrying with ${nextCandidate.provider}/${nextCandidate.model} after ${provider}/${model} failed auth.`,
+              params.deps.log.warn(
+                `[lcm] PROVIDER FALLBACK: ${provider}/${model} auth failed on retry → trying ${nextCandidate.provider}/${nextCandidate.model}`,
               );
+              const backoffMs = Math.min(500 * Math.pow(2, index), 8000);
+              await new Promise((r) => setTimeout(r, backoffMs));
               continue;
             }
             throw lastAuthError;
@@ -1441,12 +1459,12 @@ export async function createLcmSummarizeFromLegacyParams(params: {
           const retryErrMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
           const isRetryTimeout = retryErrMsg.includes("summarizer timeout");
           if (nextCandidate) {
-            console.warn(
+            params.deps.log.warn(
               `[lcm] retry ${isRetryTimeout ? "timed out" : "failed"}; provider=${provider}; model=${model}; timeout=${summarizerTimeoutMs}ms; error=${retryErrMsg}; retrying with ${nextCandidate.provider}/${nextCandidate.model}`,
             );
             continue;
           }
-          console.warn(
+          params.deps.log.warn(
             `[lcm] retry ${isRetryTimeout ? "timed out" : "failed"}; provider=${provider}; model=${model}; timeout=${summarizerTimeoutMs}ms; error=${retryErrMsg}; falling back to truncation`,
           );
           summary = initialSummary;
@@ -1455,14 +1473,14 @@ export async function createLcmSummarizeFromLegacyParams(params: {
 
       if (!summary) {
         summarySource = "fallback";
-        console.error(
+        params.deps.log.error(
           `[lcm] all extraction attempts exhausted; provider=${provider}; model=${model}; source=fallback`,
         );
         return buildDeterministicFallbackSummary(text, targetTokens);
       }
 
       if (summarySource !== "content") {
-        console.error(
+        params.deps.log.info(
           `[lcm] summary resolved via non-content path; provider=${provider}; model=${model}; source=${summarySource}`,
         );
       }
@@ -1470,10 +1488,13 @@ export async function createLcmSummarizeFromLegacyParams(params: {
       return summary;
     }
 
+    params.deps.log.error(
+      `[lcm] ALL PROVIDERS EXHAUSTED: ${resolvedCandidates.length} candidate(s) tried, none succeeded. Compaction falling back to deterministic truncation. Check provider keys and quotas.`,
+    );
     if (lastAuthError) {
       throw lastAuthError;
     }
-    return "";
+    return buildDeterministicFallbackSummary(text, targetTokens);
   };
 
   return {

package/src/tools/lcm-describe-tool.ts

@@ -57,7 +57,8 @@ function normalizeRequestedTokenCap(value: unknown): number | undefined {
 
 export function createLcmDescribeTool(input: {
   deps: LcmDependencies;
-  lcm: LcmContextEngine;
+  lcm?: LcmContextEngine;
+  getLcm?: () => Promise<LcmContextEngine>;
   sessionId?: string;
   sessionKey?: string;
 }): AnyAgentTool {
@@ -71,12 +72,16 @@ export function createLcmDescribeTool(input: {
       "token counts, and file exploration results.",
     parameters: LcmDescribeSchema,
     async execute(_toolCallId, params) {
-      const retrieval = input.lcm.getRetrieval();
-      const timezone = input.lcm.timezone;
+      const lcm = input.lcm ?? (await input.getLcm?.());
+      if (!lcm) {
+        throw new Error("LCM engine is unavailable.");
+      }
+      const retrieval = lcm.getRetrieval();
+      const timezone = lcm.timezone;
       const p = params as Record<string, unknown>;
       const id = (p.id as string).trim();
       const conversationScope = await resolveLcmConversationScope({
-        lcm: input.lcm,
+        lcm,
         deps: input.deps,
         sessionId: input.sessionId,
         sessionKey: input.sessionKey,