@martian-engineering/lossless-claw 0.7.0 → 0.8.0

package/README.md

@@ -34,12 +34,14 @@ The plugin now ships a bundled `lossless-claw` skill plus a small plugin command
 
 - `/lcm` shows version, enablement/selection state, DB path and size, summary counts, and summary-health status
 - `/lcm doctor` scans for broken or truncated summaries
+- `/lcm doctor clean` shows read-only high-confidence junk diagnostics for archived subagents, cron sessions, and NULL-key orphaned subagent runs
 - `/lossless` is an alias for `/lcm` on supported native command surfaces
 
 These are plugin slash/native commands, not root shell CLI subcommands. Supported examples:
 
 - `/lcm`
 - `/lcm doctor`
+- `/lcm doctor clean`
 - `/lossless`
 
 Not currently supported as root CLI commands:
@@ -125,8 +127,8 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
           "ignoreSessionPatterns": [
             "agent:*:cron:**"
           ],
-          "summaryModel": "anthropic/claude-haiku-4-5",
-          "expansionModel": "anthropic/claude-haiku-4-5",
+          "summaryModel": "openai/gpt-5.4-mini",
+          "expansionModel": "openai/gpt-5.4-mini",
           "delegationTimeoutMs": 300000,
           "summaryTimeoutMs": 60000
         }
@@ -164,7 +166,7 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
 | `LCM_SUMMARY_MODEL` | `""` | Model override for compaction summarization; falls back to OpenClaw's default model when unset |
 | `LCM_SUMMARY_PROVIDER` | `""` | Provider override for compaction summarization; falls back to `OPENCLAW_PROVIDER` or the provider embedded in the model ref |
 | `LCM_SUMMARY_BASE_URL` | *(from OpenClaw / provider default)* | Base URL override for summarization API calls |
-| `LCM_EXPANSION_MODEL` | *(from OpenClaw)* | Model override for `lcm_expand_query` sub-agent (e.g. `anthropic/claude-haiku-4-5`) |
+| `LCM_EXPANSION_MODEL` | *(from OpenClaw)* | Model override for `lcm_expand_query` sub-agent (e.g. `openai/gpt-5.4-mini`) |
 | `LCM_EXPANSION_PROVIDER` | *(from OpenClaw)* | Provider override for `lcm_expand_query` sub-agent |
 | `LCM_DELEGATION_TIMEOUT_MS` | `120000` | Max time to wait for delegated `lcm_expand_query` sub-agent completion |
 | `LCM_SUMMARY_TIMEOUT_MS` | `60000` | Max time to wait for a single model-backed LCM summarizer call |
@@ -174,6 +176,8 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
 
 If you want `lcm_expand_query` to run on a dedicated model via `expansionModel` or `LCM_EXPANSION_MODEL`, OpenClaw must explicitly trust the plugin to request sub-agent model overrides.
 
+For most setups, `openai/gpt-5.4-mini` is a better starting point than Anthropic Haiku because it is cheap, fast, and does not depend on Anthropic quota remaining.
+
 Add a `subagent` policy under `plugins.entries.lossless-claw` and allowlist the canonical `provider/model` target you want the plugin to use:
 
 ```json
@@ -227,6 +231,8 @@ For compaction summarization, lossless-claw resolves the model in this order:
 
 If `summaryModel` already includes a provider prefix such as `anthropic/claude-sonnet-4-20250514`, `summaryProvider` is ignored for that choice. Otherwise, the provider falls back to the matching override, then `OPENCLAW_PROVIDER`, then the provider inferred by the caller.
 
+Runtime-managed OAuth providers are supported here too. In particular, `openai-codex` and `github-copilot` auth profiles can be used for summary and expansion calls without a separate API key.
+
 ### Recommended starting configuration
 
 ```
@@ -234,6 +240,8 @@ LCM_FRESH_TAIL_COUNT=64
 LCM_LEAF_CHUNK_TOKENS=20000
 LCM_INCREMENTAL_MAX_DEPTH=1
 LCM_CONTEXT_THRESHOLD=0.75
+LCM_SUMMARY_MODEL=openai/gpt-5.4-mini
+LCM_EXPANSION_MODEL=openai/gpt-5.4-mini
 ```
 
 - **freshTailCount=64** protects the last 64 messages from compaction, giving the model more recent context for continuity.

package/docs/agent-tools.md

@@ -24,7 +24,7 @@ Summaries are lossy by design. The "Expand for details about:" footer at the end
 - Tool call sequences and their outputs
 - Verbatim quotes or specific data points
 
-`lcm_expand_query` is bounded (~120s, scoped sub-agent) and relatively cheap. Don't ration it.
+`lcm_expand_query` is bounded (~120s, scoped sub-agent) and relatively cheap. Don't ration it, but use `lcm_grep` first when you need broad discovery across many sessions.
 
 ## Tool reference
 
@@ -114,6 +114,8 @@ lcm_describe(id: "file_789abc012345")
 
 Answer a focused question by expanding summaries through the DAG. Spawns a bounded sub-agent that walks parent links down to source material and returns a compact answer.
 
+When `allConversations: true` is set, `lcm_expand_query` can now synthesize one answer across multiple conversations. That cross-conversation mode is bounded, not exhaustive: it ranks conversation buckets, expands only the top few, and marks the result truncated when lower-ranked buckets are skipped or fail.
+
 **Parameters:**
 
 | Param | Type | Required | Default | Description |
@@ -130,9 +132,11 @@ Answer a focused question by expanding summaries through the DAG. Spawns a bound
 **Returns:**
 - `answer` — The focused answer text
 - `citedIds` — Summary IDs that contributed to the answer
+- `sourceConversationIds` — Conversations that were successfully expanded
 - `expandedSummaryCount` — How many summaries were expanded
 - `totalSourceTokens` — Total tokens read from the DAG
 - `truncated` — Whether the answer was truncated to fit maxTokens
+- `conversationBreakdown` — Optional per-conversation success/failure diagnostics for bounded multi-conversation runs
 
 **Examples:**
 
@@ -149,7 +153,7 @@ lcm_expand_query(
   prompt: "What were the exact file changes?"
 )
 
-# Cross-conversation search
+# Cross-conversation synthesis
 lcm_expand_query(
   query: "deployment procedure",
   prompt: "What's the current deployment process?",
@@ -175,7 +179,7 @@ Add instructions to your agent's system prompt so it knows when to use LCM tools
 Use LCM tools for recall:
 1. `lcm_grep` — Search all conversations by keyword/regex. Prefer `mode: "full_text"` for topic recall, quote exact phrases, use `sort: "relevance"` for older-topic lookups, and `sort: "hybrid"` when recency should still matter.
 2. `lcm_describe` — Inspect a specific summary (cheap, no sub-agent)
-3. `lcm_expand_query` — Deep recall with sub-agent expansion
+3. `lcm_expand_query` — Deep recall with bounded sub-agent expansion
 
 When summaries in context have an "Expand for details about:" footer
 listing something you need, use `lcm_expand_query` to get the full detail.
@@ -183,7 +187,7 @@ listing something you need, use `lcm_expand_query` to get the full detail.
 
 ### Conversation scoping
 
-By default, tools operate on the current conversation. Use `allConversations: true` to search across all of them (all agents, all sessions). Use `conversationId` to target a specific conversation you already know about (from previous grep results).
+By default, tools operate on the current conversation. Use `lcm_grep(..., allConversations: true)` when you need broad global discovery. Use `lcm_expand_query(..., allConversations: true)` when you want bounded synthesis across sessions. Use `conversationId` when you already know the exact conversation to inspect or expand.
 
 ### Performance considerations
 
@@ -191,3 +195,4 @@ By default, tools operate on the current conversation. Use `allConversations: tr
 - `lcm_expand_query` spawns a sub-agent and takes ~30–120 seconds
 - The sub-agent has a 120-second timeout with cleanup guarantees
 - Token caps (`LCM_MAX_EXPAND_TOKENS`) prevent runaway expansion
+- Cross-conversation `lcm_expand_query` expands only a bounded set of top-ranked conversations

package/docs/configuration.md

@@ -191,6 +191,15 @@ Compaction summarization resolves candidates in this order:
 
 If `summaryModel` already contains a provider prefix such as `anthropic/claude-sonnet-4-20250514`, `summaryProvider` is ignored for that candidate.
 
+Runtime-managed OAuth providers are supported here too. In particular, `openai-codex` and `github-copilot` auth profiles can be used for summary and expansion calls without a separate API key.
+
+A practical starting point for cost-sensitive setups is:
+
+```env
+LCM_SUMMARY_MODEL=openai/gpt-5.4-mini
+LCM_EXPANSION_MODEL=openai/gpt-5.4-mini
+```
+
 ### Session pattern matching
 
 `ignoreSessionPatterns` and `statelessSessionPatterns` use full session keys.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@martian-engineering/lossless-claw",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Lossless Context Management plugin for OpenClaw — DAG-based conversation summarization with incremental compaction",
   "type": "module",
   "main": "index.ts",

package/skills/lossless-claw/SKILL.md

@@ -12,8 +12,9 @@ Start here:
 1. Confirm whether the user needs configuration help, diagnostics, recall-tool guidance, or session-lifecycle guidance.
 2. If they need a quick health check, tell them to run `/lossless` (`/lcm` is the shorter alias).
 3. If they suspect summary corruption or truncation, use `/lossless doctor`.
-4. If they ask how `/new` or `/reset` interacts with LCM, read the session-lifecycle reference before answering.
-5. Load the relevant reference file instead of improvising details from memory.
+4. If they want high-confidence junk/session cleanup guidance, use `/lossless doctor clean` before recommending any deletes.
+5. If they ask how `/new` or `/reset` interacts with LCM, read the session-lifecycle reference before answering.
+6. Load the relevant reference file instead of improvising details from memory.
 
 Reference map:
 

package/skills/lossless-claw/references/architecture.md

@@ -50,3 +50,15 @@ It looks for known summary-health markers that indicate:
 - truncated summary artifacts near the end of stored content
 
 This gives users one place to answer the question “is my summary graph healthy?” without introducing a broader mutation surface.
+
+## What `/lcm doctor clean` tells you
+
+The cleaners flow is also diagnostic first.
+
+It reports high-confidence junk patterns that are structurally safe to review as standalone cleanup candidates, including:
+
+- archived subagent sessions
+- cron sessions
+- NULL-key orphaned subagent context conversations
+
+This keeps cleanup discovery separate from summary-health diagnostics while still using the same native command surface.

package/skills/lossless-claw/references/diagnostics.md

@@ -29,6 +29,19 @@ What it should help confirm:
 - whether truncation markers exist
 - which conversations are affected most
 
+### `/lossless doctor clean`
+
+Use this when the user wants read-only diagnostics for high-confidence junk patterns before any cleanup.
+
+It should help confirm:
+
+- whether archived subagent sessions are present
+- whether cron sessions are accumulating unexpectedly
+- whether NULL-key orphaned subagent conversations are present
+- which high-confidence filters match the most conversations and messages
+
+This command is read-only. Use it to identify likely cleanup candidates before taking any separate cleanup action.
+
 ## Interpreting common states
 
 ### `/lossless` tokens vs `/status` context

package/src/assembler.ts

@@ -6,6 +6,7 @@ import type {
   MessageRole,
 } from "./store/conversation-store.js";
 import type { SummaryStore, ContextItemRecord, SummaryRecord } from "./store/summary-store.js";
+import { estimateTokens } from "./estimate-tokens.js";
 
 type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
 
@@ -46,10 +47,6 @@ export interface AssembleContextResult {
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
-/** Simple token estimate: ~4 chars per token, same as VoltCode's Token.estimate */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
 
 type SummaryPromptSignal = Pick<SummaryRecord, "kind" | "depth" | "descendantCount">;
 
@@ -98,6 +95,7 @@ function buildSystemPromptAddition(summarySignals: SummaryPromptSignal[]): strin
       "Keep raw summary IDs in tool context for follow-up; do not include them in the user-facing answer unless the user asks for sources or IDs.",
       "",
       "`lcm_grep` tips: prefer `mode: \"full_text\"` for keyword/topic lookup, quote exact multi-word phrases, use `sort: \"relevance\"` for older-topic retrieval, and use `sort: \"hybrid\"` when recency should still influence ranking.",
+      "`lcm_expand_query(query: ...)` uses the same FTS5 full-text search rules as `lcm_grep`: terms are ANDed by default, so extra query words narrow results. Keep `query` to 1-3 distinctive terms or a quoted phrase, and put the natural-language question in `prompt`.",
       "",
       "**Uncertainty checklist (run before answering):**",
       "- Am I making an exact factual claim from a compressed or condensed summary?",
@@ -1106,6 +1104,16 @@ export class ContextAssembler {
     }
 
     const parts = await this.conversationStore.getMessageParts(msg.messageId);
+
+    // Skip empty assistant messages left by error/aborted responses.
+    // These waste context tokens and can confuse models that reject
+    // consecutive empty assistant turns.  Only skip when both the stored
+    // content text AND the message_parts table are empty — assistant
+    // messages that contain tool calls have empty text content but
+    // non-empty parts and must be preserved.
+    if (msg.role === "assistant" && !msg.content.trim() && parts.length === 0) {
+      return null;
+    }
     const roleFromStore = toRuntimeRole(msg.role, parts);
     const isToolResult = roleFromStore === "toolResult";
     const toolCallId = isToolResult ? pickToolCallId(parts) : undefined;

package/src/compaction.ts

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import type { ConversationStore, CreateMessagePartInput } from "./store/conversation-store.js";
 import type { SummaryStore, SummaryRecord, ContextItemRecord } from "./store/summary-store.js";
+import { estimateTokens, truncateTextToEstimatedTokens } from "./estimate-tokens.js";
 import { extractFileIdsFromContent } from "./large-files.js";
 import { NOOP_LCM_LOGGER, type LcmLogger } from "./lcm-log.js";
 import { LcmProviderAuthError } from "./summarize.js";
@@ -93,10 +94,6 @@ type CondensedPhaseCandidate = {
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
-/** Estimate token count from character length (~4 chars per token). */
-function estimateTokens(content: string): number {
-  return Math.ceil(content.length / 4);
-}
 
 /** Deterministically cap summary text so the persisted output stays within maxTokens. */
 function capSummaryText(
@@ -112,14 +109,14 @@ function capSummaryText(
   ];
 
   for (const suffix of suffixes) {
-    const maxChars = Math.max(0, maxTokens * 4 - suffix.length);
-    const capped = `${content.slice(0, maxChars)}${suffix}`;
+    const contentBudget = Math.max(0, maxTokens - estimateTokens(suffix));
+    const capped = `${truncateTextToEstimatedTokens(content, contentBudget)}${suffix}`;
     if (estimateTokens(capped) <= maxTokens) {
       return capped;
     }
   }
 
-  return content.slice(0, Math.max(0, maxTokens * 4));
+  return truncateTextToEstimatedTokens(content, maxTokens);
 }
 
 /** Format a timestamp as `YYYY-MM-DD HH:mm TZ` for prompt source text. */
@@ -176,8 +173,8 @@ function generateSummaryId(content: string): string {
   );
 }
 
-/** Maximum characters for the deterministic fallback truncation (512 tokens * 4 chars). */
-const FALLBACK_MAX_CHARS = 512 * 4;
+/** Maximum estimated tokens for the deterministic fallback truncation. */
+const FALLBACK_MAX_TOKENS = 512;
 const DEFAULT_LEAF_CHUNK_TOKENS = 20_000;
 
 /**
@@ -1301,13 +1298,13 @@ export class CompactionEngine {
     }
     const inputTokens = Math.max(1, estimateTokens(sourceText));
     const buildDeterministicFallback = (): { content: string; level: CompactionLevel } => {
-      const truncated =
-        sourceText.length > FALLBACK_MAX_CHARS
-          ? sourceText.slice(0, FALLBACK_MAX_CHARS)
-          : sourceText;
+      const suffix = `\n[Truncated from ${inputTokens} tokens]`;
+      const truncated = truncateTextToEstimatedTokens(
+        sourceText,
+        Math.max(0, FALLBACK_MAX_TOKENS - estimateTokens(suffix)),
+      );
       return {
-        content: `${truncated}
-[Truncated from ${inputTokens} tokens]`,
+        content: `${truncated}${suffix}`,
         level: "fallback",
       };
     };

package/src/db/connection.ts

@@ -8,24 +8,34 @@ const SQLITE_BUSY_TIMEOUT_MS = 5_000;
 const connectionsByPath = new Map<ConnectionKey, Set<DatabaseSync>>();
 const connectionIndex = new Map<DatabaseSync, ConnectionKey>();
 
-function isInMemoryPath(dbPath: string): boolean {
+export function isInMemoryPath(dbPath: string): boolean {
   const normalized = dbPath.trim();
   return normalized === ":memory:" || normalized.startsWith("file::memory:");
 }
 
+export function getFileBackedDatabasePath(dbPath: string): string | null {
+  const trimmed = dbPath.trim();
+  if (!trimmed || isInMemoryPath(trimmed)) {
+    return null;
+  }
+  return resolve(trimmed);
+}
+
 export function normalizePath(dbPath: string): ConnectionKey {
-  if (isInMemoryPath(dbPath)) {
+  const fileBackedDatabasePath = getFileBackedDatabasePath(dbPath);
+  if (!fileBackedDatabasePath) {
     const trimmed = dbPath.trim();
     return trimmed.length > 0 ? trimmed : ":memory:";
   }
-  return resolve(dbPath);
+  return fileBackedDatabasePath;
 }
 
 function ensureDbDirectory(dbPath: string): void {
-  if (isInMemoryPath(dbPath)) {
+  const fileBackedDatabasePath = getFileBackedDatabasePath(dbPath);
+  if (!fileBackedDatabasePath) {
     return;
   }
-  mkdirSync(dirname(dbPath), { recursive: true });
+  mkdirSync(dirname(fileBackedDatabasePath), { recursive: true });
 }
 
 function configureConnection(db: DatabaseSync): DatabaseSync {

package/src/db/features.ts

@@ -2,19 +2,20 @@ import type { DatabaseSync } from "node:sqlite";
 
 export type LcmDbFeatures = {
   fts5Available: boolean;
+  trigramTokenizerAvailable: boolean;
 };
 
 const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
 
-function probeFts5(db: DatabaseSync): boolean {
+function probeVirtualTable(db: DatabaseSync, sql: string): boolean {
   try {
-    db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
-    db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
-    db.exec("DROP TABLE temp.__lcm_fts5_probe");
+    db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe");
+    db.exec(sql);
+    db.exec("DROP TABLE temp.__lcm_virtual_table_probe");
     return true;
   } catch {
     try {
-      db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
+      db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe");
     } catch {
       // Ignore cleanup failures after a failed probe.
     }
@@ -22,6 +23,20 @@ function probeFts5(db: DatabaseSync): boolean {
   }
 }
 
+function probeFts5(db: DatabaseSync): boolean {
+  return probeVirtualTable(
+    db,
+    "CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content)",
+  );
+}
+
+function probeTrigramTokenizer(db: DatabaseSync): boolean {
+  return probeVirtualTable(
+    db,
+    "CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content, tokenize='trigram')",
+  );
+}
+
 /**
  * Detect SQLite features exposed by the current Node runtime.
  *
@@ -36,7 +51,11 @@ export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
 
   const detected: LcmDbFeatures = {
     fts5Available: probeFts5(db),
+    trigramTokenizerAvailable: false,
   };
+  if (detected.fts5Available) {
+    detected.trigramTokenizerAvailable = probeTrigramTokenizer(db);
+  }
   featureCache.set(db, detected);
   return detected;
 }