@martian-engineering/lossless-claw 0.8.0 → 0.8.2

package/src/retrieval.ts

@@ -1,360 +0,0 @@
-import type {
-  ConversationStore,
-  MessageRecord,
-  MessageSearchResult,
-} from "./store/conversation-store.js";
-import type {
-  SummaryStore,
-  SummaryRecord,
-  SummarySearchResult,
-  LargeFileRecord,
-} from "./store/summary-store.js";
-import type { SearchSort } from "./store/full-text-sort.js";
-import { estimateTokens } from "./estimate-tokens.js";
-
-// ── Public interfaces ────────────────────────────────────────────────────────
-
-export interface DescribeResult {
-  id: string;
-  type: "summary" | "file";
-  /** Summary-specific fields */
-  summary?: {
-    conversationId: number;
-    kind: "leaf" | "condensed";
-    content: string;
-    depth: number;
-    tokenCount: number;
-    descendantCount: number;
-    descendantTokenCount: number;
-    sourceMessageTokenCount: number;
-    fileIds: string[];
-    parentIds: string[];
-    childIds: string[];
-    messageIds: number[];
-    earliestAt: Date | null;
-    latestAt: Date | null;
-    subtree: Array<{
-      summaryId: string;
-      parentSummaryId: string | null;
-      depthFromRoot: number;
-      kind: "leaf" | "condensed";
-      depth: number;
-      tokenCount: number;
-      descendantCount: number;
-      descendantTokenCount: number;
-      sourceMessageTokenCount: number;
-      earliestAt: Date | null;
-      latestAt: Date | null;
-      childCount: number;
-      path: string;
-    }>;
-    createdAt: Date;
-  };
-  /** File-specific fields */
-  file?: {
-    conversationId: number;
-    fileName: string | null;
-    mimeType: string | null;
-    byteSize: number | null;
-    storageUri: string;
-    explorationSummary: string | null;
-    createdAt: Date;
-  };
-}
-
-export interface GrepInput {
-  query: string;
-  mode: "regex" | "full_text";
-  scope: "messages" | "summaries" | "both";
-  conversationId?: number;
-  since?: Date;
-  before?: Date;
-  limit?: number;
-  /** Sort order for results. Default "recency" (newest first).
-   *  "relevance" sorts by FTS5 BM25 rank (full_text mode only).
-   *  "hybrid" blends relevance with recency. */
-  sort?: SearchSort;
-}
-
-export interface GrepResult {
-  messages: MessageSearchResult[];
-  summaries: SummarySearchResult[];
-  totalMatches: number;
-}
-
-export interface ExpandInput {
-  summaryId: string;
-  /** Max traversal depth (default 1) */
-  depth?: number;
-  /** Include raw source messages at leaf level */
-  includeMessages?: boolean;
-  /** Max tokens to return before truncating */
-  tokenCap?: number;
-}
-
-export interface ExpandResult {
-  /** Child summaries found */
-  children: Array<{
-    summaryId: string;
-    kind: "leaf" | "condensed";
-    content: string;
-    tokenCount: number;
-  }>;
-  /** Source messages (only if includeMessages=true and hitting leaf summaries) */
-  messages: Array<{
-    messageId: number;
-    role: string;
-    content: string;
-    tokenCount: number;
-  }>;
-  /** Total estimated tokens in result */
-  estimatedTokens: number;
-  /** Whether result was truncated due to tokenCap */
-  truncated: boolean;
-}
-
-// ── Helpers ──────────────────────────────────────────────────────────────────
-
-
-// ── RetrievalEngine ──────────────────────────────────────────────────────────
-
-export class RetrievalEngine {
-  constructor(
-    private conversationStore: ConversationStore,
-    private summaryStore: SummaryStore,
-  ) {}
-
-  // ── describe ─────────────────────────────────────────────────────────────
-
-  /**
-   * Describe an LCM item by ID.
-   *
-   * - IDs starting with "sum_" are looked up as summaries (with lineage).
-   * - IDs starting with "file_" are looked up as large files.
-   * - Returns null if the item is not found.
-   */
-  async describe(id: string): Promise<DescribeResult | null> {
-    if (id.startsWith("sum_")) {
-      return this.describeSummary(id);
-    }
-    if (id.startsWith("file_")) {
-      return this.describeFile(id);
-    }
-    return null;
-  }
-
-  private async describeSummary(id: string): Promise<DescribeResult | null> {
-    const summary = await this.summaryStore.getSummary(id);
-    if (!summary) {
-      return null;
-    }
-
-    // Fetch lineage in parallel
-    const [parents, children, messageIds, subtree] = await Promise.all([
-      this.summaryStore.getSummaryParents(id),
-      this.summaryStore.getSummaryChildren(id),
-      this.summaryStore.getSummaryMessages(id),
-      this.summaryStore.getSummarySubtree(id),
-    ]);
-
-    return {
-      id,
-      type: "summary",
-      summary: {
-        conversationId: summary.conversationId,
-        kind: summary.kind,
-        content: summary.content,
-        depth: summary.depth,
-        tokenCount: summary.tokenCount,
-        descendantCount: summary.descendantCount,
-        descendantTokenCount: summary.descendantTokenCount,
-        sourceMessageTokenCount: summary.sourceMessageTokenCount,
-        fileIds: summary.fileIds,
-        parentIds: parents.map((p) => p.summaryId),
-        childIds: children.map((c) => c.summaryId),
-        messageIds,
-        earliestAt: summary.earliestAt,
-        latestAt: summary.latestAt,
-        subtree: subtree.map((node) => ({
-          summaryId: node.summaryId,
-          parentSummaryId: node.parentSummaryId,
-          depthFromRoot: node.depthFromRoot,
-          kind: node.kind,
-          depth: node.depth,
-          tokenCount: node.tokenCount,
-          descendantCount: node.descendantCount,
-          descendantTokenCount: node.descendantTokenCount,
-          sourceMessageTokenCount: node.sourceMessageTokenCount,
-          earliestAt: node.earliestAt,
-          latestAt: node.latestAt,
-          childCount: node.childCount,
-          path: node.path,
-        })),
-        createdAt: summary.createdAt,
-      },
-    };
-  }
-
-  private async describeFile(id: string): Promise<DescribeResult | null> {
-    const file = await this.summaryStore.getLargeFile(id);
-    if (!file) {
-      return null;
-    }
-
-    return {
-      id,
-      type: "file",
-      file: {
-        conversationId: file.conversationId,
-        fileName: file.fileName,
-        mimeType: file.mimeType,
-        byteSize: file.byteSize,
-        storageUri: file.storageUri,
-        explorationSummary: file.explorationSummary,
-        createdAt: file.createdAt,
-      },
-    };
-  }
-
-  // ── grep ─────────────────────────────────────────────────────────────────
-
-  /**
-   * Search compacted history using regex or full-text search.
-   *
-   * Depending on `scope`, searches messages, summaries, or both (in parallel).
-   */
-  async grep(input: GrepInput): Promise<GrepResult> {
-    const { query, mode, scope, conversationId, since, before, limit, sort } = input;
-
-    const searchInput = { query, mode, conversationId, since, before, limit, sort };
-
-    let messages: MessageSearchResult[] = [];
-    let summaries: SummarySearchResult[] = [];
-
-    if (scope === "messages") {
-      messages = await this.conversationStore.searchMessages(searchInput);
-    } else if (scope === "summaries") {
-      summaries = await this.summaryStore.searchSummaries(searchInput);
-    } else {
-      // scope === "both" — run in parallel
-      [messages, summaries] = await Promise.all([
-        this.conversationStore.searchMessages(searchInput),
-        this.summaryStore.searchSummaries(searchInput),
-      ]);
-    }
-
-    return {
-      messages,
-      summaries,
-      totalMatches: messages.length + summaries.length,
-    };
-  }
-
-  // ── expand ───────────────────────────────────────────────────────────────
-
-  /**
-   * Expand a summary to its children and/or source messages.
-   *
-   * - Condensed summaries: returns child summaries, recursing up to `depth`.
-   * - Leaf summaries with `includeMessages`: fetches the source messages.
-   * - Respects `tokenCap` and sets `truncated` when the cap is exceeded.
-   */
-  async expand(input: ExpandInput): Promise<ExpandResult> {
-    const depth = input.depth ?? 1;
-    const includeMessages = input.includeMessages ?? false;
-    const tokenCap = input.tokenCap ?? Infinity;
-
-    const result: ExpandResult = {
-      children: [],
-      messages: [],
-      estimatedTokens: 0,
-      truncated: false,
-    };
-
-    await this.expandRecursive(input.summaryId, depth, includeMessages, tokenCap, result);
-
-    return result;
-  }
-
-  private async expandRecursive(
-    summaryId: string,
-    depth: number,
-    includeMessages: boolean,
-    tokenCap: number,
-    result: ExpandResult,
-  ): Promise<void> {
-    if (depth <= 0) {
-      return;
-    }
-    if (result.truncated) {
-      return;
-    }
-
-    const summary = await this.summaryStore.getSummary(summaryId);
-    if (!summary) {
-      return;
-    }
-
-    if (summary.kind === "condensed") {
-      // IMPORTANT: a condensed summary is linked to the summaries that were
-      // compacted into it via summary_parents(summary_id, parent_summary_id).
-      // For expansion/replay we need to walk those source summaries, not newer
-      // summaries that may later derive from this node.
-      const children = await this.summaryStore.getSummaryParents(summaryId);
-
-      for (const child of children) {
-        if (result.truncated) {
-          break;
-        }
-
-        // Check if adding this child would exceed the token cap
-        if (result.estimatedTokens + child.tokenCount > tokenCap) {
-          result.truncated = true;
-          break;
-        }
-
-        result.children.push({
-          summaryId: child.summaryId,
-          kind: child.kind,
-          content: child.content,
-          tokenCount: child.tokenCount,
-        });
-        result.estimatedTokens += child.tokenCount;
-
-        // Recurse into children if depth allows
-        if (depth > 1) {
-          await this.expandRecursive(child.summaryId, depth - 1, includeMessages, tokenCap, result);
-        }
-      }
-    } else if (summary.kind === "leaf" && includeMessages) {
-      // Leaf summary — fetch source messages
-      const messageIds = await this.summaryStore.getSummaryMessages(summaryId);
-
-      for (const msgId of messageIds) {
-        if (result.truncated) {
-          break;
-        }
-
-        const msg = await this.conversationStore.getMessageById(msgId);
-        if (!msg) {
-          continue;
-        }
-
-        const tokenCount = msg.tokenCount || estimateTokens(msg.content);
-
-        if (result.estimatedTokens + tokenCount > tokenCap) {
-          result.truncated = true;
-          break;
-        }
-
-        result.messages.push({
-          messageId: msg.messageId,
-          role: msg.role,
-          content: msg.content,
-          tokenCount,
-        });
-        result.estimatedTokens += tokenCount;
-      }
-    }
-  }
-}

package/src/session-patterns.ts

@@ -1,23 +0,0 @@
-/**
- * Compile a session glob into a regex.
- *
- * `*` matches any non-colon characters, while `**` can span colons.
- */
-export function compileSessionPattern(pattern: string): RegExp {
-  const escaped = pattern
-    .replace(/[.+^${}()|[\]\\]/g, "\\$&")
-    .replace(/\*\*/g, "\u0000")
-    .replace(/\*/g, "[^:]*")
-    .replace(/\u0000/g, ".*");
-  return new RegExp(`^${escaped}$`);
-}
-
-/** Compile all configured ignore patterns once at startup. */
-export function compileSessionPatterns(patterns: string[]): RegExp[] {
-  return patterns.map((pattern) => compileSessionPattern(pattern));
-}
-
-/** Check whether a session key matches any compiled ignore pattern. */
-export function matchesSessionPattern(sessionKey: string, patterns: RegExp[]): boolean {
-  return patterns.some((pattern) => pattern.test(sessionKey));
-}

package/src/startup-banner-log.ts

@@ -1,49 +0,0 @@
-type StartupBannerKey =
-  | "plugin-loaded"
-  | "compaction-model"
-  | "fallback-providers"
-  | "ignore-session-patterns"
-  | "stateless-session-patterns";
-
-type StartupBannerLogState = {
-  emitted: Set<StartupBannerKey>;
-};
-
-const STARTUP_BANNER_LOG_STATE = Symbol.for(
-  "@martian-engineering/lossless-claw/startup-banner-log-state",
-);
-
-/** Return the process-global startup banner log state. */
-function getStartupBannerLogState(): StartupBannerLogState {
-  const globalState = globalThis as typeof globalThis & {
-    [STARTUP_BANNER_LOG_STATE]?: StartupBannerLogState;
-  };
-
-  if (!globalState[STARTUP_BANNER_LOG_STATE]) {
-    globalState[STARTUP_BANNER_LOG_STATE] = {
-      emitted: new Set<StartupBannerKey>(),
-    };
-  }
-
-  return globalState[STARTUP_BANNER_LOG_STATE];
-}
-
-/** Emit a startup/config banner only once per process. */
-export function logStartupBannerOnce(params: {
-  key: StartupBannerKey;
-  log: (message: string) => void;
-  message: string;
-}): void {
-  const state = getStartupBannerLogState();
-  if (state.emitted.has(params.key)) {
-    return;
-  }
-
-  state.emitted.add(params.key);
-  params.log(params.message);
-}
-
-/** Reset startup/config banner dedupe state for tests. */
-export function resetStartupBannerLogsForTests(): void {
-  getStartupBannerLogState().emitted.clear();
-}

package/src/store/compaction-telemetry-store.ts

@@ -1,156 +0,0 @@
-import type { DatabaseSync } from "node:sqlite";
-import { withDatabaseTransaction } from "../transaction-mutex.js";
-import { parseUtcTimestampOrNull } from "./parse-utc-timestamp.js";
-
-export type CacheState = "hot" | "cold" | "unknown";
-export type ActivityBand = "low" | "medium" | "high";
-
-export type ConversationCompactionTelemetryRecord = {
-  conversationId: number;
-  lastObservedCacheRead: number | null;
-  lastObservedCacheWrite: number | null;
-  lastObservedCacheHitAt: Date | null;
-  lastObservedCacheBreakAt: Date | null;
-  cacheState: CacheState;
-  retention: string | null;
-  lastLeafCompactionAt: Date | null;
-  turnsSinceLeafCompaction: number;
-  tokensAccumulatedSinceLeafCompaction: number;
-  lastActivityBand: ActivityBand;
-  updatedAt: Date;
-};
-
-export type UpsertConversationCompactionTelemetryInput = {
-  conversationId: number;
-  lastObservedCacheRead?: number | null;
-  lastObservedCacheWrite?: number | null;
-  lastObservedCacheHitAt?: Date | null;
-  lastObservedCacheBreakAt?: Date | null;
-  cacheState: CacheState;
-  retention?: string | null;
-  lastLeafCompactionAt?: Date | null;
-  turnsSinceLeafCompaction?: number;
-  tokensAccumulatedSinceLeafCompaction?: number;
-  lastActivityBand?: ActivityBand;
-};
-
-type ConversationCompactionTelemetryRow = {
-  conversation_id: number;
-  last_observed_cache_read: number | null;
-  last_observed_cache_write: number | null;
-  last_observed_cache_hit_at: string | null;
-  last_observed_cache_break_at: string | null;
-  cache_state: CacheState;
-  retention: string | null;
-  last_leaf_compaction_at: string | null;
-  turns_since_leaf_compaction: number | null;
-  tokens_accumulated_since_leaf_compaction: number | null;
-  last_activity_band: ActivityBand | null;
-  updated_at: string;
-};
-
-function toConversationCompactionTelemetryRecord(
-  row: ConversationCompactionTelemetryRow,
-): ConversationCompactionTelemetryRecord {
-  return {
-    conversationId: row.conversation_id,
-    lastObservedCacheRead: row.last_observed_cache_read,
-    lastObservedCacheWrite: row.last_observed_cache_write,
-    lastObservedCacheHitAt: parseUtcTimestampOrNull(row.last_observed_cache_hit_at),
-    lastObservedCacheBreakAt: parseUtcTimestampOrNull(row.last_observed_cache_break_at),
-    cacheState: row.cache_state,
-    retention: row.retention,
-    lastLeafCompactionAt: parseUtcTimestampOrNull(row.last_leaf_compaction_at),
-    turnsSinceLeafCompaction: row.turns_since_leaf_compaction ?? 0,
-    tokensAccumulatedSinceLeafCompaction: row.tokens_accumulated_since_leaf_compaction ?? 0,
-    lastActivityBand: row.last_activity_band ?? "low",
-    updatedAt: parseUtcTimestampOrNull(row.updated_at) ?? new Date(0),
-  };
-}
-
-/**
- * Persist and query per-conversation prompt-cache telemetry used by
- * cache-aware incremental compaction.
- */
-export class CompactionTelemetryStore {
-  constructor(private readonly db: DatabaseSync) {}
-
-  /** Execute multiple telemetry writes atomically. */
-  withTransaction<T>(fn: () => Promise<T>): Promise<T> {
-    return withDatabaseTransaction(this.db, "BEGIN", fn);
-  }
-
-  /** Load the latest persisted telemetry for a conversation. */
-  async getConversationCompactionTelemetry(
-    conversationId: number,
-  ): Promise<ConversationCompactionTelemetryRecord | null> {
-    const row = this.db
-      .prepare(
-        `SELECT
-           conversation_id,
-           last_observed_cache_read,
-           last_observed_cache_write,
-           last_observed_cache_hit_at,
-           last_observed_cache_break_at,
-           cache_state,
-           retention,
-           last_leaf_compaction_at,
-           turns_since_leaf_compaction,
-           tokens_accumulated_since_leaf_compaction,
-           last_activity_band,
-           updated_at
-         FROM conversation_compaction_telemetry
-         WHERE conversation_id = ?`,
-      )
-      .get(conversationId) as ConversationCompactionTelemetryRow | undefined;
-    return row ? toConversationCompactionTelemetryRecord(row) : null;
-  }
-
-  /** Upsert the current cache telemetry snapshot for a conversation. */
-  async upsertConversationCompactionTelemetry(
-    input: UpsertConversationCompactionTelemetryInput,
-  ): Promise<void> {
-    this.db
-      .prepare(
-        `INSERT INTO conversation_compaction_telemetry (
-           conversation_id,
-           last_observed_cache_read,
-           last_observed_cache_write,
-           last_observed_cache_hit_at,
-           last_observed_cache_break_at,
-           cache_state,
-           retention,
-           last_leaf_compaction_at,
-           turns_since_leaf_compaction,
-           tokens_accumulated_since_leaf_compaction,
-           last_activity_band,
-           updated_at
-         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
-         ON CONFLICT(conversation_id) DO UPDATE SET
-           last_observed_cache_read = excluded.last_observed_cache_read,
-           last_observed_cache_write = excluded.last_observed_cache_write,
-           last_observed_cache_hit_at = excluded.last_observed_cache_hit_at,
-           last_observed_cache_break_at = excluded.last_observed_cache_break_at,
-           cache_state = excluded.cache_state,
-           retention = excluded.retention,
-           last_leaf_compaction_at = excluded.last_leaf_compaction_at,
-           turns_since_leaf_compaction = excluded.turns_since_leaf_compaction,
-           tokens_accumulated_since_leaf_compaction = excluded.tokens_accumulated_since_leaf_compaction,
-           last_activity_band = excluded.last_activity_band,
-           updated_at = datetime('now')`,
-      )
-      .run(
-        input.conversationId,
-        input.lastObservedCacheRead ?? null,
-        input.lastObservedCacheWrite ?? null,
-        input.lastObservedCacheHitAt?.toISOString() ?? null,
-        input.lastObservedCacheBreakAt?.toISOString() ?? null,
-        input.cacheState,
-        input.retention ?? null,
-        input.lastLeafCompactionAt?.toISOString() ?? null,
-        input.turnsSinceLeafCompaction ?? 0,
-        input.tokensAccumulatedSinceLeafCompaction ?? 0,
-        input.lastActivityBand ?? "low",
-      );
-  }
-}