@adhdev/daemon-core 0.9.82-rc.27 → 0.9.82-rc.270

package/src/config/chat-history.ts

@@ -58,6 +58,40 @@ const savedHistoryFileSummaryCache = new Map<string, SavedHistoryFileSummaryCach
 const savedHistoryBackgroundRefresh = new Set<string>();
 const savedHistoryRollupInFlight = new Set<string>();
 
+// Bounded-tail read cache. The dashboard re-subscribes and polls hot sessions
+// every ~2.5s; without a cache each poll re-reads/parses/sorts the whole
+// conversation just to slice a small tail. We key on (type, sessionId,
+// pagination args) plus the on-disk size+mtime signature so an UNCHANGED
+// session returns the previously computed tail in O(1) and only re-reads when a
+// new message is appended (signature changes). The map is bounded by a small
+// LRU to keep memory flat regardless of how many sessions are touched.
+interface BoundedTailCacheEntry {
+    signature: string;
+    result: { messages: HistoryMessage[]; hasMore: boolean };
+}
+
+const BOUNDED_TAIL_CACHE_MAX_ENTRIES = 64;
+const boundedTailReadCache = new Map<string, BoundedTailCacheEntry>();
+
+function readBoundedTailCache(key: string, signature: string): { messages: HistoryMessage[]; hasMore: boolean } | null {
+    const cached = boundedTailReadCache.get(key);
+    if (!cached || cached.signature !== signature) return null;
+    // Refresh LRU recency.
+    boundedTailReadCache.delete(key);
+    boundedTailReadCache.set(key, cached);
+    return cached.result;
+}
+
+function writeBoundedTailCache(key: string, signature: string, result: { messages: HistoryMessage[]; hasMore: boolean }): void {
+    boundedTailReadCache.delete(key);
+    boundedTailReadCache.set(key, { signature, result });
+    while (boundedTailReadCache.size > BOUNDED_TAIL_CACHE_MAX_ENTRIES) {
+        const oldest = boundedTailReadCache.keys().next().value;
+        if (oldest === undefined) break;
+        boundedTailReadCache.delete(oldest);
+    }
+}
+
 interface HistoryMessage {
     ts: string;           // ISO timestamp
     receivedAt: number;   // epoch ms
@@ -1180,6 +1214,79 @@ function pageHistoryRecords(
     return { messages: sliced, hasMore: startInclusive > 0 };
 }
 
+// A finite tail request can be served by reading only the newest files instead
+// of the whole conversation. Treat very large limits (e.g. MAX_SAFE_INTEGER, or
+// anything past a generous ceiling) as a full-history request so restore/seed
+// callers keep their existing behavior.
+const BOUNDED_TAIL_MAX_LIMIT = 5_000;
+// Slack added to the requested window before sorting/dedup/collapse so the
+// boundary message at the top of the tail dedupes/collapses identically to a
+// full read. Modest and bounded — it only widens the parse window, not output.
+const BOUNDED_TAIL_SLACK = 50;
+
+function isBoundedTailRequest(limit: number, offset: number, excludeRecentCount: number): boolean {
+    const numericLimit = Number(limit);
+    if (!Number.isFinite(numericLimit) || numericLimit <= 0) return false;
+    if (numericLimit > BOUNDED_TAIL_MAX_LIMIT) return false;
+    const numericOffset = Number(offset);
+    const numericExclude = Number(excludeRecentCount);
+    if (!Number.isFinite(numericOffset) || !Number.isFinite(numericExclude)) return false;
+    return true;
+}
+
+// Read newest-first only as many files as needed to cover the requested window
+// plus slack. listHistoryFiles already returns files reversed (newest-first), so
+// we accumulate (de-duped) candidates from the end and stop once we have enough,
+// then hand the bounded window to pageHistoryRecords in chronological order.
+function readBoundedTailRecords(
+    agentType: string,
+    dir: string,
+    files: string[],
+    needed: number,
+): { records: HistoryMessage[]; readAllFiles: boolean } {
+    const collected: HistoryMessage[] = [];
+    const seen = new Set<string>();
+    let readAllFiles = true;
+
+    for (let f = 0; f < files.length; f++) {
+        const filePath = path.join(dir, files[f]);
+        let content: string;
+        try {
+            content = fs.readFileSync(filePath, 'utf-8');
+        } catch {
+            continue;
+        }
+        const lines = content.trim().split('\n').filter(Boolean);
+        // Walk this file's lines newest-first so we fill the tail window from the
+        // bottom. seen-dedup keeps the same first-wins-by-newest semantics the
+        // full read produced (files are processed newest-first there too).
+        for (let i = lines.length - 1; i >= 0; i--) {
+            try {
+                const parsed = JSON.parse(lines[i]) as HistoryMessage;
+                const sanitizedMessage = sanitizeHistoryMessage(agentType, parsed);
+                if (!sanitizedMessage) continue;
+                const hash = buildHistoryMessageHash(agentType, sanitizedMessage);
+                if (seen.has(hash)) continue;
+                seen.add(hash);
+                collected.push(sanitizedMessage);
+            } catch { /* skip invalid lines */ }
+        }
+        // Stop once we have the window AND there is at least one more file (so a
+        // potential older boundary message exists). If this is the last file we
+        // fall through and mark the whole history as read.
+        if (collected.length >= needed && f < files.length - 1) {
+            readAllFiles = false;
+            break;
+        }
+    }
+
+    // collected is newest-first across the bounded window; restore chronological
+    // (oldest-first) order before paging. pageHistoryRecords re-sorts by
+    // receivedAt regardless, so this is purely for stable input ordering.
+    collected.reverse();
+    return { records: collected, readAllFiles };
+}
+
 export function readChatHistory(
     agentType: string,
     offset: number = 0,
@@ -1196,6 +1303,33 @@ export function readChatHistory(
  // JSONL file list — filter by persistent history key when specified
         const files = listHistoryFiles(dir, historySessionId);
 
+        const bounded = isBoundedTailRequest(limit, offset, excludeRecentCount);
+
+        if (bounded) {
+            const fileSignatures = buildSavedHistoryFileSignatureMap(dir, files);
+            const cacheKey = `${sanitized}\0${historySessionId || ''}\0${offset}\0${limit}\0${excludeRecentCount}\0${historyBehavior?.collapseConsecutiveAssistantTurns ? '1' : '0'}`;
+            const signature = buildSavedHistoryCacheSignature(files, fileSignatures);
+            const cached = readBoundedTailCache(cacheKey, signature);
+            if (cached) return cached;
+
+            // Window large enough that the top boundary dedupes/collapses the same
+            // as a full read. hasMore reflects whether older messages exist beyond
+            // the window we actually read.
+            const numericLimit = Math.max(1, Number(limit));
+            const numericOffset = Math.max(0, Number(offset));
+            const numericExclude = Math.max(0, Number(excludeRecentCount));
+            const needed = numericLimit + numericOffset + numericExclude + Math.max(BOUNDED_TAIL_SLACK, numericLimit);
+            const { records, readAllFiles } = readBoundedTailRecords(agentType, dir, files, needed);
+            const result = pageHistoryRecords(agentType, records, offset, limit, excludeRecentCount, historyBehavior);
+            // If we read every file, the conversation is fully represented in the
+            // window and pageHistoryRecords' hasMore is authoritative. If we
+            // stopped early there are older messages we never read, so hasMore
+            // must stay true regardless of the in-window slice position.
+            const boundedResult = readAllFiles ? result : { messages: result.messages, hasMore: true };
+            writeBoundedTailCache(cacheKey, signature, boundedResult);
+            return boundedResult;
+        }
+
         const allMessages: HistoryMessage[] = [];
         const seen = new Set<string>();
 
@@ -1345,7 +1479,16 @@ function rewriteCanonicalSavedHistory(agentType: string, historySessionId: strin
 
 export type ProviderNativeHistoryScripts = Record<string, ((input: any) => any) | undefined>;
 
-type ProviderNativeHistoryReadResult = { records: HistoryMessage[]; sourcePath: string; sourceMtimeMs: number };
+type ProviderNativeHistoryReadResult = {
+    records: HistoryMessage[];
+    sourcePath: string;
+    sourceMtimeMs: number;
+    providerSessionId?: string;
+    workspace?: string;
+    nativeHistoryCoverage?: string;
+    partialReason?: string;
+    unavailableReason?: string;
+};
 
 function getNativeHistoryScriptName(canonicalHistory: ProviderCanonicalHistoryConfig | undefined, key: 'readSession' | 'listSessions'): string {
     const configured = canonicalHistory?.scripts?.[key];
@@ -1367,19 +1510,42 @@ function normalizeProviderNativeHistoryRecords(agentType: string, historySession
     if (!Array.isArray(records)) return [];
     const normalizedSessionId = normalizeSavedHistorySessionId(historySessionId);
     return records
-        .map((record: any) => sanitizeHistoryMessage(agentType, {
-            ts: typeof record?.ts === 'string' ? record.ts : new Date(Number(record?.receivedAt) || Date.now()).toISOString(),
-            receivedAt: Number(record?.receivedAt) || Date.parse(record?.ts || '') || Date.now(),
-            role: record?.role,
-            content: String(record?.content || ''),
-            kind: record?.kind || (record?.role === 'system' ? 'session_start' : 'standard'),
-            senderName: record?.senderName,
-            agent: agentType,
-            instanceId: record?.instanceId,
-            historySessionId: normalizeSavedHistorySessionId(record?.historySessionId || normalizedSessionId),
-            sessionTitle: record?.sessionTitle,
-            workspace: record?.workspace,
-        } as HistoryMessage))
+        .map((record: any) => {
+            const base: HistoryMessage = {
+                ts: typeof record?.ts === 'string' ? record.ts : new Date(Number(record?.receivedAt) || Date.now()).toISOString(),
+                receivedAt: Number(record?.receivedAt) || Date.parse(record?.ts || '') || Date.now(),
+                role: record?.role,
+                content: String(record?.content || ''),
+                kind: record?.kind || (record?.role === 'system' ? 'session_start' : 'standard'),
+                senderName: record?.senderName,
+                agent: agentType,
+                instanceId: record?.instanceId,
+                historySessionId: normalizeSavedHistorySessionId(record?.historySessionId || normalizedSessionId),
+                sessionTitle: record?.sessionTitle,
+                workspace: record?.workspace,
+            } as HistoryMessage;
+            // (A2.3 v2 identity passthrough) — if the producer (native_history.js)
+            // emitted v2 stable identity, keep it across the sanitize layer so
+            // downstream (chat-commands.ts normalizeNativeHistoryMessages) sees
+            // the producer's contract output instead of recomputing from index
+            // and content hash. v1 producers without these fields are unaffected.
+            if (typeof record?.providerUnitKey === 'string' && record.providerUnitKey) {
+                (base as any).providerUnitKey = record.providerUnitKey;
+            }
+            if (typeof record?.bubbleId === 'string' && record.bubbleId) {
+                (base as any).bubbleId = record.bubbleId;
+            }
+            if (typeof record?.sequence === 'number' && Number.isFinite(record.sequence)) {
+                (base as any).sequence = record.sequence;
+            }
+            if (typeof record?._turnKey === 'string' && record._turnKey) {
+                (base as any)._turnKey = record._turnKey;
+            }
+            if (typeof record?.bubbleState === 'string' && record.bubbleState) {
+                (base as any).bubbleState = record.bubbleState;
+            }
+            return sanitizeHistoryMessage(agentType, base);
+        })
         .filter(Boolean) as HistoryMessage[];
 }
 
@@ -1387,27 +1553,41 @@ function callProviderNativeHistoryRead(
     agentType: string,
     canonicalHistory: ProviderCanonicalHistoryConfig | undefined,
     scripts: ProviderNativeHistoryScripts | undefined,
-    historySessionId: string,
+    historySessionId: string | undefined,
     workspace?: string,
+    excludeInProgressTurn?: boolean,
+    sessionStartedAtMs?: number,
+    envOverrides?: Record<string, string>,
+    forceRefresh?: boolean,
 ): ProviderNativeHistoryReadResult | null {
     const fn = getProviderNativeHistoryScript(scripts, canonicalHistory, 'readSession');
     if (!fn) return null;
+    const normalizedSessionId = normalizeSavedHistorySessionId(historySessionId || '');
     const result = fn({
         agentType,
-        sessionId: historySessionId,
-        historySessionId,
+        sessionId: normalizedSessionId,
+        historySessionId: normalizedSessionId,
         workspace,
         format: canonicalHistory?.format,
         watchPath: canonicalHistory?.watchPath,
-        args: { sessionId: historySessionId, historySessionId, workspace },
+        excludeInProgressTurn: excludeInProgressTurn === true,
+        sessionStartedAtMs,
+        envOverrides,
+        forceRefresh: forceRefresh === true,
+        args: { sessionId: normalizedSessionId, historySessionId: normalizedSessionId, workspace, excludeInProgressTurn: excludeInProgressTurn === true, sessionStartedAtMs, envOverrides, forceRefresh: forceRefresh === true },
     });
     if (!result || typeof result !== 'object') return null;
-    const records = normalizeProviderNativeHistoryRecords(agentType, historySessionId, (result as any).messages || (result as any).records);
+    const records = normalizeProviderNativeHistoryRecords(agentType, normalizedSessionId, (result as any).messages || (result as any).records);
     if (records.length === 0) return null;
     return {
         records,
         sourcePath: typeof (result as any).sourcePath === 'string' ? (result as any).sourcePath : '',
         sourceMtimeMs: Number((result as any).sourceMtimeMs) || 0,
+        providerSessionId: typeof (result as any).providerSessionId === 'string' ? (result as any).providerSessionId.trim() : undefined,
+        workspace: typeof (result as any).workspace === 'string' ? (result as any).workspace.trim() : undefined,
+        nativeHistoryCoverage: typeof (result as any).nativeHistoryCoverage === 'string' ? (result as any).nativeHistoryCoverage.trim() : undefined,
+        partialReason: typeof (result as any).partialReason === 'string' ? (result as any).partialReason.trim() : undefined,
+        unavailableReason: typeof (result as any).unavailableReason === 'string' ? (result as any).unavailableReason.trim() : undefined,
     };
 }
 
@@ -1417,10 +1597,15 @@ function buildNativeHistoryReadResult(
     scripts: ProviderNativeHistoryScripts | undefined,
     historySessionId: string | undefined,
     workspace?: string,
+    excludeInProgressTurn?: boolean,
+    sessionStartedAtMs?: number,
+    envOverrides?: Record<string, string>,
+    forceRefresh?: boolean,
 ): ProviderNativeHistoryReadResult | null {
     const normalizedSessionId = normalizeSavedHistorySessionId(historySessionId || '');
-    if (!canonicalHistory || !normalizedSessionId || !isNativeSourceCanonicalHistory(canonicalHistory)) return null;
-    return callProviderNativeHistoryRead(agentType, canonicalHistory, scripts, normalizedSessionId, workspace);
+    const normalizedWorkspace = typeof workspace === 'string' ? workspace.trim() : '';
+    if (!canonicalHistory || (!normalizedSessionId && !normalizedWorkspace) || !isNativeSourceCanonicalHistory(canonicalHistory)) return null;
+    return callProviderNativeHistoryRead(agentType, canonicalHistory, scripts, normalizedSessionId, workspace, excludeInProgressTurn, sessionStartedAtMs, envOverrides, forceRefresh);
 }
 
 function materializeNativeHistoryToMirror(
@@ -1476,16 +1661,36 @@ export function readProviderChatHistory(
         excludeRecentCount?: number;
         historyBehavior?: ProviderHistoryBehavior;
         scripts?: ProviderNativeHistoryScripts;
+        excludeInProgressTurn?: boolean;
+        sessionStartedAtMs?: number;
+        envOverrides?: Record<string, string>;
+        forceRefresh?: boolean;
     } = {},
-): { messages: HistoryMessage[]; hasMore: boolean; source: 'provider-native' | 'adhdev-mirror' | 'native-unavailable'; sourcePath?: string; sourceMtimeMs?: number } {
-    if (isNativeSourceCanonicalHistory(options.canonicalHistory) && options.historySessionId) {
-        const nativeResult = buildNativeHistoryReadResult(agentType, options.canonicalHistory, options.scripts, options.historySessionId, options.workspace);
+): {
+    messages: HistoryMessage[];
+    hasMore: boolean;
+    source: 'provider-native' | 'adhdev-mirror' | 'native-unavailable';
+    sourcePath?: string;
+    sourceMtimeMs?: number;
+    providerSessionId?: string;
+    workspace?: string;
+    nativeHistoryCoverage?: string;
+    partialReason?: string;
+    unavailableReason?: string;
+} {
+    if (isNativeSourceCanonicalHistory(options.canonicalHistory) && (options.historySessionId || options.workspace)) {
+        const nativeResult = buildNativeHistoryReadResult(agentType, options.canonicalHistory, options.scripts, options.historySessionId, options.workspace, options.excludeInProgressTurn, options.sessionStartedAtMs, options.envOverrides, options.forceRefresh);
         if (!nativeResult) return { messages: [], hasMore: false, source: 'native-unavailable' };
         return {
             ...pageHistoryRecords(agentType, nativeResult.records, options.offset || 0, options.limit || 30, options.excludeRecentCount || 0, options.historyBehavior),
             source: 'provider-native',
             sourcePath: nativeResult.sourcePath,
             sourceMtimeMs: nativeResult.sourceMtimeMs,
+            providerSessionId: nativeResult.providerSessionId,
+            workspace: nativeResult.workspace,
+            nativeHistoryCoverage: nativeResult.nativeHistoryCoverage,
+            partialReason: nativeResult.partialReason,
+            unavailableReason: nativeResult.unavailableReason,
         };
     }
     return {

package/src/config/config.ts

@@ -272,6 +272,18 @@ export function getConfigDir(): string {
     return dir;
 }
 
+/**
+ * Get the daemon runtime data directory (~/.adhdev/daemon/).
+ * Distinct from the user-config dir so runtime state can be cleared independently.
+ */
+export function getDaemonDataDir(): string {
+    const dir = join(getConfigDir(), 'daemon');
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    return dir;
+}
+
 /**
  * Get the config file path
  */