ocwatch 0.4.0 → 0.5.0

package/src/server/services/pollService.ts

@@ -1,25 +1,57 @@
 import { createHash } from "node:crypto";
-import type { PollResponse, SessionMetadata, MessageMeta, PlanProgress, ActivitySession, PartMeta } from "../../shared/types";
-import { listAllSessions, checkStorageExists } from "../storage/sessionParser";
-import { listMessages } from "../storage/messageParser";
+import type {
+  PollResponse,
+  SessionSummary,
+  SessionDetail,
+  SessionMetadata,
+  MessageMeta,
+  PlanProgress,
+  ActivitySession,
+  PartMeta,
+  TodoItem,
+} from "../../shared/types";
 import { parseBoulder, calculatePlanProgress } from "../storage/boulderParser";
-import { getPartsForSession, getSessionActivityState, isPendingToolCall, generateActivityMessage, deriveActivityType } from "../storage/partParser";
-import { getSessionStatusInfo } from "../utils/sessionStatus";
+import {
+  checkDbExists,
+  queryMaxTimestamp,
+  queryMessages,
+  queryParts,
+  querySessions,
+  queryTodos,
+} from "../storage";
+import {
+  getSessionActivityState,
+  generateActivityMessage,
+  deriveActivityType,
+  getSessionStatusInfo,
+  isAssistantFinished,
+} from "../logic";
 import { resolveProjectDirectory } from "../utils/projectResolver";
-import { isAssistantFinished, getSessionHierarchy } from "./sessionService";
+import { getSessionHierarchy } from "./sessionService";
 import { aggregateSessionStats } from "./statsService";
-import { TWENTY_FOUR_HOURS_MS, MAX_SESSIONS_LIMIT, MAX_MESSAGES_LIMIT, POLL_CACHE_TTL_MS } from "../../shared/constants";
+import {
+  toSessionMetadata,
+  toMessageMeta,
+  toPartMeta,
+  getLatestAssistantMessage,
+  getMostRecentPendingPart,
+} from "./parsing";
+import {
+  TWENTY_FOUR_HOURS_MS,
+  MAX_SESSIONS_LIMIT,
+  MAX_MESSAGES_LIMIT,
+  POLL_CACHE_TTL_MS,
+} from "../../shared/constants";
+/** Max sessions to scan when building incremental state (internal upper bound) */
+const SESSION_SCAN_LIMIT = 50_000;
+/** Max messages per session for incremental poll cache (high to avoid missing updates) */
+const MESSAGE_SCAN_LIMIT = 10_000;
 
-function getLatestAssistantMessage(messages: MessageMeta[]): MessageMeta | undefined {
-  return messages
-    .filter((message) => message.role === "assistant")
-    .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())[0];
-}
-
-function getMostRecentPendingPart(parts: PartMeta[]): PartMeta | undefined {
-  return parts
-    .filter((part) => isPendingToolCall(part))
-    .sort((a, b) => (b.startedAt?.getTime() || 0) - (a.startedAt?.getTime() || 0))[0];
+interface IncrementalPollState {
+  sessionsById: Map<string, SessionMetadata>;
+  messagesBySessionId: Map<string, MessageMeta[]>;
+  partsBySessionId: Map<string, PartMeta[]>;
+  lastTimestamp: number;
 }
 
 function sessionPriority(session: SessionMetadata): number {
@@ -31,27 +63,25 @@ function sessionPriority(session: SessionMetadata): number {
 }
 
 export function generateETag(data: PollResponse): string {
-   const dataForHash = {
-     sessions: data.sessions,
-     activeSession: data.activeSession,
-     planProgress: data.planProgress,
-     messages: data.messages,
-     activitySessions: data.activitySessions,
-   };
-   const hash = createHash("sha256")
-     .update(JSON.stringify(dataForHash))
-     .digest("hex");
-   return `"${hash.substring(0, 16)}"`;
+  const dataForHash = {
+    sessions: data.sessions,
+    activeSessionId: data.activeSessionId,
+    planProgress: data.planProgress,
+  };
+  const hash = createHash("sha256").update(JSON.stringify(dataForHash)).digest("hex");
+  return `"${hash.substring(0, 16)}"`;
 }
 
 type PollCacheEntry = { data: PollResponse; etag: string; timestamp: number };
 
 const pollCacheMap = new Map<string, PollCacheEntry>();
 const pollInProgressMap = new Map<string, Promise<PollResponse>>();
+const incrementalPollStateMap = new Map<string, IncrementalPollState>();
+const MAX_INCREMENTAL_STATE_ENTRIES = 10;
 let pollCacheEpoch = 0;
 
 function cacheKey(projectId?: string): string {
-  return projectId ?? '';
+  return projectId ?? "";
 }
 
 export function getPollCache(projectId?: string): PollCacheEntry | null {
@@ -93,76 +123,179 @@ export function getPollCacheTTL() {
   return POLL_CACHE_TTL_MS;
 }
 
-export async function fetchPollData(sessionId?: string, projectId?: string): Promise<PollResponse> {
-  const storageExists = await checkStorageExists();
-  if (!storageExists) {
+function getIncrementalState(projectId?: string): IncrementalPollState {
+  const key = cacheKey(projectId);
+  const existing = incrementalPollStateMap.get(key);
+  if (existing) {
+    return existing;
+  }
+
+  // Evict oldest entry if at capacity
+  if (incrementalPollStateMap.size >= MAX_INCREMENTAL_STATE_ENTRIES) {
+    let oldestKey: string | undefined;
+    let oldestTimestamp = Infinity;
+    for (const [k, v] of incrementalPollStateMap) {
+      if (v.lastTimestamp < oldestTimestamp) {
+        oldestTimestamp = v.lastTimestamp;
+        oldestKey = k;
+      }
+    }
+    if (oldestKey !== undefined) {
+      incrementalPollStateMap.delete(oldestKey);
+    }
+  }
+
+  const created: IncrementalPollState = {
+    sessionsById: new Map(),
+    messagesBySessionId: new Map(),
+    partsBySessionId: new Map(),
+    lastTimestamp: 0,
+  };
+  incrementalPollStateMap.set(key, created);
+  return created;
+}
+
+function sortMessagesDescending(messages: MessageMeta[]): MessageMeta[] {
+  return [...messages].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+}
+
+function loadMessagesForSession(state: IncrementalPollState, sessionId: string, forceRefresh = false): MessageMeta[] {
+  if (!forceRefresh) {
+    const cached = state.messagesBySessionId.get(sessionId);
+    if (cached) {
+      return cached;
+    }
+  }
+
+  const messages = queryMessages(sessionId, MESSAGE_SCAN_LIMIT).map(toMessageMeta);
+  const sorted = sortMessagesDescending(messages);
+  state.messagesBySessionId.set(sessionId, sorted);
+  return sorted;
+}
+
+function loadPartsForSession(state: IncrementalPollState, sessionId: string, forceRefresh = false): PartMeta[] {
+  if (!forceRefresh) {
+    const cached = state.partsBySessionId.get(sessionId);
+    if (cached) {
+      return cached;
+    }
+  }
+
+  const parts = queryParts(sessionId).map(toPartMeta);
+  state.partsBySessionId.set(sessionId, parts);
+  return parts;
+}
+
+function updateIncrementalState(state: IncrementalPollState, projectId?: string): void {
+  const currentMaxTimestamp = queryMaxTimestamp();
+  const hasBaseline = state.sessionsById.size > 0;
+  const hasIncrementalWindow = hasBaseline && currentMaxTimestamp > state.lastTimestamp;
+
+  let refreshAllSessions = !hasBaseline;
+  const changedSessionIds = new Set<string>();
+
+  if (hasIncrementalWindow) {
+    const changedSessions = querySessions(projectId, state.lastTimestamp, SESSION_SCAN_LIMIT);
+    if (changedSessions.length === 0) {
+      refreshAllSessions = true;
+    } else {
+      for (const row of changedSessions) {
+        const session = toSessionMetadata(row);
+        state.sessionsById.set(session.id, session);
+        changedSessionIds.add(session.id);
+      }
+    }
+  }
+
+  if (refreshAllSessions) {
+    state.sessionsById.clear();
+    const allRows = querySessions(projectId, undefined, SESSION_SCAN_LIMIT);
+    for (const row of allRows) {
+      const session = toSessionMetadata(row);
+      state.sessionsById.set(session.id, session);
+      changedSessionIds.add(session.id);
+    }
+
+    const activeSessionIds = new Set(state.sessionsById.keys());
+    for (const sessionKey of state.messagesBySessionId.keys()) {
+      if (!activeSessionIds.has(sessionKey)) {
+        state.messagesBySessionId.delete(sessionKey);
+      }
+    }
+    for (const sessionKey of state.partsBySessionId.keys()) {
+      if (!activeSessionIds.has(sessionKey)) {
+        state.partsBySessionId.delete(sessionKey);
+      }
+    }
+  }
+
+  if (changedSessionIds.size > 0) {
+    for (const sessionId of changedSessionIds) {
+      loadMessagesForSession(state, sessionId, true);
+      loadPartsForSession(state, sessionId, true);
+    }
+  }
+
+  state.lastTimestamp = currentMaxTimestamp;
+}
+
+export async function fetchPollData(projectId?: string): Promise<PollResponse> {
+  const dbExists = checkDbExists();
+  if (!dbExists) {
+    incrementalPollStateMap.delete(cacheKey(projectId));
     return {
       sessions: [],
-      activeSession: null,
+      activeSessionId: null,
       planProgress: null,
-      messages: [],
-      activitySessions: [],
       lastUpdate: Date.now(),
     };
   }
 
-  const allSessions = await listAllSessions();
-  let scopedSessions = allSessions;
+  const state = getIncrementalState(projectId);
+  updateIncrementalState(state, projectId);
+
+  const scopedSessions = Array.from(state.sessionsById.values());
   let selectedProjectDirectory: string | null = null;
 
   if (projectId) {
-    selectedProjectDirectory = await resolveProjectDirectory(projectId, allSessions);
-    scopedSessions = allSessions.filter((session) => session.projectID === projectId);
+    selectedProjectDirectory = await resolveProjectDirectory(projectId, scopedSessions);
   }
 
   const now = Date.now();
   const twentyFourHoursAgo = now - TWENTY_FOUR_HOURS_MS;
 
-  const recentSessions = scopedSessions.filter(
-    (s) => s.updatedAt.getTime() >= twentyFourHoursAgo
-  );
-
-  const sortedSessions = recentSessions.sort(
-    (a, b) => b.updatedAt.getTime() - a.updatedAt.getTime()
-  );
-
+  const recentSessions = scopedSessions.filter((session) => session.updatedAt.getTime() >= twentyFourHoursAgo);
+  const sortedSessions = recentSessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
   const limitedSessions = sortedSessions.slice(0, MAX_SESSIONS_LIMIT);
-  const rootSessions = limitedSessions.filter(s => !s.parentID);
+  const rootSessions = limitedSessions.filter((session) => !session.parentID);
 
-  const messagesCache = new Map<string, MessageMeta[]>();
-  const partsCache = new Map<string, PartMeta[]>();
-
-  async function getCachedMessages(id: string): Promise<MessageMeta[]> {
-    if (!messagesCache.has(id)) {
-      messagesCache.set(id, await listMessages(id));
-    }
-    return messagesCache.get(id)!;
+  function getCachedMessages(id: string): MessageMeta[] {
+    const cached = state.messagesBySessionId.get(id);
+    if (cached) return cached;
+    return loadMessagesForSession(state, id);
   }
 
-  async function getCachedParts(id: string): Promise<PartMeta[]> {
-    if (!partsCache.has(id)) {
-      partsCache.set(id, await getPartsForSession(id));
-    }
-    return partsCache.get(id)!;
+  function getCachedParts(id: string): PartMeta[] {
+    return loadPartsForSession(state, id);
   }
 
   const sessionsWithAgent = await Promise.all(
     rootSessions.map(async (session) => {
-      const messages = await getCachedMessages(session.id);
+      const messages = getCachedMessages(session.id);
       const latestAssistantMsg = getLatestAssistantMessage(messages);
-      const parts = await getCachedParts(session.id);
+      const parts = getCachedParts(session.id);
       const activityState = getSessionActivityState(parts);
       const lastAssistantFinished = isAssistantFinished(messages);
-      
+
       const statusInfo = getSessionStatusInfo(
         messages,
         activityState.hasPendingToolCall,
         activityState.lastToolCompletedAt || undefined,
         undefined,
-        lastAssistantFinished
+        lastAssistantFinished,
       );
       const status = statusInfo.status;
-      
+
       const pendingPart = getMostRecentPendingPart(parts);
       const currentAction = generateActivityMessage(
         activityState,
@@ -170,11 +303,17 @@ export async function fetchPollData(sessionId?: string, projectId?: string): Pro
         false,
         status,
         pendingPart,
-        statusInfo.waitingReason
+        statusInfo.waitingReason,
+      );
+
+      const activityType = deriveActivityType(
+        activityState,
+        lastAssistantFinished,
+        false,
+        status,
+        statusInfo.waitingReason,
       );
-      
-      const activityType = deriveActivityType(activityState, lastAssistantFinished, false, status, statusInfo.waitingReason);
-      
+
       return {
         ...session,
         agent: latestAssistantMsg?.agent || null,
@@ -184,22 +323,19 @@ export async function fetchPollData(sessionId?: string, projectId?: string): Pro
         activityType,
         currentAction,
       };
-    })
+    }),
   );
 
-   let activeSession: SessionMetadata | null = null;
-   const activeEnriched = [...sessionsWithAgent].sort((a, b) => {
-     const priorityDiff = sessionPriority(b) - sessionPriority(a);
-     if (priorityDiff !== 0) {
-       return priorityDiff;
-     }
-     return b.updatedAt.getTime() - a.updatedAt.getTime();
-   })[0];
-   if (activeEnriched && sessionPriority(activeEnriched) === 0) {
-     activeSession = null;
-   } else if (activeEnriched) {
-     activeSession = rootSessions.find(s => s.id === activeEnriched.id) ?? null;
-   }
+  const activeEnriched = [...sessionsWithAgent].sort((a, b) => {
+    const priorityDiff = sessionPriority(b) - sessionPriority(a);
+    if (priorityDiff !== 0) {
+      return priorityDiff;
+    }
+    return b.updatedAt.getTime() - a.updatedAt.getTime();
+  })[0];
+
+  const activeSessionId =
+    activeEnriched && sessionPriority(activeEnriched) > 0 ? activeEnriched.id : null;
 
   let planProgress: PlanProgress | null = null;
   let planName: string | undefined;
@@ -212,43 +348,85 @@ export async function fetchPollData(sessionId?: string, projectId?: string): Pro
         planName = boulder.planName || undefined;
       }
     } catch (err) {
-      console.debug('Failed to parse boulder.json:', err instanceof Error ? err.message : err);
+      console.debug("Failed to parse boulder.json:", err instanceof Error ? err.message : err);
     }
   }
 
-  let messages: MessageMeta[] = [];
-  let activitySessions: ActivitySession[] = [];
-  let sessionStats = undefined;
-  const scopedSessionIds = new Set(scopedSessions.map((session) => session.id));
-  const requestedSessionId =
-    sessionId && (!projectId || scopedSessionIds.has(sessionId))
-      ? sessionId
-      : undefined;
-  const targetSessionId = requestedSessionId || activeSession?.id;
-  if (targetSessionId) {
-    const fetchedMessages = await getCachedMessages(targetSessionId);
-    const sortedMessages = fetchedMessages.sort(
-      (a, b) => b.createdAt.getTime() - a.createdAt.getTime()
-    );
-    messages = sortedMessages.slice(0, MAX_MESSAGES_LIMIT);
-    
-    activitySessions = await getSessionHierarchy(targetSessionId, scopedSessions);
-    
-    for (const session of activitySessions) {
-      await getCachedMessages(session.id);
-    }
-    
-    sessionStats = aggregateSessionStats(activitySessions, messagesCache);
-  }
+  const sessions: SessionSummary[] = sessionsWithAgent.map((s) => ({
+    id: s.id,
+    projectID: s.projectID,
+    title: s.title,
+    status: s.status,
+    activityType: s.activityType,
+    currentAction: s.currentAction,
+    agent: s.agent,
+    modelID: s.modelID,
+    providerID: s.providerID,
+    updatedAt: s.updatedAt,
+    createdAt: s.createdAt,
+  }));
 
   return {
-    sessions: sessionsWithAgent,
-    activeSession,
+    sessions,
+    activeSessionId,
     planProgress,
     planName,
-    messages,
-    activitySessions,
-    sessionStats,
     lastUpdate: now,
   };
 }
+
+export async function fetchSessionDetail(sessionId: string): Promise<SessionDetail> {
+  // Get all sessions for hierarchy context
+  const allSessions = querySessions(undefined, undefined, SESSION_SCAN_LIMIT).map(toSessionMetadata);
+
+  // Get messages for this session
+  const messages = queryMessages(sessionId, MAX_MESSAGES_LIMIT).map(toMessageMeta);
+
+  // Get activity tree (hierarchy)
+  const activity: ActivitySession[] = await getSessionHierarchy(sessionId, allSessions);
+
+  // Build messages cache for stats
+  const messagesCache = new Map<string, MessageMeta[]>();
+  messagesCache.set(sessionId, messages);
+  for (const activitySession of activity) {
+    if (!messagesCache.has(activitySession.id) && !activitySession.id.includes("-phase-")) {
+      const childMessages = queryMessages(activitySession.id, MAX_MESSAGES_LIMIT).map(toMessageMeta);
+      messagesCache.set(activitySession.id, childMessages);
+    }
+  }
+
+  // Get todos
+  const todos: TodoItem[] = queryTodos(sessionId).map((row) => ({
+    content: row.content,
+    status: row.status,
+    priority: row.priority,
+    position: row.position,
+  }));
+
+  // Compute stats
+  const stats = aggregateSessionStats(activity, messagesCache);
+
+  // Find the session metadata
+  const sessionMeta = allSessions.find((s) => s.id === sessionId);
+
+  // Find the root activity entry for derived status (direct match or last phase)
+  const rootActivity =
+    activity.find((a) => a.id === sessionId) ??
+    activity.filter((a) => a.id.startsWith(`${sessionId}-phase-`)).pop();
+
+  const session: SessionSummary = {
+    id: sessionId,
+    projectID: sessionMeta?.projectID ?? "",
+    title: sessionMeta?.title ?? sessionId,
+    status: rootActivity?.status,
+    activityType: rootActivity?.activityType,
+    currentAction: rootActivity?.currentAction,
+    agent: rootActivity?.agent ?? null,
+    modelID: rootActivity?.modelID ?? null,
+    providerID: rootActivity?.providerID ?? null,
+    updatedAt: sessionMeta?.updatedAt ?? new Date(),
+    createdAt: sessionMeta?.createdAt ?? new Date(),
+  };
+
+  return { session, messages, activity, todos, stats };
+}