@tokentop/ttop 0.3.0 → 0.5.0

package/src/storage/retention.ts

@@ -0,0 +1,146 @@
+/**
+ * Data retention policy — prevents unbounded database growth.
+ *
+ * Detailed snapshot data (agent_session_snapshots, agent_session_stream_snapshots,
+ * provider_snapshots) is pruned after a retention window. Aggregated data in
+ * usage_events, hourly_aggregates, and daily_aggregates is kept longer since
+ * those tables grow much more slowly.
+ */
+
+import { getDatabase, isDatabaseInitialized } from "./db.ts";
+
+/** Keep detailed snapshots for 7 days */
+const SNAPSHOT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** Keep provider snapshots for 7 days */
+const PROVIDER_SNAPSHOT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** Keep usage events for 90 days */
+const USAGE_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
+
+/** Maximum rows to delete in a single batch (prevents long-running transactions) */
+const PRUNE_BATCH_SIZE = 10_000;
+
+export interface PruneResult {
+  agentSessionSnapshots: number;
+  providerSnapshots: number;
+  usageEvents: number;
+  durationMs: number;
+}
+
+/**
+ * Prune old data from the database. Deletes in batches to avoid
+ * holding a write lock for too long.
+ */
+export function pruneOldData(nowMs: number = Date.now()): PruneResult {
+  if (!isDatabaseInitialized()) {
+    return { agentSessionSnapshots: 0, providerSnapshots: 0, usageEvents: 0, durationMs: 0 };
+  }
+
+  const db = getDatabase();
+  const start = performance.now();
+
+  let agentSessionSnapshots = 0;
+  let providerSnapshots = 0;
+  let usageEvents = 0;
+
+  const snapshotCutoff = nowMs - SNAPSHOT_RETENTION_MS;
+  const providerCutoff = nowMs - PROVIDER_SNAPSHOT_RETENTION_MS;
+  const usageEventCutoff = nowMs - USAGE_EVENT_RETENTION_MS;
+
+  // Prune agent_session_snapshots in batches.
+  // CASCADE delete handles agent_session_stream_snapshots automatically.
+  {
+    const stmt = db.prepare(`
+      DELETE FROM agent_session_snapshots
+      WHERE id IN (
+        SELECT id FROM agent_session_snapshots
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(snapshotCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      agentSessionSnapshots += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  // Prune old provider_snapshots in batches
+  {
+    const stmt = db.prepare(`
+      DELETE FROM provider_snapshots
+      WHERE id IN (
+        SELECT id FROM provider_snapshots
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(providerCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      providerSnapshots += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  // Prune old usage_events in batches
+  {
+    const stmt = db.prepare(`
+      DELETE FROM usage_events
+      WHERE id IN (
+        SELECT id FROM usage_events
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(usageEventCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      usageEvents += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  const durationMs = Math.round(performance.now() - start);
+
+  return { agentSessionSnapshots, providerSnapshots, usageEvents, durationMs };
+}
+
+/**
+ * Run incremental auto_vacuum if supported (WAL mode).
+ * This reclaims free pages without blocking readers.
+ */
+export function incrementalVacuum(pages = 1000): void {
+  if (!isDatabaseInitialized()) return;
+
+  const db = getDatabase();
+  try {
+    db.exec(`PRAGMA incremental_vacuum(${pages})`);
+  } catch {
+    // Not all builds support incremental vacuum — silently skip
+  }
+}
+
+/**
+ * Get the database file size in bytes. Returns 0 if unable to determine.
+ */
+export function getDatabaseSizeBytes(): number {
+  if (!isDatabaseInitialized()) return 0;
+
+  const db = getDatabase();
+  try {
+    const row = db.prepare("PRAGMA page_count").get() as { page_count: number } | null;
+    const sizeRow = db.prepare("PRAGMA page_size").get() as { page_size: number } | null;
+    if (row && sizeRow) {
+      return row.page_count * sizeRow.page_size;
+    }
+  } catch {
+    // ignore
+  }
+  return 0;
+}

package/src/tui/App.tsx

@@ -1,7 +1,7 @@
 import { useKeyboard } from "@opentui/react";
+import type { ThemePlugin } from "@tokentop/plugin-sdk";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import * as builtinThemeExports from "@/plugins/themes/index.ts";
-import type { ThemePlugin } from "@/plugins/types/theme.ts";
 import { resolveTheme, ThemeProvider, useColors, useTheme } from "./contexts/ThemeContext.tsx";
 import {
   type BurstRecorder,

package/src/tui/components/DebugPanel.tsx

@@ -9,7 +9,7 @@ import { useColors } from "../contexts/ThemeContext.tsx";
 import { ModalBackdrop, Z_INDEX } from "./ModalBackdrop.tsx";
 
 type DebugTab = "logs" | "inspector" | "plugins";
-type PluginSort = "name" | "type" | "official";
+type PluginSort = "name" | "type" | "source";
 
 interface DebugInspectorData {
   sessions: Array<{
@@ -193,7 +193,7 @@ export function DebugPanel({ onClose, inspectorData }: DebugPanelProps) {
     if (activeTab === "plugins") {
       if (key.name === "s") {
         setPluginSort((prev) => {
-          const order: PluginSort[] = ["name", "type", "official"];
+          const order: PluginSort[] = ["name", "type", "source"];
           return order[(order.indexOf(prev) + 1) % order.length] ?? "name";
         });
         return;
@@ -656,12 +656,12 @@ interface PluginsTabProps {
 
 const TYPE_ORDER: Record<string, number> = { provider: 0, agent: 1, theme: 2, notification: 3 };
 
-function isOfficialPlugin(pluginId: string, pluginType: string): boolean {
+function getSourcePriority(pluginId: string, pluginType: string): number {
   const source = pluginRegistry.getSource(
     pluginType as "provider" | "agent" | "theme" | "notification",
     pluginId,
   );
-  return source === "builtin";
+  return source === "local" ? 0 : source === "npm" ? 1 : 2;
 }
 
 function PluginsTab({ colors, scrollboxRef, sortBy }: PluginsTabProps) {
@@ -671,10 +671,10 @@ function PluginsTab({ colors, scrollboxRef, sortBy }: PluginsTabProps) {
       if (sortBy === "name") {
         return a.name.localeCompare(b.name);
       }
-      if (sortBy === "official") {
-        const aOfficial = isOfficialPlugin(a.id, a.type) ? 0 : 1;
-        const bOfficial = isOfficialPlugin(b.id, b.type) ? 0 : 1;
-        if (aOfficial !== bOfficial) return aOfficial - bOfficial;
+      if (sortBy === "source") {
+        const aPriority = getSourcePriority(a.id, a.type);
+        const bPriority = getSourcePriority(b.id, b.type);
+        if (aPriority !== bPriority) return aPriority - bPriority;
         return a.name.localeCompare(b.name);
       }
       const typeA = TYPE_ORDER[a.type] ?? 99;
@@ -733,6 +733,9 @@ function PluginsTab({ colors, scrollboxRef, sortBy }: PluginsTabProps) {
         <text width={14} fg={colors.textSubtle}>
           {padRight("TYPE", 14)}
         </text>
+        <text width={10} fg={colors.textSubtle}>
+          {padRight("SOURCE", 10)}
+        </text>
         <text width={14} fg={colors.textSubtle}>
           {padRight("VERSION", 14)}
         </text>
@@ -743,7 +746,14 @@ function PluginsTab({ colors, scrollboxRef, sortBy }: PluginsTabProps) {
           {allPlugins.map((plugin) => {
             const key = `${plugin.type}-${plugin.id}`;
             const updateInfo = updates.get(key);
-            const official = isOfficialPlugin(plugin.id, plugin.type);
+            const source = pluginRegistry.getSource(plugin.type, plugin.id) ?? "builtin";
+            const official = pluginRegistry.isOfficial(plugin.type, plugin.id);
+
+            const sourceColors: Record<string, string> = {
+              builtin: colors.textSubtle,
+              local: colors.success,
+              npm: colors.info,
+            };
 
             let versionSuffix = "";
             let versionColor = colors.textMuted;
@@ -758,12 +768,15 @@ function PluginsTab({ colors, scrollboxRef, sortBy }: PluginsTabProps) {
             return (
               <box key={key} flexDirection="row" height={1}>
                 <text width={22}>
-                  {official && <span fg={colors.success}>❖ </span>}
+                  {official && <span fg={colors.warning}>✦ </span>}
                   <span fg={colors.text}>{padRight(plugin.name, official ? 20 : 22)}</span>
                 </text>
                 <text width={14} fg={typeColors[plugin.type] ?? colors.textMuted}>
                   {padRight(plugin.type, 14)}
                 </text>
+                <text width={10} fg={sourceColors[source] ?? colors.textMuted}>
+                  {padRight(source, 10)}
+                </text>
                 <text width={14} fg={versionColor}>
                   {padRight(plugin.version + versionSuffix, 14)}
                 </text>

package/src/tui/components/ProviderCard.tsx

@@ -1,7 +1,7 @@
 import type { BoxRenderable } from "@opentui/core";
 import { useTerminalDimensions } from "@opentui/react";
+import type { ProviderUsageData } from "@tokentop/plugin-sdk";
 import { forwardRef, useCallback, useEffect, useState } from "react";
-import type { ProviderUsageData } from "@/plugins/types/provider.ts";
 import { useColors } from "../contexts/ThemeContext.tsx";
 import { SkeletonProviderContent } from "./Skeleton.tsx";
 import { useSpinner } from "./Spinner.tsx";

package/src/tui/components/ProviderDetailPanel.tsx

@@ -1,4 +1,4 @@
-import type { UsageLimit } from "@/plugins/types/provider.ts";
+import type { UsageLimit } from "@tokentop/plugin-sdk";
 import type { ProviderState } from "../contexts/PluginContext.tsx";
 import { useColors } from "../contexts/ThemeContext.tsx";
 import { Sparkline } from "./Sparkline.tsx";

package/src/tui/components/SessionDetailsDrawer.tsx

@@ -314,6 +314,16 @@ export function SessionDetailsDrawer({ session, onClose: _onClose }: SessionDeta
             </text>
           </box>
 
+          {session.metadata?.isEstimated === true && (
+            <box flexDirection="row" height={1} marginBottom={0}>
+              <text height={1} overflow="hidden">
+                <span fg={colors.warning}>
+                  ≈ Token data is estimated — server enrichment pending
+                </span>
+              </text>
+            </box>
+          )}
+
           {hasCacheData && (
             <box flexDirection="row" height={1} marginBottom={0}>
               <text height={1} overflow="hidden">

package/src/tui/components/SessionsTable.tsx

@@ -154,6 +154,7 @@ const SessionRow = memo(function SessionRow({
   const effectiveTokens = session.totals.input + session.totals.output;
   const cacheRead = session.totals.cacheRead ?? 0;
   const hasCacheData = cacheRead > 0;
+  const isEstimated = session.metadata?.isEstimated === true;
   const costUsd = session.totalCostUsd ?? 0;
 
   const animatedTokens = useAnimatedValue(effectiveTokens, { durationMs: 300, precision: 0 });
@@ -260,7 +261,10 @@ const SessionRow = memo(function SessionRow({
           {String(session.requestCount).padStart(4)}
         </text>
         <text width={14} height={1} {...bgProp}>
-          <span fg={tokenColor}>{formatTokensVal(animatedTokens).padStart(7)}</span>
+          <span fg={tokenColor}>
+            {isEstimated ? "≈" : " "}
+            {formatTokensVal(animatedTokens).padStart(7)}
+          </span>
           <span fg={colors.textMuted}>
             {hasCacheData ? ` ↯${formatCacheBadge(cacheRead)}` : ""}
           </span>
@@ -315,7 +319,10 @@ const SessionRow = memo(function SessionRow({
           : modelWithCount.padEnd(narrowModelMaxChars)}
       </text>
       <text width={narrowTokenW} height={1} {...bgProp}>
-        <span fg={tokenColor}>{formatTokensVal(animatedTokens).padStart(7)}</span>
+        <span fg={tokenColor}>
+          {isEstimated ? "≈" : " "}
+          {formatTokensVal(animatedTokens).padStart(7)}
+        </span>
         <span fg={colors.textMuted}>
           {hasCacheData && showCacheBadge ? ` ↯${formatCacheBadge(cacheRead)}` : ""}
         </span>
@@ -386,6 +393,10 @@ export const SessionsTable = forwardRef(function SessionsTable(
     getKey: getSessionKey,
   });
 
+  // Track bulk changes to force scrollbox remount (clears ghost text)
+  const bulkCountRef = useRef(0);
+  if (isBulkChange) bulkCountRef.current++;
+
   let activeIndex = 0;
   const selectedSession = sessions[selectedRow] ?? null;
   const inspector = selectedSession ? getInspectorData(selectedSession) : null;
@@ -489,8 +500,8 @@ export const SessionsTable = forwardRef(function SessionsTable(
         </box>
       )}
 
-      <scrollbox ref={ref} flexGrow={1}>
-        <box flexDirection="column">
+      <scrollbox key={`sessions-sb-${bulkCountRef.current}`} ref={ref} flexGrow={1}>
+        <box flexDirection="column" flexGrow={1}>
           {sessions.length === 0 && animatedSessions.length === 0 && (
             <box paddingLeft={2}>
               <text fg={colors.textMuted}>