@hybridaione/hybridclaw 0.2.3 → 0.2.7

package/src/session-export.ts

@@ -0,0 +1,196 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { agentWorkspaceDir, ensureAgentDirs } from './ipc.js';
+import { logger } from './logger.js';
+import type { StoredMessage } from './types.js';
+
+const SESSION_EXPORTS_DIR_NAME = '.session-exports';
+
+interface ExportedMessageRow {
+  type: 'message';
+  section: 'full' | 'compacted' | 'retained';
+  sessionId: string;
+  channelId: string | null;
+  id: number;
+  role: string;
+  userId: string;
+  username: string | null;
+  content: string;
+  createdAt: string;
+}
+
+function safeFilePart(raw: string): string {
+  const normalized = raw.trim().replace(/[^a-zA-Z0-9_-]/g, '_');
+  return normalized || 'session';
+}
+
+function compactText(text: string, maxChars = 4_000): string {
+  const compact = text.replace(/\s+/g, ' ').trim();
+  if (compact.length <= maxChars) return compact;
+  return `${compact.slice(0, maxChars)}...`;
+}
+
+function exportBaseDir(agentId: string, sessionId: string): string {
+  ensureAgentDirs(agentId);
+  return path.join(
+    agentWorkspaceDir(agentId),
+    SESSION_EXPORTS_DIR_NAME,
+    safeFilePart(sessionId),
+  );
+}
+
+function exportFilePath(baseDir: string, reason: string): string {
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const filename = `${stamp}-${safeFilePart(reason)}.jsonl`;
+  return path.join(baseDir, filename);
+}
+
+function writeJsonlFile(filePath: string, rows: unknown[]): boolean {
+  try {
+    const lines = rows.map((row) => JSON.stringify(row)).join('\n');
+    fs.writeFileSync(filePath, `${lines}\n`, 'utf8');
+    return true;
+  } catch (err) {
+    logger.warn({ filePath, err }, 'Failed to write session export JSONL');
+    return false;
+  }
+}
+
+export function exportSessionSnapshotJsonl(params: {
+  agentId: string;
+  sessionId: string;
+  channelId?: string | null;
+  summary?: string | null;
+  messages: StoredMessage[];
+  reason: string;
+}): { path: string; lineCount: number } | null {
+  const agentId = params.agentId.trim();
+  const sessionId = params.sessionId.trim();
+  if (!agentId || !sessionId) return null;
+  try {
+    const baseDir = exportBaseDir(agentId, sessionId);
+    fs.mkdirSync(baseDir, { recursive: true });
+    const filePath = exportFilePath(baseDir, params.reason);
+    const now = new Date().toISOString();
+    const rows: unknown[] = [
+      {
+        type: 'meta',
+        reason: params.reason,
+        exportedAt: now,
+        agentId,
+        sessionId,
+        channelId: params.channelId || null,
+        messageCount: params.messages.length,
+        hasSummary: Boolean((params.summary || '').trim()),
+      },
+    ];
+    const summary = (params.summary || '').trim();
+    if (summary) {
+      rows.push({
+        type: 'summary',
+        sessionId,
+        content: compactText(summary, 12_000),
+      });
+    }
+    rows.push(
+      ...params.messages.map<ExportedMessageRow>((message) => ({
+        type: 'message',
+        section: 'full',
+        sessionId,
+        channelId: params.channelId || null,
+        id: message.id,
+        role: message.role,
+        userId: message.user_id,
+        username: message.username,
+        content: message.content,
+        createdAt: message.created_at,
+      })),
+    );
+    if (!writeJsonlFile(filePath, rows)) return null;
+    return { path: filePath, lineCount: rows.length };
+  } catch (err) {
+    logger.warn(
+      { agentId, sessionId, err },
+      'Failed to export session snapshot',
+    );
+    return null;
+  }
+}
+
+export function exportCompactedSessionJsonl(params: {
+  agentId: string;
+  sessionId: string;
+  channelId: string;
+  summary: string;
+  compactedMessages: StoredMessage[];
+  retainedMessages: StoredMessage[];
+  deletedCount: number;
+  cutoffId: number;
+}): { path: string; lineCount: number } | null {
+  const agentId = params.agentId.trim();
+  const sessionId = params.sessionId.trim();
+  if (!agentId || !sessionId) return null;
+  try {
+    const baseDir = exportBaseDir(agentId, sessionId);
+    fs.mkdirSync(baseDir, { recursive: true });
+    const filePath = exportFilePath(baseDir, 'compaction');
+    const exportedAt = new Date().toISOString();
+    const rows: unknown[] = [
+      {
+        type: 'meta',
+        reason: 'compaction',
+        exportedAt,
+        agentId,
+        sessionId,
+        channelId: params.channelId,
+        cutoffId: params.cutoffId,
+        deletedCount: params.deletedCount,
+        compactedMessageCount: params.compactedMessages.length,
+        retainedMessageCount: params.retainedMessages.length,
+      },
+      {
+        type: 'summary',
+        sessionId,
+        content: compactText(params.summary, 12_000),
+      },
+    ];
+
+    rows.push(
+      ...params.compactedMessages.map<ExportedMessageRow>((message) => ({
+        type: 'message',
+        section: 'compacted',
+        sessionId,
+        channelId: params.channelId,
+        id: message.id,
+        role: message.role,
+        userId: message.user_id,
+        username: message.username,
+        content: message.content,
+        createdAt: message.created_at,
+      })),
+    );
+    rows.push(
+      ...params.retainedMessages.map<ExportedMessageRow>((message) => ({
+        type: 'message',
+        section: 'retained',
+        sessionId,
+        channelId: params.channelId,
+        id: message.id,
+        role: message.role,
+        userId: message.user_id,
+        username: message.username,
+        content: message.content,
+        createdAt: message.created_at,
+      })),
+    );
+    if (!writeJsonlFile(filePath, rows)) return null;
+    return { path: filePath, lineCount: rows.length };
+  } catch (err) {
+    logger.warn(
+      { agentId, sessionId, err },
+      'Failed to export compacted session JSONL',
+    );
+    return null;
+  }
+}

package/src/session-maintenance.ts

@@ -1,28 +1,41 @@
+import { runAgent } from './agent.js';
 import {
   PRE_COMPACTION_MEMORY_FLUSH_ENABLED,
   PRE_COMPACTION_MEMORY_FLUSH_MAX_CHARS,
   PRE_COMPACTION_MEMORY_FLUSH_MAX_MESSAGES,
+  SESSION_COMPACTION_BUDGET_RATIO,
   SESSION_COMPACTION_ENABLED,
   SESSION_COMPACTION_KEEP_RECENT,
   SESSION_COMPACTION_SUMMARY_MAX_CHARS,
   SESSION_COMPACTION_THRESHOLD,
+  SESSION_COMPACTION_TOKEN_BUDGET,
 } from './config.js';
-import {
-  deleteMessagesBeforeId,
-  getCompactionCandidateMessages,
-  getSessionById,
-  markSessionMemoryFlush,
-  updateSessionSummary,
-} from './db.js';
 import { logger } from './logger.js';
+import { memoryService } from './memory-service.js';
+import { buildSystemPromptFromHooks } from './prompt-hooks.js';
+import { exportCompactedSessionJsonl } from './session-export.js';
 import { loadSkills } from './skills.js';
+import {
+  estimateTokenCountFromMessages,
+  estimateTokenCountFromText,
+} from './token-efficiency.js';
 import type { ChatMessage, StoredMessage } from './types.js';
-import { runAgent } from './agent.js';
-import { buildSystemPromptFromHooks } from './prompt-hooks.js';
 
 const COMPACTION_SOURCE_MAX_MESSAGES = 240;
 const COMPACTION_SOURCE_MAX_CHARS = 80_000;
 
+function normalizeStoredMessageRole(role: string): ChatMessage['role'] {
+  if (
+    role === 'system' ||
+    role === 'user' ||
+    role === 'assistant' ||
+    role === 'tool'
+  ) {
+    return role;
+  }
+  return 'user';
+}
+
 function formatDateStampInLocalTimezone(now: Date): string {
   const parts = new Intl.DateTimeFormat('en-US', {
     year: 'numeric',
@@ -41,7 +54,10 @@ function formatDateStampInLocalTimezone(now: Date): string {
 function normalizeSummary(summary: string): string {
   let text = summary.trim();
   if (text.startsWith('```')) {
-    text = text.replace(/^```[a-z0-9_-]*\s*/i, '').replace(/```$/i, '').trim();
+    text = text
+      .replace(/^```[a-z0-9_-]*\s*/i, '')
+      .replace(/```$/i, '')
+      .trim();
   }
   if (text.length > SESSION_COMPACTION_SUMMARY_MAX_CHARS) {
     text = `${text.slice(0, SESSION_COMPACTION_SUMMARY_MAX_CHARS)}\n\n...[truncated]`;
@@ -61,7 +77,10 @@ function formatMessagesForPrompt(
   for (const msg of selected) {
     const role = (msg.role || 'unknown').toUpperCase();
     const compact = msg.content.replace(/\r/g, '').trim();
-    const bounded = compact.length > 1_200 ? `${compact.slice(0, 1_200)}\n...[truncated]` : compact;
+    const bounded =
+      compact.length > 1_200
+        ? `${compact.slice(0, 1_200)}\n...[truncated]`
+        : compact;
     const entry = `[${role}] ${bounded}`;
     const bytes = entry.length + 2;
     if (usedChars + bytes > maxChars) break;
@@ -72,7 +91,11 @@ function formatMessagesForPrompt(
   return lines.join('\n\n');
 }
 
-function buildSystemPrompt(agentId: string, sessionSummary?: string | null, extra?: string): string {
+function buildSystemPrompt(
+  agentId: string,
+  sessionSummary?: string | null,
+  extra?: string,
+): string {
   return buildSystemPromptFromHooks({
     agentId,
     sessionSummary,
@@ -143,12 +166,18 @@ async function runPreCompactionMemoryFlush(params: {
       ['memory'],
     );
     if (output.status === 'error') {
-      logger.warn({ sessionId: params.sessionId, error: output.error }, 'Pre-compaction memory flush failed');
+      logger.warn(
+        { sessionId: params.sessionId, error: output.error },
+        'Pre-compaction memory flush failed',
+      );
       return;
     }
-    markSessionMemoryFlush(params.sessionId);
+    memoryService.markSessionMemoryFlush(params.sessionId);
   } catch (err) {
-    logger.warn({ sessionId: params.sessionId, err }, 'Pre-compaction memory flush crashed');
+    logger.warn(
+      { sessionId: params.sessionId, err },
+      'Pre-compaction memory flush crashed',
+    );
   }
 }
 
@@ -204,7 +233,10 @@ async function generateCompactionSummary(params: {
   );
 
   if (output.status === 'error' || !output.result) {
-    logger.warn({ sessionId: params.sessionId, error: output.error }, 'Session compaction summary failed');
+    logger.warn(
+      { sessionId: params.sessionId, error: output.error },
+      'Session compaction summary failed',
+    );
     return null;
   }
 
@@ -223,15 +255,60 @@ export async function maybeCompactSession(params: {
 }): Promise<void> {
   if (!SESSION_COMPACTION_ENABLED) return;
 
-  const session = getSessionById(params.sessionId);
+  const session = memoryService.getSessionById(params.sessionId);
   if (!session) return;
 
   const threshold = Math.max(SESSION_COMPACTION_THRESHOLD, 20);
-  const keepRecent = Math.max(1, Math.min(SESSION_COMPACTION_KEEP_RECENT, threshold - 1));
+  const tokenBudget = Math.max(1_000, SESSION_COMPACTION_TOKEN_BUDGET);
+  const budgetRatio = Math.max(
+    0.05,
+    Math.min(1, SESSION_COMPACTION_BUDGET_RATIO),
+  );
+  const budget = Math.max(1, Math.floor(tokenBudget * budgetRatio));
+  const allMessages = memoryService.getRecentMessages(params.sessionId);
+  const keepRecent = Math.max(
+    1,
+    Math.min(
+      SESSION_COMPACTION_KEEP_RECENT,
+      Math.max(1, threshold - 1),
+      Math.max(1, allMessages.length - 1),
+    ),
+  );
+  const msgTokens = estimateTokenCountFromMessages(
+    allMessages.map((message) => ({
+      role: normalizeStoredMessageRole(message.role),
+      content: message.content,
+    })),
+  );
+  const summaryTokens = estimateTokenCountFromText(session.session_summary);
+  const totalTokens = msgTokens + summaryTokens;
+  const shouldCompactForTokens = totalTokens >= budget;
+  const shouldCompactForMessageCount = session.message_count >= threshold;
+
+  logger.debug(
+    {
+      sessionId: params.sessionId,
+      messageCount: session.message_count,
+      loadedMessages: allMessages.length,
+      msgTokens,
+      summaryTokens,
+      totalTokens,
+      tokenBudget,
+      budgetRatio,
+      triggerBudget: budget,
+      triggerThreshold: threshold,
+      shouldCompactForTokens,
+      shouldCompactForMessageCount,
+    },
+    'Session compaction budget check',
+  );
 
-  if (session.message_count < threshold) return;
+  if (!shouldCompactForTokens && !shouldCompactForMessageCount) return;
 
-  const candidate = getCompactionCandidateMessages(params.sessionId, keepRecent);
+  const candidate = memoryService.getCompactionCandidateMessages(
+    params.sessionId,
+    keepRecent,
+  );
   if (!candidate || candidate.olderMessages.length === 0) return;
 
   await runPreCompactionMemoryFlush({
@@ -247,10 +324,27 @@ export async function maybeCompactSession(params: {
   });
   if (!summary) return;
 
-  const deleted = deleteMessagesBeforeId(params.sessionId, candidate.cutoffId);
+  const deleted = memoryService.deleteMessagesBeforeId(
+    params.sessionId,
+    candidate.cutoffId,
+  );
   if (deleted <= 0) return;
 
-  updateSessionSummary(params.sessionId, summary);
+  memoryService.updateSessionSummary(params.sessionId, summary);
+  const retainedMessages = memoryService.getRecentMessages(
+    params.sessionId,
+    keepRecent,
+  );
+  const exported = exportCompactedSessionJsonl({
+    agentId: params.agentId,
+    sessionId: params.sessionId,
+    channelId: params.channelId,
+    summary,
+    compactedMessages: candidate.olderMessages,
+    retainedMessages,
+    deletedCount: deleted,
+    cutoffId: candidate.cutoffId,
+  });
   logger.info(
     {
       sessionId: params.sessionId,
@@ -258,6 +352,15 @@ export async function maybeCompactSession(params: {
       cutoffId: candidate.cutoffId,
       threshold,
       keepRecent,
+      msgTokens,
+      summaryTokens,
+      totalTokens,
+      tokenBudget,
+      budgetRatio,
+      triggerBudget: budget,
+      shouldCompactForTokens,
+      shouldCompactForMessageCount,
+      exportPath: exported?.path || null,
     },
     'Session compacted',
   );

package/src/session-transcripts.ts

@@ -21,14 +21,20 @@ function safeSessionFilename(sessionId: string): string {
   return normalized || 'session';
 }
 
-export function appendSessionTranscript(agentId: string, entry: TranscriptEntry): void {
+export function appendSessionTranscript(
+  agentId: string,
+  entry: TranscriptEntry,
+): void {
   try {
     ensureAgentDirs(agentId);
     const workspace = agentWorkspaceDir(agentId);
     const transcriptDir = path.join(workspace, TRANSCRIPTS_DIR_NAME);
     fs.mkdirSync(transcriptDir, { recursive: true });
 
-    const filePath = path.join(transcriptDir, `${safeSessionFilename(entry.sessionId)}.jsonl`);
+    const filePath = path.join(
+      transcriptDir,
+      `${safeSessionFilename(entry.sessionId)}.jsonl`,
+    );
     const row = {
       sessionId: entry.sessionId,
       channelId: entry.channelId,
@@ -40,6 +46,9 @@ export function appendSessionTranscript(agentId: string, entry: TranscriptEntry)
     };
     fs.appendFileSync(filePath, JSON.stringify(row) + '\n', 'utf-8');
   } catch (err) {
-    logger.debug({ agentId, sessionId: entry.sessionId, err }, 'Failed to append session transcript');
+    logger.debug(
+      { agentId, sessionId: entry.sessionId, err },
+      'Failed to append session transcript',
+    );
   }
 }

package/src/side-effects.ts

@@ -15,7 +15,8 @@ export function processSideEffects(
 ): void {
   const schedules = output.sideEffects?.schedules;
   const delegations = output.sideEffects?.delegations || [];
-  if ((!schedules || schedules.length === 0) && delegations.length === 0) return;
+  if ((!schedules || schedules.length === 0) && delegations.length === 0)
+    return;
 
   let changed = false;
 
@@ -31,11 +32,24 @@ export function processSideEffects(
             effect.runAt,
             effect.everyMs,
           );
-          logger.info({ taskId, sessionId, channelId, cronExpr: effect.cronExpr, runAt: effect.runAt, everyMs: effect.everyMs }, 'Side-effect: created task');
+          logger.info(
+            {
+              taskId,
+              sessionId,
+              channelId,
+              cronExpr: effect.cronExpr,
+              runAt: effect.runAt,
+              everyMs: effect.everyMs,
+            },
+            'Side-effect: created task',
+          );
           changed = true;
         } else if (effect.action === 'remove') {
           deleteTask(effect.taskId);
-          logger.info({ taskId: effect.taskId, sessionId }, 'Side-effect: removed task');
+          logger.info(
+            { taskId: effect.taskId, sessionId },
+            'Side-effect: removed task',
+          );
           changed = true;
         }
       } catch (err) {
@@ -54,7 +68,13 @@ export function processSideEffects(
             {
               sessionId,
               channelId,
-              mode: effect.mode || (effect.chain?.length ? 'chain' : effect.tasks?.length ? 'parallel' : 'single'),
+              mode:
+                effect.mode ||
+                (effect.chain?.length
+                  ? 'chain'
+                  : effect.tasks?.length
+                    ? 'parallel'
+                    : 'single'),
               prompt: effect.prompt,
               label: effect.label,
               tasks: effect.tasks?.length,
@@ -64,7 +84,10 @@ export function processSideEffects(
           );
         }
       } catch (err) {
-        logger.error({ effect, err }, 'Failed to process delegation side-effect');
+        logger.error(
+          { effect, err },
+          'Failed to process delegation side-effect',
+        );
       }
     }
   }