clawlet 0.2.1 → 0.4.0

package/src/memory.ts

@@ -1,8 +1,15 @@
 import { createStorage, type Storage } from "unstorage";
 import fsDriver from "unstorage/drivers/fs";
-import { type ModelMessage } from "ai";
+import { generateText, type LanguageModel, type ModelMessage } from "ai";
 import path from "path";
-import { LibSqlKeyValueStorage, LibSqlListStorage, SkillHistoryStorage } from "./storage.js";
+import { LibSqlKeyValueStorage, LibSqlListStorage } from "./storage.js";
+import { logger } from './logger.js';
+import memoryDriver from 'unstorage/drivers/memory';
+
+
+// --- COMPACTION CONFIG ---
+const COMPACT_THRESHOLD = 25;  // Trigger compaction when history reaches this many items
+const COMPACT_RANGE = 10;      // Number of messages to summarize (items 1..10, skipping system prompt at 0)
 
 export class AgentMemory {
   // 1. Secrets (libSQL - file:secrets.db)
@@ -11,37 +18,98 @@ export class AgentMemory {
   // 2. History (libSQL - file:history.db)
   public history: LibSqlListStorage<ModelMessage>;
 
-  // 3. Skill History (libSQL - file:history.db, table: skill_history)
-  public skillHistory: SkillHistoryStorage<ModelMessage>;
-
-  // 4. Workspace (Unstorage - ./workspace)
+  // 3. Workspace (Unstorage - ./workspace)
   public workspace: Storage;
 
-  constructor() {
-    // A. Init Secrets DB
-    // In Production: process.env.SECRETS_DB_URL (libsql://...)
-    this.secrets = new LibSqlKeyValueStorage(
-      process.env.SECRETS_DB_URL || "file:secrets.db", 
-      process.env.SECRETS_AUTH_TOKEN
-    );
+  private constructor(secrets: LibSqlKeyValueStorage, history: LibSqlListStorage<ModelMessage>, workspace: Storage) {
+    this.secrets = secrets;
+    this.history = history;
+    this.workspace = workspace
+  }
 
-    // B. Init History DB
-    // In Production: process.env.HISTORY_DB_URL (libsql://...)
-    this.history = new LibSqlListStorage<ModelMessage>(
-      process.env.HISTORY_DB_URL || "file:history.db",
-      process.env.HISTORY_AUTH_TOKEN
+  static async createInMemory() {
+    return new AgentMemory(
+      await LibSqlKeyValueStorage.create(':memory:'),
+      await LibSqlListStorage.create<ModelMessage>(':memory:'),
+      createStorage({ driver: memoryDriver() })
     );
+  }
 
-    // C. Init Skill History (same DB as history, different table)
-    this.skillHistory = new SkillHistoryStorage<ModelMessage>(
-      process.env.HISTORY_DB_URL || "file:history.db",
-      process.env.HISTORY_AUTH_TOKEN
+  static async create() {
+    return new AgentMemory(
+      await LibSqlKeyValueStorage.create(
+        process.env.SECRETS_DB_URL || "file:secrets.db", 
+        process.env.SECRETS_AUTH_TOKEN
+      ),
+      await LibSqlListStorage.create<ModelMessage>(
+        process.env.HISTORY_DB_URL || "file:history.db",
+        process.env.HISTORY_AUTH_TOKEN
+      ),
+      createStorage({
+        driver: fsDriver({ base: path.join(process.cwd(), "workspace") })
+      })
     );
+  }
+
+
+  /**
+   * Compacts history when it reaches COMPACT_THRESHOLD items.
+   * Summarizes items 1..COMPACT_RANGE (the system prompt is not included) into a single message
+   * using the LLM, then replaces in-memory + persisted history.
+   * Result: summary message + remaining messages.
+   */
+  async compactHistory(name:string, model: LanguageModel): Promise<ModelMessage[]> {
+    const messages = await this.history.getAll(name);
+    if (messages.length < COMPACT_THRESHOLD) return messages;
+
+    logger.info({count: messages.length}, `messages to be compacted.`);
+
+    const toSummarize = messages.slice(0, COMPACT_RANGE); 
+    const remaining = messages.slice(COMPACT_RANGE);
+
+    // Build a transcript for the LLM to summarize
+    const transcript = toSummarize.map(m => {
+      const role = m.role ?? 'unknown';
+      const content = typeof m.content === 'string'
+        ? m.content
+        : JSON.stringify(m.content);
+      return `[${role}]: ${content}`;
+    }).join('\n\n');
+
+    try {
+      const { text: summary } = await generateText({
+        model,
+        messages: [
+          {
+            role: 'system',
+            content: 'You are a conversation summarizer. Summarize the following conversation transcript concisely, preserving key facts, decisions, tool results, and context that would be needed to continue the conversation. Be factual and dense. Do not add commentary.',
+          },
+          {
+            role: 'user',
+            content: `Summarize this conversation transcript:\n\n${transcript}`,
+          },
+        ],
+        temperature: 0.3,
+      });
+
+      const summaryMessage: ModelMessage = {
+        role: 'assistant',
+        content: `[Conversation Summary — compacted ${COMPACT_RANGE} messages]\n\n${summary}`,
+      };
+
+      // Rebuild in-memory messages: summary + remaining
+      const compactedMessages = [summaryMessage, ...remaining];
+
+      // Persist: clear DB and re-write all messages
+      await this.history.replaceAll(name, compactedMessages);
 
-    // D. Init Workspace (Filesystem)
-    // Unstorage abstrahiert hier nur das "Wie", aber es bleibt lokal im Ordner.
-    this.workspace = createStorage({
-      driver: fsDriver({ base: path.join(process.cwd(), "workspace") })
-    });
+      logger.info({count: compactedMessages.length}, `  Compacted messages.`);
+      return compactedMessages;
+    } catch (e) {
+      logger.error({ err: e }, 'Compaction failed, keeping original history');
+      await this.history.replaceAll(name, messages);
+      return messages;
+    }
   }
+  
 }

package/src/storage.ts

@@ -5,10 +5,16 @@ export class LibSqlKeyValueStorage {
   private client: Client;
   private tableName: string;
 
-  constructor(url: string, authToken?: string, tableName = 'kv_store') {
+
+  private constructor(url: string, authToken?: string, tableName = 'kv_store') {
+    this.client = authToken ? createClient({ url, authToken }) : createClient({ url });
     this.tableName = tableName;
-    this.client = createClient({ url, authToken });
-    this.init();
+  }
+
+  static async create<T>(url: string, authToken?: string, tableName = 'kv_store') {
+    const s = new LibSqlKeyValueStorage(url, authToken, tableName);
+    await s.init();
+    return s;
   }
 
   private async init() {
@@ -61,130 +67,176 @@ export class LibSqlKeyValueStorage {
 // --- B. List Storage (für History/Logs) ---
 export class LibSqlListStorage<T = any> {
   private client: Client;
-  private tableName: string;
-
-  constructor(url: string, authToken?: string, tableName = 'list_store') {
-    this.tableName = tableName;
-    this.client = createClient({ url, authToken });
-    this.init();
-  }
-
-  private async init() {
-    await this.client.execute(`
-      CREATE TABLE IF NOT EXISTS ${this.tableName} (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        item TEXT,
-        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-      )
-    `);
-  }
-
-  async push(item: T) {
-    await this.client.execute({
-      sql: `INSERT INTO ${this.tableName} (item) VALUES (?)`,
-      args: [JSON.stringify(item)]
-    });
-  }
-
-  // Bulk Insert für Performance
-  async pushMany(items: T[]) {
-    const promises = items.map(item => this.client.execute({
-        sql: `INSERT INTO ${this.tableName} (item) VALUES (?)`,
-        args: [JSON.stringify(item)]
-    }));
-    await Promise.all(promises);
-  }
-
-  async getAll(): Promise<T[]> {
-    const rs = await this.client.execute(
-      `SELECT item FROM ${this.tableName} ORDER BY id ASC`
-    );
-    return rs.rows.map(row => JSON.parse(row.item as string));
-  }
-  
-  // Optional: Nur die letzten N Nachrichten holen (Kontext-Fenster!)
-  async getRecent(limit: number): Promise<T[]> {
-     // Trick: Erst sortieren DESC (neueste), limitieren, dann wieder ASC sortieren
-     const rs = await this.client.execute({
-        sql: `SELECT * FROM (
-                SELECT item, id FROM ${this.tableName} ORDER BY id DESC LIMIT ?
-              ) ORDER BY id ASC`,
-        args: [limit]
-     });
-     return rs.rows.map(row => JSON.parse(row.item as string));
-  }
-
-  async count(): Promise<number> {
-    const rs = await this.client.execute(
-      `SELECT COUNT(*) as cnt FROM ${this.tableName}`
-    );
-    return Number(rs.rows[0]?.cnt ?? 0);
-  }
+  private tableName = 'list_items';
 
-  async clear() {
-    await this.client.execute(`DELETE FROM ${this.tableName}`);
-    // Reset autoincrement so IDs stay clean
-    await this.client.execute(
-      `DELETE FROM sqlite_sequence WHERE name = '${this.tableName}'`
-    );
+  private constructor(url: string, authToken?: string) {
+    this.client = authToken ? createClient({ url, authToken }) : createClient({ url });
   }
-}
 
-// --- C. Skill History Storage (single table, partitioned by skill name) ---
-export class SkillHistoryStorage<T = any> {
-  private client: Client;
-
-  constructor(url: string, authToken?: string) {
-    this.client = createClient({ url, authToken });
-    this.init();
+  static async create<T>(url: string, authToken?: string) {
+    const s = new LibSqlListStorage(url, authToken);
+    await s.init();
+    return s;
   }
 
   private async init() {
     await this.client.execute(`
-      CREATE TABLE IF NOT EXISTS skill_history (
+      CREATE TABLE IF NOT EXISTS ${this.tableName} (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         name TEXT NOT NULL,
-        item TEXT,
+        item TEXT NOT NULL,
         created_at DATETIME DEFAULT CURRENT_TIMESTAMP
       )
     `);
   }
 
-  async push(name: string, item: T) {
+  async push(name: string, item: T): Promise<void> {
     await this.client.execute({
-      sql: `INSERT INTO skill_history (name, item) VALUES (?, ?)`,
+      sql: `INSERT INTO ${this.tableName} (name, item) VALUES (?, ?)`,
       args: [name, JSON.stringify(item)]
     });
   }
 
-  async pushMany(name: string, items: T[]) {
-    const promises = items.map(item => this.client.execute({
-      sql: `INSERT INTO skill_history (name, item) VALUES (?, ?)`,
-      args: [name, JSON.stringify(item)]
-    }));
-    await Promise.all(promises);
+  async pushMany(name: string, items: T[]): Promise<void> {
+    for (const item of items) {
+      this.push(name, item);
+    }
+  }
+
+  async replaceAll(name: string, items: T[]): Promise<void> {
+    const tx = await this.client.transaction();
+    try {
+        await tx.execute({
+            sql: `DELETE FROM ${this.tableName} WHERE name = ?`,
+            args: [name],
+        });
+        for (const item of items) {
+            await tx.execute({
+                sql: `INSERT INTO ${this.tableName} (name, item) VALUES (?, ?)`,
+                args: [name, JSON.stringify(item)]
+            });
+        }
+        await tx.commit();
+    } catch (e) {
+        await tx.rollback();
+        throw e;
+    }
   }
 
   async getAll(name: string): Promise<T[]> {
     const rs = await this.client.execute({
-      sql: `SELECT item FROM skill_history WHERE name = ? ORDER BY id ASC`,
+      sql: `SELECT item FROM ${this.tableName} WHERE name = ? ORDER BY id ASC`,
       args: [name]
     });
     return rs.rows.map(row => JSON.parse(row.item as string));
   }
-
-  async clear(name: string) {
-    await this.client.execute({
-      sql: `DELETE FROM skill_history WHERE name = ?`,
+  
+  async count(name: string): Promise<number> {
+    const rs = await this.client.execute({
+      sql: `SELECT COUNT(*) as cnt FROM ${this.tableName} WHERE name = ?`,
       args: [name]
     });
+    return Number(rs.rows[0]?.cnt ?? 0);
   }
 
-  async count(name: string): Promise<number> {
-    const rs = await this.client.execute({
-      sql: `SELECT COUNT(*) as cnt FROM skill_history WHERE name = ?`,
+  async clear(name: string): Promise<void> {
+    await this.client.execute({
+      sql: `DELETE FROM ${this.tableName} WHERE name = ?`,
       args: [name]
     });
-    return Number(rs.rows[0]?.cnt ?? 0);
   }
 }
+
+
+export class LibSqlFiFoStorage<T> {
+    private client: Client;
+    private tableName = 'queue_items';
+
+    constructor(url: string, authToken?: string) {
+        this.client = authToken ? createClient({ url, authToken }) : createClient({ url });
+        this.init();
+    }
+
+    private async init(): Promise<void> {
+        await this.client.execute(`
+            CREATE TABLE IF NOT EXISTS ${this.tableName} (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                queue_name TEXT NOT NULL,
+                value TEXT NOT NULL,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            );
+        `);
+    }
+
+    public async push(queue: string, item: T): Promise<void> {
+        await this.client.execute({
+            sql: `INSERT INTO ${this.tableName} (queue_name, value) VALUES (?, ?)`,
+            args: [queue, JSON.stringify(item)],
+        });
+    }
+
+    public async pushMany(queue: string, items: T[]): Promise<void> {
+        const tx = await this.client.transaction();
+        try {
+            for (const item of items) {
+                await tx.execute({
+                    sql: `INSERT INTO ${this.tableName} (queue_name, value) VALUES (?, ?)`,
+                    args: [queue, JSON.stringify(item)],
+                });
+            }
+            await tx.commit();
+        } catch (e) {
+            await tx.rollback();
+            throw e;
+        }
+    }
+
+    public async empty(queue: string): Promise<boolean> {
+        return (await this.count(queue)) === 0;
+    }
+
+    public async pop(queue: string): Promise<T | null> {
+        const tx = await this.client.transaction();
+        try {
+            const rs = await tx.execute({
+                sql: `SELECT id, value FROM ${this.tableName} WHERE queue_name = ? ORDER BY id ASC LIMIT 1`,
+                args: [queue],
+            });
+
+            if (rs.rows.length === 0) {
+                await tx.commit();
+                return null;
+            }
+
+            const row = rs.rows[0] as any;
+            const id = row.id;
+
+            await tx.execute({
+                sql: `DELETE FROM ${this.tableName} WHERE id = ?`,
+                args: [id!],
+            });
+
+            await tx.commit();
+
+            return JSON.parse(row.value as string) as T;
+        } catch (e) {
+            await tx.rollback();
+            throw e;
+        }
+    }
+
+    public async count(queue: string): Promise<number> {
+        const rs = await this.client.execute({
+            sql: `SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue_name = ?`,
+            args: [queue],
+        });
+        return (rs.rows[0]?.count as number) ?? 0;
+    }
+
+    public async clear(queue: string): Promise<void> {
+        await this.client.execute({
+            sql: `DELETE FROM ${this.tableName} WHERE queue_name = ?`,
+            args: [queue],
+        });
+    }
+}