@aigne/afs-history 1.2.0-beta.4 → 1.2.0-beta.6

package/lib/esm/storage/index.d.ts

@@ -1,7 +1,6 @@
 import type { AFSEntry, AFSModule } from "@aigne/afs";
 import { initDatabase } from "@aigne/sqlite";
-import { entriesTable } from "./models/entries.js";
-import type { AFSStorage, AFSStorageCreatePayload, AFSStorageListOptions } from "./type.js";
+import type { AFSStorage, AFSStorageCreateOptions, AFSStorageCreatePayload, AFSStorageDeleteOptions, AFSStorageListOptions, AFSStorageReadOptions } from "./type.js";
 export * from "./type.js";
 export interface SharedAFSStorageOptions {
     url?: string;
@@ -17,11 +16,16 @@ export declare class SharedAFSStorage {
 }
 export declare class AFSStorageWithModule implements AFSStorage {
     private db;
-    private table;
-    constructor(db: ReturnType<typeof initDatabase>, table: Promise<ReturnType<typeof entriesTable>>);
+    private module;
+    constructor(db: ReturnType<typeof initDatabase>, module: AFSModule);
+    private _tables?;
+    private get tables();
     list(options?: AFSStorageListOptions): Promise<{
         data: AFSEntry[];
     }>;
-    read(path: string): Promise<AFSEntry | undefined>;
-    create(entry: AFSStorageCreatePayload): Promise<AFSEntry>;
+    read(id: string, options?: AFSStorageReadOptions): Promise<AFSEntry | undefined>;
+    create(entry: AFSStorageCreatePayload, options?: AFSStorageCreateOptions): Promise<AFSEntry>;
+    delete(id: string, options?: AFSStorageDeleteOptions): Promise<{
+        deletedCount: number;
+    }>;
 }

package/lib/esm/storage/index.js

@@ -1,6 +1,8 @@
-import { and, asc, desc, eq, initDatabase, sql } from "@aigne/sqlite";
+import { and, asc, desc, eq, gt, initDatabase, lt, sql } from "@aigne/sqlite";
 import { migrate } from "./migrate.js";
+import { compactTable } from "./models/compact.js";
 import { entriesTable } from "./models/entries.js";
+import { memoryTable } from "./models/memory.js";
 export * from "./type.js";
 const DEFAULT_AFS_STORAGE_LIST_LIMIT = 10;
 export class SharedAFSStorage {
@@ -14,55 +16,99 @@ export class SharedAFSStorage {
         return this._db;
     }
     withModule(module) {
-        return new AFSStorageWithModule(this.db, migrate(this.db, module).then(() => entriesTable(module)));
+        return new AFSStorageWithModule(this.db, module);
     }
 }
 export class AFSStorageWithModule {
     db;
-    table;
-    constructor(db, table) {
+    module;
+    constructor(db, module) {
         this.db = db;
-        this.table = table;
+        this.module = module;
+    }
+    _tables;
+    get tables() {
+        if (!this._tables) {
+            this._tables = this.db.then((db) => migrate(db, this.module).then(() => ({
+                db,
+                entries: entriesTable(this.module),
+                compact: compactTable(this.module),
+                memory: memoryTable(this.module),
+            })));
+        }
+        return this._tables;
     }
     async list(options = {}) {
-        const { filter, limit = DEFAULT_AFS_STORAGE_LIST_LIMIT } = options;
-        const db = await this.db;
-        const table = await this.table;
+        const { type = "history", scope, filter, limit = DEFAULT_AFS_STORAGE_LIST_LIMIT } = options;
+        const { db, entries, compact, memory } = await this.tables;
+        // Select table based on type
+        const table = type === "compact" ? compact : type === "memory" ? memory : entries;
         const data = await db
             .select()
             .from(table)
-            .where(and(filter?.agentId ? eq(table.agentId, filter.agentId) : undefined, filter?.userId ? eq(table.userId, filter.userId) : undefined, filter?.sessionId ? eq(table.sessionId, filter.sessionId) : undefined))
-            .orderBy(...(options.orderBy ?? []).map((item) => (item[1] === "asc" ? asc : desc)(sql.identifier(item[0]))))
+            .where(and(
+        // Add scope filter for compact/memory tables
+        scope && type !== "history"
+            ? eq(sql `json_extract(${table.metadata}, '$.scope')`, scope)
+            : undefined, filter?.agentId ? eq(table.agentId, filter.agentId) : undefined, filter?.userId ? eq(table.userId, filter.userId) : undefined, filter?.sessionId ? eq(table.sessionId, filter.sessionId) : undefined, filter?.before ? lt(table.createdAt, new Date(filter.before)) : undefined, filter?.after ? gt(table.createdAt, new Date(filter.after)) : undefined))
+            .orderBy(...(options.orderBy ?? [["createdAt", "desc"]]).map((item) => (item[1] === "asc" ? asc : desc)(sql.identifier(item[0]))))
             .limit(limit)
             .execute();
         return { data };
     }
-    async read(path) {
-        const db = await this.db;
-        const table = await this.table;
+    async read(id, options) {
+        const { type = "history", scope, filter } = options ?? {};
+        const { db, entries, compact, memory } = await this.tables;
+        // Select table based on type
+        const table = type === "compact" ? compact : type === "memory" ? memory : entries;
         return db
             .select()
             .from(table)
-            .where(eq(table.path, path))
+            .where(and(eq(table.id, id), 
+        // Add scope filter for compact/memory tables
+        scope && type !== "history"
+            ? eq(sql `json_extract(${table.metadata}, '$.scope')`, scope)
+            : undefined, filter?.agentId ? eq(table.agentId, filter.agentId) : undefined, filter?.userId ? eq(table.userId, filter.userId) : undefined, filter?.sessionId ? eq(table.sessionId, filter.sessionId) : undefined))
             .limit(1)
             .execute()
-            .then((memory) => memory.at(0));
+            .then((rows) => rows.at(0));
     }
-    async create(entry) {
-        const db = await this.db;
-        const table = await this.table;
+    async create(entry, options) {
+        const { type = "history", scope } = options ?? {};
+        const { db, entries, compact, memory } = await this.tables;
+        // Select table based on type
+        const table = type === "compact" ? compact : type === "memory" ? memory : entries;
+        // Prepare entry data - only add scope to metadata for compact/memory types
+        const entryData = { ...entry, metadata: { ...entry.metadata, scope } };
+        // Try update first, then insert if not found (upsert)
         let result = await db
             .update(table)
-            .set(entry)
-            .where(eq(table.path, entry.path))
+            .set(entryData)
+            .where(eq(table.id, entry.id))
             .returning()
             .execute();
         if (!result.length) {
-            result = await db.insert(table).values(entry).returning().execute();
+            result = await db.insert(table).values(entryData).returning().execute();
         }
         const [res] = result;
         if (!res)
-            throw new Error("Failed to create AFS entry, no result");
+            throw new Error(`Failed to create ${type} entry, no result`);
         return res;
     }
+    async delete(id, options) {
+        const { type = "history", scope, filter } = options ?? {};
+        const { db, entries, compact, memory } = await this.tables;
+        // Select table based on type
+        const table = type === "compact" ? compact : type === "memory" ? memory : entries;
+        const result = await db
+            .delete(table)
+            .where(and(eq(table.id, id), 
+        // Add scope filter for compact/memory tables
+        scope && type !== "history"
+            ? eq(sql `json_extract(${table.metadata}, '$.scope')`, scope)
+            : undefined, filter?.agentId ? eq(table.agentId, filter.agentId) : undefined, filter?.userId ? eq(table.userId, filter.userId) : undefined, filter?.sessionId ? eq(table.sessionId, filter.sessionId) : undefined))
+            .returning()
+            .execute();
+        return { deletedCount: result.length };
+    }
 }

package/lib/esm/storage/migrate.d.ts

@@ -1,3 +1,3 @@
 import type { AFSModule } from "@aigne/afs";
 import { type initDatabase } from "@aigne/sqlite";
-export declare function migrate(db: ReturnType<typeof initDatabase>, module: AFSModule): Promise<void>;
+export declare function migrate(db: Awaited<ReturnType<typeof initDatabase>>, module: AFSModule): Promise<void>;

package/lib/esm/storage/migrate.js

@@ -2,8 +2,17 @@ import { sql } from "@aigne/sqlite";
 import { v7 } from "@aigne/uuid";
 import { init } from "./migrations/001-init.js";
 import { addAgentId } from "./migrations/002-add-agent-id.js";
+import { addCompactTable } from "./migrations/003-add-compact-table.js";
+import { addMemoryTable } from "./migrations/004-add-memory-table.js";
+import { addIndexes } from "./migrations/005-add-indexes.js";
 export async function migrate(db, module) {
-    const migrations = [init, addAgentId];
+    const migrations = [
+        init,
+        addAgentId,
+        addCompactTable,
+        addMemoryTable,
+        addIndexes,
+    ];
     const migrationsTable = "__drizzle_migrations";
     const migrationTableCreate = sql `
     CREATE TABLE IF NOT EXISTS ${sql.identifier(migrationsTable)} (
@@ -12,18 +21,18 @@ export async function migrate(db, module) {
       "hash" TEXT NOT NULL
     )
   `;
-    await (await db).run(migrationTableCreate).execute();
-    const dbMigrations = await (await db)
+    await db.run(migrationTableCreate).execute();
+    const dbMigrations = await db
         .values(sql `SELECT "id", "moduleId", "hash" FROM ${sql.identifier(migrationsTable)} WHERE "moduleId" = ${sql.param(module.name)} ORDER BY id DESC LIMIT 1`)
         .execute();
     const lastDbMigration = dbMigrations[0];
     const queriesToRun = [];
     for (const migration of migrations) {
-        if (!lastDbMigration || lastDbMigration[1] < migration.hash) {
+        if (!lastDbMigration || lastDbMigration[2] < migration.hash) {
             queriesToRun.push(...migration.sql(module), sql `INSERT INTO ${sql.identifier(migrationsTable)} ("id", "moduleId", "hash") VALUES(${sql.param(v7())}, ${sql.param(module.name)}, ${sql.param(migration.hash)})`);
         }
     }
     for (const query of queriesToRun) {
-        await (await db).run(query).execute();
+        await db.run(query).execute();
     }
 }

package/lib/esm/storage/migrations/003-add-compact-table.d.ts

@@ -0,0 +1,2 @@
+import type { AFSStorageMigrations } from "../type.js";
+export declare const addCompactTable: AFSStorageMigrations;

package/lib/esm/storage/migrations/003-add-compact-table.js

@@ -0,0 +1,20 @@
+import { sql } from "@aigne/sqlite";
+import { compactTableName } from "../models/compact.js";
+export const addCompactTable = {
+    hash: "003-add-compact-table",
+    sql: (module) => [
+        sql `\
+CREATE TABLE IF NOT EXISTS ${sql.identifier(compactTableName(module))} (
+  "id" TEXT NOT NULL PRIMARY KEY,
+  "createdAt" DATETIME NOT NULL,
+  "updatedAt" DATETIME NOT NULL,
+  "path" TEXT NOT NULL,
+  "sessionId" TEXT,
+  "userId" TEXT,
+  "agentId" TEXT,
+  "metadata" JSON,
+  "content" JSON
+)
+`,
+    ],
+};

package/lib/esm/storage/migrations/004-add-memory-table.d.ts

@@ -0,0 +1,2 @@
+import type { AFSStorageMigrations } from "../type.js";
+export declare const addMemoryTable: AFSStorageMigrations;

package/lib/esm/storage/migrations/004-add-memory-table.js

@@ -0,0 +1,20 @@
+import { sql } from "@aigne/sqlite";
+import { memoryTableName } from "../models/memory.js";
+export const addMemoryTable = {
+    hash: "004-add-memory-table",
+    sql: (module) => [
+        sql `\
+CREATE TABLE IF NOT EXISTS ${sql.identifier(memoryTableName(module))} (
+  "id" TEXT NOT NULL PRIMARY KEY,
+  "createdAt" DATETIME NOT NULL,
+  "updatedAt" DATETIME NOT NULL,
+  "path" TEXT NOT NULL,
+  "sessionId" TEXT,
+  "userId" TEXT,
+  "agentId" TEXT,
+  "metadata" JSON,
+  "content" JSON
+)
+`,
+    ],
+};

package/lib/esm/storage/migrations/005-add-indexes.d.ts

@@ -0,0 +1,2 @@
+import type { AFSStorageMigrations } from "../type.js";
+export declare const addIndexes: AFSStorageMigrations;

package/lib/esm/storage/migrations/005-add-indexes.js

@@ -0,0 +1,28 @@
+import { sql } from "@aigne/sqlite";
+import { compactTableName } from "../models/compact.js";
+import { entriesTableName } from "../models/entries.js";
+import { memoryTableName } from "../models/memory.js";
+export const addIndexes = {
+    hash: "005-add-indexes",
+    sql: (module) => {
+        const entriesTable = entriesTableName(module);
+        const memoryTable = memoryTableName(module);
+        const compactTable = compactTableName(module);
+        return [
+            // Entries table indexes
+            sql `CREATE INDEX IF NOT EXISTS "idx_entries_session_created" ON ${sql.identifier(entriesTable)} ("sessionId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_entries_user_created" ON ${sql.identifier(entriesTable)} ("userId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_entries_agent_created" ON ${sql.identifier(entriesTable)} ("agentId", "createdAt" DESC)`,
+            // Memory table indexes
+            sql `CREATE INDEX IF NOT EXISTS "idx_memory_session_created" ON ${sql.identifier(memoryTable)} ("sessionId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_memory_user_created" ON ${sql.identifier(memoryTable)} ("userId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_memory_agent_created" ON ${sql.identifier(memoryTable)} ("agentId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_memory_scope" ON ${sql.identifier(memoryTable)} (json_extract("metadata", '$.scope'))`,
+            // Compact table indexes
+            sql `CREATE INDEX IF NOT EXISTS "idx_compact_session_created" ON ${sql.identifier(compactTable)} ("sessionId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_compact_user_created" ON ${sql.identifier(compactTable)} ("userId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_compact_agent_created" ON ${sql.identifier(compactTable)} ("agentId", "createdAt" DESC)`,
+            sql `CREATE INDEX IF NOT EXISTS "idx_compact_scope" ON ${sql.identifier(compactTable)} (json_extract("metadata", '$.scope'))`,
+        ];
+    },
+};

package/lib/esm/storage/models/compact.d.ts

@@ -0,0 +1,183 @@
+import type { AFSModule } from "@aigne/afs";
+export declare const compactTableName: (module: AFSModule) => string;
+export declare const compactTable: (module: AFSModule) => import("@aigne/sqlite").SQLiteTableWithColumns<{
+    name: string;
+    schema: undefined;
+    columns: {
+        id: import("@aigne/sqlite").SQLiteColumn<{
+            name: "id";
+            tableName: string;
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: true;
+            isAutoincrement: false;
+            hasRuntimeDefault: true;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        createdAt: import("@aigne/sqlite").SQLiteColumn<{
+            name: "createdAt";
+            tableName: string;
+            dataType: "custom";
+            columnType: "SQLiteCustomColumn";
+            data: Date;
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: true;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
+        }>;
+        updatedAt: import("@aigne/sqlite").SQLiteColumn<{
+            name: "updatedAt";
+            tableName: string;
+            dataType: "custom";
+            columnType: "SQLiteCustomColumn";
+            data: Date;
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: true;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
+        }>;
+        path: import("@aigne/sqlite").SQLiteColumn<{
+            name: "path";
+            tableName: string;
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        userId: import("@aigne/sqlite").SQLiteColumn<{
+            name: "userId";
+            tableName: string;
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        sessionId: import("@aigne/sqlite").SQLiteColumn<{
+            name: "sessionId";
+            tableName: string;
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        agentId: import("@aigne/sqlite").SQLiteColumn<{
+            name: "agentId";
+            tableName: string;
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
+        metadata: import("@aigne/sqlite").SQLiteColumn<{
+            name: string;
+            tableName: string;
+            dataType: "custom";
+            columnType: "SQLiteCustomColumn";
+            data: {
+                scope?: string;
+                latestEntryId?: string;
+            } & Record<string, unknown>;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
+        }>;
+        content: import("@aigne/sqlite").SQLiteColumn<{
+            name: string;
+            tableName: string;
+            dataType: "custom";
+            columnType: "SQLiteCustomColumn";
+            data: unknown;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
+        }>;
+    };
+    dialect: "sqlite";
+}>;

package/lib/esm/storage/models/compact.js

@@ -0,0 +1,22 @@
+import { datetime, json, sqliteTable, text } from "@aigne/sqlite";
+import { v7 } from "@aigne/uuid";
+export const compactTableName = (module) => `Entries_${module.name}_compact`;
+export const compactTable = (module) => sqliteTable(compactTableName(module), {
+    id: text("id")
+        .notNull()
+        .primaryKey()
+        .$defaultFn(() => v7()),
+    createdAt: datetime("createdAt")
+        .notNull()
+        .$defaultFn(() => new Date()),
+    updatedAt: datetime("updatedAt")
+        .notNull()
+        .$defaultFn(() => new Date())
+        .$onUpdateFn(() => new Date()),
+    path: text("path").notNull(),
+    userId: text("userId"),
+    sessionId: text("sessionId"),
+    agentId: text("agentId"),
+    metadata: json("metadata"),
+    content: json("content"),
+});