@agent-diaries/core 1.1.1 → 1.1.2

package/dist/src/adapters/mongo.d.ts

@@ -0,0 +1,16 @@
+import { StorageAdapter } from '../storage';
+import { Collection } from 'mongodb';
+interface MongoStorageOptions {
+    collection: Collection;
+}
+export declare class MongoStorage<T> implements StorageAdapter<T> {
+    private collection;
+    private static indexedCollections;
+    constructor(options: MongoStorageOptions);
+    private ensureIndex;
+    private hashString;
+    get(key: string): Promise<T | null>;
+    set(key: string, data: T): Promise<void>;
+    withLock<R>(key: string, fn: () => Promise<R>): Promise<R>;
+}
+export {};

package/dist/src/adapters/mongo.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MongoStorage = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+class MongoStorage {
+    collection;
+    static indexedCollections = new Set();
+    constructor(options) {
+        this.collection = options.collection;
+    }
+    async ensureIndex() {
+        const ns = this.collection.namespace;
+        if (MongoStorage.indexedCollections.has(ns))
+            return;
+        try {
+            await this.collection.createIndex({ lockedAt: 1 }, { expireAfterSeconds: 10, partialFilterExpression: { lockedAt: { $exists: true } } });
+        }
+        catch (e) {
+            console.warn('[MongoStorage] Failed to create TTL index:', e);
+        }
+        MongoStorage.indexedCollections.add(ns);
+    }
+    hashString(str) {
+        return crypto_1.default.createHash('sha256').update(str).digest('hex');
+    }
+    async get(key) {
+        const hash = this.hashString(key);
+        const doc = await this.collection.findOne({ _id: hash });
+        if (!doc || !doc.data)
+            return null;
+        return JSON.parse(doc.data);
+    }
+    async set(key, data) {
+        const hash = this.hashString(key);
+        await this.collection.updateOne({ _id: hash }, { $set: { data: JSON.stringify(data) } }, { upsert: true });
+    }
+    async withLock(key, fn) {
+        await this.ensureIndex();
+        const hash = this.hashString(key);
+        const lockId = `lock:${hash}`;
+        const acquireLock = async () => {
+            try {
+                await this.collection.insertOne({ _id: lockId, lockedAt: new Date() });
+                return true;
+            }
+            catch (error) {
+                if (error.code === 11000)
+                    return false;
+                throw error;
+            }
+        };
+        let attempt = 0;
+        while (!(await acquireLock())) {
+            const backoff = Math.min(10 * Math.pow(2, attempt), 500);
+            const jitter = Math.random() * 50;
+            await new Promise(resolve => setTimeout(resolve, backoff + jitter));
+            attempt++;
+            if (attempt > 150) {
+                throw new Error(`[MongoStorage] Lock timeout on key: ${key}`);
+            }
+        }
+        try {
+            return await fn();
+        }
+        finally {
+            await this.collection.deleteOne({ _id: lockId });
+        }
+    }
+}
+exports.MongoStorage = MongoStorage;

package/dist/src/adapters/redis.d.ts

@@ -0,0 +1,14 @@
+import { StorageAdapter } from '../storage';
+import Redis from 'ioredis';
+export declare class RedisStorage<T> implements StorageAdapter<T> {
+    private redis;
+    private prefix;
+    constructor(options: {
+        redis: Redis;
+        prefix?: string;
+    });
+    private getKey;
+    get(key: string): Promise<T | null>;
+    set(key: string, value: T): Promise<void>;
+    withLock<R>(key: string, fn: () => Promise<R>): Promise<R>;
+}

package/dist/src/adapters/redis.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStorage = void 0;
+const crypto_1 = require("crypto");
+class RedisStorage {
+    redis;
+    prefix;
+    constructor(options) {
+        this.redis = options.redis;
+        this.prefix = options.prefix || 'agent-diaries:';
+    }
+    getKey(key) {
+        return `${this.prefix}${key}`;
+    }
+    async get(key) {
+        const data = await this.redis.get(this.getKey(key));
+        if (!data)
+            return null;
+        try {
+            return JSON.parse(data);
+        }
+        catch (e) {
+            console.error(`[RedisStorage] Failed to parse JSON for key ${key}`);
+            return null;
+        }
+    }
+    async set(key, value) {
+        const data = JSON.stringify(value);
+        await this.redis.set(this.getKey(key), data);
+    }
+    async withLock(key, fn) {
+        const lockKey = `${this.getKey(key)}:lock`;
+        const lockValue = (0, crypto_1.randomUUID)();
+        const lockTtlMs = 10000; // 10 seconds max lock
+        const acquireLock = async () => {
+            const result = await this.redis.set(lockKey, lockValue, 'PX', lockTtlMs, 'NX');
+            return result === 'OK';
+        };
+        let attempt = 0;
+        while (!(await acquireLock())) {
+            const backoff = Math.min(10 * Math.pow(2, attempt), 500);
+            const jitter = Math.random() * 50;
+            await new Promise(resolve => setTimeout(resolve, backoff + jitter));
+            attempt++;
+            if (attempt > 60)
+                throw new Error(`[RedisStorage] Lock timeout on key: ${key}`);
+        }
+        try {
+            return await fn();
+        }
+        finally {
+            // Safe release: ensure we only delete the lock if we still own it
+            const luaScript = `
+        if redis.call("get", KEYS[1]) == ARGV[1] then
+          return redis.call("del", KEYS[1])
+        else
+          return 0
+        end
+      `;
+            await this.redis.eval(luaScript, 1, lockKey, lockValue);
+        }
+    }
+}
+exports.RedisStorage = RedisStorage;

package/dist/src/adapters/sqlite.d.ts

@@ -0,0 +1,16 @@
+import { StorageAdapter } from '../storage';
+import type { Database } from 'better-sqlite3';
+export interface SqliteStorageOptions {
+    db: Database;
+    tableName?: string;
+    locksTableName?: string;
+}
+export declare class SqliteStorage<T> implements StorageAdapter<T> {
+    private db;
+    private tableName;
+    private locksTableName;
+    constructor(options: SqliteStorageOptions);
+    get(key: string): Promise<T | null>;
+    set(key: string, value: T): Promise<void>;
+    withLock<R>(key: string, fn: () => Promise<R>): Promise<R>;
+}

package/dist/src/adapters/sqlite.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqliteStorage = void 0;
+class SqliteStorage {
+    db;
+    tableName;
+    locksTableName;
+    constructor(options) {
+        if (!options.db) {
+            throw new Error('[SqliteStorage] database instance (db) is required.');
+        }
+        this.db = options.db;
+        this.tableName = options.tableName || 'agent_diaries_storage';
+        this.locksTableName = options.locksTableName || 'agent_diaries_locks';
+        // Initialize tables synchronously as better-sqlite3 is fully synchronous
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${this.tableName} (
+        key TEXT PRIMARY KEY,
+        value TEXT
+      );
+      CREATE TABLE IF NOT EXISTS ${this.locksTableName} (
+        key TEXT PRIMARY KEY,
+        locked_at INTEGER
+      );
+    `);
+    }
+    async get(key) {
+        try {
+            const row = this.db
+                .prepare(`SELECT value FROM ${this.tableName} WHERE key = ?`)
+                .get(key);
+            if (!row)
+                return null;
+            return JSON.parse(row.value);
+        }
+        catch (e) {
+            console.error(`[SqliteStorage] Failed to get key ${key}:`, e);
+            return null;
+        }
+    }
+    async set(key, value) {
+        try {
+            const serialized = JSON.stringify(value);
+            this.db
+                .prepare(`
+          INSERT INTO ${this.tableName} (key, value)
+          VALUES (?, ?)
+          ON CONFLICT(key) DO UPDATE SET value = excluded.value
+        `)
+                .run(key, serialized);
+        }
+        catch (e) {
+            console.error(`[SqliteStorage] Failed to set key ${key}:`, e);
+            throw e;
+        }
+    }
+    async withLock(key, fn) {
+        const lockKey = `lock:${key}`;
+        const lockTtlMs = 10000; // 10 seconds max lock
+        const acquireLock = () => {
+            const now = Date.now();
+            try {
+                this.db
+                    .prepare(`INSERT INTO ${this.locksTableName} (key, locked_at) VALUES (?, ?)`)
+                    .run(lockKey, now);
+                return true;
+            }
+            catch (error) {
+                if (error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' ||
+                    error.code === 'SQLITE_CONSTRAINT' ||
+                    (error.message && error.message.includes('UNIQUE constraint failed'))) {
+                    // Check if the lock has expired
+                    const existing = this.db
+                        .prepare(`SELECT locked_at FROM ${this.locksTableName} WHERE key = ?`)
+                        .get(lockKey);
+                    if (existing && now - existing.locked_at > lockTtlMs) {
+                        // Attempt to clear the expired lock and acquire it
+                        try {
+                            // Wrap cleanup and acquire in a transaction for atomicity
+                            const runCleanupAndAcquire = this.db.transaction(() => {
+                                this.db
+                                    .prepare(`DELETE FROM ${this.locksTableName} WHERE key = ? AND locked_at = ?`)
+                                    .run(lockKey, existing.locked_at);
+                                this.db
+                                    .prepare(`INSERT INTO ${this.locksTableName} (key, locked_at) VALUES (?, ?)`)
+                                    .run(lockKey, now);
+                            });
+                            runCleanupAndAcquire();
+                            return true;
+                        }
+                        catch (retryError) {
+                            // Conflict during retry, lock is still held or acquired by another concurrent process
+                            return false;
+                        }
+                    }
+                    return false;
+                }
+                throw error;
+            }
+        };
+        let attempt = 0;
+        while (!acquireLock()) {
+            const backoff = Math.min(10 * Math.pow(2, attempt), 500);
+            const jitter = Math.random() * 50;
+            await new Promise(resolve => setTimeout(resolve, backoff + jitter));
+            attempt++;
+            if (attempt > 60) {
+                throw new Error(`[SqliteStorage] Lock timeout on key: ${key}`);
+            }
+        }
+        try {
+            return await fn();
+        }
+        finally {
+            try {
+                this.db
+                    .prepare(`DELETE FROM ${this.locksTableName} WHERE key = ?`)
+                    .run(lockKey);
+            }
+            catch (e) {
+                console.error(`[SqliteStorage] Failed to release lock on key ${key}:`, e);
+            }
+        }
+    }
+}
+exports.SqliteStorage = SqliteStorage;

package/dist/src/diary.d.ts

@@ -0,0 +1,59 @@
+import { StorageAdapter } from './storage';
+export interface TaskRecord {
+    title: string;
+    signature: string;
+    result?: string;
+    timestamp: number;
+}
+export interface AgentState {
+    lastRun: number;
+    seenSignatures: string[];
+    runCount: number;
+    history: TaskRecord[];
+}
+export interface AgentDiaryOptions {
+    agentId: string;
+    storage?: StorageAdapter<AgentState>;
+    maxHistory?: number;
+}
+export declare class AgentDiary {
+    private agentId;
+    private storage;
+    private maxHistory;
+    constructor(options: AgentDiaryOptions);
+    private emptyState;
+    /**
+     * Generates a normalized signature for a task title.
+     */
+    static normalizeSignature(title: string): string;
+    /**
+     * Reads the current diary state (without locking).
+     */
+    readDiary(): Promise<AgentState>;
+    /**
+     * Atomically attempts to claim a task.
+     * Returns true if successfully claimed (first time seen), false if already claimed/processed.
+     */
+    claimTask(title: string): Promise<boolean>;
+    /**
+     * Checks if a task has already been processed by the agent.
+     */
+    hasProcessedTask(title: string): Promise<boolean>;
+    /**
+     * Retrieves the stored result of a previously processed task, if available.
+     */
+    getTaskResult(title: string): Promise<string | undefined>;
+    /**
+     * Filters out items that the agent has already processed.
+     *
+     * ⚠️ WARNING: filterNewTasks() is a non-atomic snapshot. Always follow it with claimTask()
+     * to atomically claim ownership. Never act on filterNewTasks() results directly without claiming them first.
+     */
+    filterNewTasks<T extends {
+        title: string;
+    }>(tasks: T[]): Promise<T[]>;
+    /**
+     * Updates a claimed task with its final result.
+     */
+    writeTaskResult(title: string, result?: string): Promise<void>;
+}

package/dist/src/diary.js

@@ -0,0 +1,116 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentDiary = void 0;
+const storage_1 = require("./storage");
+class AgentDiary {
+    agentId;
+    storage;
+    maxHistory;
+    constructor(options) {
+        this.agentId = options.agentId;
+        // Use local file storage by default if none provided
+        this.storage = options.storage || new storage_1.LocalFileStorage();
+        this.maxHistory = options.maxHistory || 500;
+    }
+    emptyState() {
+        return {
+            lastRun: 0,
+            seenSignatures: [],
+            runCount: 0,
+            history: [],
+        };
+    }
+    /**
+     * Generates a normalized signature for a task title.
+     */
+    static normalizeSignature(title) {
+        return (title || '').toLowerCase().trim().replace(/\s+/g, ' ');
+    }
+    /**
+     * Reads the current diary state (without locking).
+     */
+    async readDiary() {
+        const state = await this.storage.get(`diary_${this.agentId}`);
+        return state ?? this.emptyState();
+    }
+    /**
+     * Atomically attempts to claim a task.
+     * Returns true if successfully claimed (first time seen), false if already claimed/processed.
+     */
+    async claimTask(title) {
+        const signature = AgentDiary.normalizeSignature(title);
+        return await this.storage.withLock(`diary_${this.agentId}`, async () => {
+            const state = await this.readDiary();
+            const seenSet = new Set(state.seenSignatures);
+            if (seenSet.has(signature)) {
+                return false; // Task already exists
+            }
+            // Claim it immediately to prevent race conditions
+            const record = {
+                title,
+                signature,
+                timestamp: Date.now()
+            };
+            state.history = [record, ...state.history].slice(0, this.maxHistory);
+            // Fix Desync Bug: seenSignatures is strictly derived from the sliced history
+            state.seenSignatures = state.history.map(r => r.signature);
+            state.runCount += 1;
+            state.lastRun = Date.now();
+            await this.storage.set(`diary_${this.agentId}`, state);
+            return true;
+        });
+    }
+    /**
+     * Checks if a task has already been processed by the agent.
+     */
+    async hasProcessedTask(title) {
+        const signature = AgentDiary.normalizeSignature(title);
+        const state = await this.readDiary();
+        const seenSet = new Set(state.seenSignatures);
+        return seenSet.has(signature);
+    }
+    /**
+     * Retrieves the stored result of a previously processed task, if available.
+     */
+    async getTaskResult(title) {
+        const signature = AgentDiary.normalizeSignature(title);
+        const state = await this.readDiary();
+        const record = state.history.find(r => r.signature === signature);
+        return record?.result;
+    }
+    /**
+     * Filters out items that the agent has already processed.
+     *
+     * ⚠️ WARNING: filterNewTasks() is a non-atomic snapshot. Always follow it with claimTask()
+     * to atomically claim ownership. Never act on filterNewTasks() results directly without claiming them first.
+     */
+    async filterNewTasks(tasks) {
+        const state = await this.readDiary();
+        const seenSet = new Set(state.seenSignatures);
+        return tasks.filter(task => {
+            const signature = AgentDiary.normalizeSignature(task.title);
+            return !seenSet.has(signature);
+        });
+    }
+    /**
+     * Updates a claimed task with its final result.
+     */
+    async writeTaskResult(title, result) {
+        const signature = AgentDiary.normalizeSignature(title);
+        await this.storage.withLock(`diary_${this.agentId}`, async () => {
+            const state = await this.readDiary();
+            const recordIndex = state.history.findIndex(r => r.signature === signature);
+            if (recordIndex !== -1) {
+                state.history[recordIndex].result = result;
+                state.history[recordIndex].timestamp = Date.now(); // update timestamp
+                state.lastRun = Date.now();
+                await this.storage.set(`diary_${this.agentId}`, state);
+            }
+            else {
+                // If not claimed first, we throw a loud error
+                throw new Error(`[AgentDiary] Task "${title}" was not claimed. Call claimTask() before writeTaskResult().`);
+            }
+        });
+    }
+}
+exports.AgentDiary = AgentDiary;

package/dist/src/index.d.ts

@@ -0,0 +1,2 @@
+export * from './storage';
+export * from './diary';

package/dist/src/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./storage"), exports);
+__exportStar(require("./diary"), exports);

package/dist/src/storage.d.ts

@@ -0,0 +1,18 @@
+export interface StorageAdapter<T> {
+    get(key: string): Promise<T | null>;
+    set(key: string, value: T): Promise<void>;
+    /**
+     * Acquires a lock on the key, executes the critical section, and releases the lock.
+     */
+    withLock<R>(key: string, fn: () => Promise<R>): Promise<R>;
+}
+export declare class LocalFileStorage<T> implements StorageAdapter<T> {
+    private baseDir;
+    constructor(options?: {
+        baseDir?: string;
+    });
+    private getFilePath;
+    get(key: string): Promise<T | null>;
+    set(key: string, value: T): Promise<void>;
+    withLock<R>(key: string, fn: () => Promise<R>): Promise<R>;
+}

package/dist/src/storage.js

@@ -0,0 +1,95 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalFileStorage = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const lockfile = __importStar(require("proper-lockfile"));
+class LocalFileStorage {
+    baseDir;
+    constructor(options = {}) {
+        this.baseDir = options.baseDir || path.join(process.cwd(), '.agent-diaries');
+        if (!fs.existsSync(this.baseDir)) {
+            fs.mkdirSync(this.baseDir, { recursive: true });
+        }
+    }
+    getFilePath(key) {
+        const safeKey = key.replace(/[^a-zA-Z0-9_-]/g, '_');
+        return path.join(this.baseDir, `${safeKey}.json`);
+    }
+    async get(key) {
+        const filePath = this.getFilePath(key);
+        if (!fs.existsSync(filePath)) {
+            return null;
+        }
+        try {
+            const data = await fs.promises.readFile(filePath, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch (e) {
+            console.error(`[LocalFileStorage] Error reading key ${key}:`, e);
+            return null;
+        }
+    }
+    async set(key, value) {
+        const filePath = this.getFilePath(key);
+        try {
+            const data = JSON.stringify(value, null, 2);
+            await fs.promises.writeFile(filePath, data, 'utf-8');
+        }
+        catch (e) {
+            console.error(`[LocalFileStorage] Error writing key ${key}:`, e);
+            throw e;
+        }
+    }
+    async withLock(key, fn) {
+        const filePath = this.getFilePath(key);
+        // Ensure the file exists so we can lock it (proper-lockfile requires the file/dir to exist)
+        if (!fs.existsSync(filePath)) {
+            await fs.promises.writeFile(filePath, "null", 'utf-8');
+        }
+        const release = await lockfile.lock(filePath, {
+            retries: { retries: 100, minTimeout: 10, maxTimeout: 100 },
+            realpath: false
+        });
+        try {
+            return await fn();
+        }
+        finally {
+            await release();
+        }
+    }
+}
+exports.LocalFileStorage = LocalFileStorage;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-diaries/core",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "The lightweight, framework-agnostic memory layer for edge AI agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -35,17 +35,16 @@
   "homepage": "https://github.com/swapwarick/agent-diaries-core#readme",
   "license": "MIT",
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.11",
-    "@types/dotenv": "^6.1.1",
-    "@types/node": "^20.0.0",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.9.1",
     "@types/proper-lockfile": "^4.1.4",
-    "@vitest/coverage-v8": "^4.1.5",
+    "@vitest/coverage-v8": "^4.1.7",
     "dotenv": "^17.4.2",
-    "ioredis": "^5.5.0",
-    "mongodb": "^6.13.0",
+    "ioredis": "^5.10.1",
+    "mongodb": "^7.2.0",
     "ts-node": "^10.9.2",
-    "typescript": "^5.0.0",
-    "vitest": "^4.1.5"
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.7"
   },
   "dependencies": {
     "proper-lockfile": "^4.1.2"
@@ -66,4 +65,4 @@
       "optional": true
     }
   }
-}
+}