@agent-diaries/core 0.1.2 → 0.1.4

package/README.md

@@ -4,6 +4,7 @@
 
   [![NPM Version](https://img.shields.io/npm/v/@agent-diaries/core?style=for-the-badge&logo=npm&color=CB3837)](https://www.npmjs.com/package/@agent-diaries/core)
   [![TypeScript](https://img.shields.io/badge/TypeScript-3178C6?style=for-the-badge&logo=typescript&logoColor=white)](https://www.typescriptlang.org/)
+  [![Cloud Tested](https://img.shields.io/badge/Tested-Cloud%20Ready-success?style=for-the-badge&logo=icloud&logoColor=white)](#-200-agent-real-world-cloud-benchmarks)
   [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge)](https://opensource.org/licenses/MIT)
 </div>
 
@@ -126,7 +127,7 @@ console.log(`   Actual Locks: ${successful}`); // Always exactly 1.
 ```
 
 ### The Results (Zero Race Conditions)
-> *Tested via WAN connection to an Upstash Serverless Redis instance*
+> *Tested via WAN connection to an Upstash Serverless Redis instance and a Free Tier MongoDB Atlas Cluster*
 
 ```text
 =================================
@@ -149,6 +150,27 @@ console.log(`   Actual Locks: ${successful}`); // Always exactly 1.
    Actual Written:   200
    Write Duration:   16267ms
    🟢 PASSED (Zero data corruption)
+
+=================================
+🌪️ INITIALIZING 200-AGENT SWARM: MongoDB Atlas (Cloud Free Tier)
+=================================
+[Test 1] The Herd Effect: 200 Agents competing for exactly ONE viral task...
+   Expected Locks: 1
+   Actual Locks:   1
+   Resolution Time: 7362ms
+   🟢 PASSED (199 race conditions prevented)
+
+[Test 2] Real World Distribution: 200 Agents processing 10 common data tasks...
+   Expected Locks: 10
+   Actual Locks:   10
+   Resolution Time: 5545ms
+   🟢 PASSED (190 duplicate LLM calls prevented)
+
+[Test 3] Extreme Write Contention: 200 Agents blasting state updates at the exact same time...
+   Expected Written: 200
+   Actual Written:   200
+   Write Duration:   9410ms
+   🟢 PASSED (Zero data corruption)
 ```
 
 **💡 Engineering Insight:** While SQL databases perform well on local network environments, relational connection poolers (like pgBouncer or Supavisor) completely buckle under the massive concurrent TCP bursts generated by serverless AI swarms. **Redis or MongoDB (via atomic upserts)** are strictly required for reliable lock management in high-concurrency serverless edge environments.
@@ -161,6 +183,7 @@ console.log(`   Actual Locks: ${successful}`); // Always exactly 1.
   Retrieves the exact string output/result from a previously completed task so your agent can instantly reuse it.
 - **`diary.filterNewTasks(tasks: T[]): Promise<T[]>`**
   Pass in an array of task objects. Returns only the tasks that the agent has *not* seen yet.
+  *⚠️ WARNING: This method returns a non-atomic snapshot. Always follow up with `claimTask()` on individual items before acting on them in a high-concurrency environment.*
 - **`diary.writeTaskResult(title: string, result: string): Promise<void>`**
   Saves the final result into the agent's memory bank after the agent finishes its work.
 

package/dist/adapters/mongo.d.ts

@@ -2,9 +2,11 @@ import { StorageAdapter } from '../storage';
 import { Collection } from 'mongodb';
 export declare class MongoStorage<T> implements StorageAdapter<T> {
     private collection;
+    private initialized;
     constructor(config: {
         collection: Collection;
     });
+    private ensureIndex;
     private hashString;
     get(key: string): Promise<T | null>;
     set(key: string, data: T): Promise<void>;

package/dist/adapters/mongo.js

@@ -7,9 +7,21 @@ exports.MongoStorage = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 class MongoStorage {
     collection;
+    initialized = false;
     constructor(config) {
         this.collection = config.collection;
     }
+    async ensureIndex() {
+        if (this.initialized)
+            return;
+        try {
+            await this.collection.createIndex({ lockedAt: 1 }, { expireAfterSeconds: 30, partialFilterExpression: { lockedAt: { $exists: true } } });
+        }
+        catch (e) {
+            console.warn('[MongoStorage] Failed to create TTL index:', e);
+        }
+        this.initialized = true;
+    }
     hashString(str) {
         return crypto_1.default.createHash('sha256').update(str).digest('hex');
     }
@@ -25,6 +37,7 @@ class MongoStorage {
         await this.collection.updateOne({ _id: hash }, { $set: { data: JSON.stringify(data) } }, { upsert: true });
     }
     async withLock(key, fn) {
+        await this.ensureIndex();
         const hash = this.hashString(key);
         const lockId = `lock:${hash}`;
         const acquireLock = async () => {
@@ -38,8 +51,14 @@ class MongoStorage {
                 throw error;
             }
         };
+        let attempt = 0;
         while (!(await acquireLock())) {
-            await new Promise(resolve => setTimeout(resolve, 50));
+            const backoff = Math.min(10 * Math.pow(2, attempt), 500);
+            const jitter = Math.random() * 50;
+            await new Promise(resolve => setTimeout(resolve, backoff + jitter));
+            attempt++;
+            if (attempt > 60)
+                throw new Error(`[MongoStorage] Lock timeout on key: ${key}`);
         }
         try {
             return await fn();

package/dist/adapters/redis.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RedisStorage = void 0;
+const crypto_1 = require("crypto");
 class RedisStorage {
     redis;
     prefix;
@@ -29,15 +30,20 @@ class RedisStorage {
     }
     async withLock(key, fn) {
         const lockKey = `${this.getKey(key)}:lock`;
-        const lockValue = Date.now().toString() + Math.random().toString();
+        const lockValue = (0, crypto_1.randomUUID)();
         const lockTtlMs = 10000; // 10 seconds max lock
         const acquireLock = async () => {
             const result = await this.redis.set(lockKey, lockValue, 'PX', lockTtlMs, 'NX');
             return result === 'OK';
         };
-        // Spin-lock until we acquire it
+        let attempt = 0;
         while (!(await acquireLock())) {
-            await new Promise((resolve) => setTimeout(resolve, 50));
+            const backoff = Math.min(10 * Math.pow(2, attempt), 500);
+            const jitter = Math.random() * 50;
+            await new Promise(resolve => setTimeout(resolve, backoff + jitter));
+            attempt++;
+            if (attempt > 60)
+                throw new Error(`[RedisStorage] Lock timeout on key: ${key}`);
         }
         try {
             return await fn();

package/dist/diary.d.ts

@@ -45,6 +45,9 @@ export declare class AgentDiary {
     getTaskResult(title: string): Promise<string | undefined>;
     /**
      * Filters out items that the agent has already processed.
+     *
+     * ⚠️ WARNING: filterNewTasks() is a non-atomic snapshot. Always follow it with claimTask()
+     * to atomically claim ownership. Never act on filterNewTasks() results directly without claiming them first.
      */
     filterNewTasks<T extends {
         title: string;

package/dist/diary.js

@@ -41,7 +41,8 @@ class AgentDiary {
         const signature = AgentDiary.normalizeSignature(title);
         return await this.storage.withLock(`diary_${this.agentId}`, async () => {
             const state = await this.readDiary();
-            if (state.seenSignatures.includes(signature)) {
+            const seenSet = new Set(state.seenSignatures);
+            if (seenSet.has(signature)) {
                 return false; // Task already exists
             }
             // Claim it immediately to prevent race conditions
@@ -65,7 +66,8 @@ class AgentDiary {
     async hasProcessedTask(title) {
         const signature = AgentDiary.normalizeSignature(title);
         const state = await this.readDiary();
-        return state.seenSignatures.includes(signature);
+        const seenSet = new Set(state.seenSignatures);
+        return seenSet.has(signature);
     }
     /**
      * Retrieves the stored result of a previously processed task, if available.
@@ -78,12 +80,16 @@ class AgentDiary {
     }
     /**
      * Filters out items that the agent has already processed.
+     *
+     * ⚠️ WARNING: filterNewTasks() is a non-atomic snapshot. Always follow it with claimTask()
+     * to atomically claim ownership. Never act on filterNewTasks() results directly without claiming them first.
      */
     async filterNewTasks(tasks) {
         const state = await this.readDiary();
+        const seenSet = new Set(state.seenSignatures);
         return tasks.filter(task => {
             const signature = AgentDiary.normalizeSignature(task.title);
-            return !state.seenSignatures.includes(signature);
+            return !seenSet.has(signature);
         });
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-diaries/core",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "The lightweight, framework-agnostic memory layer for edge AI agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -14,11 +14,15 @@
     "prepublishOnly": "npm run build"
   },
   "keywords": [
-    "ai",
-    "agent",
-    "memory",
-    "diary",
-    "local"
+    "ai-agent",
+    "agent-memory",
+    "deduplication",
+    "distributed-lock",
+    "redis",
+    "mongodb",
+    "serverless",
+    "idempotency",
+    "task-queue"
   ],
   "author": "swapwarick_n",
   "repository": {
@@ -31,6 +35,7 @@
   "homepage": "https://github.com/swapwarick/agent-diaries-core#readme",
   "license": "MIT",
   "devDependencies": {
+    "dotenv": "^17.4.2",
     "@types/dotenv": "^6.1.1",
     "@types/node": "^20.0.0",
     "@types/proper-lockfile": "^4.1.4",
@@ -40,9 +45,18 @@
     "vitest": "^4.1.5"
   },
   "dependencies": {
-    "dotenv": "^17.4.2",
-    "ioredis": "^5.10.1",
-    "mongodb": "^7.2.0",
     "proper-lockfile": "^4.1.2"
+  },
+  "peerDependencies": {
+    "ioredis": ">=5",
+    "mongodb": ">=6"
+  },
+  "peerDependenciesMeta": {
+    "ioredis": {
+      "optional": true
+    },
+    "mongodb": {
+      "optional": true
+    }
   }
 }