agentic-api 1.0.3 → 1.0.5

package/README.md

@@ -27,7 +27,8 @@ Super simple API for intelligent agent orchestration with automatic sequences an
 - Applications requiring specialized agent orchestration
 - Projects needing reliable content extraction
 
-[![](https://mermaid.ink/img/pako:eNpVkd1OwjAUx1-lOYmJJkDYhxvswoRB0AtvdCQmbsTU7bAt2dqltCogz2K89uV8BMs-AM9F199__3N6TruDmCcIHqwK_h5nVEiymEWM6Aikpsuw_iyvGm0S3gpEiWLZ8OwyjOD36-eb3KmSsgg6Y7Ou1WsqaJWRAMVbHuPLJEUm1yRsmTTcFjtEkguMZc4ZuX88qb4RBrTAc6NvhlO1lrxEQQJVVVw3efbXCh8Ul8cMZEnEmu00fFrMlx3V05F-_4ZM2glr8I1_ZP4jq0u-uCBzLmLdAS3ylGGphyEZF_mWM0mLxuQbpD84K-KbLVstWw3Puqptwuec5oUS-EmmbeOH00_q0Q890JdQ0jzRz7g7aBHITDcTgae3Ca6oKmQEEdtrK1WSBxsWgyeFwh4IrtIMvBUt1ppUlVCJs5zqZyuPakXZM-dll5KKw1Ftur5ZFFOumATPdN3aDN4OPsC7HpkDwzRcczw0DEtHDzbgucPBtWPajuXYY9twbGffg21dfTgYufb4PPZ_wB3E1Q?type=png)](https://mermaid.live/edit#pako:eNpVkd1OwjAUx1-lOYmJJkDYhxvswoRB0AtvdCQmbsTU7bAt2dqltCogz2K89uV8BMs-AM9F199__3N6TruDmCcIHqwK_h5nVEiymEWM6Aikpsuw_iyvGm0S3gpEiWLZ8OwyjOD36-eb3KmSsgg6Y7Ou1WsqaJWRAMVbHuPLJEUm1yRsmTTcFjtEkguMZc4ZuX88qb4RBrTAc6NvhlO1lrxEQQJVVVw3efbXCh8Ul8cMZEnEmu00fFrMlx3V05F-_4ZM2glr8I1_ZP4jq0u-uCBzLmLdAS3ylGGphyEZF_mWM0mLxuQbpD84K-KbLVstWw3Puqptwuec5oUS-EmmbeOH00_q0Q890JdQ0jzRz7g7aBHITDcTgae3Ca6oKmQEEdtrK1WSBxsWgyeFwh4IrtIMvBUt1ppUlVCJs5zqZyuPakXZM-dll5KKw1Ftur5ZFFOumATPdN3aDN4OPsC7HpkDwzRcczw0DEtHDzbgucPBtWPajuXYY9twbGffg21dfTgYufb4PPZ_wB3E1Q)
+
+[![](https://mermaid.ink/img/pako:eNpVkcluwjAQhl_FGgmJSoDIQoAcKhEQ7aGXNkiVmqDKTYYkUmJHxu5GeZaq575cH6EmC8sc7PnG_4xn7B1EPEZwYZPztyilQpLVImREmy81dYNqW1_VsVlwIxAlinXNi24Qwt_37w-5VQVlIbTCet2ql0TQMiU-itcswudZgkxuSdAwqbkpdrA4ExjJjDNy93CKekbg0xzPhZ4ZzNVW8gIF8VVZct3k2akV3CsujxnI4pDV7jx4XC3XLVXTkX7_msyaCSvwjAsyL8hqkzsdsuQi0h3QPEsYFnoYknKRfXImaV6LPIP0B2dFPLNhq2Gr5kVbtUn4WtIsVwK_yLxp_HD7KXrUQw_0IxQ0i_U37g6xEGSqmwnB1W6MG6pyGULI9lpKleT-B4vAlUJhDwRXSQruhuZbTaqMqcRFRvW3Fa2kpOyJ8yMm4nBTk60fFsWcKybBHVVScHfwrmFiDgzTGJvToWFY2nrwAe54OBg5pu1Yjj21Dcd29j34rGoPB5OxPT23_T90g8Qf?type=png)](https://mermaid.live/edit#pako:eNpVkcluwjAQhl_FGgmJSoDIQoAcKhEQ7aGXNkiVmqDKTYYkUmJHxu5GeZaq575cH6EmC8sc7PnG_4xn7B1EPEZwYZPztyilQpLVImREmy81dYNqW1_VsVlwIxAlinXNi24Qwt_37w-5VQVlIbTCet2ql0TQMiU-itcswudZgkxuSdAwqbkpdrA4ExjJjDNy93CKekbg0xzPhZ4ZzNVW8gIF8VVZct3k2akV3CsujxnI4pDV7jx4XC3XLVXTkX7_msyaCSvwjAsyL8hqkzsdsuQi0h3QPEsYFnoYknKRfXImaV6LPIP0B2dFPLNhq2Gr5kVbtUn4WtIsVwK_yLxp_HD7KXrUQw_0IxQ0i_U37g6xEGSqmwnB1W6MG6pyGULI9lpKleT-B4vAlUJhDwRXSQruhuZbTaqMqcRFRvW3Fa2kpOyJ8yMm4nBTk60fFsWcKybBHVVScHfwrmFiDgzTGJvToWFY2nrwAe54OBg5pu1Yjj21Dcd29j34rGoPB5OxPT23_T90g8Qf)
 
 ## 📦 Installation
 
@@ -150,6 +151,51 @@ Agent transfer is automatically managed with:
 - Conversation context preservation
 - Automatic system instruction updates
 
+## 💾 Pull-based Data Digestion
+
+This feature enables agents to process large documents मोहब्बत chunk by chunk. It's designed for scenarios where the entire document cannot fit into the agent's context window.
+
+- **Chunked Processing**: The `pullContentDigestor` tool allows an agent to request and process content in manageable chunks.
+- **Stateful Digestion**: The agent receives the current chunk and relevant instructions, including the previous processing results, to maintain context throughout the digestion of the entire document.
+- **Flexible Templates**: Different processing templates (e.g., "facts", "compress", "semantic", "minutes") can be applied to each chunk, guiding the agent on how to digest the information.
+- **EOF Handling**: The tool signals the end of the document with an "EOF" message, allowing the agent to finalize its processing.
+
+```typescript
+// Example of how an agent might use the pullContentDigestor tool
+
+// Agent receives a call to process a large document
+// Initial call to the tool
+const initialChunkData = await pullContentDigestor({
+  path: "path/to/large/document.pdf",
+  template: "facts", // or any other relevant template
+  position: 0 // Initial position
+});
+
+let currentChunk = initialChunkData.content;
+let nextPosition = initialChunkData.nextPosition;
+let accumulatedResults = ""; // Agent will accumulate results here
+
+while (!currentChunk.includes("EOF")) {
+  // Process the currentChunk based on instructions and template
+  // The agent would use its LLM capabilities here, 
+  // using the provided instructions, previous results, and the new chunk.
+  const processingResult = await processMyChunk(currentChunk, initialChunkData.instructions, accumulatedResults);
+  accumulatedResults += processingResult; // Append to overall result
+
+  // Request the next chunk
+  const nextChunkData = await pullContentDigestor({
+    path: "path/to/large/document.pdf",
+    template: "facts",
+    position: nextPosition
+  });
+  currentChunk = nextChunkData.content;
+  nextPosition = nextChunkData.nextPosition;
+}
+
+// Final processing of accumulatedResults
+const finalDigestedDocument = finalizeProcessing(accumulatedResults);
+```
+
 ## 🧪 Testing
 
 ```bash

package/dist/src/agents/digestor.d.ts

@@ -1,7 +1,17 @@
-import { AgenticContext, Tool } from "../types";
+import { AgenticCache, AgenticContext, Tool } from "../types";
 export declare const Digestor: {
-    createTools: () => {
-        pullContentDigestor: (args?: any, session?: AgenticContext) => Promise<void>;
+    createTools: (cache: AgenticCache, chunkSize: number, templates: {
+        [key: string]: string;
+    }) => {
+        pullContentDigestor: (args?: any, session?: AgenticContext) => Promise<{
+            content: string;
+            role?: undefined;
+            name?: undefined;
+        } | {
+            role: string;
+            name: string;
+            content: string;
+        }>;
         toolPullContentDigestor: Tool;
     };
 };

package/dist/src/agents/digestor.js

@@ -1,23 +1,24 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Digestor = void 0;
+const fs_1 = require("fs");
 const promptPullContentDigestor = `
-This tool allows you to load the content of a large file, chunk by chunk, for progressive processing.
+This tool allows you to load the content of a large content, chunk by chunk, for progressive processing.
 
 # Usage Instructions:
 - On each call, the tool returns the next chunk of the specified file, along with instructions for processing this content.
-- The "position" parameter indicates where to resume reading (e.g., the last 20 characters of the previous chunk).
-- When all content has been processed, the tool returns "EOF" to signal the end of the file.
+- The "position" parameter indicates where to resume reading.
+- When all content has been processed, the tool returns "\nEOF" to signal the end of the file.
 
 # Parameters:
-- path (str): Absolute or relative path of the file to read.
-- template (enum): Indicates the type of processing to apply to the chunk (e.g., "facts", "compress", "semantic", "minutes").
-- position (str): Allows you to continue reading from the last known position (e.g., last 20 characters of the previous chunk).
+- path (str): Absolute,relative, or inline content.
+- template (enum): Indicates the type of processing to apply to the chunk (e.g., "facts", "compress", "semantic", "minutes", etc).
+- position (str): Allows you to continue reading from the last known position.
 
 # Best Practices:
-- Use this tool to process files that are too large to be read in a single pass.
-- Never skip a chunk: process each piece in order.
-- Stop calling the tool as soon as "EOF" is returned.
+- Use this tool to process contents that are too large to be read in a single pass.
+- Never skip a chunk: process each piece in order and always use the last position parameter.
+- Stop calling the tool as soon as "EOF" (or "\nEOF") is returned.
 `;
 const toolPullContentDigestor = {
     type: "function",
@@ -30,16 +31,16 @@ const toolPullContentDigestor = {
             properties: {
                 path: {
                     type: "string",
-                    description: "Absolute or relative path of the file to read."
+                    description: "Absolute or relative path, or a unique identifier of the file to read (e.g., 'https:....html', 'file://...', 'https://.../file.pdf')."
                 },
                 template: {
                     type: "string",
                     description: "Type of processing to apply to the chunk (e.g., 'facts', 'compress', 'semantic', 'minutes').",
-                    enum: ["facts", "compress", "semantic", "minutes"]
+                    enum: ["facts", "compress", "semantic", "minutes", "custom"]
                 },
                 position: {
-                    type: "string",
-                    description: "Allows you to continue reading from the last 20 characters of the previous chunk."
+                    type: "number",
+                    description: "position of the last loaded chunk. (0 for the first call)"
                 }
             },
             required: ["path", "template", "position"],
@@ -47,11 +48,98 @@ const toolPullContentDigestor = {
         }
     }
 };
-const createTools = () => {
+/**
+ * Loads a content chunk from a file or URL.
+ *
+ * @param {string} path - The path to the content file or URL.
+ * @param {number} position - The position in the content to start loading from.
+ * @param {number} chunkSize - The size of each content chunk to load.
+ * @param {AgenticCache} cache - The cache instance used to store and retrieve content chunks.
+ * @param {string} userid - The user ID for caching purposes.
+ * @returns {Promise<Content>} A promise that resolves to the content chunk.
+ */
+const loadContentChunk = async (path, position, chunkSize, cache, userid) => {
+    //FIXME: multiple users access to the cache with different position!
+    const loader = async (path, cache, userid) => {
+        const cacheKey = `content:${path}`;
+        const value = await cache.get(cacheKey);
+        if (value) {
+            return value;
+        }
+        if ((0, fs_1.existsSync)(path)) {
+            const fileContent = (0, fs_1.readFileSync)(path, "utf8");
+            const value = { content: fileContent, position: { [userid]: 0 }, key: cacheKey };
+            await cache.set(cacheKey, value);
+            return value;
+        }
+        else if (path.startsWith("http://") || path.startsWith("https://")) {
+            const response = await fetch(path);
+            const fileContent = await response.text();
+            const value = { content: fileContent, position: { [userid]: 0 }, key: cacheKey };
+            await cache.set(cacheKey, value);
+            return value;
+        }
+        //
+        // if not a file or url, return the path as content
+        return { content: path, position: { [userid]: 0 }, key: cacheKey };
+    };
+    //
+    // load asset content
+    const value = await loader(path, cache, userid);
+    const hasNextLN = value.content.indexOf('\n', value.position[userid] + chunkSize - 2);
+    // Determine the start position for the next chunk.
+    // If there is no next newline, use the current position; otherwise, start after the next newline.
+    const startPos = value.position[userid];
+    const nextPos = (hasNextLN == -1) ? startPos + chunkSize : startPos + hasNextLN + 1;
+    value.position[userid] = startPos + nextPos;
+    await cache.set(value.key, value);
+    if (startPos >= value.content.length) {
+        return { content: 'EOF', position: nextPos };
+    }
+    if (startPos + chunkSize + 1 > value.content.length) {
+        return { content: value.content.slice(startPos) + "\nEOF", position: nextPos };
+    }
+    const endPos = Math.min(startPos + chunkSize, value.content.length);
+    return { content: value.content.slice(startPos, endPos), position: endPos };
+};
+/**
+ * Creates tools for chunked content loading and processing.
+ *
+ * @param {AgenticCache} cache - The cache instance used to store and retrieve content chunks.
+ * @param {number} chunkSize - The size of each content chunk to load.
+ * @param {{[key: string]: string}} templates - An object mapping template names to their instruction strings.
+ * @returns {{ pullContentDigestor: Function, toolPullContentDigestor: any }} An object containing the pullContentDigestor function and its tool definition.
+ */
+const createTools = (cache, chunkSize, templates) => {
     const pullContentDigestor = async (args, session) => {
         const { path, template, position } = args;
-        // const {memory} = session;
-        // const {content,title} = await parse(html, temp);
+        const userid = session?.user?.id;
+        const chunk = await loadContentChunk(path, position, chunkSize, cache, userid);
+        if (!chunk || !chunk.content) {
+            throw new Error("Load content chunk failed");
+        }
+        //
+        // check template and ask for a valid template name
+        if (!templates[template]) {
+            return { content: 'Missing template name, recall with a valid template name' };
+        }
+        //
+        // get the prompt for the template
+        const promptPullContentDigestorFollowUp = templates[template] || '';
+        // Construit le message formaté pour l'agent LLM
+        console.log("-- Digest DBG chunk", position, chunk);
+        const templateInstructions = !position ? `${promptPullContentDigestorFollowUp}\nTemplate: ${template}\nChunk:\n` : '';
+        const content = JSON.stringify({
+            instructions: `${templateInstructions}`,
+            chunk: `${chunk.content}`,
+            nextPosition: chunk.position,
+            eof: chunk.content.includes('EOF')
+        });
+        return {
+            role: "assistant",
+            name: "pullContentDigestor",
+            content
+        };
     };
     return { pullContentDigestor, toolPullContentDigestor };
 };

package/dist/src/agents/digestor.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/agents/digestor.test.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const execute_1 = require("../execute");
+const digestor_1 = require("./digestor");
+// Implémentation simple du cache pour les tests
+class TestCache {
+    constructor() {
+        this.cache = new Map();
+    }
+    async get(key) {
+        return this.cache.get(key) || null;
+    }
+    async set(key, value, ttl) {
+        this.cache.set(key, value);
+        return value;
+    }
+}
+describe("Digestor Agent", () => {
+    it("should merge two chunks of 50 characters", async () => {
+        // Création d'un contenu de test de 100 caractères
+        const testContent = "A".repeat(50) + "B".repeat(50);
+        // Configuration de l'agent avec une instruction simple de fusion
+        const agentConfig = {
+            name: "chunkMerger",
+            publicDescription: "Agent that merges chunks of text",
+            instructions: `You are a simple agent that merges chunks of text.
+When you receive a chunk, you should:
+1. If it's the first chunk, store it
+2. If it's the second chunk, merge it with the first one
+3. If you receive "EOF", return the final merged result`,
+            tools: [digestor_1.Digestor.createTools(new TestCache(), 50).toolPullContentDigestor]
+        };
+        // Mock du cache pour simuler le contenu
+        const cache = new TestCache();
+        await cache.set("content:test.txt", testContent);
+        // Exécution de l'agent
+        const result = await (0, execute_1.executeAgentSet)([agentConfig], {
+            messages: [],
+            cache,
+            user: { id: "test-user" }
+        });
+        // Vérification du résultat
+        expect(result.messages[result.messages.length - 1].content).toBe("A".repeat(50) + "B".repeat(50));
+    });
+});

package/dist/src/execute.js

@@ -58,7 +58,7 @@ function sendFeedback(params) {
         usage,
         state
     };
-    console.log('--- DBG feedback --:', description);
+    //console.log('--- DBG feedback --:',description);
     //
     // send agent state and description
     stdout.write(`\n<step>${JSON.stringify(feedback)}</step>\n`);

package/dist/src/types.d.ts

@@ -75,6 +75,10 @@ export interface AgenticMemorySession {
     messages: any[];
     usage: Usage;
 }
+export type AgenticCache = {
+    get: <T>(key: string) => Promise<T | null>;
+    set: <T>(key: string, value: T, ttl?: number) => Promise<T>;
+};
 export interface AgenticContext {
     memory?: AgenticMemorySession;
     user: UserNano;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentic-api",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "API pour l'orchestration d'agents intelligents avec séquences et escalades automatiques",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",