agentic-api 1.0.1

package/dist/src/scrapper.js

@@ -0,0 +1,294 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractCaptcha = extractCaptcha;
+exports.callGPTForParsingPDF = callGPTForParsingPDF;
+exports.callGPTForParsingHTML = callGPTForParsingHTML;
+exports.html2markdown = html2markdown;
+exports.pdf2markdown = pdf2markdown;
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const jsdom_1 = require("jsdom");
+const readability_1 = require("@mozilla/readability");
+const princing_openai_1 = require("./princing.openai");
+const prompts_1 = require("./prompts");
+const utils_1 = require("./utils");
+// Promisify exec for easier async/await usage
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
+const randomFile = (ext = '') => {
+    const random = () => Math.random() * 1000 | 0;
+    return `temp-${random()}-${random()}${ext}`;
+};
+async function extractCaptcha(base64Image, openai) {
+    const content = [
+        { type: 'text', text: "Extrais uniquement le nombre" },
+        { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64Image}` } },
+    ];
+    // Cost per captcha $0.0000696
+    const model = "gpt-4o-mini";
+    const response = await openai.chat.completions.create({
+        model,
+        messages: [{ role: "user", content }],
+        max_completion_tokens: 50,
+    });
+    const cost = (0, princing_openai_1.calculateCost)(model, response.usage);
+    // Récupérer la réponse markdown
+    const number = response.choices[0].message.content;
+    return { number, cost };
+}
+/**
+ * Calls GPT to parse a PDF file and convert it to markdown format.
+ *
+ * @param {string} inputfile - The name of the PDF file being processed
+ * @param {any} pdfData - The extracted content from the PDF file
+ * @param {any} openai - The OpenAI client instance
+ * @param {any[]} links - Optional array of links extracted from the PDF to be integrated into the markdown
+ * @returns {Promise<{markdown: string, cost: number}>} - The parsed markdown content and the cost of the API call
+ */
+async function callGPTForParsingPDF(inputfile, pdfData, openai, links = []) {
+    // Convertir le contenu en chaîne de caractères (attention à la taille potentielle !)
+    const pdfDataAsString = JSON.stringify(pdfData, null, 2);
+    // Format: YYYY-MM-DD
+    const today = new Date().toISOString().substring(0, 10);
+    const linkLabel = ' une liste de liens que tu dois placer dans le document. Si le libellé d\'un lien correspond à un texte existant dans le document, intègre le lien directement dans le texte. Sinon, place ces liens sous forme de liste à puces dans une section annexe (# Références externes) à la fin du document.';
+    const linkPrefix = linkLabel + links.reduce((acc, link) => {
+        return acc + `- [${link.text}](${link.href})\n`;
+    }, '');
+    // Créer le prompt pour décrire la tâche au LLM
+    const messages = [
+        { role: "system",
+            content: prompts_1.textToMarkdownPrompt },
+        { role: "user",
+            content: `Nous sommes le ${today}, voici le nom du document "${inputfile}", et ${linkPrefix}` },
+        { role: "user",
+            content: `Voici le document, pour commencer, tu DOIS extrais uniquement la structure  (titres et sous-titres) selon les instructions et sans les contenus détaillés:\n\n${pdfDataAsString}` }
+    ];
+    // WARNING: o3-mini is buggy with "Marche à suivre nouveau bail.pdf"
+    const model = "o3-mini"; // "gpt-4o-mini";
+    let response = await openai.chat.completions.create({
+        model: model,
+        messages,
+        max_completion_tokens: 15192,
+        reasoning_effort: "low",
+        stop: "|<|james|>|"
+    });
+    // response_format: { type: "json_object" }
+    let cost = (0, princing_openai_1.calculateCost)(model, response.usage);
+    console.log(`Markdown 💰 cost: ${cost}`);
+    messages.push({
+        role: "user",
+        content: `Maintenant génère le contenu Markdown détaillé et exhaustif correspondant à chaque section avec les liens intégrés correctement.`
+    });
+    response = await openai.chat.completions.create({
+        model: model,
+        messages,
+        max_completion_tokens: 15192,
+        reasoning_effort: "low",
+        stop: "|<|james|>|"
+    });
+    // Récupérer la réponse markdown
+    let markdown = response.choices[0].message.content;
+    cost += (0, princing_openai_1.calculateCost)(model, response.usage);
+    //
+    // add a regex to extract the markdown content between <thinking></thinking> tags
+    let markdownWithoutThinking = markdown.replace(/<thinking>[\s\S]*?<\/thinking>/g, '');
+    // const hragPrompt = await extractHRAGSections(markdownWithoutThinking, openai);
+    // messages.push({
+    //   role: "user",
+    //   content: hragPrompt
+    // });
+    // response = await openai.chat.completions.create({
+    //   model: model,
+    //   messages,
+    //   max_completion_tokens: 15192,
+    //   reasoning_effort:"low",
+    //   stop:"|<|james|>|"
+    // });
+    // // Récupérer la réponse markdown
+    // markdown = response.choices[0].message.content;
+    // cost += calculateCost(model, response.usage);
+    // //
+    // // add a regex to extract the markdown content between <thinking></thinking> tags
+    // markdownWithoutThinking = markdown.replace(/<thinking>[\s\S]*?<\/thinking>/g, '');
+    return { markdown: markdownWithoutThinking, cost };
+}
+/**
+ * Extracts hyperlinks from a PDF file by converting it to HTML and parsing the links.
+ *
+ * @param {string} pdfPath - The file path to the PDF document to extract links from
+ * @param {string} output - The directory output where temporary files will be created
+ * @returns {Promise<Array<{text: string, href: string}>>} - A promise that resolves to an array of link objects,
+ *                                                          each containing the link text and href attributes
+ */
+async function extractLinksFromPDF(pdfPath, output) {
+    const tempOut = path_1.default.join(output, `${randomFile()}`);
+    try {
+        // STEP 2: Convert the cleaned PDF to XML using pdftohtml.
+        // La commande génère un fichier XML à partir du PDF nettoyé.
+        const pdftohtmlCommand = `pdftohtml -s -nodrm -c "${pdfPath}" "${tempOut}"`;
+        await execAsync(pdftohtmlCommand);
+        const html = fs_1.default.readFileSync(tempOut + '-html.html', "utf8");
+        const dom = new jsdom_1.JSDOM(html);
+        const links = Array.from(dom.window.document.querySelectorAll('a')).map(link => ({
+            text: link.textContent?.trim() || link.href,
+            href: link.href
+        }));
+        process.stdout.write("Extracting links: " + links.length + " ");
+        return links;
+    }
+    catch (error) {
+        console.error('❌ Error extracting links from PDF:', error);
+        return [];
+    }
+    finally {
+        if (fs_1.default.existsSync(tempOut + '-html.html')) {
+            fs_1.default.unlinkSync(tempOut + '-html.html');
+            // Clean up any PNG files that might have been generated
+            const pngFiles = fs_1.default.readdirSync(path_1.default.dirname(tempOut))
+                .filter(file => file.startsWith(path_1.default.basename(tempOut)) && file.endsWith('.png'));
+            for (const pngFile of pngFiles) {
+                const pngPath = path_1.default.join(path_1.default.dirname(tempOut), pngFile);
+                if (fs_1.default.existsSync(pngPath))
+                    fs_1.default.unlinkSync(pngPath);
+            }
+        }
+    }
+}
+/**
+ * Convertit un document HTML en markdown en appelant le modèle GPT (ex: gpt-4o-mini)
+ * pour analyser et reformater le document.
+ *
+ * @param {any} htmlData - Le document HTML à transformer en markdown.
+ * @param {any} openai - L'instance OpenAI configurée pour appeler l'API.
+ * @param {boolean} simple - Si true, utilise un prompt simplifié.
+ * @returns {Promise<Object>} - Le contenu markdown structuré créé par le modèle LLM.
+ */
+async function callGPTForParsingHTML(html, openai) {
+    const htmlDataAsString = html;
+    // Créer le prompt pour décrire la tâche au LLM
+    const messages = [
+        {
+            role: "system",
+            content: prompts_1.htmlToMarkdownPrompt
+        },
+        {
+            role: "user",
+            content: `Voici le document HTML à transformer en markdown : \n${htmlDataAsString}`
+        }
+    ];
+    // Appel à l'API ChatCompletion
+    const response = await openai.chat.completions.create({
+        model: "gpt-4o-mini",
+        messages,
+        max_completion_tokens: 15192,
+        temperature: 0,
+        frequency_penalty: 0.0,
+        presence_penalty: 0.0,
+        stop: "|<|james|>|"
+    });
+    const cost = (0, princing_openai_1.calculateCost)("gpt-4o-mini", response.usage);
+    console.log(`Markdown 💰 cost: ${cost}`);
+    // Récupérer la réponse markdown
+    const markdown = response.choices[0].message.content;
+    if (!markdown)
+        throw new Error("No markdown found");
+    // Extraction et suppression des balises <thinking></thinking>
+    const markdownWithoutThinking = markdown.replace(/<thinking>[\s\S]*?<\/thinking>/g, '');
+    return markdownWithoutThinking;
+}
+function cleanHTML(html) {
+    const dom = new jsdom_1.JSDOM(html);
+    // Instancie Readability avec le document
+    const reader = new readability_1.Readability(dom.window.document);
+    const article = reader.parse();
+    return article?.content || '';
+}
+/**
+ * Parses an HTML file and converts it to markdown using GPT.
+ *
+ * @param {string} output - The directory path where the output markdown file will be saved.
+ * @param {string} file - The path to the HTML file to be parsed.
+ * @param {string} service - The service name used as part of the output filename output.
+ * @param {any} openai - The OpenAI instance to use for the GPT API calls.
+ * @returns {Promise<{markdown: string, cost: number}>} - The generated markdown content and the cost of the GPT API call.
+ */
+async function html2markdown(output, file, service, openai) {
+    const filename = (0, utils_1.slug)(path_1.default.basename(file, path_1.default.extname(file)));
+    const htmlraw = fs_1.default.readFileSync(file, "utf8");
+    const html = cleanHTML(htmlraw);
+    const outputfile = html.indexOf('Please sign in') > -1 ? 'unauthorized-' : (service.toLocaleLowerCase() + '-');
+    const { markdown, cost } = await callGPTForParsingPDF(file, html, openai);
+    fs_1.default.writeFileSync(path_1.default.join(output, `${outputfile + filename}.md`), markdown);
+    return { markdown, cost };
+}
+/**
+ * Parse un PDF en effectuant :
+ * 1. Le nettoyage du PDF avec Ghostscript.
+ * 2. Sa conversion en XML via pdftohtml.
+ * 3. (Optionnellement) Le passage du contenu converti au modèle GPT pour analyser la structure.
+ *
+ * @param {string} file - Chemin vers le fichier PDF à analyser.
+ * @param {any} openai - L'instance configurée pour appeler l'API OpenAI.
+ * @returns {Promise<any>} - Le markdown structuré (issue de GPT) ou tout autre traitement souhaité.
+ */
+async function pdf2markdown(output, file, service, openai) {
+    const filename = (0, utils_1.slug)(path_1.default.basename(file, path_1.default.extname(file)));
+    // Créez des noms de fichiers temporaires pour le PDF nettoyé et le XML généré.
+    const tempPdf = path_1.default.join(output, `cleaned-${randomFile()}.pdf`);
+    const tempOut = path_1.default.join(output, `${filename}.txt`);
+    //
+    // generated output path
+    const outputPath = path_1.default.join(output, `${service.toLocaleLowerCase()}-${filename}.md`);
+    try {
+        //
+        // replace pdftotext with python script PyMuPDF
+        // Ca ne marche pas mieux que pdftotext
+        // const { stdout } = await execFileAsync("python3", ["./bin/extract_text_with_links.py", file]);
+        // const { text, links } = JSON.parse(stdout);
+        await execAsync(`pdftotext -nodiag -nopgbrk "${file}" "${outputPath}"`);
+        const links = await extractLinksFromPDF(file, output);
+        const text = fs_1.default.readFileSync(outputPath, "utf8");
+        const { markdown, cost } = await callGPTForParsingPDF(file, text, openai, links);
+        fs_1.default.writeFileSync(outputPath, markdown);
+        return { markdown, cost };
+        /**
+    
+        // STEP 1: Clean the PDF using Ghostscript.
+        // La commande utilise -o pour spécifier le fichier de sortie et applique des options de mise en forme.
+        const gsCommand = `gs -o "${tempPdf}" -sDEVICE=pdfwrite -dFIXEDMEDIA -dDEVICEWIDTHPOINTS=595 -dDEVICEHEIGHTPOINTS=752 -dORIGINY=100 -dFILTERVECTOR "${file}"`;
+        console.log("Executing Ghostscript command:", gsCommand);
+        await execAsync(gsCommand);
+    
+        // STEP 2: Convert the cleaned PDF to XML using pdftohtml.
+        // La commande génère un fichier XML à partir du PDF nettoyé.
+        const pdftohtmlCommand = `pdftohtml -xml -nodrm -s -c "${tempPdf}" "${tempOut}"`;
+        console.log("Executing pdftohtml command:", pdftohtmlCommand);
+        await execAsync(pdftohtmlCommand);
+    
+        // Lecture du contenu XML généré
+        const xmlContent = fs.readFileSync(tempOut, "utf8");
+    
+        // (OPTIONNEL) STEP 3: Utilisez GPT pour analyser la structure du contenu XML.
+        // Vous pouvez adapter le traitement en fonction du contenu généré par pdftohtml.
+        const {markdown,cost} = await callGPTForParsingPDF(file, xmlContent, openai);
+        fs.writeFileSync(outputPath, markdown);
+        return {markdown,cost}; */
+    }
+    catch (error) {
+        console.error("Error during PDF parsing: ", error);
+        throw error;
+    }
+    finally {
+        if (fs_1.default.existsSync(tempPdf)) {
+            fs_1.default.unlinkSync(tempPdf);
+        }
+        if (fs_1.default.existsSync(tempOut)) {
+            fs_1.default.unlinkSync(tempOut);
+        }
+    }
+}

package/dist/src/types.d.ts

@@ -0,0 +1,126 @@
+import { Writable } from "stream";
+export type SessionStatus = "DISCONNECTED" | "CONNECTING" | "CONNECTED";
+export interface ToolParameterProperty {
+    type: string;
+    description?: string;
+    enum?: string[];
+    pattern?: string;
+    properties?: Record<string, ToolParameterProperty>;
+    required?: string[];
+    additionalProperties?: boolean;
+    items?: ToolParameterProperty;
+}
+export interface ToolParameters {
+    type: string;
+    properties: Record<string, ToolParameterProperty>;
+    required?: string[];
+    additionalProperties?: boolean;
+}
+export interface Tool {
+    type: "function";
+    function: {
+        name: string;
+        description: string;
+        parameters: ToolParameters;
+        strict: boolean;
+    };
+}
+export interface AgentModelMapping {
+    [key: string]: string;
+}
+export interface AgentModel {
+    model: string;
+    temperature: null;
+    maxTokens: number;
+    topP?: number;
+    parallel_tool_calls: boolean;
+    frequencyPenalty?: number;
+    presencePenalty?: number;
+    stop?: string[] | string;
+    tool_choice?: "auto" | "none" | "required";
+}
+export interface AgentConfig {
+    name: string;
+    model?: AgentModel;
+    human?: boolean;
+    publicDescription: string;
+    instructions: string;
+    maxSteps?: number;
+    tools: Tool[];
+    toolLogic?: Record<string, (args: any, session?: AgenticContext) => Promise<any> | any>;
+    downstreamAgents?: AgentConfig[] | {
+        name: string;
+        publicDescription: string;
+        human?: boolean;
+    }[];
+    ready?: boolean;
+}
+export interface UserNano {
+    id: string;
+    isAnonymous?: boolean;
+    role: string;
+}
+export declare abstract class AgenticMemory {
+    abstract get currentAgent(): AgentConfig;
+    abstract set currentAgent(agent: AgentConfig);
+    abstract agents: AgentConfig[];
+    abstract messages: any[];
+    abstract usage: Usage;
+    abstract state?: string;
+}
+export interface AgenticMemorySession {
+    state: string;
+    user: UserNano;
+    currentAgent: string;
+    messages: any[];
+    usage: Usage;
+}
+export interface AgenticContext {
+    memory?: AgenticMemorySession;
+    user: UserNano;
+    [key: string]: any;
+}
+export declare class AgenticMemoryManager extends AgenticMemory {
+    #private;
+    constructor(session: {
+        memory?: AgenticMemorySession;
+        user?: UserNano;
+    }, initialAgent: string, agents: AgentConfig[]);
+    static createOrLoad(session: {
+        memory?: AgenticMemorySession;
+        user: UserNano;
+    }, agents: any[], initialAgent: string): AgenticMemoryManager;
+    get session(): AgenticMemorySession;
+    get agents(): AgentConfig[];
+    get state(): string;
+    set state(state: string);
+    get currentAgent(): AgentConfig;
+    set currentAgent(agent: AgentConfig);
+    get usage(): Usage;
+    get messages(): any[];
+    set messages(messages: any[]);
+    set usage(usage: Usage);
+}
+export type AllAgentConfigsType = Record<string, AgentConfig[]>;
+export interface Usage {
+    prompt: number;
+    completion: number;
+    total: number;
+    cost: number;
+}
+export interface Feedback {
+    agent: string;
+    stdout: Writable;
+    description?: string;
+    usage: Usage;
+    state: string;
+}
+export interface TranscriptItem {
+    id: string;
+    type: "MESSAGE" | "BREADCRUMB";
+    role?: "user" | "assistant";
+    title?: string;
+    content?: string;
+    timestamp: string;
+    status: "IN_PROGRESS" | "DONE";
+}

package/dist/src/types.js

@@ -0,0 +1,93 @@
+"use strict";
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _AgenticMemoryManager_memory, _AgenticMemoryManager_initialAgent, _AgenticMemoryManager_agents;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgenticMemoryManager = exports.AgenticMemory = void 0;
+class AgenticMemory {
+}
+exports.AgenticMemory = AgenticMemory;
+class AgenticMemoryManager extends AgenticMemory {
+    constructor(session, initialAgent, agents) {
+        super();
+        _AgenticMemoryManager_memory.set(this, void 0);
+        _AgenticMemoryManager_initialAgent.set(this, void 0);
+        _AgenticMemoryManager_agents.set(this, void 0);
+        __classPrivateFieldSet(this, _AgenticMemoryManager_initialAgent, initialAgent, "f");
+        __classPrivateFieldSet(this, _AgenticMemoryManager_agents, agents, "f");
+        //
+        // create sessionmemory if not exists
+        if (!session.memory) {
+            if (!session.user || !session.user.id) {
+                throw new Error('User ID is required');
+            }
+            session.memory = __classPrivateFieldSet(this, _AgenticMemoryManager_memory, {
+                state: '',
+                user: session.user,
+                messages: [],
+                currentAgent: initialAgent,
+                usage: { prompt: 0, completion: 0, total: 0, cost: 0 }
+            }, "f");
+        }
+        __classPrivateFieldSet(this, _AgenticMemoryManager_memory, session.memory, "f");
+        // this.#memory.state = session.memory.state,
+        // this.#memory.messages = session.memory.messages,
+        // this.#memory.currentAgent = session.memory.currentAgent,  
+        // this.#memory.usage = session.memory.usage    
+    }
+    //
+    // control and bind agent memory for one session
+    static createOrLoad(session, agents, initialAgent) {
+        return new AgenticMemoryManager(session, initialAgent, agents);
+    }
+    //
+    // get serializable session memory
+    get session() {
+        return {
+            state: __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").state,
+            user: __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").user,
+            messages: __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").messages,
+            currentAgent: __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").currentAgent,
+            usage: __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").usage,
+        };
+    }
+    get agents() {
+        return __classPrivateFieldGet(this, _AgenticMemoryManager_agents, "f");
+    }
+    get state() {
+        return __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").state;
+    }
+    set state(state) {
+        __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").state = state;
+    }
+    get currentAgent() {
+        const name = __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").currentAgent || __classPrivateFieldGet(this, _AgenticMemoryManager_initialAgent, "f");
+        return __classPrivateFieldGet(this, _AgenticMemoryManager_agents, "f").find((a) => a.name == name);
+    }
+    set currentAgent(agent) {
+        __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").currentAgent = agent.name;
+    }
+    get usage() {
+        return __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").usage;
+    }
+    get messages() {
+        return __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").messages;
+    }
+    set messages(messages) {
+        __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").messages = messages;
+    }
+    set usage(usage) {
+        __classPrivateFieldGet(this, _AgenticMemoryManager_memory, "f").usage = usage;
+    }
+}
+exports.AgenticMemoryManager = AgenticMemoryManager;
+_AgenticMemoryManager_memory = new WeakMap(), _AgenticMemoryManager_initialAgent = new WeakMap(), _AgenticMemoryManager_agents = new WeakMap();

package/dist/src/utils.d.ts

@@ -0,0 +1,27 @@
+import { ParsedFunctionToolCall } from "openai/resources/beta/chat/completions";
+import { AgentConfig, AgenticContext, AgenticMemory } from "./types";
+import OpenAI from "openai";
+export declare const openaiInstance: (openai?: OpenAI) => any;
+export declare const slug: (text: string) => string;
+/**
+ * This defines and adds "transferAgents" tool dynamically based on the specified downstreamAgents on each agent.
+ */
+export declare function injectTransferTools(agentDefs: AgentConfig[]): AgentConfig[];
+export declare function handleTransferCall(memory: AgenticMemory, functionCallParams: ParsedFunctionToolCall, session: AgenticContext): Promise<any>;
+/**
+ * Calls the system review agent to analyze a prompt
+ *
+ * This function sends a prompt to the system review agent and returns its analysis.
+ * The system review agent evaluates prompts based on various criteria like clarity,
+ * identity, scope, decision logic, etc.
+ *
+ * @param {string} prompt - The prompt text to be analyzed by the system review agent
+ * @param {Object} params - Parameters for the API call
+ * @param {Object} params.openai - The OpenAI client instance
+ * @param {Object} params.defaultOptions - Default options to be merged with agent model options
+ * @returns {Promise<string>} The analysis response from the system review agent
+ */
+export declare function callForSystemReview(prompt: string, params: any): Promise<{
+    content: any;
+    cost: number;
+}>;