goblin-agents 0.1.1

package/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.{js,jsx,json}]
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+max_line_length = 80

package/.zou-flow

@@ -0,0 +1,2 @@
+[update-version]
+	package-json = package.json

package/aiAgent.js

@@ -0,0 +1,4 @@
+const {Elf} = require('xcraft-core-goblin');
+const {AiAgent, AiAgentLogic} = require('goblin-agents/lib/llm/aiAgent.js');
+
+exports.xcraftCommands = Elf.birth(AiAgent, AiAgentLogic);

package/config.js

@@ -0,0 +1,34 @@
+'use strict';
+
+module.exports = [
+  {
+    type: 'input',
+    name: 'version',
+    message: 'Agents version',
+    default: 2,
+  },
+  {
+    type: 'input',
+    name: 'defaultProfile',
+    message: 'Agents default profile',
+    default: null,
+  },
+  {
+    type: 'input',
+    name: 'defaultSettings',
+    message: 'Agents default setting',
+    default: null,
+  },
+  {
+    type: 'input',
+    name: 'profiles',
+    message: 'Agents profiles for overriding settings',
+    default: {},
+  },
+  {
+    type: 'input',
+    name: 'settings',
+    message: 'Settings by name',
+    default: {},
+  },
+];

package/lib/llm/aiAgent.js

@@ -0,0 +1,677 @@
+// @ts-check
+const {Elf} = require('xcraft-core-goblin');
+const {id, MetaShape} = require('xcraft-core-goblin/lib/types.js');
+const {
+  string,
+  array,
+  enumeration,
+  option,
+  object,
+  boolean,
+  number,
+  record,
+} = require('xcraft-core-stones');
+const LLMProvider = require('./providers.js');
+const {OCR_SYSTEM_PROMPT, getPDFImages} = require('./utils.js');
+
+class MessageShape {
+  role = enumeration('assistant', 'user', 'tool');
+  content = string;
+  images = option(array(string));
+  tool_calls = option(array(object));
+  tool_call_id = option(string);
+  name = option(string);
+}
+
+class OptionsShape {
+  // Active NUMA (Non-Uniform Memory Access) pour optimiser la gestion mémoire sur les architectures multi-sockets.
+  numa = option(boolean);
+  // Détermine la taille maximale du contexte utilisé par le modèle (en tokens).
+  num_ctx = option(number);
+  // Nombre de requêtes traitées simultanément par lots (batch processing).
+  num_batch = option(number);
+  // Nombre de GPU à utiliser pour l'inférence (si pris en charge).
+  num_gpu = option(number);
+  // Identifiant du GPU principal utilisé pour l'inférence.
+  main_gpu = option(number);
+  // Active le mode faible consommation de mémoire vidéo (utile pour les GPU avec peu de VRAM).
+  low_vram = option(boolean);
+  // Utilise des valeurs en float16 pour les clés et valeurs des caches d'attention (optimisation mémoire et performance).
+  f16_kv = option(boolean);
+  // Active la sortie des logits pour chaque token généré (utile pour le débogage et certaines analyses).
+  logits_all = option(boolean);
+  // Charge uniquement le vocabulaire du modèle, sans les poids (utile pour l'analyse des tokens).
+  vocab_only = option(boolean);
+  // Utilise mmap pour mapper les poids du modèle en mémoire au lieu de les charger en RAM (économie de mémoire).
+  use_mmap = option(boolean);
+  // Utilise mlock pour empêcher la mise en cache des poids en swap (utile pour les systèmes à faible latence).
+  use_mlock = option(boolean);
+  // Ne génère pas de texte, mais renvoie uniquement les embeddings des tokens (pour des cas d'usage spécifiques comme la recherche vectorielle).
+  embedding_only = option(boolean);
+  // Nombre de threads utilisés pour l'inférence sur CPU.
+  num_thread = option(number);
+  // Nombre de tokens à conserver avant le début d'une nouvelle génération.
+  num_keep = option(number);
+  // Graine pour la génération aléatoire, permet de reproduire les mêmes résultats.
+  seed = option(number);
+  // Nombre de tokens à prédire lors de la génération.
+  num_predict = option(number);
+  // Filtre les tokens les moins probables en ne conservant que les `top_k` plus probables.
+  top_k = option(number);
+  // Filtre les tokens en fonction d'une probabilité cumulative `top_p` (nucleus sampling).
+  top_p = option(number);
+  // Facteur de filtrage basé sur la fonction TFS-z pour la diversité dans la génération.
+  tfs_z = option(number);
+  // Probabilité typique utilisée pour équilibrer la diversité et la cohérence de la génération.
+  typical_p = option(number);
+  // Nombre de tokens récents pris en compte pour éviter la répétition dans la génération.
+  repeat_last_n = option(number);
+  // Contrôle la température du modèle (plus haut = génération plus créative, plus bas = plus déterministe).
+  temperature = option(number);
+  // Applique une pénalité sur la répétition de mots/tokens déjà générés.
+  repeat_penalty = option(number);
+  // Applique une pénalité sur les tokens déjà générés pour encourager la diversité.
+  presence_penalty = option(number);
+  // Applique une pénalité en fonction de la fréquence d’apparition des tokens (évite les répétitions excessives).
+  frequency_penalty = option(number);
+  // Active le mode Mirostat, un algorithme adaptatif pour le contrôle de la perplexité.
+  mirostat = option(number);
+  // Paramètre `tau` du mode Mirostat, ajuste le niveau de perplexité cible.
+  mirostat_tau = option(number);
+  // Paramètre `eta` du mode Mirostat, contrôle la vitesse d'adaptation.
+  mirostat_eta = option(number);
+  // Applique une pénalisation spécifique aux nouvelles lignes pour limiter leur apparition.
+  penalize_newline = option(boolean);
+  // Liste de tokens ou mots qui stoppent immédiatement la génération lorsqu’ils sont rencontrés.
+  stop = option(array(string));
+}
+
+class AiAgentShape {
+  id = id('aiAgent');
+  version = number;
+  name = string; //agent name
+  role = string; //agent role
+  prompt = string; //default prompt
+  provider = enumeration('ollama', 'open-ai');
+  model = string;
+  host = string;
+  headers = record(string, string);
+  messages = record(string, array(MessageShape)); //for keeping chat history by context
+  options = option(OptionsShape); //for tweeking
+  format = option(object);
+  tools = option(array(object));
+  toolServiceId = option(string);
+  usability = enumeration('disabled', 'experimental', 'stable', 'deprecated');
+  meta = MetaShape;
+}
+
+class AiAgentState extends Elf.Sculpt(AiAgentShape) {}
+
+/** @type {Partial<AiAgentState>} */
+const defaultValues = {
+  name: 'agent',
+  role: 'assistant',
+  prompt: '',
+  provider: 'ollama',
+  model: 'mistral-small',
+  options: {},
+  host: 'http://127.0.0.1:11434',
+  headers: {},
+  format: null,
+  tools: null,
+  usability: 'disabled',
+};
+
+class AiAgentLogic extends Elf.Archetype {
+  static db = 'agents';
+  state = new AiAgentState({
+    id: undefined,
+    version: 1,
+    name: defaultValues.name,
+    role: defaultValues.role,
+    prompt: defaultValues.prompt,
+    provider: defaultValues.provider,
+    model: defaultValues.model,
+    messages: {},
+    options: defaultValues.options,
+    host: defaultValues.host,
+    headers: defaultValues.headers,
+    format: defaultValues.format,
+    tools: defaultValues.tools,
+    usability: defaultValues.usability,
+    meta: {
+      status: 'published',
+    },
+  });
+
+  create(id, agentState) {
+    const {state} = this;
+    state.id = id;
+    this.patch(agentState);
+  }
+
+  patch(patch) {
+    const {state} = this;
+    for (const [k, v] of Object.entries(defaultValues)) {
+      state[k] = patch[k] ? patch[k] : v;
+    }
+  }
+
+  upgrade(version, patch) {
+    const {state} = this;
+    this.patch(patch);
+    state.version = version;
+  }
+
+  reset(contextId) {
+    const {state} = this;
+    state.messages[contextId] = [];
+  }
+
+  set(contextId, messages) {
+    const {state} = this;
+    state.messages[contextId] = messages;
+  }
+
+  chat(contextId, role, content) {
+    const {state} = this;
+    if (!state.messages[contextId]) {
+      state.messages[contextId] = [];
+    }
+    state.messages[contextId].push({role, content});
+  }
+
+  ask(contextId, question, answer) {
+    const {state} = this;
+    if (!state.messages[contextId]) {
+      state.messages[contextId] = [];
+    }
+    state.messages[contextId].push({role: 'user', content: question});
+    state.messages[contextId].push({role: 'assistant', content: answer});
+  }
+
+  addAssistantMessage(contextId, answer) {
+    const {state} = this;
+    if (!state.messages[contextId]) {
+      state.messages[contextId] = [];
+    }
+    state.messages[contextId].push({role: 'assistant', content: answer});
+  }
+
+  callTools(contextId, toolCalls) {
+    const {state} = this;
+    if (!state.messages[contextId]) {
+      state.messages[contextId] = [];
+    }
+    state.messages[contextId].push({
+      role: 'assistant',
+      tool_calls: toolCalls,
+      content: '',
+    });
+  }
+
+  addToolMessage(contextId, toolCallId, toolName, answer) {
+    const {state} = this;
+    if (!state.messages[contextId]) {
+      state.messages[contextId] = [];
+    }
+    state.messages[contextId].push({
+      role: 'tool',
+      tool_call_id: toolCallId,
+      name: toolName,
+      content: answer,
+    });
+  }
+
+  change(path, newValue) {
+    const {state} = this;
+    state._state.set(path, newValue);
+  }
+
+  trash() {
+    const {state} = this;
+    state.meta.status = 'trashed';
+  }
+}
+
+class AiAgent extends Elf {
+  logic = Elf.getLogic(AiAgentLogic);
+  state = new AiAgentState();
+
+  /**
+   * @param {string} id
+   * @param {string} desktopId
+   * @param {Partial<AiAgentState>} [agentState]
+   * @returns {Promise<this>}
+   */
+  async create(id, desktopId, agentState) {
+    this.logic.create(id, agentState);
+    await this.persist();
+    return this;
+  }
+
+  /**
+   * Restore the last known 'persist' when the changes provided by the client
+   * are using a different version.
+   */
+  async beforePersistOnServer() {
+    const {state} = this;
+    const {version} = require('xcraft-core-etc')().load('goblin-agents');
+    if (state.version === version) {
+      return; /* It's fine, we keep the changes */
+    }
+
+    /* Restore the last 'persist' */
+    const prev = await this.cryo.getState(
+      AiAgentLogic.db,
+      this.id,
+      AiAgentShape
+    );
+    this.logic.patch(prev);
+  }
+
+  /**
+   * @param {Partial<AiAgentState>} [agentState]
+   */
+  async patch(agentState) {
+    this.logic.patch(agentState);
+    await this.persist();
+  }
+
+  /**
+   * @param {number} version
+   * @param {Partial<AiAgentState>} [agentState]
+   */
+  async upgrade(version, agentState) {
+    const {state} = this;
+    if (state.version >= version) {
+      return;
+    }
+    this.logic.upgrade(version, agentState);
+    await this.persist();
+  }
+
+  async gen(prompt) {
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+
+    let options;
+    if (this.state.options) {
+      options = this.state.options.toJS();
+    }
+    return await provider.gen({
+      model: this.state.model,
+      prompt,
+      system: this.state.prompt,
+      format: this.state.format?.toJS(),
+      options,
+      stream: false,
+    });
+  }
+
+  async reset(contextId, save = false) {
+    if (!contextId) {
+      for (const contextId of Object.keys(this.state.messages)) {
+        await this.reset(contextId);
+      }
+    } else {
+      this.logic.reset(contextId);
+    }
+    if (save) {
+      await this.persist();
+    }
+  }
+
+  async readPDFpages(pdfPath) {
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+
+    const images = await getPDFImages(pdfPath);
+    const descriptions = [];
+    for (const image of images) {
+      const messages = [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: OCR_SYSTEM_PROMPT,
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                url: image,
+              },
+            },
+          ],
+        },
+      ];
+      const result = await provider.chat({
+        model: this.state.model,
+        messages,
+      });
+      descriptions.push(result.message.content);
+    }
+    return descriptions;
+  }
+
+  async embed(rawText) {
+    if (!rawText) {
+      return null;
+    }
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    return await provider.embed({
+      model: this.state.model,
+      input: rawText,
+    });
+  }
+
+  async embedInBatch(rawTexts = []) {
+    if (rawTexts.length === 0) {
+      return [];
+    }
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    return await provider.embedInBatch({
+      model: this.state.model,
+      input: rawTexts,
+    });
+  }
+
+  async set(contextId, messages) {
+    this.logic.set(contextId, messages);
+  }
+
+  async chat(contextId, question, userDesktopId, sessionId = null) {
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    const contextMessages = this.state.messages[contextId] || [];
+    const messages = [
+      {role: 'system', content: this.state.prompt},
+      ...Array.from(contextMessages).map((m) => m._state.toJS()), //history
+    ];
+    //add new question
+    messages.push({role: 'user', content: question});
+
+    let options;
+    if (this.state.options) {
+      const pState = this.state._state.toJS();
+      options = pState?.options;
+    }
+
+    if (!sessionId) {
+      this.logic.chat(contextId, 'user', question);
+      sessionId = Elf.uuid();
+      this.log.dbg(`🐈 Chat session started : ${sessionId}`);
+    }
+
+    const result = await provider.chat({
+      model: this.state.model,
+      messages,
+      options,
+      format: this.state.format?.toJS(),
+      tools: this.state.tools?.toJS(),
+    });
+
+    if (this.state.toolServiceId && this.state.tools) {
+      const used = await this.callTools(
+        contextId,
+        result.message,
+        userDesktopId,
+        sessionId
+      );
+      if (used) {
+        return await this.chat(contextId, question, userDesktopId, sessionId);
+      }
+    }
+
+    //keep agent response
+    this.logic.chat(contextId, 'assistant', result.message.content);
+
+    return result.message.content;
+  }
+
+  async callTools(contextId, message, userDesktopId, sessionId) {
+    const {tool_calls} = message;
+    if (!tool_calls || !userDesktopId) {
+      return false;
+    }
+    this.logic.callTools(contextId, tool_calls);
+    for (const tool of tool_calls) {
+      const {
+        id,
+        function: {name},
+      } = tool;
+      let result = '';
+      try {
+        const payload = {
+          id: this.state.toolServiceId,
+          desktopId: userDesktopId,
+          ...JSON.parse(tool.function.arguments),
+        };
+        const toolNamespace = this.state.toolServiceId?.split('@', 1)[0];
+        const toolCmd = `${toolNamespace}.${name}`;
+        this.log.dbg(
+          `🔧 Tool call: ${toolCmd}(${JSON.stringify(
+            payload
+          )}) sid: ${sessionId}`
+        );
+        result = await this.quest.cmd(toolCmd, payload);
+        await this.addToolMessage(contextId, id, name, result);
+      } catch (err) {
+        this.log.dbg(`🔧 Tool error : ${err} sid: ${sessionId}`);
+      }
+    }
+    return true;
+  }
+
+  async addToolMessage(contextId, toolId, toolName, answer) {
+    this.logic.addToolMessage(contextId, toolId, toolName, answer);
+  }
+
+  async ask(contextId, question, questionId) {
+    if (this.state.provider !== 'ollama') {
+      throw new Error('Streaming API not impl.');
+    }
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    const feedId = await this.newQuestFeed();
+
+    const contextMessages = this.state.messages[contextId] || [];
+    const messages = [
+      {role: 'system', content: this.state.prompt},
+      ...Array.from(contextMessages).map((m) => m._state.toJS()), //history
+    ];
+    //add new question
+    messages.push({role: 'user', content: question});
+
+    let options;
+    if (this.state.options) {
+      const pState = this.state._state.toJS();
+      options = pState?.options;
+    }
+    const response = await provider.chat({
+      model: this.state.model,
+      messages,
+      options,
+      stream: true,
+    });
+
+    let agentResponse = '';
+    for await (const part of response) {
+      agentResponse += part.message.content;
+      this.quest.evt(`<${questionId}>`, {part: part.message.content});
+    }
+
+    //keep agent response
+    this.logic.ask(contextId, question, agentResponse);
+  }
+
+  async resumeExchange(resumePrompt, contextId) {
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    const feedId = await this.newQuestFeed();
+    const contextMessages = this.state.messages[contextId] || [];
+
+    let options;
+    if (this.state.options) {
+      options = this.state.options.toJS();
+    }
+    const prompt = Array.from(contextMessages)
+      .map((m) => m._state.toJS())
+      .filter((m) => m.role !== 'system')
+      .map((m) => `${m.role}: ${m.content}`)
+      .join('\n');
+
+    return await provider.gen({
+      model: this.state.model,
+      prompt,
+      system: resumePrompt,
+      options,
+      stream: false,
+    });
+  }
+
+  async react(reactPrompt, contextId, question) {
+    const Instance = LLMProvider.get(this.state.provider);
+    const provider = new Instance(this.state.host, this.state.headers);
+    const feedId = await this.newQuestFeed();
+    const contextMessages = this.state.messages[contextId] || [];
+    const messages = [
+      {role: 'system', content: reactPrompt},
+      ...Array.from(contextMessages).map((m) => m._state.toJS()), //history
+      {role: 'user', content: question},
+    ];
+    this.log.dbg('ReAct Question:', question);
+
+    let options;
+    if (this.state.options) {
+      options = this.state.options.toJS();
+    }
+
+    const answer = await provider.chat({
+      model: this.state.model,
+      messages,
+      options,
+      format: {
+        type: 'object',
+        properties: {
+          observation: {type: 'string'},
+          thought: {type: 'string'},
+          action: {type: 'string'},
+          agentId: {type: 'string'},
+          result: {type: 'string'},
+        },
+        required: ['observation', 'thought', 'action', 'agentId', 'result'],
+      },
+    });
+    if (!answer.message.content) {
+      await this.addAssistantMessage(
+        contextId,
+        'Génération stoppée. Impossible de répondre.'
+      );
+      return;
+    }
+    const {agentId, action, result, observation} = answer.message.content;
+    this.log.dbg('ReAct Observation:', observation);
+    this.log.dbg('ReAct Action:', action);
+    // Ajoute l'observation
+    await this.addAssistantMessage(contextId, observation);
+    // Si un agent est nécessaire, appel de l'agent externe
+    if (agentId) {
+      const result = await this.callAgent(
+        contextId,
+        agentId,
+        `${observation} -> ${action}`,
+        feedId
+      );
+      return {agentId, action, result, observation};
+    }
+
+    return {agentId, action, result, observation};
+  }
+
+  async callAgent(contextId, agentId, action, feedId) {
+    this.log.dbg(`🥸 calling "${agentId}" with action: "${action}"`);
+    const exist = await this.cryo.isPersisted(AiAgentLogic.db, agentId);
+    if (!exist) {
+      this.log.dbg(`🥸 "${agentId}" is not available, stopping"`);
+      await this.addAssistantMessage(contextId, 'Agent non disponible.');
+      return null;
+    }
+    const toolAgent = await new AiAgent(this).create(agentId, feedId);
+    const agentResponse = await toolAgent.chat(contextId, action);
+    if (typeof agentResponse !== 'string') {
+      await this.addAssistantMessage(
+        contextId,
+        '```json' + JSON.stringify(agentResponse, null, 2) + '```'
+      );
+    } else {
+      await this.addAssistantMessage(contextId, agentResponse);
+    }
+    return agentResponse;
+  }
+
+  async addAssistantMessage(contextId, message) {
+    this.logic.addAssistantMessage(contextId, message);
+  }
+
+  async save() {
+    await this.persist();
+  }
+
+  /**
+   * @template {keyof this['state']} K
+   * @param {K} path
+   * @param {this['state'][K]} newValue
+   */
+  async change(path, newValue) {
+    if (path === 'options.temperature') {
+      newValue = parseFloat(newValue);
+    }
+    if (path === 'options.num_ctx') {
+      newValue = parseInt(newValue);
+    }
+    this.logic.change(path, newValue);
+    await this.persist();
+  }
+
+  async getUserChatContextHistory(contextId, filterTool = true) {
+    const contextMessages = this.state.messages[contextId] || [];
+    return [
+      ...Array.from(contextMessages)
+        .map((m) => m._state.toJS())
+        .filter((m) => {
+          if (!filterTool) {
+            return m;
+          }
+          if (m.role !== 'tool' && !m.tool_calls) {
+            return m;
+          }
+        }),
+    ];
+  }
+
+  async getBaseSettings() {
+    const {provider, host, headers, model} = this.state.toJS();
+    return {provider, host, headers, model};
+  }
+
+  async getUsability() {
+    return this.state.usability;
+  }
+
+  async trash() {
+    this.logic.trash();
+    await this.persist();
+  }
+
+  delete() {}
+}
+
+module.exports = {
+  AiAgent,
+  AiAgentLogic,
+  AiAgentShape,
+};

package/lib/llm/providers.js

@@ -0,0 +1,198 @@
+const {Ollama} = require('ollama');
+const {RestAPI} = require('xcraft-core-utils');
+const {Semaphore} = require('xcraft-core-utils/lib/locks.js');
+
+class LLMProvider {
+  static get(name) {
+    switch (name) {
+      default:
+      case 'ollama':
+        return OllamaProvider;
+      case 'open-ai':
+        return OpenAIProvider;
+    }
+  }
+
+  constructor() {
+    if (new.target === LLMProvider) {
+      throw new Error('Cannot instantiate an abstract class.');
+    }
+  }
+
+  async chatStream() {
+    throw new Error('not impl.');
+  }
+
+  async chat() {
+    throw new Error('not impl.');
+  }
+
+  async gen() {
+    throw new Error('not impl.');
+  }
+
+  async embed() {
+    throw new Error('not impl.');
+  }
+}
+
+class OllamaProvider extends LLMProvider {
+  #semEmbed = new Semaphore(4);
+  #ollama;
+
+  constructor(host, headers) {
+    super();
+    this.#ollama = new Ollama({
+      host,
+      headers,
+    });
+  }
+
+  async chat(chatRequest) {
+    const {model, messages, options, format, tools, stream} = chatRequest;
+    const useFormat = !!format;
+    const chat = await this.#ollama.chat({
+      model,
+      messages,
+      options,
+      format,
+      tools,
+      stream,
+    });
+    if (useFormat) {
+      chat.message.content = JSON.parse(chat.message.content);
+      return chat;
+    } else {
+      return chat;
+    }
+  }
+
+  async gen(genRequest) {
+    const {model, system, prompt, format, options, stream} = genRequest;
+    const useFormat = !!format;
+    const gen = await this.#ollama.generate({
+      model,
+      prompt,
+      system,
+      format,
+      options,
+      stream,
+    });
+    if (useFormat) {
+      return JSON.parse(gen.response);
+    } else {
+      return gen.response;
+    }
+  }
+
+  async embed(embedRequest) {
+    try {
+      await this.#semEmbed.wait();
+      const {model, input} = embedRequest;
+      const {embeddings} = await this.#ollama.embed({
+        model,
+        input,
+      });
+      return embeddings[0];
+    } finally {
+      this.#semEmbed.signal();
+    }
+  }
+
+  async embedInBatch(embedRequest) {
+    const {model, input} = embedRequest;
+    const {embeddings} = await this.#ollama.embed({
+      model,
+      input,
+    });
+    return embeddings;
+  }
+}
+
+class OpenAIProvider extends LLMProvider {
+  #baseUrl;
+  #openAI;
+
+  constructor(host = 'https://openrouter.ai/api/v1', headers) {
+    super();
+    this.#baseUrl = host;
+    this.#openAI = new RestAPI(300000, headers);
+  }
+
+  async chat(chatRequest) {
+    if (chatRequest.options) {
+      chatRequest = {...chatRequest, ...chatRequest.options};
+      delete chatRequest.options;
+    }
+    if (!chatRequest.tools) {
+      delete chatRequest.tools;
+    }
+    if (!chatRequest.format) {
+      delete chatRequest.format;
+    }
+    const useFormat = chatRequest?.format !== undefined;
+    if (useFormat) {
+      chatRequest.response_format = {
+        type: 'json_schema',
+        json_schema: {
+          name: 'default',
+          schema: {
+            type: 'object',
+            properties: {
+              results: {...chatRequest.format},
+            },
+            required: ['results'],
+          },
+        },
+      };
+      delete chatRequest.format;
+    }
+    const query = `${this.#baseUrl}/chat/completions`;
+    const chat = await this.#openAI._post(query, chatRequest);
+    const {choices} = chat;
+    if (chat.error) {
+      const err = new Error(chat.error.message);
+      err.code = chat.error.code;
+      throw err;
+    }
+    if (useFormat) {
+      choices[0].message.content = JSON.parse(
+        choices[0].message.content
+      ).results;
+      return choices[0];
+    } else {
+      return choices[0];
+    }
+  }
+
+  async gen(genRequest) {
+    const {model, system, prompt, format, tools, options, stream} = genRequest;
+    const chatRequest = {
+      model,
+      messages: [
+        {role: 'system', content: system},
+        {role: 'user', content: prompt},
+      ],
+      options,
+      format,
+      tools,
+      stream,
+    };
+    const chat = await this.chat(chatRequest);
+    return chat.message.content;
+  }
+
+  async embed(embedRequest) {
+    const query = `${this.#baseUrl}/v1/embeddings`;
+    const {data} = await this.#openAI._post(query, embedRequest);
+    return data[0].embedding;
+  }
+
+  async embedInBatch(embedRequest) {
+    const query = `${this.#baseUrl}/v1/embeddings`;
+    const {data} = await this.#openAI._post(query, embedRequest);
+    return data;
+  }
+}
+
+module.exports = LLMProvider;

package/lib/llm/utils.js

@@ -0,0 +1,291 @@
+const HtmlEntities = require('html-entities');
+const sanitizeHtml = require('sanitize-html');
+const {RecursiveCharacterTextSplitter} = require('@langchain/textsplitters');
+
+const MAX_SEGMENT_LENGTH = 500;
+
+const OCR_SYSTEM_PROMPT = `Effectue une reconnaissance optique de caractères (OCR) sur cette image et extraits-en le texte lisible.
+Respecte la mise en page autant que possible.
+Si le texte contient des tableaux ou des colonnes, préserve leur structure.
+Ignore les artefacts visuels inutiles.
+Corrige les erreurs évidentes et restitue un texte propre et fidèle à l'original.
+Ta réponse contient uniquement le texte lisible.`;
+
+const getPDFImages = async (pdfPath) => {
+  const {pdfToPng} = require('pdf-to-png-converter');
+  // The function accepts PDF file path or a Buffer
+  const pages = await pdfToPng(pdfPath, {
+    disableFontFace: true, // When `false`, fonts will be rendered using a built-in font renderer that constructs the glyphs with primitive path commands. Default value is true.
+    useSystemFonts: true, // When `true`, fonts that aren't embedded in the PDF document will fallback to a system font. Default value is false.
+    enableXfa: true, // Render Xfa forms if any. Default value is false.
+    viewportScale: 0.8, // The desired scale of PNG viewport. Default value is 1.0 which means to display page on the existing canvas with 100% scale.
+    //outputFolder: 'output/folder', // Folder to write output PNG files. If not specified, PNG output will be available only as a Buffer content, without saving to a file.
+    //outputFileMaskFunc: (pageNumber) => `page_${pageNumber}.png`, // Output filename mask function. Example: (pageNumber) => `page_${pageNumber}.png`
+    //pdfFilePassword: password, // Password for encrypted PDF.
+    //pagesToProcess: [1, 3, 11], // Subset of pages to convert (first page = 1), other pages will be skipped if specified.
+    //strictPagesToProcess: false, // When `true`, will throw an error if specified page number in pagesToProcess is invalid, otherwise will skip invalid page. Default value is false.
+    verbosityLevel: 0, // Verbosity level. ERRORS: 0, WARNINGS: 1, INFOS: 5. Default value is 0.
+  });
+  const images = [];
+  for (const page of pages) {
+    const {content} = page;
+    const base64Image = content.toString('base64');
+    images.push(`data:image/png;base64,${base64Image}`);
+  }
+  return images;
+};
+
+const cleanHtmlContent = (html) => {
+  const decodedHtml = HtmlEntities.decode(html);
+  const textOnly = decodedHtml.replace(/<[^>]+>/g, '');
+  const cleanedText = textOnly.replace(/\s+/g, ' ').trim();
+  return cleanedText;
+};
+
+const simplifyHtml = (html) => {
+  return sanitizeHtml(HtmlEntities.decode(html));
+};
+
+const splitLongText = (text, maxLength, baseId) => {
+  let segments = [];
+  let words = text.split(/\s+/); // Découper en mots
+  let currentSegment = [];
+  let currentLength = 0;
+  let partIndex = 1;
+
+  words.forEach((word) => {
+    if (currentLength + word.length + 1 > maxLength) {
+      segments.push({
+        id: `${baseId}_part${partIndex}`,
+        content: currentSegment.join(' '),
+      });
+      currentSegment = [];
+      currentLength = 0;
+      partIndex++;
+    }
+    currentSegment.push(word);
+    currentLength += word.length + 1;
+  });
+
+  if (currentSegment.length > 0) {
+    segments.push({
+      id: `${baseId}_part${partIndex}`,
+      content: currentSegment.join(' '),
+    });
+  }
+
+  return segments;
+};
+
+const html2chunks = async (html, chunkSize = 1000, chunkOverlap = 0) => {
+  const textSplitter = RecursiveCharacterTextSplitter.fromLanguage('html', {
+    chunkSize,
+    chunkOverlap,
+    separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage('html'),
+  });
+  return await textSplitter.splitText(html);
+};
+
+const extractTitlesAndContent = (html) => {
+  const cleanHtml = simplifyHtml(html);
+
+  const titleRegex = /<h([1-6])>(.*?)<\/h\1>/g;
+  const paragraphRegex = /<p>(.*?)<\/p>/g;
+
+  let result = [];
+  let match;
+  let lastIndex = 0;
+  let segmentId = 1;
+
+  while ((match = titleRegex.exec(cleanHtml)) !== null) {
+    const titleText = match[2].trim();
+    const titleStartIndex = match.index;
+
+    const contentBeforeTitle = cleanHtml.slice(lastIndex, titleStartIndex);
+    if (contentBeforeTitle && contentBeforeTitle.length > 3) {
+      result.push(
+        ...splitLongText(
+          contentBeforeTitle,
+          MAX_SEGMENT_LENGTH,
+          `segment_${segmentId}`
+        )
+      );
+      segmentId++;
+    }
+
+    result.push({
+      id: `segment_${segmentId}`,
+      content: titleText,
+    });
+
+    lastIndex = titleRegex.lastIndex;
+    segmentId++;
+  }
+
+  while ((match = paragraphRegex.exec(cleanHtml)) !== null) {
+    const paragraphText = match[1].trim();
+    if (paragraphText.length > 3) {
+      if (paragraphText.length > MAX_SEGMENT_LENGTH) {
+        result.push(
+          ...splitLongText(
+            paragraphText,
+            MAX_SEGMENT_LENGTH,
+            `segment_${segmentId}`
+          )
+        );
+      } else {
+        result.push({
+          id: `segment_${segmentId}`,
+          content: paragraphText,
+        });
+      }
+    }
+
+    segmentId++;
+  }
+
+  const remainingContent = cleanHtml.slice(lastIndex).trim();
+  if (remainingContent && remainingContent.length > 3) {
+    result.push(
+      ...splitLongText(
+        remainingContent,
+        MAX_SEGMENT_LENGTH,
+        `segment_${segmentId}`
+      )
+    );
+  }
+
+  return result;
+};
+
+const estimateTokenCount = (text) => {
+  return Math.ceil(text.split(/\s+/).length * 1.33);
+};
+
+function sliceText(text, chunkSize) {
+  const chunks = [];
+  let currentChunk = '';
+
+  const sentences = text.match(/[^.!?]+[.!?]*/g) || [text]; // Découpe en phrases
+
+  for (const sentence of sentences) {
+    if ((currentChunk + sentence).length <= chunkSize) {
+      currentChunk += sentence;
+    } else {
+      if (currentChunk) chunks.push(currentChunk.trim());
+      currentChunk = sentence;
+    }
+  }
+
+  if (currentChunk) chunks.push(currentChunk.trim());
+
+  return chunks;
+}
+
+function groupSentences(text, maxSize) {
+  const chunks = [];
+  let currentChunk = '';
+
+  // Découpe le texte en phrases en gardant la ponctuation
+  const sentences = text.match(/[^.!?]+[.!?]*/g) || [text];
+
+  for (const sentence of sentences) {
+    if ((currentChunk + ' ' + sentence).trim().length > maxSize) {
+      // Si la phrase est trop longue, on découpe par mot
+      if (sentence.length > maxSize) {
+        const words = sentence.split(' ');
+        let tempChunk = '';
+
+        for (const word of words) {
+          if ((tempChunk + ' ' + word).trim().length > maxSize) {
+            chunks.push(tempChunk.trim());
+            tempChunk = word;
+          } else {
+            tempChunk += (tempChunk ? ' ' : '') + word;
+          }
+        }
+
+        if (tempChunk) chunks.push(tempChunk.trim());
+        currentChunk = ''; // Réinitialiser après un découpage forcé
+      } else {
+        // Ajouter le chunk en attente et commencer un nouveau
+        if (currentChunk) chunks.push(currentChunk.trim());
+        currentChunk = sentence.trim();
+      }
+    } else {
+      currentChunk += (currentChunk ? ' ' : '') + sentence.trim();
+    }
+  }
+
+  // Ajoute le dernier bloc s'il contient du texte
+  if (currentChunk) chunks.push(currentChunk.trim());
+
+  return chunks;
+}
+
+const buildPromptFromArticles = (articles, question, maxTokens) => {
+  let context = '[CONTEXT]\n\n';
+  let currentTokenCount = estimateTokenCount(context);
+
+  articles.forEach((article) => {
+    const {kind, title, link, locale} = article;
+    const articleContext = `# ${kind}: ${title} #\nLien: ${link}\n Langue: ${locale}\n`;
+
+    context += articleContext;
+
+    if (article.segments && article.segments.length > 0) {
+      context += '## Segments pertinents: ##\n';
+      article.segments
+        .sort((a, b) => b.score - a.score) // Trier par pertinence
+        .forEach((s, i) => {
+          let segmentContext = `  ${i + 1}. ${s.segment} (Pertinence: ${
+            s.score
+          })\n`;
+          currentTokenCount += estimateTokenCount(segmentContext);
+
+          if (currentTokenCount > maxTokens) {
+            return;
+          }
+
+          context += segmentContext;
+
+          // Ajouter du contexte avant et après le segment
+          const beforeContext = article.content.substring(
+            Math.max(0, s.start - 50),
+            s.start
+          );
+          const afterContext = article.content.substring(
+            s.end,
+            Math.min(article.content.length, s.end + 50)
+          );
+
+          const additionalContext = `### Contexte avant: ###\n${beforeContext}\n### Contexte après: ###\n${afterContext}\n`;
+          currentTokenCount += estimateTokenCount(additionalContext);
+
+          if (currentTokenCount > maxTokens) {
+            return;
+          }
+
+          context += additionalContext;
+        });
+    }
+  });
+
+  const questionContext = '[QUESTION]\n\n' + question;
+  context += questionContext;
+  return context.trim();
+};
+
+module.exports = {
+  extractTitlesAndContent,
+  html2chunks,
+  splitLongText,
+  cleanHtmlContent,
+  simplifyHtml,
+  estimateTokenCount,
+  OCR_SYSTEM_PROMPT,
+  getPDFImages,
+  sliceText,
+  groupSentences,
+  buildPromptFromArticles,
+};

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "goblin-agents",
+  "version": "0.1.1",
+  "description": "AI Agents framework",
+  "author": "Samuel Loup",
+  "license": "MIT",
+  "config": {
+    "xcraft": {
+      "commands": true
+    }
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "@langchain/core": "^0.3.42",
+    "@langchain/textsplitters": "^0.1.0",
+    "fs-extra": "^9.1.0",
+    "html-entities": "^2.4.0",
+    "ollama": "^0.5.10",
+    "pdf-to-png-converter": "^3.6.5",
+    "sanitize-html": "^2.12.1",
+    "xcraft-core-goblin": "^5.8.0",
+    "xcraft-core-utils": "^4.14.0"
+  },
+  "devDependencies": {
+    "xcraft-dev-prettier": "^2.0.0",
+    "xcraft-dev-rules": "^4.4.2"
+  },
+  "prettier": "xcraft-dev-prettier"
+}