agentlang 0.9.9 → 0.9.11

package/src/runtime/agents/common.ts

@@ -1,7 +1,7 @@
 import { IfPattern, LiteralPattern } from '../../language/syntax.js';
 import { trimQuotes } from '../util.js';
 
-export const PlannerInstructions = `Agentlang is a very-high-level declarative language that makes it easy to define business applications as 'models'.
+export const PlannerDataModelInstructions = `Agentlang is a very-high-level declarative language that makes it easy to define business applications as 'models'.
 The model of a business application consists of entity definitions and workflows defined in "modules". 
 A module is be encoded in a syntax inspired by JavaScript and JSON. Example of a simple module follows:
 
@@ -29,6 +29,36 @@ record EmailMessage {
     body String
 }
 
+Entities in a module can be connected together in relationships. There are two types of relationships - 'contains' and 'between'.
+'Contains' relationship is for hierarchical data, as in a Library entity containing Books. 'Between' relationship is for graph-like data,
+like two Profiles in a social media app is connected as friends. A 'between' relationship can be one of the following three types - 'one_one' (one-to-one),
+'one_many' (one-to-many) and 'many_many' (many-to-many), which is the default.
+
+The following example shows how additional profile data for an employee could be defined as a new entity and attached to the Employee entity as a between-relationship:
+
+entity Profile {
+    id UUID @id @default(uuid()),
+    address String @optional,
+    photo URL @optional,
+    dateOfBirth DateTime @optional
+}
+
+relationship EmployeeProfile between (Erp/Employee, Erp/Profile) @one_one
+
+The '@one_one' annotation means exactly one Employee and Profile can be related to each other via 'EmployeeProfile'.
+
+As an example of 'contains' relaionships, consider modelling task-assignments for an Employee as folllows:
+
+entity TaskAssignment {
+    id UUID @id @default(uuid()),
+    description String,
+    assignmentDate DateTime @default(now())
+}
+
+relationship EmployeeTaskAssignment contains (Erp/Employee, Erp/TaskAssignment)
+`;
+
+export const PlannerWorkflowInstructions = `
 Another major construct in Agentlang is the 'workflow'. Workflows contains JSON "patterns" that perform CRUD operations on entities. 
 For example, here's is a workflow that creates a new instance of the Employee entity:
 
@@ -179,25 +209,7 @@ A pattern may execute asynchronously and its eventual result can be handled by p
 If you are instructed that a particular event will be called asynchronously, always provide the patterns that follows in its '@then' clause. You must add the 
 '@then' clause only if an event's documentation or instruction explicitly requires to do so.
 
-Entities in a module can be connected together in relationships. There are two types of relationships - 'contains' and 'between'.
-'Contains' relationship is for hierarchical data, as in a Library entity containing Books. 'Between' relationship is for graph-like data,
-like two Profiles in a social media app is connected as friends. A 'between' relationship can be one of the following three types - 'one_one' (one-to-one),
-'one_many' (one-to-many) and 'many_many' (many-to-many), which is the default.
-
-The following example shows how additional profile data for an employee could be defined as a new entity and attached to the Employee entity as a between-relationship:
-
-entity Profile {
-    id UUID @id @default(uuid()),
-    address String @optional,
-    photo URL @optional,
-    dateOfBirth DateTime @optional
-}
-
-relationship EmployeeProfile between (Erp/Employee, Erp/Profile) @one_one
-
-The '@one_one' annotation means exactly one Employee and Profile can be related to each other via 'EmployeeProfile'.
-
-Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
+Earlier we discussed teh concept of 'between' relationship. Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
 
 workflow CreateEmployee {
     {Erp/Employee {firstName CreateEmployee.firstName,
@@ -214,17 +226,7 @@ The following pattern can be user to query an Employee along with his Profile:
 {Erp/Employee {employeeId? "56392e13-0d9a-42f7-b556-0d7cd9468a24"},
  Erp/EmployeeProfile {Erp/Profile? {}}}
 
-As an example of 'contains' relaionships, consider modelling task-assignments for an Employee as folllows:
-
-entity TaskAssignment {
-    id UUID @id @default(uuid()),
-    description String,
-    assignmentDate DateTime @default(now())
-}
-
-relationship EmployeeTaskAssignment contains (Erp/Employee, Erp/TaskAssignment)
-
-The following workflow shows how to assign a new task to an Employee:
+We also discussed the concept of 'contains' relationship and showed an example of modelling employee-task-assignment using it. The following workflow shows how to assign a new task to an Employee:
 
 workflow AssignNewTask {
     {Erp/Employee {employeeId? AssignNewTask.employeeId},
@@ -264,6 +266,10 @@ As an example, if the user request is "send an email to employee 101 with this m
  {email {to emp.email, body "please call me as soon as possible"}}]
 
 You MUST separate each pattern in the array with a semi-colon (;)  and never use a comma (,) for this purpose.
+`;
+
+export const PlannerInstructions = `${PlannerDataModelInstructions}
+${PlannerWorkflowInstructions}
 
 Now consider the following module definition and generate appropriate patterns in response to the user instructions. You must return only valid patterns or workflows,
 no other descriptive text or comments are needed.

package/src/runtime/docs.ts

@@ -44,7 +44,7 @@ async function getPDFParse() {
   return PDFParse;
 }
 
-async function parsePdfBuffer(buffer: Uint8Array): Promise<string> {
+export async function parsePdfBuffer(buffer: Uint8Array): Promise<string> {
   try {
     const PDFParseClass = await getPDFParse();
     const parser = new PDFParseClass({
@@ -166,3 +166,19 @@ async function fetchTextFile(path: string): Promise<string> {
 registerDocFetcher('http', httpFetcher);
 registerDocFetcher('https', httpFetcher);
 registerDocFetcher('file', fetchFile);
+
+async function documentServiceFetcher(url: string): Promise<string | undefined> {
+  try {
+    const { documentFetcher } = await import('./services/documentFetcher.js');
+    const result = await documentFetcher.fetchDocument({
+      title: 'temp', // Title will be updated by handleDocEvent
+      url: url,
+    });
+    return result?.content;
+  } catch (error: any) {
+    logger.error(`Failed to fetch from document service ${url}: ${error.message}`);
+    return undefined;
+  }
+}
+
+registerDocFetcher('document-service', documentServiceFetcher);

package/src/runtime/interpreter.ts

@@ -87,6 +87,7 @@ import {
   AgentInstance,
   findAgentByName,
   normalizeGeneratedCode,
+  saveFlowStepResult,
 } from './modules/ai.js';
 import { logger } from './logger.js';
 import {
@@ -103,7 +104,7 @@ import {
   flushMonitoringData,
   triggerTimer,
 } from './modules/core.js';
-import { invokeModuleFn } from './jsmodules.js';
+import { getModuleDef, invokeModuleFn } from './jsmodules.js';
 import { invokeOpenApiEvent, isOpenApiEventInstance } from './openapi.js';
 import { fetchDoc } from './docs.js';
 import { FlowSpec, FlowStep, getAgentFlow } from './agents/flows.js';
@@ -111,6 +112,7 @@ import { isMonitoringEnabled } from './state.js';
 import { Monitor, MonitorEntry } from './monitor.js';
 import { detailedDiff } from 'deep-object-diff';
 import { callMcpTool, mcpClientNameFromToolEvent } from './mcpclient.js';
+import { isNodeEnv } from '../utils/runtime.js';
 
 export type Result = any;
 
@@ -441,6 +443,11 @@ export class Environment extends Instance {
     }
   }
 
+  softSuspend(): string {
+    this.suspensionId = this.preGeneratedSuspensionId;
+    return this.suspensionId;
+  }
+
   releaseSuspension(): Environment {
     this.suspensionId = undefined;
     this.preGeneratedSuspensionId = crypto.randomUUID();
@@ -1752,7 +1759,39 @@ export function isMcpEventInstance(inst: Instance): boolean {
 }
 
 async function handleDocEvent(inst: Instance, env: Environment): Promise<void> {
-  const s = await fetchDoc(inst.lookup('url'));
+  const url = inst.lookup('url');
+  if (typeof url === 'string' && url.startsWith('s3://')) {
+    if (!isNodeEnv) {
+      throw new Error('Document fetching is only available in Node.js environment');
+    }
+    const title = inst.lookup('title');
+    const retrievalConfig = inst.lookup('retrievalConfig');
+    const embeddingConfig = inst.lookup('embeddingConfig');
+    const { documentFetcher } = await import('./services/documentFetcher.js');
+    await documentFetcher.fetchDocument({
+      title,
+      url,
+      retrievalConfig,
+      embeddingConfig,
+    });
+    return;
+  }
+
+  if (typeof url === 'string' && url.startsWith('document-service://')) {
+    const title = inst.lookup('title');
+    const retrievalConfig = inst.lookup('retrievalConfig');
+    const embeddingConfig = inst.lookup('embeddingConfig');
+    const { documentFetcher } = await import('./services/documentFetcher.js');
+    await documentFetcher.fetchDocument({
+      title,
+      url,
+      retrievalConfig,
+      embeddingConfig,
+    });
+    return;
+  }
+
+  const s = await fetchDoc(url);
   if (s) {
     const title = inst.lookup('title');
     const doc = makeInstance(
@@ -1998,7 +2037,7 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
             isWf = true;
           }
           if (isWf) {
-            const wf = await parseWorkflow(rs);
+            const wf = await parseWorkflow(normalizeGeneratedCode(rs));
             if (stmtsExec) {
               await stmtsExec(wf.statements, env);
             } else {
@@ -2010,7 +2049,9 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
               const r = await stmtsExec([stmt], env);
               env.setLastResult(r);
             } else {
-              env.setLastResult(await parseAndEvaluateStatement(rs, undefined, env));
+              env.setLastResult(
+                await parseAndEvaluateStatement(normalizeGeneratedCode(rs), undefined, env)
+              );
             }
           }
           agent.maybeAddScratchData(env);
@@ -2139,7 +2180,7 @@ export async function restartFlow(
   const rootAgent: AgentInstance = await findAgentByName(agentName, env);
   const flow = getAgentFlow(agentName, rootAgent.moduleName);
   if (flow) {
-    const newCtx = `${ctx}\n${step} --> ${userData}\n`;
+    const newCtx = `${ctx}\nRestart the flow at ${step} using the following user-input as additional guidance:\n${userData}\n`;
     env.setScratchPad(JSON.parse(spad));
     await iterateOnFlow(flow, rootAgent, newCtx, env);
   }
@@ -2167,7 +2208,8 @@ async function iterateOnFlow(
   let needAgentProcessing = preprocResult.needAgentProcessing;
   let context = initContext;
   let stepc = 0;
-  const iterId = crypto.randomUUID();
+  const chatId = env.getActiveEventInstance()?.lookup('chatId');
+  const iterId = chatId || crypto.randomUUID();
   console.debug(`Starting iteration ${iterId} on flow: ${flow}`);
   const executedSteps = new Set<string>();
   const monitoringEnabled = isMonitoringEnabled();
@@ -2215,6 +2257,12 @@ async function iterateOnFlow(
         `\n----> Completed execution of step ${step}, iteration id ${iterId} with result:\n${rs}`
       );
       context = `${context}\n${step} --> ${rs}\n`;
+      if (chatId) {
+        const suspEnv = new Environment(env.name, env);
+        suspEnv.softSuspend();
+        await saveFlowSuspension(rootAgent, context, step, suspEnv);
+        await saveFlowStepResult(chatId, step, rs, suspEnv.getSuspensionId(), env);
+      }
       if (isfxc) {
         preprocResult = await preprocessStep(rs, rootModuleName, env);
       } else {
@@ -2462,10 +2510,19 @@ async function followReference(env: Environment, s: string): Promise<Result> {
   for (let i = 0; i < refs.length; ++i) {
     const r: string = refs[i];
     const v: Result | undefined = await getRef(r, src, env);
-    if (v === undefined) return EmptyResult;
+    if (v === undefined || v === null) {
+      result = EmptyResult;
+      break;
+    }
     result = v;
     src = result;
   }
+  if (result === EmptyResult) {
+    result = getModuleDef(s);
+    if (result === undefined) {
+      result = EmptyResult;
+    }
+  }
   return result;
 }
 

package/src/runtime/jsmodules.ts

@@ -303,7 +303,7 @@ export async function invokeModuleFn(
   }
 }
 
-export function getModuleFn(fqFnName: string): Function | undefined {
+export function getModuleDef(fqFnName: string): any {
   const refs: string[] = splitRefs(fqFnName);
   const m = importedModules.get(refs[0]);
   if (m !== undefined) {
@@ -311,4 +311,6 @@ export function getModuleFn(fqFnName: string): Function | undefined {
   } else return undefined;
 }
 
+export const getModuleFn = getModuleDef;
+
 setModuleFnFetcher(getModuleFn);

package/src/runtime/loader.ts

@@ -819,7 +819,8 @@ async function addAgentDefinition(
       }
       const ov = v;
       if (apdef.value.id || apdef.value.ref || apdef.value.array) {
-        v = `"${v}"`;
+        if (!(apdef.name === 'instruction' || apdef.name === 'role' || apdef.name === 'llm'))
+          v = `"${v}"`;
       } else if (apdef.value.str) {
         v = `"${escapeSpecialChars(v)}"`;
       }
@@ -837,7 +838,7 @@ async function addAgentDefinition(
   // Create a copy of attrsStrs for the database operation
   const dbAttrsStrs = [...attrsStrs];
   // Only add llm to database attributes if we have one
-  if (llmName) {
+  if (llmName && createDefaultLLM) {
     dbAttrsStrs.push(`llm "${llmName}"`);
   }
 

package/src/runtime/module.ts

@@ -3731,6 +3731,10 @@ export function newInstanceAttributes(): InstanceAttributes {
   return new Map<string, any>();
 }
 
+export function objectToInstanceAttributes(obj: any): InstanceAttributes {
+  return new Map(Object.entries(obj));
+}
+
 const EmptyInstanceAttributes: InstanceAttributes = newInstanceAttributes();
 
 export class Instance {

package/src/runtime/modules/ai.ts

@@ -1,4 +1,5 @@
 import {
+  DefaultModuleName,
   escapeSpecialChars,
   isFqName,
   isString,
@@ -29,9 +30,10 @@ import {
   isModule,
   makeInstance,
   newInstanceAttributes,
+  objectToInstanceAttributes,
   Record,
-  Retry,
   resolveDocumentAliases,
+  Retry,
 } from '../module.js';
 import { provider } from '../agents/registry.js';
 import {
@@ -66,6 +68,8 @@ import { FlowStep } from '../agents/flows.js';
 import Handlebars from 'handlebars';
 import { Statement } from '../../language/generated/ast.js';
 import { isMonitoringEnabled, TtlCache } from '../state.js';
+import { isNodeEnv } from '../../utils/runtime.js';
+import { getFileSystem } from '../../utils/fs-utils.js';
 
 export const CoreAIModuleName = makeCoreModuleName('ai');
 export const AgentEntityName = 'Agent';
@@ -74,27 +78,6 @@ export const AgentLearnerType = 'learner';
 
 const AgentEvalType = 'eval';
 
-function buildEmbeddingConfig(): object {
-  const config: any = {
-    provider: process.env.AGENTLANG_EMBEDDING_PROVIDER || 'openai',
-    model: process.env.AGENTLANG_EMBEDDING_MODEL || 'text-embedding-3-small',
-    chunkSize: 1000,
-    chunkOverlap: 200,
-  };
-
-  if (process.env.AGENTLANG_EMBEDDING_CHUNKSIZE) {
-    config.chunkSize = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKSIZE, 1000);
-  }
-
-  if (process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP) {
-    config.chunkOverlap = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP, 200);
-  }
-
-  return config;
-}
-
-const embeddingConfig = JSON.stringify(buildEmbeddingConfig());
-
 export default `module ${CoreAIModuleName}
 
 import "./modules/ai.js" @as ai
@@ -139,12 +122,24 @@ workflow saveAgentChatSession {
 entity Document {
   title String @id,
   content String,
-  @meta {"fullTextSearch": "*", "embeddingConfig": ${embeddingConfig}}
+  embeddingConfig Map @optional,
+  @meta {"fullTextSearch": "*"}
 }
 
 event doc {
   title String,
-  url String
+  url String,
+  config Map @optional,
+  retrievalConfig Map @optional,
+  embeddingConfig Map @optional
+}
+
+workflow processDoc {
+  // Fetch document from URL and create Document entity
+  // Supports: local paths, http/https URLs, s3:// URLs
+  // S3 config can be provided via retrievalConfig or env vars
+  // Embedding config can customize chunking behavior
+  await ai.fetchAndCreateDocument(doc.title, doc.url, doc.retrievalConfig, doc.embeddingConfig)
 }
 
 entity Directive {
@@ -193,6 +188,27 @@ entity AgentLearningResult {
 workflow agentLearning {
     await ai.processAgentLearning(agentLearning.agentModuleName, agentLearning.agentName, agentLearning.instruction)
 }
+
+entity AgentFlowStep {
+    id UUID @id @default(uuid()),
+    chatId String @indexed,
+    step String @indexed,
+    result String,
+    suspensionId String
+}
+
+@public event restartFlow {
+  chatId String,
+  step String,
+  userInput String
+}
+
+workflow restartFlow {
+  await ai.loadFlowStep(restartFlow.chatId, restartFlow.step) @as fs;
+  if (fs) {
+    {${DefaultModuleName}/restartSuspension {id fs.suspensionId, data restartFlow.userInput}}
+  }
+}
 `;
 
 enum AgentCacheType {
@@ -622,7 +638,7 @@ export class AgentInstance {
 Only return a pure JSON object with no extra text, annotations etc.`;
     }
     const spad = env.getScratchPad();
-    if (spad !== undefined) {
+    if (spad !== undefined && Object.keys(spad).length > 0) {
       if (finalInstruction.indexOf('{{') > 0) {
         return AgentInstance.maybeRewriteTemplatePatterns(spad, finalInstruction, env);
       } else {
@@ -1282,17 +1298,38 @@ function processScenarioResponse(resp: string): string {
   return resp;
 }
 
+type ExtractedCode = {
+  language: string | null;
+  code: string;
+};
+
+export function extractFencedCodeBlocks(markdown: string): ExtractedCode[] {
+  const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+  const blocks: ExtractedCode[] = [];
+  let match;
+
+  while ((match = codeBlockRegex.exec(markdown)) !== null) {
+    blocks.push({
+      language: match[1] || null,
+      code: match[2],
+    });
+  }
+
+  return blocks;
+}
+
 export function normalizeGeneratedCode(code: string | undefined): string {
   if (code !== undefined) {
-    let s = code.trim();
-    if (s.startsWith('```')) {
-      const idx = s.indexOf('\n');
-      s = s.substring(idx).trimStart();
-    }
-    if (s.endsWith('```')) {
-      s = s.substring(0, s.length - 3);
+    const blocks = extractFencedCodeBlocks(code);
+    if (blocks.length > 0) {
+      return blocks
+        .map((v: ExtractedCode) => {
+          return v.code;
+        })
+        .join('\n\n');
+    } else {
+      return code;
     }
-    return s;
   } else {
     return '';
   }
@@ -1392,3 +1429,203 @@ export async function processAgentLearning(
   await parseAndInternAgentLearning(moduleName, agentName, learning, env);
   return { agentLearning: { result: learning } };
 }
+
+const LocalAgentgFlow = true;
+const LocalFlowStepsRootDirName = 'flows';
+const AgentFlowStep = 'AgentFlowStep';
+
+export async function saveFlowStepResultLocally(
+  chatId: string,
+  step: string,
+  result: string,
+  suspensionId: string
+): Promise<Instance | undefined> {
+  const fs = await getFileSystem();
+  const attrs = newInstanceAttributes()
+    .set('chatId', chatId)
+    .set('step', step)
+    .set('result', result)
+    .set('suspensionId', suspensionId);
+  const inst = makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
+  const rootDirName = LocalFlowStepsRootDirName;
+  if (!(await fs.exists(rootDirName))) {
+    await fs.mkdir(rootDirName);
+  }
+  const dirName = `${rootDirName}/${chatId}`;
+  if (!(await fs.exists(dirName))) {
+    await fs.mkdir(dirName);
+  }
+  const fileName = `${dirName}/${step}`;
+  if (await fs.exists(fileName)) {
+    await fs.unlink(fileName);
+  }
+  await fs.writeFile(fileName, JSON.stringify(inst.attributesAsObject(true)));
+  return inst;
+}
+
+export async function saveFlowStepResult(
+  chatId: string,
+  step: string,
+  result: string,
+  suspensionId: string,
+  env: Environment
+): Promise<Instance | undefined> {
+  if (LocalAgentgFlow) {
+    return await saveFlowStepResultLocally(chatId, step, result, suspensionId);
+  } else {
+    const t = `${CoreAIModuleName}/${AgentFlowStep}`;
+    try {
+      await parseAndEvaluateStatement(
+        `purge {${t} {chatId? "${chatId}", step? "${step}"}}`,
+        undefined,
+        env
+      );
+      const inst: Instance = await parseAndEvaluateStatement(
+        `{${t} {chatId "${chatId}", step "${step}", result "${escapeSpecialChars(result)}", suspensionId "${suspensionId}"}}`,
+        undefined,
+        env
+      );
+      if (isInstanceOfType(inst, t)) return inst;
+      else return undefined;
+    } catch (reason: any) {
+      logger.error(`failed to save flow result for step ${step} - ${reason}`);
+      return undefined;
+    }
+  }
+}
+
+export async function loadLocalFlowStepResults(chatId: string): Promise<Instance[]> {
+  const fs = await getFileSystem();
+  const dirName = `${LocalFlowStepsRootDirName}/${chatId}`;
+  if (await fs.exists(dirName)) {
+    const fileNames = await fs.readdir(dirName);
+    const result: Instance[] = new Array<Instance>();
+    for (let i = 0; i < fileNames.length; ++i) {
+      const fileName = fileNames[i];
+      const attrs = objectToInstanceAttributes(
+        JSON.parse(await fs.readFile(`${dirName}/${fileName}`))
+      );
+      result.push(makeInstance(CoreAIModuleName, AgentFlowStep, attrs));
+    }
+    return result;
+  } else {
+    return [];
+  }
+}
+
+export async function loadFlowStepResults(chatId: string): Promise<Instance[]> {
+  if (LocalAgentgFlow) {
+    return await loadLocalFlowStepResults(chatId);
+  } else {
+    try {
+      return await parseAndEvaluateStatement(`{${CoreAIModuleName}/${AgentFlowStep} {
+        chatId? "${chatId}"}}`);
+    } catch (reason: any) {
+      logger.error(`failed to query flow-steps for ${chatId} - ${reason}`);
+      return [];
+    }
+  }
+}
+
+export async function loadLocalFlowStep(
+  chatId: string,
+  step: string
+): Promise<Instance | undefined> {
+  const fs = await getFileSystem();
+  const dirName = `${LocalFlowStepsRootDirName}/${chatId}`;
+  if (await fs.exists(dirName)) {
+    const fileNames = await fs.readdir(dirName);
+    for (let i = 0; i < fileNames.length; ++i) {
+      const fileName = fileNames[i];
+      if (fileName === step) {
+        const attrs = objectToInstanceAttributes(
+          JSON.parse(await fs.readFile(`${dirName}/${fileName}`))
+        );
+        return makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
+      }
+    }
+    return undefined;
+  } else {
+    return undefined;
+  }
+}
+
+export async function loadFlowStep(chatId: string, step: string): Promise<Instance | undefined> {
+  if (LocalAgentgFlow) {
+    return await loadLocalFlowStep(chatId, step);
+  } else {
+    try {
+      const insts: Instance[] =
+        await parseAndEvaluateStatement(`{${CoreAIModuleName}/${AgentFlowStep} {
+        chatId? "${chatId}", step? "${step}"}}`);
+      if (insts.length > 0) {
+        return insts[0];
+      } else {
+        return undefined;
+      }
+    } catch (reason: any) {
+      logger.error(`failed to lookup step ${step} for flow ${chatId} - ${reason}`);
+      return undefined;
+    }
+  }
+}
+
+export async function fetchAndCreateDocument(
+  title: string,
+  url: string,
+  retrievalConfig: Map<string, any> | undefined,
+  embeddingConfig: Map<string, any> | undefined,
+  _env: Environment
+): Promise<any> {
+  if (!isNodeEnv) {
+    throw new Error('Document fetching is only available in Node.js environment');
+  }
+
+  const { documentFetcher } = await import('../services/documentFetcher.js');
+
+  const config: any = {
+    title,
+    url,
+  };
+
+  if (retrievalConfig) {
+    const provider = retrievalConfig.get('provider');
+    const providerConfig = retrievalConfig.get('config');
+
+    config.retrievalConfig = {
+      provider: provider || 's3',
+      config: providerConfig
+        ? {
+            region: providerConfig.get('region'),
+            endpoint: providerConfig.get('endpoint'),
+            accessKeyId: providerConfig.get('accessKeyId'),
+            secretAccessKey: providerConfig.get('secretAccessKey'),
+            forcePathStyle: providerConfig.get('forcePathStyle'),
+          }
+        : {},
+    };
+  }
+
+  if (embeddingConfig) {
+    config.embeddingConfig = {
+      provider: embeddingConfig.get('provider'),
+      model: embeddingConfig.get('model'),
+      chunkSize: embeddingConfig.get('chunkSize'),
+      chunkOverlap: embeddingConfig.get('chunkOverlap'),
+    };
+  }
+
+  const document = await documentFetcher.fetchDocument(config);
+
+  if (document) {
+    return {
+      document: {
+        title: document.title,
+        url: document.url,
+        format: document.format,
+      },
+    };
+  } else {
+    throw new Error(`Failed to fetch document: ${title} from ${url}`);
+  }
+}