@ugm/desiagent 0.1.37 → 0.2.1

package/dist/core/execution/dags.js

@@ -16,6 +16,7 @@ import { DecomposerJobSchema } from '../../types/dag.js';
 import { createLLMProvider } from '../providers/factory.js';
 import { LlmExecuteTool } from '../tools/llmExecute.js';
 import { DAGExecutor } from './dagExecutor.js';
+import { MinimalSkillDetector } from '../skills/detector.js';
 export function generateDAGId() {
     return `dag_${nanoid(21)}`;
 }
@@ -32,6 +33,12 @@ export class DAGsService {
     agentsService;
     scheduler;
     artifactsDir;
+    staleExecutionMinutes;
+    apiKey;
+    ollamaBaseUrl;
+    skipGenerationStats;
+    skillRegistry;
+    statsQueue;
     logger = getLogger();
     constructor(deps) {
         this.db = deps.db;
@@ -39,7 +46,13 @@ export class DAGsService {
         this.toolRegistry = deps.toolRegistry;
         this.agentsService = deps.agentsService;
         this.scheduler = deps.scheduler;
-        this.artifactsDir = deps.artifactsDir || process.env.ARTIFACTS_DIR || './artifacts';
+        this.artifactsDir = deps.artifactsDir;
+        this.staleExecutionMinutes = deps.staleExecutionMinutes ?? 5;
+        this.apiKey = deps.apiKey;
+        this.ollamaBaseUrl = deps.ollamaBaseUrl;
+        this.skipGenerationStats = deps.skipGenerationStats;
+        this.skillRegistry = deps.skillRegistry;
+        this.statsQueue = deps.statsQueue;
     }
     buildGlobalContext(job) {
         const entitiesStr = job.entities.length > 0
@@ -157,7 +170,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
         let activeLLMProvider;
         if (activeProvider && activeModel) {
             this.logger.info({ requestedProvider: activeProvider, requestedModel: activeModel }, 'Creating custom LLM provider');
-            activeLLMProvider = createLLMProvider({ provider: activeProvider, model: activeModel });
+            activeLLMProvider = createLLMProvider({ provider: activeProvider, model: activeModel, apiKey: this.apiKey, baseUrl: this.ollamaBaseUrl, skipGenerationStats: this.skipGenerationStats });
             const validationResult = await activeLLMProvider.validateToolCallSupport(activeModel);
             if (!validationResult.supported) {
                 this.logger.warn({ model: activeModel, reason: validationResult.message }, 'Model does not support tool calling');
@@ -167,10 +180,20 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             activeLLMProvider = this.llmProvider;
             this.logger.debug('Using default LLM provider');
         }
+        // Detect relevant skills and build skills list for prompt injection
+        const skillsList = this.skillRegistry ? this.skillRegistry.getFormattedList() : '';
+        if (this.skillRegistry) {
+            const detector = new MinimalSkillDetector();
+            const detectedSkills = detector.detect(goalText, this.skillRegistry.getAll());
+            if (detectedSkills.length > 0) {
+                this.logger.info({ detectedSkills }, 'Skills detected for goal');
+            }
+        }
         const toolDefinitions = this.toolRegistry.getAllDefinitions();
         const systemPrompt = agent.systemPrompt
             .replace(/\{\{tools\}\}/g, JSON.stringify(toolDefinitions))
-            .replace(/\{\{currentDate\}\}/g, new Date().toLocaleString());
+            .replace(/\{\{currentDate\}\}/g, new Date().toLocaleString())
+            .replace(/\{\{skills\}\}/g, skillsList);
         if (systemPrompt.length < 100) {
             this.logger.warn('System prompt is empty after replacement');
             throw new ValidationError('System prompt seems short! ', 'systemPrompt', systemPrompt);
@@ -195,10 +218,13 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 temperature,
                 maxTokens,
                 abortSignal: options.abortSignal,
+                deferGenerationStats: true,
             });
             const attemptUsage = response.usage;
             const attemptCost = response.costUsd;
             const attemptGenStats = response.generationStats;
+            const attemptGenStatsPromise = response.generationStatsPromise;
+            const attemptGenerationId = response.generationId;
             if (attemptUsage) {
                 planningUsageTotal.promptTokens += attemptUsage.promptTokens ?? 0;
                 planningUsageTotal.completionTokens += attemptUsage.completionTokens ?? 0;
@@ -273,7 +299,6 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 continue;
             }
             const usage = response.usage ?? null;
-            const generationStats = response.generationStats ?? null;
             const validatedResult = DecomposerJobSchema.safeParse(result);
             if (!validatedResult.success) {
                 planningAttempts.push({
@@ -343,13 +368,12 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 const dagId = generateDAGId();
                 const now = new Date();
                 this.logger.info({ dagId }, 'DagID Generated for clarification');
-                const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
                 const baseInsertData = {
                     id: dagId,
                     status: 'pending',
                     result: dag,
                     usage: usage,
-                    generationStats: generationStats,
+                    generationStats: null,
                     attempts: attempt,
                     agentName,
                     dagTitle: null,
@@ -371,28 +395,6 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                     createdAt: now,
                     updatedAt: now,
                 };
-                const titleResult = await titleMasterPromise;
-                if (titleResult) {
-                    this.logger.info('TitleMaster generated Result for clarification');
-                    baseInsertData.dagTitle = titleResult.title;
-                    planningAttempts.push({
-                        attempt,
-                        reason: 'title_master',
-                        usage: titleResult.usage,
-                        costUsd: titleResult.costUsd,
-                        generationStats: titleResult.generationStats,
-                    });
-                    if (titleResult.usage) {
-                        planningUsageTotal.promptTokens += titleResult.usage.promptTokens ?? 0;
-                        planningUsageTotal.completionTokens += titleResult.usage.completionTokens ?? 0;
-                        planningUsageTotal.totalTokens += titleResult.usage.totalTokens ?? 0;
-                    }
-                    if (titleResult.costUsd != null) {
-                        planningCostTotal += titleResult.costUsd;
-                    }
-                    baseInsertData.planningTotalUsage = planningUsageTotal;
-                    baseInsertData.planningTotalCostUsd = planningCostTotal.toString();
-                }
                 try {
                     await this.db.insert(dags).values(baseInsertData);
                 }
@@ -402,6 +404,9 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                     throw new Error(`Database insert failed for clarification DAG ${dagId}: ${errorMessage}`);
                 }
                 this.logger.info({ dagId, agentName, clarificationQuery: dag.clarification_query }, 'Clarification DAG saved to database');
+                // Fire-and-forget: update title and generation stats in the background
+                const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
+                this.backgroundUpdateDag(dagId, attemptGenStatsPromise, attemptGenerationId, titleMasterPromise, [...planningAttempts], { ...planningUsageTotal }, planningCostTotal, attempt);
                 return {
                     status: 'clarification_required',
                     dagId,
@@ -414,15 +419,12 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 const dagId = generateDAGId();
                 const now = new Date();
                 this.logger.info({ dagId }, "DagID Generated");
-                // Run TitleMaster generation in parallel with preparing insert data
-                const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
-                // Prepare base insert data (doesn't need title yet)
                 const baseInsertData = {
                     id: dagId,
                     status: 'success',
                     result: dag,
                     usage: usage,
-                    generationStats: generationStats,
+                    generationStats: null,
                     attempts: attempt,
                     agentName,
                     dagTitle: null,
@@ -444,31 +446,6 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                     createdAt: now,
                     updatedAt: now,
                 };
-                // Wait for title generation (runs in parallel with data prep above)
-                const titleResult = await titleMasterPromise;
-                if (titleResult) {
-                    this.logger.info('TitleMaster generated Result');
-                    baseInsertData.dagTitle = titleResult.title;
-                    planningAttempts.push({
-                        attempt,
-                        reason: 'title_master',
-                        usage: titleResult.usage,
-                        costUsd: titleResult.costUsd,
-                        generationStats: titleResult.generationStats,
-                    });
-                    if (titleResult.usage) {
-                        planningUsageTotal.promptTokens += titleResult.usage.promptTokens ?? 0;
-                        planningUsageTotal.completionTokens += titleResult.usage.completionTokens ?? 0;
-                        planningUsageTotal.totalTokens += titleResult.usage.totalTokens ?? 0;
-                    }
-                    if (titleResult.costUsd != null) {
-                        planningCostTotal += titleResult.costUsd;
-                    }
-                    // Update the totals in insert data
-                    baseInsertData.planningTotalUsage = planningUsageTotal;
-                    baseInsertData.planningTotalCostUsd = planningCostTotal.toString();
-                }
-                this.logger.info('Base insert data prepared');
                 try {
                     await this.db.insert(dags).values(baseInsertData);
                 }
@@ -477,14 +454,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                     this.logger.error({ err: dbError, dagId, goalText: showGoalText }, 'Failed to insert DAG into database');
                     throw new Error(`Database insert failed for DAG ${dagId}: ${errorMessage}`);
                 }
-                this.logger.info({
-                    dagId,
-                    agentName,
-                    goalText: showGoalText,
-                    cronSchedule,
-                    scheduleActive,
-                    planningCost: planningCostTotal,
-                }, 'DAG saved to database');
+                this.logger.info({ dagId, agentName, goalText: showGoalText, cronSchedule, scheduleActive }, 'DAG saved to database');
                 if (this.scheduler && cronSchedule && scheduleActive) {
                     this.scheduler.registerDAGSchedule({
                         id: dagId,
@@ -494,6 +464,9 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                     });
                     this.logger.info({ dagId, cronSchedule, timezone }, 'DAG schedule registered');
                 }
+                // Fire-and-forget: update title and generation stats in the background
+                const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
+                this.backgroundUpdateDag(dagId, attemptGenStatsPromise, attemptGenerationId, titleMasterPromise, [...planningAttempts], { ...planningUsageTotal }, planningCostTotal, attempt);
                 return { status: 'success', dagId };
             }
             if (dag.validation.gaps && dag.validation.gaps.length > 0) {
@@ -508,13 +481,12 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             const dagId = generateDAGId();
             const now = new Date();
             this.logger.info({ dagId }, 'DagID Generated for low-coverage DAG');
-            const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
             const baseInsertData = {
                 id: dagId,
                 status: 'success',
                 result: dag,
                 usage: usage,
-                generationStats: generationStats,
+                generationStats: null,
                 attempts: attempt,
                 agentName,
                 dagTitle: null,
@@ -536,28 +508,6 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 createdAt: now,
                 updatedAt: now,
             };
-            const titleResult = await titleMasterPromise;
-            if (titleResult) {
-                this.logger.info('TitleMaster generated Result for low-coverage DAG');
-                baseInsertData.dagTitle = titleResult.title;
-                planningAttempts.push({
-                    attempt,
-                    reason: 'title_master',
-                    usage: titleResult.usage,
-                    costUsd: titleResult.costUsd,
-                    generationStats: titleResult.generationStats,
-                });
-                if (titleResult.usage) {
-                    planningUsageTotal.promptTokens += titleResult.usage.promptTokens ?? 0;
-                    planningUsageTotal.completionTokens += titleResult.usage.completionTokens ?? 0;
-                    planningUsageTotal.totalTokens += titleResult.usage.totalTokens ?? 0;
-                }
-                if (titleResult.costUsd != null) {
-                    planningCostTotal += titleResult.costUsd;
-                }
-                baseInsertData.planningTotalUsage = planningUsageTotal;
-                baseInsertData.planningTotalCostUsd = planningCostTotal.toString();
-            }
             try {
                 await this.db.insert(dags).values(baseInsertData);
             }
@@ -567,6 +517,9 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 throw new Error(`Database insert failed for low-coverage DAG ${dagId}: ${errorMessage}`);
             }
             this.logger.info({ dagId, agentName }, 'Low-coverage DAG saved to database');
+            // Fire-and-forget: update title and generation stats in the background
+            const titleMasterPromise = this.generateTitleAsync(activeLLMProvider, goalText, options.abortSignal);
+            this.backgroundUpdateDag(dagId, attemptGenStatsPromise, attemptGenerationId, titleMasterPromise, [...planningAttempts], { ...planningUsageTotal }, planningCostTotal, attempt);
             return { status: 'success', dagId };
         }
         throw new ValidationError(`Failed to create DAG after ${maxAttempts} attempts`, 'attempts', maxAttempts);
@@ -597,7 +550,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
         this.logger.info({ dagId, originalGoalText: truncateForLog(originalGoalText) }, 'Resuming clarification DAG');
         const planningResult = await this.createFromGoal({
             goalText: augmentedGoalText,
-            agentName: params?.agentName || existingDag.agentName || 'Decomposer',
+            agentName: params?.agentName || existingDag.agentName || process.env.DEFAULT_DECOMPOSER_AGENT || 'DecomposerV8',
             provider: params?.provider,
             model: params?.model,
             temperature: params?.temperature ?? 0.7,
@@ -694,6 +647,11 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             llmProvider: this.llmProvider,
             toolRegistry: this.toolRegistry,
             artifactsDir: this.artifactsDir,
+            apiKey: this.apiKey,
+            ollamaBaseUrl: this.ollamaBaseUrl,
+            skipGenerationStats: this.skipGenerationStats,
+            skillRegistry: this.skillRegistry,
+            statsQueue: this.statsQueue,
         });
         // Start execution in background - don't await
         dagExecutor.execute(job, executionId, dagId, originalGoalText, _options?.executionConfig).catch((error) => {
@@ -711,7 +669,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
         }
         // For 'running' executions, only allow resume if they are stale (no recent update)
         if (execution.status === 'running') {
-            const staleMinutes = parseInt(process.env.STALE_EXECUTION_MINUTES || '5', 10);
+            const staleMinutes = this.staleExecutionMinutes;
             const staleThreshold = new Date(Date.now() - staleMinutes * 60 * 1000);
             const lastActivity = execution.updatedAt || execution.startedAt;
             if (lastActivity && new Date(lastActivity) > staleThreshold) {
@@ -751,6 +709,11 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             llmProvider: this.llmProvider,
             toolRegistry: this.toolRegistry,
             artifactsDir: this.artifactsDir,
+            apiKey: this.apiKey,
+            ollamaBaseUrl: this.ollamaBaseUrl,
+            skipGenerationStats: this.skipGenerationStats,
+            skillRegistry: this.skillRegistry,
+            statsQueue: this.statsQueue,
         });
         // Start execution in background - don't await
         dagExecutor.execute(job, executionId, execution.dagId, originalGoalText, executionConfig).catch((error) => {
@@ -810,7 +773,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             taskResults.set(taskId, snapshot.result);
         }
         const globalContext = this.buildGlobalContext(job);
-        const llmExecuteTool = new LlmExecuteTool();
+        const llmExecuteTool = new LlmExecuteTool({ apiKey: this.apiKey, baseUrl: this.ollamaBaseUrl, skipGenerationStats: this.skipGenerationStats });
         const overrideProvider = params?.provider;
         const overrideModel = params?.model;
         try {
@@ -865,6 +828,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                         runId: `redo-inference-${Date.now()}`,
                         executionId,
                         subStepId: pendingSubStepId,
+                        artifactsDir: this.artifactsDir,
                     });
                     await this.db.update(dagSubSteps)
                         .set({
@@ -1118,6 +1082,7 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
                 temperature: 0.7,
                 maxTokens: 100,
                 abortSignal,
+                deferGenerationStats: true,
             });
             const title = titleResponse.content.trim();
             this.logger.info({ dagTitle: title }, 'Generated DAG title from TitleMaster');
@@ -1133,6 +1098,120 @@ Respond with ONLY the expected output format. Build upon dependencies for cohere
             return null;
         }
     }
+    /**
+     * Fire-and-forget: resolves generation stats + title in the background and updates the DAG row.
+     */
+    backgroundUpdateDag(dagId, genStatsPromise, generationId, titlePromise, planningAttempts, planningUsageTotal, planningCostTotal, attempt) {
+        // When statsQueue is available, offload generation stats to the background worker
+        if (this.statsQueue && generationId) {
+            this.statsQueue.enqueue({
+                table: 'dags',
+                id: dagId,
+                generationId,
+                attemptIndex: planningAttempts.length - 1,
+            });
+            // Still handle title in main thread (per requirement)
+            const run = async () => {
+                const updateData = {};
+                try {
+                    const titleResult = await titlePromise;
+                    if (titleResult) {
+                        updateData.dagTitle = titleResult.title;
+                        planningAttempts.push({
+                            attempt,
+                            reason: 'title_master',
+                            usage: titleResult.usage,
+                            costUsd: titleResult.costUsd,
+                            generationStats: titleResult.generationStats,
+                        });
+                        if (titleResult.usage) {
+                            planningUsageTotal.promptTokens += titleResult.usage.promptTokens ?? 0;
+                            planningUsageTotal.completionTokens += titleResult.usage.completionTokens ?? 0;
+                            planningUsageTotal.totalTokens += titleResult.usage.totalTokens ?? 0;
+                        }
+                        if (titleResult.costUsd != null) {
+                            planningCostTotal += titleResult.costUsd;
+                        }
+                    }
+                }
+                catch (err) {
+                    this.logger.warn({ err, dagId }, 'Background: failed to generate title');
+                }
+                updateData.planningAttempts = planningAttempts;
+                updateData.planningTotalUsage = planningUsageTotal;
+                updateData.planningTotalCostUsd = planningCostTotal.toString();
+                updateData.updatedAt = new Date();
+                try {
+                    await this.db.update(dags).set(updateData).where(eq(dags.id, dagId));
+                    this.logger.info({ dagId, dagTitle: updateData.dagTitle }, 'Background: DAG updated with title (stats deferred to worker)');
+                }
+                catch (err) {
+                    this.logger.error({ err, dagId }, 'Background: failed to update DAG row');
+                }
+            };
+            run().catch(err => this.logger.error({ err, dagId }, 'Background DAG update failed'));
+            return;
+        }
+        // Fallback: original inline behavior when no statsQueue
+        const run = async () => {
+            const updateData = {};
+            // Resolve main generation stats
+            if (genStatsPromise) {
+                try {
+                    const stats = await genStatsPromise;
+                    if (stats.generationStats) {
+                        updateData.generationStats = stats.generationStats;
+                    }
+                    if (stats.costUsd != null) {
+                        planningAttempts[planningAttempts.length - 1].costUsd = stats.costUsd;
+                        planningAttempts[planningAttempts.length - 1].generationStats = stats.generationStats;
+                        planningCostTotal += stats.costUsd;
+                    }
+                }
+                catch (err) {
+                    this.logger.warn({ err, dagId }, 'Background: failed to fetch generation stats');
+                }
+            }
+            // Resolve title
+            try {
+                const titleResult = await titlePromise;
+                if (titleResult) {
+                    updateData.dagTitle = titleResult.title;
+                    planningAttempts.push({
+                        attempt,
+                        reason: 'title_master',
+                        usage: titleResult.usage,
+                        costUsd: titleResult.costUsd,
+                        generationStats: titleResult.generationStats,
+                    });
+                    if (titleResult.usage) {
+                        planningUsageTotal.promptTokens += titleResult.usage.promptTokens ?? 0;
+                        planningUsageTotal.completionTokens += titleResult.usage.completionTokens ?? 0;
+                        planningUsageTotal.totalTokens += titleResult.usage.totalTokens ?? 0;
+                    }
+                    if (titleResult.costUsd != null) {
+                        planningCostTotal += titleResult.costUsd;
+                    }
+                }
+            }
+            catch (err) {
+                this.logger.warn({ err, dagId }, 'Background: failed to generate title');
+            }
+            // Update DB row
+            updateData.planningAttempts = planningAttempts;
+            updateData.planningTotalUsage = planningUsageTotal;
+            updateData.planningTotalCostUsd = planningCostTotal.toString();
+            updateData.updatedAt = new Date();
+            try {
+                await this.db.update(dags).set(updateData).where(eq(dags.id, dagId));
+                this.logger.info({ dagId, dagTitle: updateData.dagTitle }, 'Background: DAG updated with title and stats');
+            }
+            catch (err) {
+                this.logger.error({ err, dagId }, 'Background: failed to update DAG row');
+            }
+        };
+        run().catch(err => this.logger.error({ err, dagId }, 'Background DAG update failed'));
+    }
     mapDAG(record) {
         return {
             id: record.id,