@smythos/sre 1.7.41 → 1.8.0

package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts

@@ -7,7 +7,8 @@ import fs from 'fs';
 
 import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
-import { TLLMParams, TLLMPreparedParams, ILLMRequestContext, ToolData, APIKeySource, TLLMEvent, LLMModelInfo } from '@sre/types/LLM.types';
+import { TLLMParams, TLLMPreparedParams, ILLMRequestContext, ToolData, APIKeySource, TLLMEvent, LLMModelInfo, TLLMFinishReason } from '@sre/types/LLM.types';
+import { LLMHelper } from '@sre/LLMManager/LLM.helper';
 import { OpenAIApiInterface, ToolConfig } from './OpenAIApiInterface';
 import { HandlerDependencies, TToolType } from '../types';
 import { SUPPORTED_MIME_TYPES_MAP } from '@sre/constants';
@@ -79,23 +80,34 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
         this.deps = deps;
     }
 
-    async createRequest(body: OpenAI.Responses.ResponseCreateParams, context: ILLMRequestContext): Promise<OpenAI.Responses.Response> {
+    async createRequest(
+        body: OpenAI.Responses.ResponseCreateParams,
+        context: ILLMRequestContext,
+        abortSignal?: AbortSignal
+    ): Promise<OpenAI.Responses.Response> {
         const openai = await this.deps.getClient(context);
-        return await openai.responses.create({
-            ...body,
-            stream: false,
-        });
+        return await openai.responses.create(
+            {
+                ...body,
+                stream: false,
+            },
+            { signal: abortSignal }
+        );
     }
 
     async createStream(
         body: OpenAI.Responses.ResponseCreateParams,
-        context: ILLMRequestContext
+        context: ILLMRequestContext,
+        abortSignal?: AbortSignal
     ): Promise<Stream<OpenAI.Responses.ResponseStreamEvent>> {
         const openai = await this.deps.getClient(context);
-        return (await openai.responses.create({
-            ...body,
-            stream: true,
-        })) as Stream<OpenAI.Responses.ResponseStreamEvent>;
+        return (await openai.responses.create(
+            {
+                ...body,
+                stream: true,
+            },
+            { signal: abortSignal }
+        )) as Stream<OpenAI.Responses.ResponseStreamEvent>;
     }
 
     public handleStream(stream: Stream<OpenAI.Responses.ResponseStreamEvent>, context: ILLMRequestContext): EventEmitter {
@@ -262,20 +274,22 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
     /**
      * Emit final events
      */
-    private emitFinalEvents(emitter: EventEmitter, toolsData: ToolData[], reportedUsage: any[], finishReason: string): void {
+    private emitFinalEvents(emitter: EventEmitter, toolsData: ToolData[], reportedUsage: any[], finishReason: string | TLLMFinishReason): void {
+        const normalizedFinishReason = typeof finishReason === 'string' ? LLMHelper.normalizeFinishReason(finishReason) : finishReason;
+        
         // Emit tool info event if tools were called
         if (toolsData.length > 0) {
             emitter.emit(TLLMEvent.ToolInfo, toolsData);
         }
 
         // Emit interrupted event if finishReason is not 'stop'
-        if (finishReason !== 'stop') {
-            emitter.emit(TLLMEvent.Interrupted, finishReason);
+        if (normalizedFinishReason !== TLLMFinishReason.Stop) {
+            emitter.emit(TLLMEvent.Interrupted, normalizedFinishReason);
         }
 
         // Emit end event with setImmediate to ensure proper event ordering
         setImmediate(() => {
-            emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
+            emitter.emit(TLLMEvent.End, toolsData, reportedUsage, normalizedFinishReason);
         });
     }
 

package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts

@@ -15,6 +15,7 @@ import {
     TLLMChatResponse,
     ILLMRequestContext,
     TLLMEvent,
+    TLLMFinishReason,
 } from '@sre/types/LLM.types';
 import { LLMHelper } from '@sre/LLMManager/LLM.helper';
 
@@ -100,21 +101,21 @@ export class xAIConnector extends LLMConnector {
     }
 
     @hookAsync('LLMConnector.request')
-    protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
+    protected async request({ acRequest, body, context, abortSignal }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
         try {
             logger.debug(`request ${this.name}`, acRequest.candidate);
             const grok = await this.getClient(context);
-            const response = await grok.post('/chat/completions', body);
+            const response = await grok.post('/chat/completions', body, { signal: abortSignal });
 
             const message = response?.data?.choices?.[0]?.message;
-            const finishReason = response?.data?.choices?.[0]?.finish_reason;
+            const finishReason = LLMHelper.normalizeFinishReason(response?.data?.choices?.[0]?.finish_reason);
             const usage = response?.data?.usage as TUsage;
             const citations = response?.data?.citations;
 
             let toolsData: ToolData[] = [];
             let useTool = false;
 
-            if (finishReason === 'tool_calls') {
+            if (finishReason === TLLMFinishReason.ToolCalls) {
                 toolsData =
                     message?.tool_calls?.map((tool, index) => ({
                         index,
@@ -157,7 +158,7 @@ export class xAIConnector extends LLMConnector {
     }
 
     @hookAsync('LLMConnector.streamRequest')
-    protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
+    protected async streamRequest({ acRequest, body, context, abortSignal }: ILLMRequestFuncParams): Promise<EventEmitter> {
         const emitter = new EventEmitter();
 
         try {
@@ -168,11 +169,12 @@ export class xAIConnector extends LLMConnector {
                 { ...body, stream: true, stream_options: { include_usage: true } },
                 {
                     responseType: 'stream',
+                    signal: abortSignal,
                 }
             );
 
             const reportedUsage: any[] = [];
-            let finishReason = 'stop';
+            let finishReason: TLLMFinishReason = TLLMFinishReason.Stop;
             let toolsData: any[] = [];
             let usage: any = {};
             let citations: any[] = [];
@@ -225,7 +227,7 @@ export class xAIConnector extends LLMConnector {
                             }
 
                             if (parsed.choices?.[0]?.finish_reason) {
-                                finishReason = parsed.choices[0].finish_reason;
+                                finishReason = LLMHelper.normalizeFinishReason(parsed.choices[0].finish_reason);
                             }
                         } catch (e) {
                             // Ignore parsing errors for incomplete chunks
@@ -257,7 +259,7 @@ export class xAIConnector extends LLMConnector {
                     reportedUsage.push(_reported);
                 }
 
-                if (finishReason !== 'stop') {
+                if (finishReason !== TLLMFinishReason.Stop) {
                     emitter.emit(TLLMEvent.Interrupted, finishReason);
                 }
 

package/src/subsystems/MemoryManager/LLMContext.ts

@@ -10,6 +10,9 @@ export class LLMContext {
     private _llmContextStore: ILLMContextStore;
     private _llmCache: LLMCache;
 
+    /** Resolves when the context store has finished loading (if any). Safe to call before using addUserMessage, getContextWindow, or other context operations. */
+    private _readyPromise: Promise<void>;
+
     public get systemPrompt() {
         return this._systemPrompt;
     }
@@ -38,19 +41,31 @@ export class LLMContext {
      */
     constructor(private llmInference, _systemPrompt: string = '', llmContextStore?: ILLMContextStore) {
         this._llmCache = new LLMCache(AccessCandidate.team(this.llmInference.teamId));
+
         //this._systemPrompt = _systemPrompt;
         this.systemPrompt = _systemPrompt;
 
         if (llmContextStore) {
             this._llmContextStore = llmContextStore;
-            this._llmContextStore.load().then((messages) => {
+            this._readyPromise = this._llmContextStore.load().then((messages) => {
                 this._messages = messages;
                 this._llmCache.set('messages', this._messages);
             });
+        } else {
+            this._readyPromise = Promise.resolve();
         }
     }
 
-    private push(...message: any[]) {
+    /**
+     * Returns a promise that resolves when the context is ready (store loaded if present).
+     * Call before pushing or reading messages to avoid race conditions.
+     */
+    public ready(): Promise<void> {
+        return this._readyPromise;
+    }
+
+    private async push(...message: any[]) {
+        await this.ready();
         this._messages.push(...message);
 
         if (this._llmContextStore) {
@@ -58,7 +73,8 @@ export class LLMContext {
         }
         this._llmCache.set('messages', this._messages);
     }
-    public addUserMessage(content: string, message_id: string, metadata?: any) {
+    public async addUserMessage(content: string, message_id: string, metadata?: any): Promise<void> {
+        await this.ready();
         //in the current implementation, we do not support forked conversations
         //we always attatch to the last message in the queue
 
@@ -75,9 +91,10 @@ export class LLMContext {
         const prev = lastMessage?.__smyth_data__?.message_id;
         const next = [];
 
-        this.push({ role: 'user', content, __smyth_data__: { message_id, ...metadata, prev, next } });
+        await this.push({ role: 'user', content, __smyth_data__: { message_id, ...metadata, prev, next } });
     }
-    public addAssistantMessage(content: string, message_id: string, metadata?: any) {
+    public async addAssistantMessage(content: string, message_id: string, metadata?: any): Promise<void> {
+        await this.ready();
         const lastMessage = this._messages[this._messages.length - 1];
 
         if (lastMessage) {
@@ -89,9 +106,10 @@ export class LLMContext {
 
         const prev = lastMessage?.__smyth_data__?.message_id;
         const next = [];
-        this.push({ role: 'assistant', content, __smyth_data__: { message_id, ...metadata, prev, next } });
+        await this.push({ role: 'assistant', content, __smyth_data__: { message_id, ...metadata, prev, next } });
     }
-    public addToolMessage(messageBlock: any, toolsData: any, message_id: string, metadata?: any) {
+    public async addToolMessage(messageBlock: any, toolsData: any, message_id: string, metadata?: any): Promise<void> {
+        await this.ready();
         const lastMessage = this._messages[this._messages.length - 1];
 
         if (lastMessage) {
@@ -103,10 +121,11 @@ export class LLMContext {
 
         const prev = lastMessage?.__smyth_data__?.message_id;
         const next = [];
-        this.push({ messageBlock, toolsData, __smyth_data__: { message_id, ...metadata, prev, next } });
+        await this.push({ messageBlock, toolsData, __smyth_data__: { message_id, ...metadata, prev, next } });
     }
 
     public async getContextWindow(maxTokens: number, maxOutputTokens: number = 1024): Promise<any[]> {
+        await this.ready();
         const messages = JSON.parse(JSON.stringify(this._messages));
         // if (messages[0]?.role === 'system') {
         //     messages[0].content = this.systemPrompt;