@aj-archipelago/cortex 1.3.5 → 1.3.7

package/helper-apps/cortex-realtime-voice-server/src/realtime/realtimeTypes.ts

@@ -0,0 +1,279 @@
+type RealtimeEvent = {
+  event_id: string,
+}
+
+type RealtimeContentResponseEvent = RealtimeEvent & {
+  response_id: string,
+  item_id: string,
+  output_index: number,
+  content_index: number,
+}
+
+type RealtimeFunctionResponseEvent = RealtimeEvent & {
+  response_id: string,
+  item_id: string,
+  output_index: number,
+  call_id: string,
+}
+
+type AudioFormat = 'pcm16' | 'g711_ulaw' | 'g711_alaw';
+export type AzureVoice = 'amuch' | 'dan' | 'elan' | 'marilyn' | 'meadow' | 'breeze' | 'cove' | 'ember' | 'jupiter' | 'alloy' | 'echo' | 'shimmer';
+export type OpenAIVoice = 'alloy' | 'echo' | 'shimmer' | 'ash' | 'ballad' | 'coral' | 'sage' | 'verse';
+export type Voice = AzureVoice | OpenAIVoice;
+type Modality = 'text' | 'audio';
+type ToolDefinition = {
+  type: string,
+  name: string,
+  description: string,
+  parameters: Record<string, any>,
+}
+type ToolChoice = 'auto' | 'none' | 'required' | { type: 'function'; name: string };
+
+export type RealtimeResponseConfig = {
+  conversation: string,
+  metadata: Record<string, any>,
+  modalities: Array<Modality>,
+  instructions: string,
+  voice: Voice,
+  output_audio_format: AudioFormat,
+  tools: Array<ToolDefinition>,
+  tool_choice: ToolChoice,
+  temperature: number,
+  max_output_tokens: number | 'inf'
+}
+
+export type RealtimeSessionConfig = {
+  modalities: Array<Modality>,
+  instructions: string,
+  voice: Voice,
+  input_audio_format: AudioFormat,
+  output_audio_format: AudioFormat,
+  input_audio_transcription: null | { model: 'whisper-1' | (string & {}) },
+  turn_detection: null | {
+    type: 'server_vad' | 'none',
+    threshold?: number,
+    prefix_padding_ms?: number,
+    silence_duration_ms?: number
+  },
+  tools: Array<ToolDefinition>,
+  tool_choice: ToolChoice,
+  temperature: number,
+  max_response_output_tokens: number | 'inf'
+}
+
+export type RealtimeSession = RealtimeSessionConfig & {
+  id: string,
+  model: string,
+}
+
+export type RealtimeItem = {
+  id: string,
+  type: 'message' | 'function_call' | 'function_call_output',
+  status?: 'in_progress' | 'completed' | 'incomplete',
+  role?: 'user' | 'assistant' | 'system',
+  content?: Array<{
+    type: 'input_text' | 'input_audio' | 'text' | 'audio',
+    audio?: string,
+    text?: string,
+    transcript?: string | null,
+  }>,
+  call_id?: string,
+  name?: string,
+  arguments?: string,
+  output?: string,
+}
+
+type RealtimeResponse = {
+  id: string,
+  status: 'in_progress' | 'completed' | 'incomplete' | 'failed',
+  status_details: null | {
+    type: 'incomplete',
+    reason: 'interruption' | 'max_output_tokens' | 'content_filter',
+  } | {
+    type: 'failed',
+    error?: Error | null,
+  },
+  output: Array<RealtimeItem>,
+  usage?: {
+    total_tokens: number
+    input_tokens: number
+    output_tokens: number
+  },
+}
+
+type RealtimeContentPart = {
+  type: 'text' | 'audio',
+  text?: string,
+  audio?: string,
+  transcript?: string | null,
+}
+
+export type RealtimeErrorEvent = RealtimeEvent & {
+  type: 'error',
+  error: {
+    type: string,
+    code: string,
+    message: string,
+    param: null,
+    event_id: string
+  }
+}
+
+export type SessionCreatedEvent = RealtimeEvent & {
+  type: 'session.created',
+  session: RealtimeSession
+}
+
+export type SessionUpdatedEvent = RealtimeEvent & {
+  type: 'session.updated',
+  session: RealtimeSession
+}
+
+export type ConversationCreatedEvent = RealtimeEvent & {
+  type: 'conversation.created',
+  conversation: {
+    id: string,
+  }
+}
+
+export type ConversationItemCreatedEvent = RealtimeEvent & {
+  type: 'conversation.item.created',
+  previous_item_id: string,
+  item: RealtimeItem,
+}
+
+export type ConversationItemInputAudioTranscriptionCompletedEvent = RealtimeEvent & {
+  type: 'conversation.item.input_audio_transcription.completed',
+  item_id: string,
+  content_index: number,
+  transcript: string,
+}
+
+export type ConversationItemInputAudioTranscriptionFailedEvent = RealtimeEvent & {
+  type: 'conversation.item.input_audio_transcription.failed',
+  item_id: string,
+  content_index: number,
+  error: {
+    type: string,
+    code: string,
+    message: string,
+    param: null | string,
+  }
+}
+
+export type ConversationItemTruncatedEvent = RealtimeEvent & {
+  type: 'conversation.item.truncated',
+  item_id: string,
+  content_index: number,
+  audio_end_ms: number,
+}
+
+export type ConversationItemDeletedEvent = RealtimeEvent & {
+  type: 'conversation.item.deleted',
+  item_id: string,
+}
+
+export type InputAudioBufferCommittedEvent = RealtimeEvent & {
+  type: 'input_audio_buffer.committed',
+  previous_item_id: string,
+  item_id: string,
+}
+
+export type InputAudioBufferClearedEvent = RealtimeEvent & {
+  type: 'input_audio_buffer.cleared',
+}
+
+export type InputAudioBufferSpeechStartedEvent = RealtimeEvent & {
+  type: 'input_audio_buffer.speech_started',
+  audio_start_ms: number,
+  item_id: string,
+}
+
+export type InputAudioBufferSpeechStoppedEvent = RealtimeEvent & {
+  type: 'input_audio_buffer.speech_stopped',
+  audio_end_ms: number,
+  item_id: string,
+}
+
+export type ResponseCreatedEvent = RealtimeEvent & {
+  type: 'response.created',
+  response: RealtimeResponse
+}
+
+export type ResponseDoneEvent = RealtimeEvent & {
+  type: 'response.done',
+  response: RealtimeResponse,
+}
+
+export type ResponseOutputItemAddedEvent = RealtimeEvent & {
+  type: 'response.output_item.added',
+  response_id: string,
+  output_index: number,
+  item: RealtimeItem,
+}
+
+export type ResponseOutputItemDoneEvent = RealtimeEvent & {
+  type: 'response.output_item.done',
+  response_id: string,
+  output_index: number,
+  item: RealtimeItem,
+}
+
+export type ResponseContentPartAddedEvent = RealtimeContentResponseEvent & {
+  type: 'response.content_part.added',
+  part: RealtimeContentPart,
+}
+
+export type ResponseContentPartDoneEvent = RealtimeContentResponseEvent & {
+  type: 'response.content_part.done',
+  part: RealtimeContentPart,
+}
+
+export type ResponseTextDeltaEvent = RealtimeContentResponseEvent & {
+  type: 'response.text.delta',
+  delta: string,
+}
+
+export type ResponseTextDoneEvent = RealtimeContentResponseEvent & {
+  type: 'response.text.done',
+  text: string,
+}
+
+export type ResponseAudioTranscriptDeltaEvent = RealtimeContentResponseEvent & {
+  type: 'response.audio_transcript.delta',
+  delta: string,
+}
+
+export type ResponseAudioTranscriptDoneEvent = RealtimeContentResponseEvent & {
+  type: 'response.audio_transcript.done',
+  transcript: string,
+}
+
+export type ResponseAudioDeltaEvent = RealtimeContentResponseEvent & {
+  type: 'response.audio.delta',
+  delta: string,
+}
+
+export type ResponseAudioDoneEvent = RealtimeContentResponseEvent & {
+  type: 'response.audio.done',
+}
+
+export type ResponseFunctionCallArgumentsDeltaEvent = RealtimeFunctionResponseEvent & {
+  type: 'response.function_call_arguments.delta',
+  delta: string,
+}
+
+export type ResponseFunctionCallArgumentsDoneEvent = RealtimeFunctionResponseEvent & {
+  type: 'response.function_call_arguments.done',
+  arguments: string,
+}
+
+export type RateLimitsUpdatedEvent = RealtimeEvent & {
+  type: 'rate_limits.updated',
+  rate_limits: Array<{
+    name: 'requests' | 'tokens' | 'input_tokens' | 'output_tokens' | (string & {})
+    limit: number
+    remaining: number
+    reset_seconds: number
+  }>
+}

package/helper-apps/cortex-realtime-voice-server/src/realtime/socket.ts

@@ -0,0 +1,27 @@
+import type {RealtimeItem} from "./realtimeTypes";
+
+type DeltaType = {
+  transcript?: string;
+  audio?: string;
+  text?: string;
+  arguments?: string;
+};
+
+export interface ServerToClientEvents {
+  error: (message: string) => void;
+  ready: () => void;
+  conversationUpdated: (item: RealtimeItem, delta: DeltaType) => void;
+  conversationInterrupted: () => void;
+  imageCreated: (imageUrl: string) => void;
+  requestScreenshot: () => void;
+}
+
+export interface ClientToServerEvents {
+  sendMessage: (message: string) => void;
+  appendAudio: (audio: string) => void;
+  cancelResponse: () => void;
+  conversationCompleted: () => void;
+  audioPlaybackComplete: (trackId: string) => void;
+  screenshotCaptured: (imageData: string) => void;
+  screenshotError: (error: string) => void;
+}

package/helper-apps/cortex-realtime-voice-server/src/realtime/transcription.ts

@@ -0,0 +1,75 @@
+import { RealtimeItem } from './realtimeTypes';
+
+export class Transcription {
+  private readonly items: Record<string, { realtimeItem: RealtimeItem, previousItemId: string }>;
+  private lastItemId: string;
+
+  constructor() {
+    this.items = {};
+    this.lastItemId = '';
+  }
+
+  public addItem(realtimeItem: RealtimeItem, previousItemId: string): void {
+    const itemCopy = this.getItemCopy(realtimeItem);
+    this.items[itemCopy.id] = {
+      realtimeItem: itemCopy,
+      previousItemId,
+    };
+    this.lastItemId = itemCopy.id;
+  }
+
+  public addTranscriptToItem(itemId: string, transcript: string): void {
+    const item = this.items[itemId];
+    if (item) {
+      item.realtimeItem.content = [{
+        type: 'input_text',
+        text: transcript,
+      }];
+    }
+  }
+
+  public updateItem(itemId: string, realtimeItem: RealtimeItem): void {
+    const newItem = this.getItemCopy(realtimeItem);
+    if (newItem.role === 'assistant' && newItem.content) {
+      newItem.content = newItem.content.map((contentPart) => {
+        if (contentPart.type === 'audio') {
+          return { type: 'text', text: contentPart.transcript || '' };
+        }
+        return contentPart;
+      });
+    }
+    this.items[itemId] = {
+      realtimeItem: newItem,
+      previousItemId: this.items[itemId]?.previousItemId || '',
+    };
+  }
+
+  public getItem(id: string): RealtimeItem | undefined {
+    return this.items[id]?.realtimeItem;
+  }
+
+  public removeItem(id: string): void {
+    delete this.items[id];
+  }
+
+  public getOrderedItems(): RealtimeItem[] {
+    const orderedItems: RealtimeItem[] = [];
+    let currentItemId = this.lastItemId;
+    while (currentItemId) {
+      const item = this.items[currentItemId];
+      if (item) {
+        orderedItems.push(item.realtimeItem);
+        currentItemId = item.previousItemId;
+      } else {
+        break;
+      }
+    }
+    return orderedItems.reverse();
+  }
+
+  protected getItemCopy(item: RealtimeItem): RealtimeItem {
+    const itemCopy: any = structuredClone(item);
+    delete itemCopy['object'];
+    return itemCopy;
+  }
+}

package/helper-apps/cortex-realtime-voice-server/src/realtime/utils.ts

@@ -0,0 +1,33 @@
+export function hasNativeWebSocket(): boolean {
+  return !!process.versions.bun || !!globalThis.WebSocket;
+}
+
+export function trimDebugEvent(event?: any): any {
+  if (!event) return event;
+
+  const maxLimit = 200;
+  const e = structuredClone(event);
+
+  // if (e.item?.content?.find((c: any) => c.audio)) {
+  //   e.item.content = e.item.content.map(({ audio, c }: any) => {
+  //     if (audio) {
+  //       return {
+  //         ...c,
+  //         audio: '(base64 redacted...)',
+  //       };
+  //     } else {
+  //       return c;
+  //     }
+  //   });
+  // }
+  //
+  // if (e.audio) {
+  //   e.audio = '(audio redacted...)';
+  // }
+
+  if (e.delta?.length > maxLimit) {
+    e.delta = e.delta.slice(0, maxLimit) + '... (truncated)';
+  }
+
+  return e;
+}

package/helper-apps/cortex-realtime-voice-server/src/utils/logger.ts

@@ -0,0 +1,45 @@
+// Logger utility for centralized logging control
+
+// Environment-based logging control
+const isProduction = process.env.NODE_ENV === 'production';
+let isLoggingEnabled = !isProduction;
+
+export const logger = {
+  enable: () => {
+    isLoggingEnabled = true;
+  },
+  
+  disable: () => {
+    isLoggingEnabled = false;
+  },
+
+  log: (...args: any[]) => {
+    if (isLoggingEnabled) {
+      console.log(...args);
+    }
+  },
+
+  // Additional logging levels if needed
+  debug: (...args: any[]) => {
+    if (isLoggingEnabled) {
+      console.debug(...args);
+    }
+  },
+
+  error: (...args: any[]) => {
+    // Always log errors, even in production
+    console.error(...args);
+  },
+
+  warn: (...args: any[]) => {
+    if (isLoggingEnabled) {
+      console.warn(...args);
+    }
+  },
+
+  info: (...args: any[]) => {
+    if (isLoggingEnabled) {
+      console.info(...args);
+    }
+  }
+}; 

package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts

@@ -0,0 +1,81 @@
+import { RealtimeVoiceClient } from "../realtime/client";
+import { createId } from "@paralleldrive/cuid2";
+import { logger } from "./logger";
+
+// Time to wait after last user message before allowing AI to speak
+const USER_SPEAKING_THRESHOLD_MS = 1500;
+
+export interface SendPromptOptions {
+  allowTools?: boolean;
+  disposable?: boolean;
+  aiResponding?: boolean;
+  audioPlaying?: boolean;
+  lastUserMessageTime?: number;
+  userSpeaking?: boolean;
+}
+
+export async function sendPrompt(
+  client: RealtimeVoiceClient,
+  prompt: string,
+  getOptions: (() => SendPromptOptions) | SendPromptOptions
+): Promise<{ skipped: boolean }> {
+  const options = typeof getOptions === 'function' ? getOptions() : getOptions;
+  
+  const {
+    allowTools = false,
+    disposable = true,
+    aiResponding = false,
+    audioPlaying = false,
+    lastUserMessageTime = 0,
+    userSpeaking = false
+  } = options;
+
+  // Check if user is currently speaking (based on active speaking or recent message)
+  const timeSinceLastMessage = Date.now() - lastUserMessageTime;
+  const recentlySpoke = timeSinceLastMessage < USER_SPEAKING_THRESHOLD_MS;
+  const isUserActive = userSpeaking || recentlySpoke;
+
+  // Don't send prompt if AI is responding, audio is playing, or user is speaking/recently spoke
+  if (aiResponding || audioPlaying || isUserActive) {
+    logger.log(`${disposable ? 'Skipping' : 'Queuing'} prompt while ${
+      userSpeaking ? 'user is actively speaking' :
+      recentlySpoke ? 'user recently finished speaking' :
+      aiResponding ? 'AI is responding' :
+      'AI audio is playing'
+    }`);
+    if (!disposable) {
+      // Try again after a short delay if the message is important
+      return new Promise((resolve) => {
+        setTimeout(() => {
+          sendPrompt(client, prompt, getOptions).then(resolve);
+        }, 1000);
+      });
+    }
+    return { skipped: true };
+  }
+
+  logger.log('Sending prompt');
+
+  const promptText = `<INSTRUCTIONS>${prompt}</INSTRUCTIONS>`;
+
+  client.createConversationItem({
+    id: createId(),
+    type: 'message',
+    role: 'user',
+    content: [
+      { type: 'input_text', text: promptText }
+    ]
+  });
+
+  /*
+  await this.realtimeClient.createConversationItem({
+    id: createId(),
+    type: 'function_call_output',
+    call_id: call.call_id,
+    output: response?.result || '',
+  });
+  */
+
+  client.createResponse({ tool_choice: allowTools ? 'auto' : 'none' });
+  return { skipped: false };
+} 

package/helper-apps/cortex-realtime-voice-server/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "node",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.5",
+  "version": "1.3.7",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/basePathway.js

@@ -21,7 +21,7 @@ export default {
     useInputSummarization: false, // true or false - instead of chunking, summarize the input and act on the summary    
     truncateFromFront: false, // true or false - if true, truncate from the front of the input instead of the back
     timeout: 120, // seconds, cancels the pathway after this many seconds
-    enableDuplicateRequests: true, // true or false - if true, duplicate requests are sent if the request is not completed after duplicateRequestAfter seconds
+    enableDuplicateRequests: false, // true or false - if true, duplicate requests are sent if the request is not completed after duplicateRequestAfter seconds
     duplicateRequestAfter: 10, // seconds, if the request is not completed after this many seconds, a backup request is sent
     // override the default execution of the pathway
     // callback signature: executeOverride({args: object, runAllPrompts: function})
@@ -32,5 +32,7 @@ export default {
     temperature: 0.9,
     // Require a valid JSON response from the model
     json: false,
+    // Manage the token length of the input for the model
+    manageTokenLength: true,
 };
 

package/pathways/system/entity/memory/sys_memory_manager.js

@@ -36,6 +36,9 @@ export default {
                 await callPathway('sys_save_memory', { ...args, aiMemory: AI_MEMORY_DEFAULTS });
             }
 
+            // Update context for the conversation turn
+            callPathway('sys_search_memory', { ...args, section: 'memoryAll',updateContext: true });
+
             // Check if this conversation turn requires memory updates
             const memoryRequired = await callPathway('sys_memory_required', { 
                 ...args,