@casual-simulation/aux-runtime 3.4.0-alpha.14070378151 → 3.4.0

package/runtime/AuxLibraryDefinitions.def

@@ -10476,6 +10476,192 @@ export interface AISloydGenerateModelOptions {
     }
 }
 
+
+/**
+ * Defines a request to create a new realtime session.
+ * 
+ * @dochash types/ai
+ * @docname RealtimeSessionRequest
+ */
+export interface CreateRealtimeSessionTokenRequest {
+    /**
+     * The default system instructions (i.e. system message) prepended to model calls. This field allows the client to guide the model on desired responses. The model can be instructed on response content and format, (e.g. "be extremely succinct", "act friendly", "here are examples of good responses") and on audio behavior (e.g. "talk quickly", "inject emotion into your voice", "laugh frequently"). The instructions are not guaranteed to be followed by the model, but they provide guidance to the model on the desired behavior.
+     *
+     * Note that the server sets default instructions which will be used if this field is not set and are visible in the session.created event at the start of the session.
+     */
+    instructions?: string;
+
+    /**
+     * The Realtime model used for this session.
+     */
+    model: string;
+
+    /**
+     * The set of modalities the model can respond with. To disable audio, set this to ["text"].
+     */
+    modalities?: ('audio' | 'text')[];
+
+    /**
+     * Maximum number of output tokens for a single assistant response, inclusive of tool calls. Provide an integer between 1 and 4096 to limit output tokens, or inf for the maximum available tokens for a given model. Defaults to inf.
+     */
+    maxResponseOutputTokens?: number;
+
+    /**
+     * The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. For `pcm16`, input audio must be 16-bit PCM at a 24kHz sample rate, single channel (mono), and little-endian byte order.
+     */
+    inputAudioFormat?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
+
+    /**
+     * Configuration for input audio noise reduction. This can be set to null to turn off. Noise reduction filters audio added to the input audio buffer before it is sent to VAD and the model. Filtering the audio can improve VAD and turn detection accuracy (reducing false positives) and model performance by improving perception of the input audio.
+     */
+    inputAudioNoiseReduction?: {
+        /**
+         * Type of noise reduction. `near_field` is for close-talking microphones such as headphones, `far_field` is for far-field microphones such as laptop or conference room microphones.
+         */
+        type?: 'near_field' | 'far_field';
+    } | null;
+
+    /**
+     * Configuration for input audio transcription, defaults to off and can be set to `null` to turn off once on. Input audio transcription is not native to the model, since the model consumes audio directly. Transcription runs asynchronously through the /audio/transcriptions endpoint and should be treated as guidance of input audio content rather than precisely what the model heard. The client can optionally set the language and prompt for transcription, these offer additional guidance to the transcription service.
+     */
+    inputAudioTranscription?: {
+        /**
+         * The language of the input audio. Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`) format will improve accuracy and latency.
+         */
+        language?: string;
+
+        /**
+         * The model to use for transcription, current options are `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`, and `whisper-1`.
+         */
+        model?: string;
+
+        /**
+         * An optional text to guide the model's style or continue a previous audio segment. For `whisper-1`, the prompt is a list of keywords. For `gpt-4o-transcribe` models, the prompt is a free text string, for example "expect words related to technology".
+         */
+        prompt?: string;
+    } | null;
+
+    /**
+     * The format of output audio. Options are pcm16, g711_ulaw, or g711_alaw. For pcm16, output audio is sampled at a rate of 24kHz.
+     */
+    outputAudioFormat?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
+
+    /**
+     * Sampling temperature for the model, limited to [0.6, 1.2]. For audio models a temperature of 0.8 is highly recommended for best performance.
+     */
+    temperature?: number;
+
+    /**
+     * How the model chooses tools. Options are `auto`, `none`, `required`, or specify a function.
+     */
+    toolChoice?: string;
+
+    /**
+     * Tools (functions) available to the model.
+     */
+    tools?: {
+        /**
+         * The description of the function, including guidance on when and how to call it, and guidance about what to tell the user when calling (if anything).
+         */
+        description?: string;
+
+        /**
+         * The name of the function.
+         */
+        name?: string;
+
+        /**
+         * Parameters of the function in JSON Schema.
+         */
+        parameters?: any;
+
+        /**
+         * The type of the tool, i.e. `function`.
+         */
+        type?: 'function';
+    }[];
+
+    /**
+     * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be set to `null` to turn off, in which case the client must manually trigger model response. Server VAD means that the model will detect the start and end of speech based on audio volume and respond at the end of user speech. Semantic VAD is more advanced and uses a turn detection model (in conjuction with VAD) to semantically estimate whether the user has finished speaking, then dynamically sets a timeout based on this probability. For example, if user audio trails off with "uhhm", the model will score a low probability of turn end and wait longer for the user to continue speaking. This can be useful for more natural conversations, but may have a higher latency.
+     */
+    turnDetection?: {
+        /**
+         * Whether or not to automatically generate a response when a VAD stop event occurs.
+         */
+        createResponse?: boolean;
+
+        /**
+         * Used only for `semantic_vad` mode. The eagerness of the model to respond. `low` will wait longer for the user to continue speaking, `high` will respond more quickly. `auto` is the default and is equivalent to `medium`.
+         */
+        eagerness?: 'low' | 'medium' | 'high';
+
+        /**
+         * Whether or not to automatically interrupt any ongoing response with output to the default conversation (i.e. `conversation` of `auto`) when a VAD start event occurs.
+         */
+        interruptResponse?: boolean;
+
+        /**
+         * Used only for `server_vad` mode. Amount of audio to include before the VAD detected speech (in milliseconds). Defaults to 300ms.
+         */
+        prefixPaddingMs?: number;
+
+        /**
+         * Used only for `server_vad` mode. Duration of silence to detect speech stop (in milliseconds). Defaults to 500ms. With shorter values the model will respond more quickly, but may jump in on short pauses from the user.
+         */
+        silenceDurationMs?: number;
+
+        /**
+         * Used only for `server_vad` mode. Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. A higher threshold will require louder audio to activate the model, and thus might perform better in noisy environments.
+         */
+        threshold?: number;
+
+        /**
+         * Type of turn detection.
+         */
+        type?: 'server_vad' | 'semantic_vad';
+    } | null;
+
+    /**
+     * The voice the model uses to respond. Voice cannot be changed during the session once the model has responded with audio at least once. Current voice options are `alloy`, `ash`, `ballad`, `coral`, `echo`, `fable`, `onyx`, `nova`, `sage`, `shimmer`, and `verse`.
+     */
+    voice?: string;
+}
+
+/**
+ * The response to a request to create a realtime session token using the OpenAI interface.
+ * @dochash types/ai
+ * @docname AICreateOpenAIRealtimeSessionTokenResult
+ */
+export type AICreateOpenAIRealtimeSessionTokenResult =
+    | AICreateOpenAIRealtimeSessionTokenSuccess
+    | AICreateOpenAIRealtimeSessionTokenFailure;
+
+/**
+ * A successful response to a request to create a realtime session token using the OpenAI interface.
+ * @dochash types/ai
+ * @docname AICreateOpenAIRealtimeSessionTokenSuccess
+ */
+export interface AICreateOpenAIRealtimeSessionTokenSuccess {
+    success: true;
+    sessionId: string;
+    clientSecret: {
+        value: string;
+        expiresAt: number;
+    };
+}
+
+/**
+ * A unsuccessful response to a request to create a realtime session token using the OpenAI interface.
+ * @dochash types/ai
+ * @docname AICreateOpenAIRealtimeSessionTokenFailure
+ */
+export interface AICreateOpenAIRealtimeSessionTokenFailure {
+    success: false;
+    errorCode: KnownErrorCodes;
+    errorMessage: string;
+}
+
+
 /**
  * Defines an interface for a generic HTTP response.
  */
@@ -11859,6 +12045,23 @@ interface Ai {
         generateModel(request: AISloydGenerateModelOptions, options?: RecordActionOptions): Promise<AISloydGenerateModelResponse>;
     }
 
+    openai: {
+        /**
+         * Creates a new OpenAI Realtime Session.
+         * @param recordName The name of the record that the session is for.
+         * @param request The request options for the session.
+         * @param options The options for the records request.
+         * 
+         * @dochash actions/ai
+         * @docname ai.openai.createRealtimeSession
+         */
+        createRealtimeSession(
+            recordName: string,
+            request: CreateRealtimeSessionTokenRequest,
+            options?: RecordActionOptions
+        ): Promise<AICreateOpenAIRealtimeSessionTokenResult>;
+    }
+
     stream: {
         /**
          * Sends a chat message to the AI and streams the response back.

package/runtime/AuxPartitionRealtimeEditModeProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"AuxPartitionRealtimeEditModeProvider.js","sourceRoot":"","sources":["AuxPartitionRealtimeEditModeProvider.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAE3E,OAAO,EAAE,kCAAkC,EAAE,MAAM,SAAS,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,OAAO,oCAAoC;IAK7C,YAAY,UAAyB;QACjC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,WAAW,CAAC,KAAa;QACrB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,kCAAkC,CACrC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,gBAAgB,CAC3C,CAAC;QACN,CAAC;QACD,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YACxB,OAAO,gBAAgB,CAAC,IAAI,CAAC;QACjC,CAAC;aAAM,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,gBAAgB,CAAC,IAAI,CAAC;QACjC,CAAC;QACD,OAAO,0BAA0B,CAAC;IACtC,CAAC;CACJ"}
+{"version":3,"file":"AuxPartitionRealtimeEditModeProvider.js","sourceRoot":"","sources":["AuxPartitionRealtimeEditModeProvider.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAE3E,OAAO,EAAE,kCAAkC,EAAE,MAAM,SAAS,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,OAAO,oCAAoC;IAK7C,YAAY,UAAyB;QACjC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,WAAW,CAAC,KAAa;QACrB,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,kCAAkC,CACrC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,gBAAgB,CAC3C,CAAC;QACN,CAAC;QACD,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YACxB,OAAO,gBAAgB,CAAC,IAAI,CAAC;QACjC,CAAC;aAAM,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,gBAAgB,CAAC,IAAI,CAAC;QACjC,CAAC;QACD,OAAO,0BAA0B,CAAC;IACtC,CAAC;CACJ"}

package/runtime/AuxRealtimeEditModeProvider.js

@@ -1,3 +1,20 @@
+/* CasualOS is a set of web-based tools designed to facilitate the creation of real-time, multi-user, context-aware interactive experiences.
+ *
+ * Copyright (c) 2019-2025 Casual Simulation, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 import { RealtimeEditMode } from './RuntimeBot';
 /**
  * The default realtime edit mode.

package/runtime/AuxRealtimeEditModeProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"AuxRealtimeEditModeProvider.js","sourceRoot":"","sources":["AuxRealtimeEditModeProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GACnC,gBAAgB,CAAC,SAAS,CAAC;AAO/B;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAC7C,IAAI,GAAG,CAAC;IACJ,CAAC,QAAQ,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACtC,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACrC,CAAC,WAAW,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACzC,CAAC,SAAS,EAAE,gBAAgB,CAAC,OAAO,CAAC;IACrC,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;IACnC,CAAC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC;IACpC,CAAC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC;CACvC,CAAC,CAAC;AAEP;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAC/B,GAA6B,EAC7B,KAAa;IAEb,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,0BAA0B,CAAC;AACxD,CAAC;AAaD,MAAM,OAAO,+BAA+B;IAKxC,YACI,MAAgC,oCAAoC;QAEpE,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,WAAW,CAAC,KAAa;QACrB,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;CACJ"}
+{"version":3,"file":"AuxRealtimeEditModeProvider.js","sourceRoot":"","sources":["AuxRealtimeEditModeProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GACnC,gBAAgB,CAAC,SAAS,CAAC;AAO/B;;GAEG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAC7C,IAAI,GAAG,CAAC;IACJ,CAAC,QAAQ,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACtC,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACrC,CAAC,WAAW,EAAE,gBAAgB,CAAC,SAAS,CAAC;IACzC,CAAC,SAAS,EAAE,gBAAgB,CAAC,OAAO,CAAC;IACrC,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;IACnC,CAAC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC;IACpC,CAAC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC;CACvC,CAAC,CAAC;AAEP;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAC/B,GAA6B,EAC7B,KAAa;IAEb,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,0BAA0B,CAAC;AACxD,CAAC;AAaD,MAAM,OAAO,+BAA+B;IAKxC,YACI,MAAgC,oCAAoC;QAEpE,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,WAAW,CAAC,KAAa;QACrB,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;CACJ"}

package/runtime/AuxResults.js.map

@@ -1 +1 @@
-{"version":3,"file":"AuxResults.js","sourceRoot":"","sources":["AuxResults.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C;QACI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC/B,CAAC;CACJ"}
+{"version":3,"file":"AuxResults.js","sourceRoot":"","sources":["AuxResults.ts"],"names":[],"mappings":"AAqBA,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C;QACI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC/B,CAAC;CACJ"}

package/runtime/AuxRuntime.js

@@ -1,10 +1,20 @@
-// CasualOS has several key components:
-//
-// 1. Simulations - These are wrapper objects that manage creating and interfacing with AUX virtual machines.
-// 2. VM - AUX Virtual Machines provide a security boundary to keep user scripts separate across multiple virtual machines.
-// 3. Channel - These are manager objects which handle the persistence and runtime aspects of an AUX.
-// 4. Partitions - These are services which manage the persistence and realtime sync of the AUX data model.
-// 5. Runtimes - These are services which manage script execution and formula precalculation.
+/* CasualOS is a set of web-based tools designed to facilitate the creation of real-time, multi-user, context-aware interactive experiences.
+ *
+ * Copyright (c) 2019-2025 Casual Simulation, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {