@casual-simulation/aux-runtime 3.3.3 → 3.3.4-alpha.9225751827

package/runtime/AuxLibraryDefinitions.def

@@ -10589,6 +10589,177 @@ interface Ai {
         negativePrompt?: string | RecordActionOptions,
         options?: RecordActionOptions
     ): Promise<string | AIGenerateImageSuccess>;
+
+    stream: {
+        /**
+         * Sends a chat message to the AI and streams the response back.
+         * Returns a promise that resolves with an [async iterable](https://javascript.info/async-iterators-generators#async-iterables) that contains the responses from the AI.
+         * 
+         * Throws a {@link CasualOSError} if an error occurs while sending the message.
+         *
+         * This function can be useful for creating chat bots, or for using an Artificial Intelligence (AI) to process a message.
+         *
+         * @param message The message that should be sent to the AI.
+         * @param options The options that should be used.
+         *
+         * @example Send a message to the AI and log the response.
+         * const response = await ai.stream.chat("Hello!");
+         * 
+         * for await (let message of response) {
+         *    console.log(message);
+         * }
+         *
+         * @dochash actions/ai
+         * @doctitle AI Actions
+         * @docsidebar AI
+         * @docdescription AI actions are functions that make it easier to work with the AI.
+         * @docname ai.stream.chat
+         * @docid ai.stream.chat-string
+         */
+        chat(message: string, options?: AIChatOptions): Promise<AsyncIterable<string>>;
+
+        /**
+         * Sends a chat message to the AI and streams the response back.
+         * Returns a promise that resolves with an [async iterable](https://javascript.info/async-iterators-generators#async-iterables) that contains the responses from the AI.
+         * 
+         * Throws a {@link CasualOSError} if an error occurs while sending the message.
+         *
+         * This function can be useful for creating chat bots, or for using an Artificial Intelligence (AI) to process a message.
+         *
+         * @param message The message that should be sent to the AI.
+         * @param options The options that should be used.
+         *
+         * @example Send a message to the AI and log the response.
+         * const response = await ai.chat({
+         *     role: "user",
+         *     content: "Hello!"
+         * });
+         * console.log(`${response.role}: ${response.content}`);
+         *
+         * @example Ask the AI to describe an uploaded image.
+         * const files = await os.showUploadFiles();
+         * const firstFile = files[0];
+         * const base64 = bytes.toBase64String(new Uint8Array(firstFile.data));
+         * const response = await ai.stream.chat({
+         *    role: 'user',
+         *    content: [
+         *        {
+         *            base64: base64,
+         *            mimeType: firstFile.mimeType,
+         *        },
+         *        {
+         *            text: 'please describe the image'
+         *        }
+         *    ]
+         * }, {
+         *    preferredModel: 'gemini-pro-vision'
+         * });
+         * 
+         * for await (let message of response) {
+             os.toast(message.content);
+        * }
+        *
+        * @dochash actions/ai
+        * @docname ai.stream.chat
+        * @docid ai.stream.chat-message
+        */
+        chat(
+            message: AIChatMessage,
+            options?: AIChatOptions
+        ): Promise<AsyncIterable<AIChatMessage>>;
+
+        /**
+         * Sends a chat message to the AI.
+         * Returns a promise that resolves with an [async iterable](https://javascript.info/async-iterators-generators#async-iterables) that contains the responses from the AI.
+         * 
+         * Throws a {@link CasualOSError} if an error occurs while sending the message.
+         *
+         * This function can be useful for creating chat bots, or for using an Artificial Intelligence (AI) to process a message.
+         *
+         * @param message The message that should be sent to the AI.
+         * @param options The options that should be used.
+         *
+         * @example Send a message to the AI and log the response.
+         * const response = await ai.stream.chat([
+         *      {
+         *          role: "system",
+         *          content: "You are a helpful assistant."
+         *      },
+         *     {
+         *          role: "user",
+         *          content: "Hello!"
+         *     }
+         * ]);
+         * 
+         * for await (let message of response) {
+         *    console.log(`${message.role}: ${message.content}`);
+         * }
+         *
+         * @example Build a basic chat bot.
+         * const messages = [
+         *      {
+         *          role: "system",
+         *          content: "You are a helpful assistant."
+         *      },
+         * ];
+         *
+         * while(true) {
+         *      const userInput = await os.showInput();
+         *      if (!userInput) {
+         *          break;
+         *      }
+         *      messages.push({
+         *          role: "user",
+         *          content: userInput
+         *      });
+         *
+         *      const response = await ai.stream.chat(messages);
+         * 
+         *      for await (let message of response) {
+         *          messages.push(message);
+         *          os.toast(message.content);
+         *      }
+         * }
+         *
+         * os.toast("Goodbye!");
+         *
+         * @example Ask the AI to describe an uploaded image.
+         * const files = await os.showUploadFiles();
+         * const firstFile = files[0];
+         * const base64 = bytes.toBase64String(new Uint8Array(firstFile.data));
+         * const response = await ai.stream.chat([{
+         *    role: 'user',
+         *    content: [
+         *        {
+         *            base64: base64,
+         *            mimeType: firstFile.mimeType,
+         *        },
+         *        {
+         *            text: 'please describe the image'
+         *        }
+         *    ]
+         * }], {
+         *    preferredModel: 'gemini-pro-vision'
+         * });
+         *
+         * for await (let message of response) {
+             os.toast(message.content);
+        * }
+        *
+        * @dochash actions/ai
+        * @docname ai.stream.chat
+        * @docid ai.stream.chat-messages
+        */
+        chat(
+            messages: AIChatMessage[],
+            options?: AIChatOptions
+        ): Promise<AsyncIterable<AIChatMessage>>;
+
+        chat(
+            messages: string | AIChatMessage | AIChatMessage[],
+            options?: AIChatOptions
+        ): Promise<AsyncIterable<AIChatMessage | string>>;
+    }
 }
 
 interface Os {
@@ -11167,6 +11338,21 @@ interface Os {
      */
     capturePhoto(options?: OpenPhotoCameraOptions): Promise<Photo>;
 
+    /**
+     * Captures a screenshot (i.e. photo/picture) from the grid portal.
+     * 
+     * Returns a promise that resolves with the captured photo.
+     * 
+     * @param portal the portal to capture the screenshot from. Defaults to the grid portal.
+     * 
+     * @example Capture a screenshot from the grid portal.
+     * const screenshot = await os.capturePortalScreenshot();
+     * 
+     * @dochash actions/os/portals
+     * @docname os.capturePortalScreenshot
+     */
+    capturePortalScreenshot(): Promise<Photo>;
+
     /**
      * Shows the given barcode.
      * @param code The code that should be shown.

package/runtime/AuxRuntime.js

@@ -1119,6 +1119,30 @@ export class AuxRuntime {
                 this._scheduleJobQueueCheck();
             }
         }
+        else if (action.type === 'iterable_next') {
+            if (!this._globalContext.iterableNext(action.taskId, action.value, false)) {
+                this._actionBatch.push(action);
+            }
+            else {
+                this._scheduleJobQueueCheck();
+            }
+        }
+        else if (action.type === 'iterable_complete') {
+            if (!this._globalContext.iterableComplete(action.taskId, false)) {
+                this._actionBatch.push(action);
+            }
+            else {
+                this._scheduleJobQueueCheck();
+            }
+        }
+        else if (action.type === 'iterable_throw') {
+            if (!this._globalContext.iterableThrow(action.taskId, action.error, false)) {
+                this._actionBatch.push(action);
+            }
+            else {
+                this._scheduleJobQueueCheck();
+            }
+        }
         else if (action.type === 'register_custom_app') {
             this._registerPortalBot(action.appId, action.botId);
             this._actionBatch.push(action);