@fluidframework/ai-collab 2.22.1 → 2.23.0

package/src/aiCollabApi.ts

@@ -7,6 +7,55 @@ import type { TreeNode } from "@fluidframework/tree";
 // eslint-disable-next-line import/no-named-as-default
 import type OpenAI from "openai";
 
+/**
+ * Core Debug event type for the ai-collab
+ * @alpha
+ */
+export interface DebugEvent {
+	/**
+	 * The unique id of the debug event.
+	 */
+	id: string;
+	/**
+	 * An id that will be shared across all debug events that originate from the same single execution of ai-collab.
+	 * @remarks This is intended to be used to correlate all debug events that originate from the same execution
+	 */
+	traceId: string;
+	/**
+	 * The name of the debug event.
+	 */
+	eventName: string;
+	/**
+	 * The date and time at which the debug event was created.
+	 */
+	timestamp: string;
+}
+
+/**
+ * A Debug event that marks the start or end of a single core logic flow, such as generated tree edits, planning prompt, etc.
+ * @alpha
+ */
+export interface EventFlowDebugEvent extends DebugEvent {
+	/**
+	 * The name of the particular event flow.
+	 */
+	eventFlowName: string;
+	/**
+	 * The status of the particular event flow.
+	 */
+	eventFlowStatus: "STARTED" | "COMPLETED" | "IN_PROGRESS";
+	/**
+	 * A unique id that will be shared across all debug events that are part of the same event flow.
+	 */
+	eventFlowTraceId: string;
+}
+
+/**
+ * A callback function that can be used to handle debug events that occur during the AI collaboration process.
+ * @alpha
+ */
+export type DebugEventLogHandler = <T extends DebugEvent>(event: T) => unknown;
+
 /**
  * OpenAI client options for the {@link AiCollabOptions} interface.
  *
@@ -99,9 +148,9 @@ export interface AiCollabOptions {
 	 */
 	readonly validator?: (newContent: TreeNode) => void;
 	/**
-	 * When enabled, the library will console.log information useful for debugging the AI collaboration.
+	 * An optional handler for debug events that occur during the AI collaboration.
 	 */
-	readonly dumpDebugLog?: boolean;
+	readonly debugEventLogHandler?: DebugEventLogHandler;
 }
 
 /**
@@ -139,12 +188,14 @@ export interface AiCollabErrorResponse {
 	 * - 'tooManyErrors' indicates that the LLM made too many errors in a row
 	 * - 'tooManyModelCalls' indicates that the LLM made too many model calls
 	 * - 'aborted' indicates that the AI collaboration was aborted by the user or a limiter
+	 * - 'unexpectedError' indicates that an unexpected error occured
 	 */
 	readonly errorMessage:
 		| "tokenLimitExceeded"
 		| "tooManyErrors"
 		| "tooManyModelCalls"
-		| "aborted";
+		| "aborted"
+		| "unexpectedError";
 	/**
 	 * {@inheritDoc TokenUsage}
 	 */

package/src/explicit-strategy/debugEvents.ts

@@ -0,0 +1,297 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { v4 as uuidv4 } from "uuid";
+
+import type { DebugEvent, EventFlowDebugEvent } from "../aiCollabApi.js";
+
+/**
+ * This file contains the types for the debug events that are emitted by the explicit strategy
+ * as well as a helper functions to help create a consistent method for producing a base {@link DebugEvent}.
+ */
+
+/**
+ * The possible values for the `eventFlowName` field in an {@link EventFlowDebugEvent}.
+ * @alpha
+ */
+export const EventFlowDebugNames = {
+	CORE_EVENT_LOOP: "CORE_EVENT_LOOP",
+	GENERATE_PLANNING_PROMPT: "GENERATE_PLANNING_PROMPT",
+	GENERATE_AND_APPLY_TREE_EDIT: "GENERATE_AND_APPLY_TREE_EDIT",
+	FINAL_REVIEW: "FINAL_REVIEW",
+} as const;
+
+/**
+ * The type for possible values for the `eventFlowName` field in an {@link EventFlowDebugEvent}.
+ * @alpha
+ */
+export type EventFlowDebugName =
+	(typeof EventFlowDebugNames)[keyof typeof EventFlowDebugNames];
+
+/**
+ * An edit generated by an LLM that can be applied to a given SharedTree.
+ * @privateremarks TODO: We need a better solution here because don't want to expose the internal TreeEdit type here, but we need to be able to type it.
+ * @alpha
+ */
+export type LlmTreeEdit = Record<string, unknown>;
+
+/**
+ * An {@link EventFlowDebugEvent} for signaling the start of the ai-collab's core event loop.
+ * Which makes various calls to the LLM to eventually apply edits to the users SharedTree which
+ * accomplish the user's provided goal. There will be exactly 1 of these events per ai-collab function execution.
+ * @alpha
+ */
+export interface CoreEventLoopStarted extends EventFlowDebugEvent {
+	eventName: "CORE_EVENT_LOOP_STARTED";
+	eventFlowName: typeof EventFlowDebugNames.CORE_EVENT_LOOP;
+	eventFlowStatus: "STARTED";
+}
+
+/**
+ * An {@link EventFlowDebugEvent} for signaling the end of the ai-collab's core event loop.
+ * There could be various reasons for the event loop to end, early exits and failures
+ * which should be captured in the status and failureReason fields. There will be exactly 1
+ * of these events per ai-collab function execution.
+ * @alpha
+ */
+export interface CoreEventLoopCompleted extends EventFlowDebugEvent {
+	eventName: "CORE_EVENT_LOOP_COMPLETED";
+	eventFlowName: typeof EventFlowDebugNames.CORE_EVENT_LOOP;
+	eventFlowStatus: "COMPLETED";
+	status: "success" | "failure";
+	failureReason?: string;
+	errorMessage?: string;
+}
+
+// #region Planning Prompt Debug events -------------------------------
+
+/**
+ * An {@link EventFlowDebugEvent} marking the initiation of the flow for prompting an LLM to generate a plan for accomplishing the user's goal.
+ * There will be exactly 1 of these events per ai-collab function execution.
+ * @alpha
+ */
+export interface PlanningPromptStarted extends EventFlowDebugEvent {
+	eventName: "GENERATE_PLANNING_PROMPT_STARTED";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_PLANNING_PROMPT;
+	eventFlowStatus: "STARTED";
+}
+
+/**
+ * An {@link EventFlowDebugEvent} marking the completion of the flow for prompting an LLM to generate a plan for accomplishing the user's goal.
+ * There will be exactly 1 of these events per ai-collab function execution.
+ * @alpha
+ */
+export interface PlanningPromptCompleted<
+	TIsLlmResponseValid = boolean,
+	TPlan = TIsLlmResponseValid extends true ? string : undefined,
+> extends EventFlowDebugEvent {
+	eventName: "GENERATE_PLANNING_PROMPT_COMPLETED";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_PLANNING_PROMPT;
+	eventFlowStatus: "COMPLETED";
+	/**
+	 * Whether the response produced by the LLM is an expected response.
+	 * In the event that the LLM fails to respond in an expected way, despite the API call to the LLM itself being successful, then this fields value will be `false`.
+	 *
+	 * For now, this case is limited to the LLM returning undefined as a response when it should have returned something. But in the future this could expand
+	 * to things such as invalid json.
+	 */
+	isLlmResponseValid: TIsLlmResponseValid;
+	/**
+	 * The plan generated by the LLM. If the `wasLlmResponseValid` field is `false` then this will be undefined, otherwise it will be a string.
+	 */
+	llmGeneratedPlan: TPlan;
+}
+
+// #endregion Planning Prompt Debug events ----------------------------------------------------------
+
+// #region Generate And Apply Tree Edit Debug events ------------------
+
+/**
+ * An {@link EventFlowDebugEvent} marking the initiation of the flow for prompting an LLM to
+ * generate an {@link LlmTreeEdit} to the users Shared Tree based on the users initial ask.
+ * It is expected that the LLM will generate multiple of these events when it must generate multiple tree edits to satisfy the user request
+ * @alpha
+ */
+export interface GenerateTreeEditStarted extends EventFlowDebugEvent {
+	eventName: "GENERATE_TREE_EDIT_STARTED";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_AND_APPLY_TREE_EDIT;
+	eventFlowStatus: "STARTED";
+	llmPrompt: string;
+}
+
+/**
+ * An {@link EventFlowDebugEvent} marking the completion of the flow for prompting an LLM to
+ * generate an {@link LlmTreeEdit} to the users Shared Tree based on the users initial ask.
+ * It is expected that the LLM will generate multiple of these events when it must generate multiple tree edits to satisfy the user request
+ * @alpha
+ */
+export interface GenerateTreeEditCompleted<
+	TIsLlmResponseValid = boolean,
+	TEdit = TIsLlmResponseValid extends true ? LlmTreeEdit | null : undefined,
+> extends EventFlowDebugEvent {
+	eventName: "GENERATE_TREE_EDIT_COMPLETED";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_AND_APPLY_TREE_EDIT;
+	eventFlowStatus: "COMPLETED";
+	/**
+	 * Whether the response produced by the LLM is an expected response.
+	 * In the event that the LLM fails to respond in an expected way, despite the API call to the LLM itself being successful, then this fields value will be `false`.
+	 *
+	 * For now, this case is limited to the LLM returning undefined as a response when it should have returned something. But in the future this could expand
+	 * to things such as invalid json.
+	 */
+	isLlmResponseValid: TIsLlmResponseValid;
+	/**
+	 * If the `isLlmResponseValid` field value is `true` then this be one of two following values returned by the LLM:
+	 * 1. An edit that can be applied to a SharedTree to further accomplish the user's goal.
+	 * 2. `null` if the LLM decides no more edits are necessary.
+	 *
+	 * If the `isLlmResponseValid` field is `false` then this field will be `undefined`.
+	 */
+	llmGeneratedEdit: TEdit;
+}
+
+/**
+ * An {@link EventFlowDebugEvent} marking the successful application of an edit generated by the LLM to a SharedTree.
+ * @alpha
+ */
+export interface ApplyEditSuccess extends EventFlowDebugEvent {
+	eventName: "APPLIED_EDIT_SUCCESS";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_AND_APPLY_TREE_EDIT;
+	eventFlowStatus: "IN_PROGRESS";
+	/**
+	 * A unique id that will be shared across all debug events that are part of the same event flow.
+	 */
+	eventFlowTraceId: string;
+	/**
+	 * The {@link LlmTreeEdit} generated by the LLM.
+	 */
+	edit: LlmTreeEdit;
+}
+
+/**
+ * An {@link EventFlowDebugEvent} marking the failure of applying an edit generated by the LLM to a SharedTree.
+ * @alpha
+ */
+export interface ApplyEditFailure extends EventFlowDebugEvent {
+	eventName: "APPLIED_EDIT_FAILURE";
+	eventFlowName: typeof EventFlowDebugNames.GENERATE_AND_APPLY_TREE_EDIT;
+	eventFlowStatus: "IN_PROGRESS";
+	/**
+	 * A unique id that will be shared across all debug events that are part of the same event flow.
+	 */
+	eventFlowTraceId: string;
+	/**
+	 * The {@link LlmTreeEdit} generated by the LLM.
+	 */
+	edit: LlmTreeEdit;
+	/**
+	 * The error message that was thrown when attempting to apply the edit.
+	 */
+	errorMessage: string;
+	/**
+	 * The total number of errors that have occurred up until this point, not including this error.
+	 */
+	sequentialErrorCount: number;
+}
+
+// #endregion Generate And Apply Tree Edit Debug events ----------------------
+
+// #region Generate Final Review Debug events -------------------------
+
+/**
+ * An {@link EventFlowDebugEvent} marking the initiation of the flow for prompting an LLM to complete a final review of its edits
+ * and determine whether the user's goal was accomplished.
+ * @alpha
+ */
+export interface FinalReviewStarted extends EventFlowDebugEvent {
+	eventName: "FINAL_REVIEW_STARTED";
+	eventFlowName: typeof EventFlowDebugNames.FINAL_REVIEW;
+	eventFlowStatus: "STARTED";
+	/**
+	 * The prompt sent to the LLM to complete its final review of the edits its made.
+	 */
+	llmPrompt: string;
+}
+
+/**
+ * An {@link EventFlowDebugEvent} marking the end of the flow for prompting an LLM to complete a final review of its edits
+ * and determine whether the user's goal was accomplished.
+ * @alpha
+ */
+export interface FinalReviewCompleted<
+	TIsLlmResponseValid = boolean,
+	TReviewResponse = TIsLlmResponseValid extends true ? "yes" | "no" : undefined,
+> extends EventFlowDebugEvent {
+	eventName: "FINAL_REVIEW_COMPLETED";
+	eventFlowName: typeof EventFlowDebugNames.FINAL_REVIEW;
+	eventFlowStatus: "COMPLETED";
+	/**
+	 * Whether the response produced by the LLM is an expected response.
+	 * In the event that the LLM fails to respond in an expected way, despite the API call to the LLM itself being successful, then this fields value will be `false`.
+	 *
+	 * For now, this case is limited to the LLM returning undefined as a response when it should have returned something. But in the future this could expand
+	 * to things such as invalid json.
+	 */
+	isLlmResponseValid: TIsLlmResponseValid;
+	/**
+	 * Whether the Llm believes the user's ask was accomplished, based on the Edits is has already generated and applied.
+	 * If the `status` field is `false` then this field value will be `undefined`, otherwise it will be either "yes" or "no"
+	 */
+	didLlmAccomplishGoal: TReviewResponse;
+}
+
+// #endregion Generate Final Review Debug events ----------------------
+
+/**
+ * An {@link DebugEvent} for an API call directly to a LLM.
+ * @alpha
+ */
+export interface LlmApiCallDebugEvent extends DebugEvent {
+	eventName: "LLM_API_CALL";
+	/**
+	 * The event flow name that made this LLM API call.
+	 */
+	triggeringEventFlowName: EventFlowDebugName;
+	/**
+	 * The unique id that will be shared across all debug events that are part of the same event flow.
+	 * @remarks This can be used to correlate all debug events that are part of the same event flow.
+	 */
+	eventFlowTraceId: string;
+	/**
+	 * The LLM model name that was used for the API call.
+	 */
+	modelName: string;
+	/**
+	 * The request parameters sent in the API call to the LLM. (Not the HTTP request parameters such as headers, etc.)
+	 */
+	requestParams: unknown;
+	/**
+	 * The raw response from the LLM.
+	 */
+	response: unknown;
+	/**
+	 * The total number of tokens used in the API call to the LLM.
+	 */
+	tokenUsage?: {
+		promptTokens: number;
+		completionTokens: number;
+	};
+}
+
+/**
+ * Creates a base {@link DebugEvent}.
+ */
+export function generateDebugEvent<Tevent extends string>(
+	eventName: Tevent,
+	traceId: string,
+): DebugEvent & { eventName: Tevent } {
+	const event = {
+		eventName,
+		id: uuidv4(),
+		traceId,
+		timestamp: new Date().toISOString(),
+	};
+	return event;
+}