@autobe/agent 0.25.7 → 0.27.0

package/src/context/AutoBeContext.ts

@@ -10,9 +10,12 @@ import {
   AutoBeAssistantMessageHistory,
   AutoBeEvent,
   AutoBeEventSource,
+  AutoBeFunctionCallingMetric,
   AutoBeHistory,
   AutoBeInterfaceHistory,
+  AutoBePhase,
   AutoBePrismaHistory,
+  AutoBeProcessAggregateCollection,
   AutoBeRealizeHistory,
   AutoBeTestHistory,
   IAutoBeCompiler,
@@ -33,12 +36,16 @@ export interface AutoBeContext<Model extends ILlmSchema.Model> {
   retry: number;
 
   // accessors
+  aggregates: AutoBeProcessAggregateCollection;
   compilerListener: IAutoBeCompilerListener;
   compiler: () => Promise<IAutoBeCompiler>;
   files: (options: IAutoBeGetFilesOptions) => Promise<Record<string, string>>;
   histories: () => Readonly<AutoBeHistory[]>;
   state: () => Readonly<AutoBeState>;
   usage: () => AutoBeTokenUsage;
+  getCurrentAggregates: (
+    phase: AutoBePhase,
+  ) => AutoBeProcessAggregateCollection;
 
   // events
   dispatch: <Event extends Exclude<AutoBeEvent, AutoBeAssistantMessageEvent>>(
@@ -79,5 +86,6 @@ export namespace AutoBeContext {
   export interface IResult<Model extends ILlmSchema.Model> {
     histories: MicroAgenticaHistory<Model>[];
     tokenUsage: IAutoBeTokenUsageJson.IComponent;
+    metric: AutoBeFunctionCallingMetric;
   }
 }

package/src/context/AutoBeTokenUsage.ts

@@ -1,7 +1,7 @@
 import { IAutoBeTokenUsageJson } from "@autobe/interface";
 
+import { IAutoBeFacadeApplication } from "../orchestrate/facade/histories/IAutoBeFacadeApplication";
 import { AutoBeTokenUsageComponent } from "./AutoBeTokenUsageComponent";
-import { IAutoBeFacadeApplication } from "./IAutoBeFacadeApplication";
 
 /**
  * Comprehensive token usage tracker for the AutoBe vibe coding system.

package/src/factory/AutoBeFunctionCallingMetricFactory.ts

@@ -0,0 +1,44 @@
+import { AutoBeFunctionCallingMetric } from "@autobe/interface";
+
+export namespace AutoBeFunctionCallingMetricFactory {
+  export const create = (): AutoBeFunctionCallingMetric => ({
+    attempt: 0,
+    success: 0,
+    consent: 0,
+    validationFailure: 0,
+    invalidJson: 0,
+  });
+
+  export const increment = (
+    x: AutoBeFunctionCallingMetric,
+    y: AutoBeFunctionCallingMetric,
+  ): void => {
+    x.attempt += y.attempt;
+    x.success += y.success;
+    x.consent += y.consent;
+    x.validationFailure += y.validationFailure;
+    x.invalidJson += y.invalidJson;
+  };
+
+  export const plus = (
+    x: AutoBeFunctionCallingMetric,
+    y: AutoBeFunctionCallingMetric,
+  ): AutoBeFunctionCallingMetric => ({
+    attempt: x.attempt + y.attempt,
+    success: x.success + y.success,
+    consent: x.consent + y.consent,
+    validationFailure: x.validationFailure + y.validationFailure,
+    invalidJson: x.invalidJson + y.invalidJson,
+  });
+
+  export const minus = (
+    x: AutoBeFunctionCallingMetric,
+    y: AutoBeFunctionCallingMetric,
+  ): AutoBeFunctionCallingMetric => ({
+    attempt: x.attempt - y.attempt,
+    success: x.success - y.success,
+    consent: x.consent - y.consent,
+    validationFailure: x.validationFailure - y.validationFailure,
+    invalidJson: x.invalidJson - y.invalidJson,
+  });
+}

package/src/factory/AutoBeProcessAggregateFactory.ts

@@ -0,0 +1,141 @@
+import {
+  AutoBeAggregateEventBase,
+  AutoBePhase,
+  AutoBeProcessAggregate,
+  AutoBeProcessAggregateCollection,
+} from "@autobe/interface";
+
+import { AutoBeTokenUsageComponent } from "../context/AutoBeTokenUsageComponent";
+import { TokenUsageComputer } from "../utils/TokenUsageComputer";
+import { AutoBeFunctionCallingMetricFactory } from "./AutoBeFunctionCallingMetricFactory";
+
+export namespace AutoBeProcessAggregateFactory {
+  export const createAggregate = (): AutoBeProcessAggregate => ({
+    metric: AutoBeFunctionCallingMetricFactory.create(),
+    tokenUsage: new AutoBeTokenUsageComponent().toJSON(),
+  });
+
+  export const createCollection = <
+    Phase extends AutoBePhase | "all",
+  >(): AutoBeProcessAggregateCollection<Phase> =>
+    ({
+      total: createAggregate(),
+    }) satisfies AutoBeProcessAggregateCollection as AutoBeProcessAggregateCollection<Phase>;
+
+  export const computeTotal = <Phase extends AutoBePhase | "all">(
+    collection: AutoBeProcessAggregateCollection<Phase>,
+  ): AutoBeProcessAggregate => {
+    const total: AutoBeProcessAggregate = createAggregate();
+    for (const [key, value] of Object.entries(collection)) {
+      if (key === "total") continue;
+      AutoBeFunctionCallingMetricFactory.increment(total.metric, value.metric);
+      TokenUsageComputer.increment(total.tokenUsage, value.tokenUsage);
+    }
+    return total;
+  };
+
+  export const emplaceEvent = <
+    Event extends AutoBeAggregateEventBase & {
+      type: string;
+    },
+  >(
+    collection: AutoBeProcessAggregateCollection,
+    event: Event,
+  ): void => {
+    (collection as any)[event.type] ??= createAggregate();
+    collection.total ??= computeTotal(collection);
+
+    const local: AutoBeProcessAggregate = (collection as any)[
+      event.type
+    ] as AutoBeProcessAggregate;
+    const total: AutoBeProcessAggregate = collection.total;
+
+    AutoBeFunctionCallingMetricFactory.increment(local.metric, event.metric);
+    AutoBeFunctionCallingMetricFactory.increment(total.metric, event.metric);
+    TokenUsageComputer.increment(local.tokenUsage, event.tokenUsage);
+    TokenUsageComputer.increment(total.tokenUsage, event.tokenUsage);
+  };
+
+  export const filterPhase = <Phase extends AutoBePhase>(
+    collection: AutoBeProcessAggregateCollection,
+    phase: Phase,
+  ): AutoBeProcessAggregateCollection<Phase> => {
+    const result: AutoBeProcessAggregateCollection<Phase> = createCollection();
+    for (const [key, value] of Object.entries(collection)) {
+      if (key === "total") continue;
+      else if (key.startsWith(phase) === false) continue;
+
+      (result as any)[key] = value;
+      AutoBeFunctionCallingMetricFactory.increment(
+        result.total.metric,
+        value.metric,
+      );
+      TokenUsageComputer.increment(result.total.tokenUsage, value.tokenUsage);
+    }
+    return result;
+  };
+
+  export const reduce = (
+    collections: AutoBeProcessAggregateCollection[],
+  ): AutoBeProcessAggregateCollection => {
+    const result: AutoBeProcessAggregateCollection = createCollection();
+    for (const collection of collections) {
+      for (const [key, value] of Object.entries(collection)) {
+        if (key === "total") continue;
+        (result as any)[key] ??= createAggregate();
+        const local: AutoBeProcessAggregate = (result as any)[
+          key
+        ] as AutoBeProcessAggregate;
+        AutoBeFunctionCallingMetricFactory.increment(
+          local.metric,
+          value.metric,
+        );
+        TokenUsageComputer.increment(local.tokenUsage, value.tokenUsage);
+        AutoBeFunctionCallingMetricFactory.increment(
+          result.total.metric,
+          value.metric,
+        );
+        TokenUsageComputer.increment(result.total.tokenUsage, value.tokenUsage);
+      }
+    }
+    result.total ??= createAggregate();
+    Object.assign(result.total, computeTotal(result));
+    return result;
+  };
+
+  export const increment = (
+    x: AutoBeProcessAggregateCollection,
+    y: AutoBeProcessAggregateCollection,
+  ): void => {
+    for (const [key, value] of Object.entries(y)) {
+      if (key === "total") continue;
+      (x as any)[key] ??= createAggregate();
+      const local: AutoBeProcessAggregate = (x as any)[
+        key
+      ] as AutoBeProcessAggregate;
+      AutoBeFunctionCallingMetricFactory.increment(local.metric, value.metric);
+    }
+    x.total ??= createAggregate();
+    Object.assign(x.total, computeTotal(x));
+  };
+
+  export const minus = (
+    x: AutoBeProcessAggregateCollection,
+    y: AutoBeProcessAggregateCollection,
+  ): AutoBeProcessAggregateCollection => {
+    const result = JSON.parse(
+      JSON.stringify(x),
+    ) as AutoBeProcessAggregateCollection;
+    for (const [key, value] of Object.entries(y)) {
+      if (key === "total") continue;
+      (result as any)[key] ??= createAggregate();
+      const local: AutoBeProcessAggregate = (result as any)[
+        key
+      ] as AutoBeProcessAggregate;
+      AutoBeFunctionCallingMetricFactory.minus(local.metric, value.metric);
+    }
+    result.total ??= createAggregate();
+    Object.assign(result.total, computeTotal(result));
+    return result;
+  };
+}

package/src/factory/consentFunctionCall.ts

@@ -16,6 +16,7 @@ import { AutoBeConfigConstant } from "../constants/AutoBeConfigConstant";
 import { AutoBeSystemPromptConstant } from "../constants/AutoBeSystemPromptConstant";
 import { IAutoBeConfig } from "../structures/IAutoBeConfig";
 import { IAutoBeVendor } from "../structures/IAutoBeVendor";
+import { getCommonPrompt } from "./getCommonPrompt";
 import { supportMistral } from "./supportMistral";
 
 export const consentFunctionCall = async (props: {
@@ -37,6 +38,9 @@ export const consentFunctionCall = async (props: {
       executor: {
         describe: null,
       },
+      systemPrompt: {
+        common: () => getCommonPrompt(props.config),
+      },
     },
     histories: [
       {

package/src/factory/createAutoBeContext.ts

@@ -5,6 +5,7 @@ import {
   AutoBeAnalyzeStartEvent,
   AutoBeAssistantMessageEvent,
   AutoBeEvent,
+  AutoBeFunctionCallingMetric,
   AutoBeHistory,
   AutoBeInterfaceCompleteEvent,
   AutoBeInterfaceHistory,
@@ -12,6 +13,8 @@ import {
   AutoBePrismaCompleteEvent,
   AutoBePrismaHistory,
   AutoBePrismaStartEvent,
+  AutoBeProcessAggregate,
+  AutoBeProcessAggregateCollection,
   AutoBeRealizeCompleteEvent,
   AutoBeRealizeHistory,
   AutoBeRealizeStartEvent,
@@ -34,12 +37,15 @@ import { AutoBeContext } from "../context/AutoBeContext";
 import { AutoBeState } from "../context/AutoBeState";
 import { AutoBeTokenUsage } from "../context/AutoBeTokenUsage";
 import { AutoBeTokenUsageComponent } from "../context/AutoBeTokenUsageComponent";
-import { IAutoBeFacadeApplication } from "../context/IAutoBeFacadeApplication";
+import { IAutoBeFacadeApplication } from "../orchestrate/facade/histories/IAutoBeFacadeApplication";
 import { IAutoBeConfig } from "../structures/IAutoBeConfig";
 import { IAutoBeVendor } from "../structures/IAutoBeVendor";
 import { AutoBeTimeoutError } from "../utils/AutoBeTimeoutError";
 import { TimedConversation } from "../utils/TimedConversation";
+import { TokenUsageComputer } from "../utils/TokenUsageComputer";
+import { AutoBeProcessAggregateFactory } from "./AutoBeProcessAggregateFactory";
 import { consentFunctionCall } from "./consentFunctionCall";
+import { getCommonPrompt } from "./getCommonPrompt";
 import { getCriticalCompiler } from "./getCriticalCompiler";
 import { supportMistral } from "./supportMistral";
 
@@ -54,6 +60,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
   histories: () => AutoBeHistory[];
   usage: () => AutoBeTokenUsage;
   dispatch: (event: AutoBeEvent) => Promise<void>;
+  aggregates: AutoBeProcessAggregateCollection;
 }): AutoBeContext<Model> => {
   const config: Required<Omit<IAutoBeConfig, "backoffStrategy" | "timezone">> =
     {
@@ -67,6 +74,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
     vendor: props.vendor,
     retry: config.retry,
     locale: config.locale,
+    aggregates: props.aggregates,
     compilerListener: props.compilerListener,
     compiler: async () => {
       const compiler = await props.compiler();
@@ -85,7 +93,37 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
       return message;
     },
     conversate: async (next, closure) => {
-      const trial = {
+      const aggregate: AutoBeProcessAggregate =
+        AutoBeProcessAggregateFactory.createAggregate();
+      const metric = (key: keyof AutoBeFunctionCallingMetric) => {
+        const accumulate = (collection: AutoBeProcessAggregateCollection) => {
+          ++collection.total.metric[key];
+          collection[next.source as "analyzeWrite"] ??=
+            AutoBeProcessAggregateFactory.createAggregate();
+          ++collection[next.source as "analyzeWrite"]!.metric[key];
+        };
+        ++aggregate.metric[key];
+        accumulate(props.aggregates);
+      };
+      const consume = (tokenUsage: IAutoBeTokenUsageJson.IComponent) => {
+        const accumulate = (collection: AutoBeProcessAggregateCollection) => {
+          TokenUsageComputer.increment(collection.total.tokenUsage, tokenUsage);
+          collection[next.source as "analyzeWrite"] ??=
+            AutoBeProcessAggregateFactory.createAggregate();
+          TokenUsageComputer.increment(
+            collection[next.source as "analyzeWrite"]!.tokenUsage,
+            tokenUsage,
+          );
+        };
+        TokenUsageComputer.increment(aggregate.tokenUsage, tokenUsage);
+        accumulate(props.aggregates);
+        props
+          .usage()
+          .record(tokenUsage, [
+            STAGES.find((stage) => next.source.startsWith(stage)) ?? "analyze",
+          ]);
+      };
+      const progress = {
         request: 0,
         response: 0,
         timeout: 0,
@@ -101,6 +139,9 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
             executor: {
               describe: null,
             },
+            systemPrompt: {
+              common: () => getCommonPrompt(props.config),
+            },
           },
           histories: next.histories,
           controllers: [next.controller],
@@ -119,7 +160,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
             ...event,
             type: "vendorRequest",
             source: next.source,
-            retry: trial.request++,
+            retry: progress.request++,
           });
         });
         agent.on("response", async (event) => {
@@ -128,11 +169,15 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
               ...event,
               type: "vendorResponse",
               source: next.source,
-              retry: trial.response++,
+              retry: progress.response++,
             })
             .catch(() => {});
         });
+        agent.on("call", () => {
+          metric("attempt");
+        });
         agent.on("jsonParseError", (event) => {
+          metric("invalidJson");
           void props
             .dispatch({
               ...event,
@@ -141,6 +186,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
             .catch(() => {});
         });
         agent.on("validate", (event) => {
+          metric("validationFailure");
           void props
             .dispatch({
               type: "jsonValidateError",
@@ -189,7 +235,16 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
           .record(tokenUsage, [
             STAGES.find((stage) => next.source.startsWith(stage)) ?? "analyze",
           ]);
+        consume(tokenUsage);
 
+        const success = (histories: MicroAgenticaHistory<Model>[]) => {
+          metric("success");
+          return {
+            histories,
+            tokenUsage: aggregate.tokenUsage,
+            metric: aggregate.metric,
+          };
+        };
         if (result.type === "error") throw result.error;
         else if (result.type === "timeout") {
           void props
@@ -198,7 +253,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
               id: v7(),
               source: next.source,
               timeout: config.timeout!,
-              retry: trial.timeout++,
+              retry: progress.timeout++,
               created_at: new Date().toISOString(),
             })
             .catch(() => {});
@@ -229,6 +284,7 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
             last?.type === "assistantMessage" &&
             last.text.trim().length !== 0
           ) {
+            metric("consent");
             const consent: string | null = await consentFunctionCall({
               source: next.source,
               dispatch: (e) => {
@@ -241,43 +297,46 @@ export const createAutoBeContext = <Model extends ILlmSchema.Model>(props: {
             if (consent !== null) {
               const newHistories: MicroAgenticaHistory<Model>[] =
                 await agent.conversate(consent);
-              const newTokenUsage: IAutoBeTokenUsageJson.IComponent = agent
-                .getTokenUsage()
-                .toJSON().aggregate;
-              props
-                .usage()
-                .record(
-                  AutoBeTokenUsageComponent.minus(
-                    new AutoBeTokenUsageComponent(newTokenUsage),
-                    new AutoBeTokenUsageComponent(tokenUsage),
+              const newTokenUsage: IAutoBeTokenUsageJson.IComponent =
+                AutoBeTokenUsageComponent.minus(
+                  new AutoBeTokenUsageComponent(
+                    agent.getTokenUsage().toJSON().aggregate,
                   ),
-                  [
-                    STAGES.find((stage) => next.source.startsWith(stage)) ??
-                      "analyze",
-                  ],
+                  new AutoBeTokenUsageComponent(tokenUsage),
                 );
-              if (
-                newHistories.some(
-                  (h) => h.type === "execute" && h.success === true,
-                )
-              )
-                return {
-                  histories: newHistories,
-                  tokenUsage: newTokenUsage,
-                };
+              consume(newTokenUsage);
+              if (newHistories.some((h) => h.type === "execute" && h.success))
+                return success(newHistories);
             }
           }
           failure();
         }
-        return {
-          histories: result.histories,
-          tokenUsage,
-        };
+        return success(result.histories);
       };
       if (next.enforceFunctionCall === true)
         return await forceRetry(execute, config.retry);
       else return await execute();
     },
+    getCurrentAggregates: (phase) => {
+      const previous: AutoBeProcessAggregateCollection =
+        AutoBeProcessAggregateFactory.reduce(
+          props
+            .histories()
+            .filter(
+              (h) =>
+                h.type === "analyze" ||
+                h.type === "prisma" ||
+                h.type === "interface" ||
+                h.type === "test" ||
+                h.type === "realize",
+            )
+            .map((h) => h.aggregates),
+        );
+      return AutoBeProcessAggregateFactory.filterPhase(
+        AutoBeProcessAggregateFactory.minus(props.aggregates, previous),
+        phase,
+      );
+    },
   };
 };
 
@@ -311,11 +370,12 @@ const createDispatch = (props: {
           type: "analyze",
           id: v7(),
           prefix: event.prefix,
-          roles: event.roles,
+          actors: event.actors,
           files: event.files,
+          aggregates: event.aggregates,
+          step: event.step,
           created_at: analyzeStart?.created_at ?? new Date().toISOString(),
           completed_at: event.created_at,
-          step: event.step,
         } satisfies AutoBeAnalyzeHistory,
       }) as AutoBeContext.DispatchHistory<Event>;
     else if (event.type === "prismaComplete")
@@ -331,9 +391,10 @@ const createDispatch = (props: {
           schemas: event.schemas,
           result: event.result,
           compiled: event.compiled,
+          aggregates: event.aggregates,
+          step: event.step,
           created_at: prismaStart?.created_at ?? new Date().toISOString(),
           completed_at: event.created_at,
-          step: event.step,
         } satisfies AutoBePrismaHistory,
       }) as AutoBeContext.DispatchHistory<Event>;
     else if (event.type === "interfaceComplete")
@@ -349,9 +410,10 @@ const createDispatch = (props: {
           authorizations: event.authorizations,
           document: event.document,
           missed: event.missed,
+          aggregates: event.aggregates,
+          step: event.step,
           created_at: interfaceStart?.created_at ?? new Date().toISOString(),
           completed_at: new Date().toISOString(),
-          step: event.step,
         } satisfies AutoBeInterfaceHistory,
       }) as AutoBeContext.DispatchHistory<Event>;
     else if (event.type === "testComplete")
@@ -366,9 +428,10 @@ const createDispatch = (props: {
           instruction: testStart?.reason ?? "",
           files: event.files,
           compiled: event.compiled,
+          aggregates: event.aggregates,
+          step: event.step,
           created_at: testStart?.created_at ?? new Date().toISOString(),
           completed_at: new Date().toISOString(),
-          step: event.step,
         } satisfies AutoBeTestHistory,
       }) as AutoBeContext.DispatchHistory<Event>;
     else if (event.type === "realizeComplete")
@@ -385,9 +448,10 @@ const createDispatch = (props: {
           functions: event.functions,
           controllers: event.controllers,
           compiled: event.compiled,
+          aggregates: event.aggregates,
+          step: event.step,
           created_at: realizeStart?.created_at ?? new Date().toISOString(),
           completed_at: new Date().toISOString(),
-          step: event.step,
         } satisfies AutoBeRealizeHistory,
       }) as AutoBeContext.DispatchHistory<Event>;
     void props.dispatch(event).catch(() => {});

package/src/factory/getCommonPrompt.ts

@@ -0,0 +1,25 @@
+import { Singleton, is_node } from "tstl";
+
+import { AutoBeSystemPromptConstant } from "../constants/AutoBeSystemPromptConstant";
+import { IAutoBeConfig } from "../structures/IAutoBeConfig";
+
+export const getCommonPrompt = (
+  config?: Pick<IAutoBeConfig, "locale" | "timezone"> | undefined,
+) =>
+  AutoBeSystemPromptConstant.COMMON.replace(
+    "${locale}",
+    config?.locale ?? locale.get(),
+  )
+    .replace("${timezone}", config?.timezone ?? timezone.get())
+    .replace("${datetime}", new Date().toISOString());
+
+const locale = new Singleton(() =>
+  is_node()
+    ? // eslint-disable-next-line node/prefer-global/process
+      (process.env.LANG?.split(".")[0] ?? "en-US")
+    : navigator.language,
+);
+
+const timezone = new Singleton(
+  () => Intl.DateTimeFormat().resolvedOptions().timeZone,
+);

package/src/orchestrate/analyze/histories/transformAnalyzeWriteHistories.ts

@@ -53,14 +53,14 @@ export const transformAnalyzeWriteHistories = <Model extends ILlmSchema.Model>(
       ## Metadata
       
       Prefix name of the service to create is ${props.scenario.prefix}
-      and here is the list of the roles to reference.
-      
+      and here is the list of the actors to reference.
+
       \`\`\`json
-      ${JSON.stringify(props.scenario.roles)}
+      ${JSON.stringify(props.scenario.actors)}
       \`\`\`
-      
+
       Here is the entire list of the documents that would be published
-      in someday, and your role is to writing a document of them:
+      in someday, and your task is to write a document of them:
       
       ## The other documents that would be published in someday
 

package/src/orchestrate/analyze/orchestrateAnalyze.ts

@@ -89,9 +89,10 @@ export const orchestrateAnalyze = async <Model extends ILlmSchema.Model>(
   return ctx.dispatch({
     type: "analyzeComplete",
     id: v7(),
-    roles: scenario.roles,
+    actors: scenario.actors,
     prefix: scenario.prefix,
     files: newFiles,
+    aggregates: ctx.getCurrentAggregates("analyze"),
     step,
     elapsed: new Date().getTime() - startTime.getTime(),
     created_at: new Date().toISOString(),

package/src/orchestrate/analyze/orchestrateAnalyzeReview.ts

@@ -28,7 +28,7 @@ export const orchestrateAnalyzeReview = async <Model extends ILlmSchema.Model>(
   const pointer: IPointer<IAutoBeAnalyzeReviewApplication.IProps | null> = {
     value: null,
   };
-  const { tokenUsage } = await ctx.conversate({
+  const { metric, tokenUsage } = await ctx.conversate({
     source: "analyzeReview",
     controller: createController({
       model: ctx.model,
@@ -57,6 +57,7 @@ export const orchestrateAnalyzeReview = async <Model extends ILlmSchema.Model>(
     review: pointer.value.review,
     content: pointer.value.content,
     tokenUsage,
+    metric,
     total: props.progress.total,
     completed: ++props.progress.completed,
     step: (ctx.state().analyze?.step ?? -1) + 1,

package/src/orchestrate/analyze/orchestrateAnalyzeScenario.ts

@@ -26,7 +26,7 @@ export const orchestrateAnalyzeScenario = async <
   const pointer: IPointer<IAutoBeAnalyzeScenarioApplication.IProps | null> = {
     value: null,
   };
-  const { histories, tokenUsage } = await ctx.conversate({
+  const { histories, tokenUsage, metric } = await ctx.conversate({
     source: "analyzeScenario",
     controller: createController<Model>({
       model: ctx.model,
@@ -35,8 +35,8 @@ export const orchestrateAnalyzeScenario = async <
     histories: transformAnalyzeSceHistories(ctx),
     enforceFunctionCall: false,
     message: StringUtil.trim`
-      Design a complete list of documents and user roles for this project.
-      Define user roles that can authenticate via API and create appropriate documentation files.
+      Design a complete list of documents and user actors for this project.
+      Define user actors that can authenticate via API and create appropriate documentation files.
       You must respect the number of documents specified by the user.
       Note that the user's locale is in ${ctx.locale}.
     `,
@@ -57,8 +57,9 @@ export const orchestrateAnalyzeScenario = async <
     id: v7(),
     prefix: pointer.value.prefix,
     language: pointer.value.language,
-    roles: pointer.value.roles,
+    actors: pointer.value.actors,
     files: pointer.value.files,
+    metric,
     tokenUsage,
     step: (ctx.state().analyze?.step ?? -1) + 1,
     created_at: start.toISOString(),

package/src/orchestrate/analyze/orchestrateAnalyzeWrite.ts

@@ -28,7 +28,7 @@ export const orchestrateAnalyzeWrite = async <Model extends ILlmSchema.Model>(
   const pointer: IPointer<IAutoBeAnalyzeWriteApplication.IProps | null> = {
     value: null,
   };
-  const { tokenUsage } = await ctx.conversate({
+  const { metric, tokenUsage } = await ctx.conversate({
     source: "analyzeWrite",
     controller: createController<Model>({
       model: ctx.model,
@@ -49,7 +49,8 @@ export const orchestrateAnalyzeWrite = async <Model extends ILlmSchema.Model>(
       ...file,
       content: pointer.value.content,
     },
-    tokenUsage: tokenUsage,
+    tokenUsage,
+    metric,
     step: (ctx.state().analyze?.step ?? -1) + 1,
     total: progress.total,
     completed: ++progress.completed,