@dxos/assistant 0.8.4-main.b97322e → 0.8.4-main.bc2380dfbc

package/src/session/session.ts

@@ -1,336 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { AiLanguageModel, AiToolkit, type AiError, type AiResponse, type AiTool } from '@effect/ai';
-import { Chunk, Context, Effect, Option, pipe, Stream, type Schema } from 'effect';
-import { Array, String } from 'effect';
-
-import {
-  AiParser,
-  AiPreprocessor,
-  getToolCalls,
-  runTool,
-  ToolExecutionService,
-  ToolResolverService,
-  type AgentStatus,
-  type AiInputPreprocessingError,
-  type AiToolNotFoundError,
-  type GenerationStream,
-} from '@dxos/ai';
-import { Event } from '@dxos/async';
-import { type Blueprint } from '@dxos/blueprints';
-import { todo } from '@dxos/debug';
-import { Obj } from '@dxos/echo';
-import { ObjectVersion } from '@dxos/echo-db';
-import { type ObjectId } from '@dxos/echo-schema';
-import { DatabaseService } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { DataType, type ContentBlock } from '@dxos/schema';
-
-import { AiAssistantError } from '../errors';
-
-export type AiSessionOptions = {};
-
-export type SessionRunOptions<Tools extends AiTool.Any> = {
-  prompt: string;
-  history: DataType.Message[];
-  systemPrompt?: string;
-  toolkit?: AiToolkit.AiToolkit<Tools>;
-  blueprints?: Blueprint.Blueprint[];
-};
-
-/**
- * Contains message history, tools, current context.
- * Current context means the state of the app, time of day, and other contextual information.
- * It makes requests to the model, its a state machine.
- * It keeps track of the current goal.
- * It manages the context window.
- * Tracks the success criteria of reaching the goal, exposing metrics (stretch).
- * Could be run locally in the app or remotely.
- * Could be personal or shared.
- */
-export class AiSession {
-  // TODO(burdon): Move to conversation.
-  private readonly _semaphore = Effect.runSync(Effect.makeSemaphore(1));
-
-  /** Pending messages (incl. the current user request). */
-  private _pending: DataType.Message[] = [];
-
-  /** Current streaming response. */
-  private _stream: GenerationStream | undefined;
-
-  /** Prior history from queue. */
-  private _history: DataType.Message[] = [];
-
-  /**
-   * New message.
-   */
-  public readonly message = new Event<DataType.Message>();
-
-  /**
-   * Complete block added to Message.
-   */
-  public readonly block = new Event<ContentBlock.Any>();
-
-  /**
-   * Update partial block (while streaming).
-   */
-  public readonly update = new Event<ContentBlock.Any>();
-
-  /**
-   * Unparsed events from the underlying generation stream.
-   */
-  public readonly streamEvent = new Event<AiResponse.Part>();
-
-  /**
-   * User prompt or tool result.
-   */
-  public readonly userMessage = new Event<DataType.Message>();
-
-  /**
-   * Agent self-reporting its status.
-   * Triggered by the model.
-   */
-  public readonly statusReport = new Event<AgentStatus>();
-
-  /**
-   * Emits when the session is done.
-   */
-  public readonly done = new Event<void>();
-
-  constructor(private readonly _options: AiSessionOptions = {}) {}
-
-  /**
-   * Runs the AI model loop interacting with tools and artifacts.
-   * @param options - The session options.
-   * @returns The messages generated by the session, including the user's prompt.
-   */
-  // TODO(dmaretskyi): Toolkit context doesn't get added to the effect type.
-  run = <Tools extends AiTool.Any>(
-    options: SessionRunOptions<Tools>,
-  ): Effect.Effect<
-    DataType.Message[],
-    AiAssistantError | AiInputPreprocessingError | AiToolNotFoundError | AiError.AiError,
-    AiLanguageModel.AiLanguageModel | ToolResolverService | ToolExecutionService | AiTool.ToHandler<Tools>
-  > =>
-    Effect.gen(this, function* () {
-      this._history = [...options.history];
-
-      const promptMessages = yield* this._formatUserPrompt(options.prompt, options.history);
-      this._pending = [promptMessages];
-      this.userMessage.emit(promptMessages);
-
-      // Potential tool use loop.
-      do {
-        log('request', {
-          pending: this._pending.length,
-          history: this._history.length,
-          tools: (Object.values(options.toolkit?.tools ?? {}) as AiTool.Any[]).map((tool: AiTool.Any) => tool.name),
-        });
-
-        const prompt = yield* AiPreprocessor.preprocessAiInput([...this._history, ...this._pending]);
-
-        const blueprints = options.blueprints ?? [];
-
-        // Build system prompt from blueprint templates.
-        // TODO(dmaretskyi): Loading BP from the Database should be done at the higher level. We need a type for the resolved blueprint.
-        const systemPrompt = yield* pipe(
-          blueprints,
-          Effect.forEach((blueprint) => DatabaseService.loadRef(blueprint.instructions)),
-          Effect.flatMap(Effect.forEach((template) => DatabaseService.loadRef(template.source))),
-          Effect.map(Array.map((template) => `\n\n<blueprint>${template.content}</blueprint>`)),
-          Effect.map(Array.reduce(options.systemPrompt ?? '', String.concat)),
-        );
-
-        // Build a combined toolkit from the blueprint tools and the provided toolkit.
-        const blueprintToolkit = yield* ToolResolverService.resolveToolkit(blueprints.flatMap((bp) => bp.tools));
-        const blueprintToolkitHandler: Context.Context<AiTool.ToHandler<AiTool.Any>> =
-          yield* blueprintToolkit.toContext(yield* ToolExecutionService.handlersFor(blueprintToolkit));
-        const toolkit = options.toolkit != null ? AiToolkit.merge(options.toolkit, blueprintToolkit) : blueprintToolkit;
-        const toolkitWithBlueprintHandlers = yield* toolkit.pipe(
-          Effect.provide(blueprintToolkitHandler) as any,
-        ) as Effect.Effect<AiToolkit.ToHandler<any>, never, AiTool.ToHandler<Tools>>;
-
-        log.info('run', {
-          systemPrompt: [systemPrompt.slice(0, 32), '...', systemPrompt.slice(-32), systemPrompt.length].join(' '),
-          tools: Object.keys(blueprintToolkit.tools),
-        });
-
-        const blocks = yield* AiLanguageModel.streamText({
-          prompt,
-          system: systemPrompt,
-          toolkit: toolkitWithBlueprintHandlers,
-          disableToolCallResolution: true,
-        }).pipe(
-          AiParser.parseGptStream({
-            onBlock: (block) =>
-              Effect.gen(this, function* () {
-                if (block.pending) {
-                  this.update.emit(block);
-                } else {
-                  this.block.emit(block);
-                }
-              }),
-            onPart: (part) =>
-              Effect.gen(this, function* () {
-                this.streamEvent.emit(part);
-              }),
-          }),
-          Stream.runCollect,
-          Effect.map(Chunk.toArray),
-        );
-
-        const response = Obj.make(DataType.Message, {
-          created: new Date().toISOString(),
-          sender: { role: 'assistant' },
-          blocks,
-        });
-        this._pending.push(response);
-        this.message.emit(response);
-
-        const toolCalls = getToolCalls(response);
-        if (toolCalls.length === 0) {
-          break;
-        }
-
-        const toolResults = yield* runTools(toolCalls, toolkitWithBlueprintHandlers as any);
-        this._pending.push(
-          Obj.make(DataType.Message, {
-            created: new Date().toISOString(),
-            sender: { role: 'user' },
-            blocks: toolResults,
-          }),
-        );
-      } while (true);
-
-      return this._pending;
-    }).pipe(this._semaphore.withPermits(1), Effect.withSpan('AiSession.run'));
-
-  async runStructured<S extends Schema.Schema.AnyNoContext>(
-    schema: S,
-    options: SessionRunOptions<AiTool.Any>,
-  ): Promise<Schema.Schema.Type<S>> {
-    return todo();
-    // const parser = structuredOutputParser(schema);
-    // const result = await this.run({
-    //   ...options,
-    //   executableTools: [...(options.executableTools ?? []), parser.tool],
-    // });
-    // return parser.getResult(result);
-  }
-
-  private _formatUserPrompt = (prompt: string, history: DataType.Message[]) =>
-    Effect.gen(function* () {
-      const prelude: ContentBlock.Any[] = [];
-
-      // TODO(dmaretskyi): Evaluate other approaches as `serviceOption` isn't represented in the type system.
-      const artifactDiffResolver = yield* Effect.serviceOption(ArtifactDiffResolver);
-      if (Option.isSome(artifactDiffResolver)) {
-        const versions = gatherObjectVersions(history);
-
-        const artifactDiff = yield* Effect.tryPromise({
-          try: () =>
-            artifactDiffResolver.value.resolve(
-              [...versions.entries()].map(([id, version]) => ({ id, lastVersion: version })),
-            ),
-          catch: AiAssistantError.wrap('Artifact diff resolution error'),
-        });
-
-        log.info('vision', {
-          artifactDiff,
-          versions,
-        });
-
-        for (const [id, { version }] of [...artifactDiff.entries()]) {
-          if (ObjectVersion.equals(version, versions.get(id)!)) {
-            artifactDiff.delete(id);
-            continue;
-          }
-
-          prelude.push({ _tag: 'anchor', objectId: id, version });
-        }
-        if (artifactDiff.size > 0) {
-          prelude.push(createArtifactUpdateBlock(artifactDiff));
-        }
-      }
-
-      return Obj.make(DataType.Message, {
-        created: new Date().toISOString(),
-        sender: { role: 'user' },
-        blocks: [...prelude, { _tag: 'text', text: prompt }],
-      });
-    });
-
-  abort(): void {
-    this._stream?.abort();
-  }
-}
-
-const gatherObjectVersions = (messages: DataType.Message[]): Map<ObjectId, ObjectVersion> => {
-  const artifactIds = new Map<ObjectId, ObjectVersion>();
-  for (const message of messages) {
-    for (const block of message.blocks) {
-      if (block._tag === 'anchor') {
-        artifactIds.set(block.objectId, block.version as ObjectVersion);
-      }
-    }
-  }
-
-  return artifactIds;
-};
-
-const createArtifactUpdateBlock = (
-  artifactDiff: Map<ObjectId, { version: ObjectVersion; diff?: string }>,
-): ContentBlock.Any => {
-  return {
-    _tag: 'text',
-    // TODO(dmaretskyi): Does this need to be a special content-block?
-    disposition: 'artifact-update',
-    text: `
-      The following artifacts have been updated since the last message:
-      ${[...artifactDiff.entries()]
-        .map(([id, { diff }]) => `<changed-artifact id="${id}">${diff ? `\n${diff}` : ''}</changed-artifact>`)
-        .join('\n')}
-    `,
-  };
-};
-
-/**
- * Resolves artifact ids to their versions.
- * Used to give the model a sense of the changes to the artifacts made by users during the conversation.
- * The artifacts versions are pinned in the history, and whenever the artifact changes in-between assistant's steps,
- * a diff is inserted into the conversation.
- *
- * Can be optionally provided to the session run call.
- */
-// TODO(dmaretskyi): Convert to Context.Reference
-export class ArtifactDiffResolver extends Context.Tag('ArtifactDiffResolver')<
-  ArtifactDiffResolver,
-  ArtifactDiffResolver.Service
->() {}
-
-export namespace ArtifactDiffResolver {
-  export type Service = {
-    resolve: (artifacts: { id: ObjectId; lastVersion: ObjectVersion }[]) => Promise<
-      Map<
-        ObjectId,
-        {
-          version: ObjectVersion;
-          diff?: string;
-        }
-      >
-    >;
-  };
-}
-
-const runTools: <Tools extends AiTool.Any>(
-  toolCalls: ContentBlock.ToolCall[],
-  toolkit: AiToolkit.AiToolkit<Tools>,
-) => Effect.Effect<ContentBlock.ToolResult[], AiError.AiError, AiTool.ToHandler<Tools>> = Effect.fn('runTools')(
-  function* (toolCalls, toolkit) {
-    const toolkitWithHandlers = Effect.isEffect(toolkit) ? yield* toolkit : toolkit;
-
-    return yield* Effect.forEach(toolCalls, (toolCall) => runTool(toolkitWithHandlers, toolCall));
-  },
-);

package/src/session/version-pin.test.ts

@@ -1,7 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { describe } from 'vitest';
-
-describe('VersionPin', () => {});

package/src/summarization/summarization.test.ts

@@ -1,208 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-// TODO(burdon): Fix.
-// @ts-nocheck
-
-import { pipe, Schema } from 'effect';
-import { beforeAll, describe, test } from 'vitest';
-
-import { AiService, MixedStreamParser } from '@dxos/ai';
-import { Obj } from '@dxos/echo';
-import { type EchoDatabase } from '@dxos/echo-db';
-import { EchoTestBuilder } from '@dxos/echo-db/testing';
-import { defineFunction, FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { type ContentBlock, DataType } from '@dxos/schema';
-import { createTestData } from '@dxos/schema/testing';
-import { trim } from '@dxos/util';
-
-const REMOTE_AI = true;
-
-const summarizationFn = defineFunction({
-  description: 'Summarize a document',
-  inputSchema: Schema.Struct({
-    document: Schema.optional(DataType.Text),
-    transcript: Schema.Array(DataType.Message),
-  }),
-  outputSchema: DataType.Text,
-  handler: async ({ data: { document, transcript }, context }) => {
-    const ai = context.getService(AiService);
-    const result = await new MixedStreamParser().parse(
-      await ai.client.execStream({
-        model: '@anthropic/claude-3-5-haiku-20241022',
-        // TODO(burdon): Factor out and test prompts.
-        systemPrompt: trim`
-          Your job is to summarize a meeting transcript.
-          Transcript Summarizer updates an existing summary within <summary> tags with a new transcript that is placed in <transcript> tags.
-          Transcript Summarizer updates only the relevant parts of the summary.
-          If there's no summary, create a new one.
-          Transcript Summarizer outputs only the summary texts and nothing else.
-          Transcript Summarizer keeps the original summary structure.
-          Transcript Summarizer keeps the unrelated parts of the summary untouched.
-          The Transcript Summarizer errs on the side of being too verbose.
-          The Transcript Summarizer outputs the entire update summary.
-          The summary is formatted as hierarchical markdown bullet points with no headings.
-          The hierarchical structure is deeply nested.
-          The Transcript Summarizer formats the bullets with "-" symbol as prefix and two spaces indentation.
-          The Transcript Summarizer knows that the content of transcript will be updated solely by its output.
-          The Transcript Summarizer outputs only the summary text.
-        `,
-        history: [
-          Obj.make(DataType.Message, {
-            created: new Date().toISOString(),
-            sender: { role: 'user' },
-            blocks: [
-              {
-                _tag: 'text',
-                text: trim`
-                  <transcript>
-                    ${JSON.stringify(transcript, null, 2)}
-                  </transcript>
-                `,
-              },
-              ...(document != null && document.content.length > 0
-                ? [
-                    {
-                      _tag: 'text',
-                      text: trim`
-                        <summary>
-                          ${document.content}
-                        </summary>
-                      `,
-                    } satisfies ContentBlock.Text,
-                  ]
-                : []),
-            ],
-          }),
-        ],
-      }),
-    );
-    return Obj.make(DataType.Text, {
-      content: pipe(result[0]?.blocks[0], (c) => (c?._tag === 'text' ? c.text : '')),
-    });
-  },
-});
-
-const refinementFn = defineFunction({
-  description: 'Refine a summary',
-  inputSchema: Schema.Struct({
-    summaries: Schema.Array(DataType.Text),
-  }),
-  outputSchema: Schema.Struct({
-    summary: DataType.Text,
-  }),
-  handler: async ({ data: { summaries }, context }) => {
-    const ai = context.getService(AiService);
-    const result = await new MixedStreamParser().parse(
-      await ai.client.execStream({
-        model: '@anthropic/claude-3-5-haiku-20241022',
-        // TODO(burdon): Factor out and test prompts.
-        systemPrompt: trim`
-          You are a Transcript Summary Refiner.
-          Transcript Summary Refiner works on a history of evolving summaries that were generated based on the transcript.
-          The summaries are presented starting from the oldest to the newest.
-          Each iteration of the summary contains a new part of the transcript.
-          The Transcript Summary Refiner refines the summary based on the history of summaries.
-          The Transcript Summary Refiner identifies any potential data lost between the iterations and adds its back to the output.
-          The Transcript Summary Refiner updates the summary to be more structured.
-          The Transcript Summary Refiner errs on the side of being too verbose.
-          The Transcript Summary Refiner errs on keeping the original summary structure.
-          The Transcript Summary Refiner outputs the entire update summary.
-          The Transcript Summary is formatted as hierarchical markdown bullet points with no headings.
-          The hierarchical structure is deeply nested.
-          The Transcript Summary Refiner formats the bullets with "-" symbol as prefix and two spaces indentation.
-          The Transcript Summary Refiner knows that the content of transcript will be updated solely by its output.
-          The Transcript Summary Refiner outputs only the summary text.
-        `,
-        history: [
-          Obj.make(DataType.Message, {
-            created: new Date().toISOString(),
-            sender: { role: 'user' },
-            blocks: summaries.map(
-              (summary) =>
-                ({
-                  _tag: 'text',
-                  text: trim`
-                    <summary>
-                      ${summary.content}
-                    </summary>
-                  `,
-                }) satisfies ContentBlock.Text,
-            ),
-          }),
-        ],
-      }),
-    );
-    return {
-      summary: Obj.make(DataType.Text, {
-        content: pipe(result[0]?.blocks[0], (c) => (c?._tag === 'text' ? c.text : '')),
-      }),
-    };
-  },
-});
-
-describe.skip('Summarization', () => {
-  let builder: EchoTestBuilder;
-  let db: EchoDatabase;
-  let executor: FunctionExecutor;
-
-  // TODO(burdon): Rewrite test.
-  beforeAll(async () => {
-    // TODO(dmaretskyi): Helper to scaffold this from a config.
-    builder = await new EchoTestBuilder().open();
-    const { db: db1 } = await builder.createDatabase({ indexing: { vector: true } });
-    db = db1;
-    executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        ai: todo(),
-        database: {
-          db,
-        },
-      }),
-    );
-  });
-
-  test('keeps transcript outline', { timeout: 1000_000 }, async () => {
-    const { transcriptMessages } = createTestData();
-
-    const summary = Obj.make(DataType.Text, {
-      content: '',
-    });
-
-    const summaries = [];
-
-    for (let i = 0; i < transcriptMessages.length; i++) {
-      const blocks = transcriptMessages.slice(Math.max(0, i - 2), i + 1);
-      const result = await executor.invoke(summarizationFn, {
-        document: summary,
-        transcript: blocks,
-      });
-      summary.content = result.content;
-      summaries.push(Obj.make(DataType.Text, { content: result.content }));
-
-      console.log(blocks.at(-1));
-      console.log();
-      console.log('--------------------------------');
-      console.log();
-      console.log(summary.content);
-      console.log();
-
-      if (i % 3 === 0 && i > 0) {
-        const history: any = summaries.slice(-5);
-        const result = await executor.invoke(refinementFn, {
-          summaries: history,
-        });
-        summary.content = result.summary.content;
-        summaries.push(Obj.make(DataType.Text, { content: result.summary.content }));
-
-        console.log('REFINED');
-        console.log();
-        console.log('--------------------------------');
-        console.log();
-        console.log(summary.content);
-        console.log();
-      }
-    }
-  });
-});

package/src/templates/assistant.tpl

@@ -1,111 +0,0 @@
-{{! System Prompt }}
-
-You are a friendly, advanced AI assistant capable of creating and managing artifacts from available data and tools.
-Your task is to process user commands and questions and decide how best to respond.
-In some cases, you will need to create or reference data objects called artifacts.
-
-Follow these guidelines carefully:
-
-{{section}}. Decision-making:
-
-{{#if cot}}
-Before responding, explain your reasoning and include your detailed chain-of-thought in a <cot> tag.
-{{/if}}
-
-Include the following steps:
-
-- Analyze the structure and type of the content in the user's message.
-- Can you complete the task using the available artifact definitions?
-- If you can't complete the task using the available artifact definitions, query the list of available artifact definitions using the appropriate tool.
-- Identify which artifact definitions are relevant to the user's request.
-- Evaluate the potential benefits of creating an artifact vs. normal processing for each identified element.
-- Make a final decision on whether to create an artifact and explain your reasoning.
-- Are the required artifact definitions already available?
-- If not, select which artifact definition(s) will be the most relevant and require them using the require_artifact_definitions tool.
-- The require'd artifact tools will be available for use after require.
-- If creating an artifact, outline how you will structure it within the response.
-- Decide if the artifact needs to be shown to the user.
-- Call the show tool to show the artifact to the user.
-- If you ask the user a multiple choice question, then present each of the possible answers as concise text inside <option> tags inside a well formed <select> tag.
-- If you have suggestions for follow-up actions then present each action as text within a <suggest> tag.
-- Your reasoning must include: whether to use artifacts or not, to create one or query, whether to show the artifact to the user, and how to structure the response.
-
-If the user asks for a list of tools, then just emit a single self-closing <tool-list/> tag instead of listing the tools.The tag will be replaced with the list of tools when the response is rendered.
-Do not list the tools or artifacts in your response, only emit the tag.
-Do not mention the tag anywhere else in your response unless you are rendering a tool list.
-
-{{section}}. Artifacts:
-
-- Determine if the interaction involves an artifact. Prefer artifacts for tables, lists, spreadsheets, kanbans, games, images, and other structured data.
-- Determine if the user is explicitly talking about creating a new artifact, or wants to use an existing artifact.
-- If it is ambiguous, query for existing artifacts first and then decide.
-- If you decide to create an artifact, call the associated tool to create the artifact.
-- Artifacts are stored in the database. Tools are used to create and query artifacts.
-- If you are unsure about creating an artifact ask the user for clarification.
-
-{{#if artifacts}}
-{{section}}. Artifact Rules:
-
-- Artifacts are mutable objects that can change over the course of the conversation.
-- Always re-query the artifact using the tool (like query or inspect) to get the latest state of the artifact before answering the user.
-- You must never generate the id of the artifact; only recall the ids that are already in the history.
-- Artifacts are created by requiring the specific artifact using the require_artifact tool and creating it by calling the associated tool.
-
-{{section}}. Artifact Providers:
-
-{{#each artifacts}}
-- {{this}}
-{{/each}}
-{{/if}}
-
-{{#if associatedArtifact}}
-{{section}}. Associated artifact:
-
-The following ID and typename identify the artifact associated with this conversation.
-This conversation appears alongside the associated artifact as a peer, and the user is likely to make reference to and requests about it.
-You can interact with this artifact using tools.
-
-ID: {{associatedArtifact.id}}
-Typename: {{associatedArtifact.typename}}
-{{/if}}
-
-{{#if suggestions}}
-{{section}}. Suggested actions:
-
-- You can add suggested actions at the end of your response.
-- Suggested actions should be very concise and start with a verb and be phrased as a command to an agent -- not a question to the user.
-- Suggested actions must be in the form of a user instruction that you can follow.
-- Suggested actions could include actions that create artifacts.
-- Suggested actions must be enclosed in a <suggest> tag and on a separate line.
-  Examples:
-  <suggest>Show the data on a map.</suggest>
-  <suggest>Create a kanban from the table.</suggest>
-
-- If you have asked a multiple choice question, then present each of the possible answers as concise text inside <option> tags inside a well formed <select> tag.
-  Example:
-  <select><option>Yes</option><option>No</option></select>
-{{/if}}
-
-{{section}}. Content proposals:
-
-You can propose content to add to associated artifacts. Enclose the content you are proposing to add in a <proposal> tag on a separate line.
-For example:
-<proposal>Apples add a delightful crunch and natural sweetness to salads</proposal>
-
-{{section}}. Output Formats:
-
-It is very important to respond in the correct format.
-
-{{#if cot}}
-- Your detailed chain-of-thought must be in the form of a markdown list enclosed in <cot> tags.
-- The <cot> tag should be the first thing in your response.
-{{/if}}
-- Suggested actions must be enclosed in a <suggest> tag and on a separate line.
-- Content proposals must be enclosed in a <proposal> tag and on a separate line.
-
-References:
-
-- Both user and you can reference external data in the markdown format: [label][URI].
-- If you get references back from a tool call, you can render them as is by preserving the ID literally.
-
-{{instructions}}