@dxos/assistant-toolkit 0.8.4-main.e8ec1fe → 0.8.4-main.fcc0d83b33

package/src/functions/agent/prompt.ts

@@ -2,113 +2,222 @@
 // Copyright 2025 DXOS.org
 //
 
+import * as Tool from '@effect/ai/Tool';
+import * as Toolkit from '@effect/ai/Toolkit';
 import * as Array from 'effect/Array';
+import * as Deferred from 'effect/Deferred';
 import * as Effect from 'effect/Effect';
 import * as Function from 'effect/Function';
-import * as Match from 'effect/Match';
+import * as Layer from 'effect/Layer';
 import * as Option from 'effect/Option';
 import * as Schema from 'effect/Schema';
 
-import { AiService, ConsolePrinter, ModelName } from '@dxos/ai';
-import { AiSession, GenerationObserver, createToolkit } from '@dxos/assistant';
-import { Prompt, Template } from '@dxos/blueprints';
-import { Obj, Ref, Type } from '@dxos/echo';
-import { DatabaseService, TracingService, defineFunction } from '@dxos/functions';
+import { AiService, OpaqueToolkit, ModelName } from '@dxos/ai';
+import {
+  AiSession,
+  getOperationFromTool,
+  makeToolExecutionService,
+  makeToolResolverFromOperations,
+} from '@dxos/assistant';
+import { Template } from '@dxos/compute';
+import { Trace } from '@dxos/compute';
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Obj, Ref } from '@dxos/echo';
+import { acquireReleaseResource } from '@dxos/effect';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
+import { trim } from '@dxos/util';
 
-const DEFAULT_MODEL: ModelName = '@anthropic/claude-opus-4-0';
-
-export default defineFunction({
-  key: 'dxos.org/function/prompt',
-  name: 'Agent',
-  description: 'Agentic worker that executes a provided prompt using blueprints and tools.',
-  inputSchema: Schema.Struct({
-    prompt: Type.Ref(Prompt.Prompt),
-    systemPrompt: Type.Ref(Prompt.Prompt).pipe(Schema.optional),
-    /**
-     * @default @anthropic/claude-opus-4-0
-     */
-    model: Schema.optional(ModelName),
-    /**
-     * Input object or data.
-     * References get auto-resolved.
-     */
-    input: Schema.Any.pipe(Schema.annotations({ title: 'Input' })),
-  }),
-  outputSchema: Schema.Any,
-  handler: Effect.fnUntraced(function* ({ data }) {
-    log.info('processing input', { input: data.input });
-
-    // TODO(wittjosiah): Support templated object as input.
-    //   Currently the object templating only supports direct pass-through or strings.
-    // const input = { ...data.input };
-    // for (const key of Object.keys(data.input)) {
-    //   const value = data.input[key];
-    //   if (Ref.isRef(value)) {
-    //     const object = yield* DatabaseService.load(value);
-    //     input[key] = Obj.toJSON(object);
-    //   } else {
-    //     input[key] = JSON.stringify(value);
-    //   }
-    // }
-    const input = yield* Match.value(data.input).pipe(
-      Match.when(
-        (value: any) => Ref.isRef(value),
-        Effect.fnUntraced(function* (ref) {
-          const object = yield* DatabaseService.load(ref);
-          return Obj.toJSON(object as Obj.Any);
-        }),
-      ),
-      Match.orElse(() => Effect.succeed(data.input)),
-    );
-
-    yield* DatabaseService.flush({ indexes: true });
-    const prompt = yield* DatabaseService.load(data.prompt);
-    const systemPrompt = data.systemPrompt ? yield* DatabaseService.load(data.systemPrompt) : undefined;
-    yield* TracingService.emitStatus({ message: `Running ${prompt.id}` });
-
-    log.info('starting agent', { prompt: prompt.id, input });
-
-    const blueprints = yield* Function.pipe(
-      prompt.blueprints,
-      Array.appendAll(systemPrompt?.blueprints ?? []),
-      Effect.forEach(DatabaseService.loadOption),
-      Effect.map(Array.filter(Option.isSome)),
-      Effect.map(Array.map((option) => option.value)),
-    );
-    const objects = yield* Function.pipe(
-      prompt.context,
-      Array.appendAll(systemPrompt?.context ?? []),
-      Effect.forEach(DatabaseService.loadOption),
-      Effect.map(Array.filter(Option.isSome)),
-      Effect.map(Array.map((option) => option.value)),
-    );
-    const toolkit = yield* createToolkit({ blueprints });
-
-    const promptInstructions = yield* DatabaseService.load(prompt.instructions.source);
-    const promptText = Template.process(promptInstructions.content, input);
-
-    const systemInstructions = systemPrompt ? yield* DatabaseService.load(systemPrompt.instructions.source) : undefined;
-    const systemText = systemInstructions ? Template.process(systemInstructions.content, {}) : undefined;
-
-    const session = new AiSession();
-    const result = yield* session
-      .run({
-        prompt: promptText,
-        system: systemText,
-        blueprints,
-        objects: objects as Obj.Any[],
-        toolkit,
-        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'agent' })),
-      })
-      .pipe(Effect.provide(AiService.model(data.model ?? DEFAULT_MODEL)));
-    const lastBlock = result
-      .at(-1)
-      ?.blocks.filter((block) => block._tag === 'text')
-      .at(-1);
-
-    return {
-      note: lastBlock?.text,
-    };
-  }),
-});
+import { PromptError } from '../../errors';
+import * as Chat from '../../types/Chat';
+import { AgentPrompt } from './definitions';
+
+const DEFAULT_MODEL: ModelName = '@anthropic/claude-opus-4-6';
+
+export default AgentPrompt.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(
+      function* (data) {
+        log.info('processing input', { input: data.input });
+
+        const input = yield* Ref.isRef(data.input)
+          ? Database.load(data.input).pipe(Effect.map(Obj.toJSON))
+          : Effect.succeed(data.input);
+
+        yield* Database.flush();
+        const prompt = yield* Database.load(data.prompt);
+        yield* Trace.emitStatus(`Running ${prompt.id}`);
+
+        log.info('starting agent', { prompt: prompt.id, input });
+
+        const blueprints = yield* Function.pipe(
+          prompt.blueprints,
+          Effect.forEach(Database.loadOption),
+          Effect.map(Array.filter(Option.isSome)),
+          Effect.map(Array.map((option) => option.value)),
+        );
+
+        const objects = yield* Function.pipe(
+          prompt.context,
+          Effect.forEach(Database.loadOption),
+          Effect.map(Array.filter(Option.isSome)),
+          Effect.map(Array.map((option) => option.value)),
+        );
+
+        const promptInstructions = yield* Database.load(prompt.instructions.source);
+        let promptText = Template.process(promptInstructions.content, input);
+
+        if (input !== undefined) {
+          promptText += `\n<input>${JSON.stringify(input)}</input>`;
+        }
+
+        let systemText = trim`
+          You are an agent running in the non-interactive mode.
+          The user is unable to see what you are doing, and cannot answer any questions.
+          Do not ask questions.
+          Complete the task before you, and at the end call [completeJob] with the output.
+          If you are unable to complete the task, call [completeJob] with the failure reason.
+          If no output is required, call [completeJob] with an empty object: {}
+          Do not stop until you call [completeJob].
+        `;
+        if (data.systemInstructions) {
+          systemText += `\n\n${data.systemInstructions}`;
+        }
+
+        const modelLayer = AiService.model(data.model ?? DEFAULT_MODEL);
+
+        let feed: Feed.Feed;
+        if (data.chat) {
+          const chat = yield* Database.load(data.chat);
+          invariant(Obj.instanceOf(Chat.Chat, chat), 'Expected Chat object.');
+          feed = yield* Database.load(chat.feed);
+        } else {
+          feed = yield* Database.add(Feed.make());
+        }
+
+        const resultSink = yield* Deferred.make<unknown, PromptError>();
+        const promptToolkit = makePromptAgentToolkit({
+          output: Schema.Any, // TODO(dmaretskyi): Use prompt's output schema.
+          resultSink,
+        });
+
+        const runtime = yield* Effect.runtime<Feed.FeedService>();
+        const session = yield* acquireReleaseResource(() => new AiSession({ feed, runtime }));
+
+        yield* Effect.promise(() =>
+          session.context.bind({
+            blueprints: blueprints.map((blueprint) => Ref.make(blueprint)),
+            objects: objects.map((object) => Ref.make(object as Obj.Unknown)),
+          }),
+        );
+
+        yield* session
+          .createRequest({
+            prompt: promptText,
+            system: systemText,
+            toolkit: promptToolkit,
+          })
+          .pipe(
+            Effect.provide(
+              Layer.mergeAll(modelLayer, ToolExecutionService({ feed }), makeToolResolverFromOperations()),
+            ),
+          );
+
+        return yield* Deferred.poll(resultSink).pipe(
+          Effect.flatten,
+          Effect.flatten,
+          Effect.catchTag('NoSuchElementException', () =>
+            Effect.gen(function* () {
+              yield* session
+                .createRequest({
+                  prompt: 'You must signal task completion by calling [completeJob] with the output or failure reason.',
+                  system: systemText,
+                  toolkit: promptToolkit,
+                })
+                .pipe(
+                  Effect.provide(
+                    Layer.mergeAll(modelLayer, ToolExecutionService({ feed }), makeToolResolverFromOperations()),
+                  ),
+                );
+
+              return yield* Deferred.poll(resultSink).pipe(
+                Effect.flatten,
+                Effect.flatten,
+                Effect.catchTag('NoSuchElementException', () =>
+                  Effect.fail(new PromptError('Agent did not signal task completion.', {})),
+                ),
+              );
+            }),
+          ),
+        );
+      },
+      Effect.scoped,
+      Effect.provide(Trace.writerLayerNoop),
+    ),
+  ),
+  Operation.opaqueHandler,
+);
+
+const makePromptAgentToolkit = (options: {
+  output: Schema.Schema.Any;
+  resultSink: Deferred.Deferred<unknown, PromptError>;
+}) => {
+  class PromptAgentToolkit extends Toolkit.make(
+    Tool.make('completeJob', {
+      parameters: {
+        success: Schema.optional(Schema.Any), // TODO(dmaretskyi): Pipe output schema here.
+        failure: Schema.optional(
+          Schema.Struct({
+            message: Schema.String.annotations({
+              description: 'Short message describing the error.',
+            }),
+            description: Schema.optional(Schema.String).annotations({
+              description: 'Optional longer message describing in detail what went wrong',
+            }),
+          }),
+        ),
+      },
+    }),
+  ) {}
+  const layer = PromptAgentToolkit.toLayer({
+    completeJob: Effect.fnUntraced(function* (result) {
+      if (result.failure) {
+        yield* Deferred.fail(
+          options.resultSink,
+          new PromptError(result.failure.message, {
+            description: result.failure.description,
+          }),
+        );
+      } else {
+        yield* Deferred.succeed(options.resultSink, result.success);
+      }
+    }),
+  });
+
+  return OpaqueToolkit.make(PromptAgentToolkit, layer);
+};
+
+interface ToolExecutionServiceOptions {
+  feed: Feed.Feed;
+}
+
+const ToolExecutionService = ({ feed }: ToolExecutionServiceOptions) =>
+  Layer.unwrapEffect(
+    Effect.gen(function* () {
+      const operationInvoker = yield* Operation.Service;
+      return makeToolExecutionService({
+        invoke: (tool, input) =>
+          Effect.gen(function* () {
+            const operationDef = getOperationFromTool(tool).pipe(Option.getOrThrow);
+            log('invoking operation', { operationDef, input });
+            const result = yield* operationInvoker
+              .invoke(operationDef, input, {
+                conversation: Obj.getDXN(feed).toString(),
+              })
+              .pipe(Effect.orDie);
+            log('result', { result });
+            return result;
+          }),
+      });
+    }),
+  );

package/src/functions/index.ts

@@ -3,9 +3,3 @@
 //
 
 export * from './agent';
-export * from './discord';
-export * from './document';
-export * from './entity-extraction';
-export * from './linear';
-export * from './research';
-export * from './tasks';

package/src/index.ts

@@ -3,8 +3,8 @@
 //
 
 export * from './blueprints';
+export * from './crud';
 export * from './functions';
-export * from './plugins';
 export * from './sync';
-export * from './crud';
 export * from './toolkits';
+export * from './types';

package/src/sync/sync.ts

@@ -5,7 +5,7 @@
 import * as Effect from 'effect/Effect';
 
 import { Filter, Obj, Query, Ref } from '@dxos/echo';
-import { DatabaseService } from '@dxos/functions';
+import { Database } from '@dxos/echo';
 import { failedInvariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
@@ -18,18 +18,27 @@ import { log } from '@dxos/log';
  * @param opts.foreignKeyId - The key to use for matching objects.
  */
 export const syncObjects: (
-  objs: Obj.Any[],
+  objs: Obj.Unknown[],
   opts: { foreignKeyId: string },
-) => Effect.Effect<Obj.Any[], never, DatabaseService> = Effect.fn('syncObjects')(function* (objs, { foreignKeyId }) {
+) => Effect.Effect<Obj.Unknown[], never, Database.Service> = Effect.fn('syncObjects')(function* (
+  objs,
+  { foreignKeyId },
+) {
   return yield* Effect.forEach(
     objs,
     Effect.fnUntraced(function* (obj) {
       // Sync referenced objects.
       for (const key of Object.keys(obj)) {
-        if (typeof key !== 'string' || key === 'id') continue;
-        if (!Ref.isRef((obj as any)[key])) continue;
-        const ref: Ref.Any = (obj as any)[key];
-        if (!ref.target) continue;
+        if (typeof key !== 'string' || key === 'id') {
+          continue;
+        }
+        if (!Ref.isRef((obj as any)[key])) {
+          continue;
+        }
+        const ref: Ref.Unknown = (obj as any)[key];
+        if (!ref.target) {
+          continue;
+        }
         if (Obj.getDXN(ref.target).isLocalObjectId()) {
           // obj not persisted to db.
           const [target] = yield* syncObjects([ref.target], { foreignKeyId });
@@ -39,9 +48,7 @@ export const syncObjects: (
 
       const schema = Obj.getSchema(obj) ?? failedInvariant('No schema.');
       const foreignId = Obj.getKeys(obj, foreignKeyId)[0]?.id ?? failedInvariant('No foreign key.');
-      const {
-        objects: [existing],
-      } = yield* DatabaseService.runQuery(
+      const [existing] = yield* Database.runQuery(
         Query.select(Filter.foreignKeys(schema, [{ source: foreignKeyId, id: foreignId }])),
       );
       log('sync object', {
@@ -50,7 +57,7 @@ export const syncObjects: (
         existing: existing ? Obj.getDXN(existing) : undefined,
       });
       if (!existing) {
-        yield* DatabaseService.add(obj);
+        yield* Database.add(obj);
         return obj;
       } else {
         copyObjectData(existing, obj);
@@ -61,27 +68,39 @@ export const syncObjects: (
   );
 });
 
-const copyObjectData = (existing: Obj.Any, newObj: Obj.Any) => {
-  for (const key of Object.keys(newObj)) {
-    if (typeof key !== 'string' || key === 'id') continue;
-    if (
-      typeof (newObj as any)[key] !== 'string' &&
-      typeof (newObj as any)[key] !== 'number' &&
-      typeof (newObj as any)[key] !== 'boolean' &&
-      !Ref.isRef((newObj as any)[key])
-    )
-      continue;
-    (existing as any)[key] = (newObj as any)[key];
-  }
-  for (const key of Object.keys(existing)) {
-    if (typeof key !== 'string' || key === 'id') continue;
-    if (!(key in newObj)) {
-      delete (existing as any)[key];
+const copyObjectData = (existing: Obj.Unknown, newObj: Obj.Unknown) => {
+  Obj.update(existing, (existing) => {
+    // Copy properties from newObj to existing.
+    for (const key of Object.keys(newObj)) {
+      if (typeof key !== 'string' || key === 'id') {
+        continue;
+      }
+      if (
+        typeof (newObj as any)[key] !== 'string' &&
+        typeof (newObj as any)[key] !== 'number' &&
+        typeof (newObj as any)[key] !== 'boolean' &&
+        !Ref.isRef((newObj as any)[key])
+      ) {
+        continue;
+      }
+      (existing as any)[key] = (newObj as any)[key];
+    }
+
+    // Delete properties that don't exist in newObj.
+    for (const key of Object.keys(existing)) {
+      if (typeof key !== 'string' || key === 'id') {
+        continue;
+      }
+      if (!(key in newObj)) {
+        delete (existing as any)[key];
+      }
+    }
+
+    // Update foreign keys.
+    for (const foreignKey of Obj.getMeta(newObj).keys) {
+      Obj.deleteKeys(existing, foreignKey.source);
+      // TODO(dmaretskyi): Doesn't work: `Obj.getMeta(existing).keys.push(foreignKey);`
+      Obj.getMeta(existing).keys.push({ ...foreignKey });
     }
-  }
-  for (const foreignKey of Obj.getMeta(newObj).keys) {
-    Obj.deleteKeys(existing, foreignKey.source);
-    // TODO(dmaretskyi): Doesn't work: `Obj.getMeta(existing).keys.push(foreignKey);`
-    Obj.getMeta(existing).keys.push({ ...foreignKey });
-  }
+  });
 };

package/src/testing/index.ts

@@ -2,4 +2,5 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './data';
+export * from '../blueprints/testing';
+export * from './plugins';

package/src/testing/plugins.tsx

@@ -0,0 +1,79 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import React from 'react';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { Surface } from '@dxos/app-framework/ui';
+import { Format, type Obj, Type } from '@dxos/echo';
+import { Card } from '@dxos/react-ui';
+import { Syntax } from '@dxos/react-ui-syntax-highlighter';
+
+export const MapSchema = Schema.Struct({
+  coordinates: Format.GeoPoint,
+}).pipe(
+  Type.object({
+    typename: 'com.example.type.map',
+    version: '0.1.0',
+  }),
+);
+
+export type MapSchema = Schema.Schema.Type<typeof MapSchema>;
+
+// TODO(burdon): Move to ECHO def.
+export type ArtifactsContext = {
+  items: Obj.Unknown[];
+  getArtifacts: () => Obj.Unknown[];
+  addArtifact: (artifact: Obj.Unknown) => void;
+};
+
+declare global {
+  interface ToolContextExtensions {
+    artifacts?: ArtifactsContext;
+  }
+}
+
+// TODO(dmaretskyi): Removed images from conductor GPT implementation.
+const isImage = (data: any): data is any => false;
+
+export const capabilities: Capability.Any[] = [
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-image',
+      role: 'card--content',
+      filter: (data: any): data is any => isImage(data.value),
+      component: ({ data }) => (
+        <Card.Content>
+          <img
+            className='grow object-cover'
+            src={`data:image/jpeg;base64,${data.value.source.data}`}
+            alt={data.value.prompt ?? `Generated image [id=${data.value.id}]`}
+          />
+        </Card.Content>
+      ),
+    }),
+  ),
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-default',
+      role: 'card--content',
+      position: 'fallback',
+      component: ({ data }) => (
+        <Card.Content>
+          <Syntax.Root data={data}>
+            <Syntax.Content>
+              <Syntax.Filter />
+              <Syntax.Viewport>
+                <Syntax.Code />
+              </Syntax.Viewport>
+            </Syntax.Content>
+          </Syntax.Root>
+        </Card.Content>
+      ),
+    }),
+  ),
+];

package/src/toolkits/WebToolkit.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as AnthropicTool from '@effect/ai-anthropic/AnthropicTool';
+import * as Tool from '@effect/ai/Tool';
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+// TODO(dmaretskyi): Testing only.
+export const WebToolkit = Toolkit.make(
+  AnthropicTool.WebSearch_20250305({}).pipe(
+    // TODO(dmaretskyi): Effect bug -- provider-defined tools don't support annotations.
+    // ToolFormatter.assign({
+    //   debugFormatResult: (result) =>
+    //     Array.isArray(result) ? result.map(Struct.pick('title', 'type', 'url', 'page_age')) : result,
+    // }),
+  ),
+  Tool.make('WebFetch', {
+    parameters: {
+      url: Schema.String,
+    },
+    success: Schema.String,
+  }),
+);
+
+export const layer = WebToolkit.toLayer({
+  WebFetch: Effect.fnUntraced(function* ({ url }) {
+    const response = yield* Effect.promise(() => fetch(url).then((response) => response.text()));
+    return response;
+  }),
+});

package/src/toolkits/index.ts

@@ -2,5 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * as AssistantToolkit from './AssistantToolkit';
-export * as SystemToolkit from './SystemToolkit';
+export * as WebToolkit from './WebToolkit';