@dxos/assistant-toolkit 0.8.4-main.03d5cd7b56

package/src/functions/agent/prompt.ts

@@ -0,0 +1,224 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Tool from '@effect/ai/Tool';
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as Array from 'effect/Array';
+import * as Deferred from 'effect/Deferred';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, OpaqueToolkit, ModelName } from '@dxos/ai';
+import {
+  AiSession,
+  getOperationFromTool,
+  makeToolExecutionService,
+  makeToolResolverFromOperations,
+} from '@dxos/assistant';
+import { Template, Trace, Operation } from '@dxos/compute';
+import { Database, Feed, Obj, Ref } from '@dxos/echo';
+import { acquireReleaseResource } from '@dxos/effect';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { trim } from '@dxos/util';
+
+import { PromptError } from '../../errors';
+import * as Chat from '../../types/Chat';
+import { AgentPrompt } from './definitions';
+
+const DEFAULT_MODEL: ModelName = '@anthropic/claude-opus-4-6';
+
+export default AgentPrompt.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(
+      function* (data) {
+        log.info('processing input', { input: data.input });
+
+        const input = yield* Ref.isRef(data.input)
+          ? Database.load(data.input).pipe(Effect.map(Obj.toJSON))
+          : Effect.succeed(data.input);
+
+        yield* Database.flush();
+        const prompt = yield* Database.load(data.prompt);
+        yield* Trace.emitStatus(`Running ${prompt.id}`);
+
+        log.info('starting agent', { prompt: prompt.id, input });
+
+        const blueprints = yield* Function.pipe(
+          prompt.blueprints,
+          Effect.forEach(Database.loadOption),
+          Effect.map(Array.filter(Option.isSome)),
+          Effect.map(Array.map((option) => option.value)),
+        );
+
+        const objects = yield* Function.pipe(
+          prompt.context,
+          Effect.forEach(Database.loadOption),
+          Effect.map(Array.filter(Option.isSome)),
+          Effect.map(Array.map((option) => option.value)),
+        );
+
+        const promptInstructions = yield* Database.load(prompt.instructions.source);
+        let promptText = Template.process(promptInstructions.content, input);
+
+        if (input !== undefined) {
+          promptText += `\n<input>${JSON.stringify(input)}</input>`;
+        }
+
+        let systemText = trim`
+          You are an agent running in the non-interactive mode.
+          The user is unable to see what you are doing, and cannot answer any questions.
+          Do not ask questions.
+          Complete the task before you, and at the end call [completeJob] with the output.
+          If you are unable to complete the task, call [completeJob] with the failure reason.
+          If no output is required, call [completeJob] with an empty object: {}
+          Do not stop until you call [completeJob].
+        `;
+        if (data.systemInstructions) {
+          systemText += `\n\n${data.systemInstructions}`;
+        }
+
+        const modelLayer = AiService.model(data.model ?? DEFAULT_MODEL);
+
+        let feed: Feed.Feed;
+        if (data.chat) {
+          const chat = yield* Database.load(data.chat);
+          invariant(Obj.instanceOf(Chat.Chat, chat), 'Expected Chat object.');
+          feed = yield* Database.load(chat.feed);
+        } else {
+          feed = yield* Database.add(Feed.make());
+        }
+
+        const resultSink = yield* Deferred.make<unknown, PromptError>();
+        const promptToolkit = makePromptAgentToolkit({
+          output: Schema.Any, // TODO(dmaretskyi): Use prompt's output schema.
+          resultSink,
+        });
+
+        const runtime = yield* Effect.runtime<Feed.FeedService>();
+        const session = yield* acquireReleaseResource(() => new AiSession({ feed, runtime }));
+
+        yield* Effect.promise(() =>
+          session.context.bind({
+            blueprints: blueprints.map((blueprint) => Ref.make(blueprint)),
+            objects: objects.map((object) => Ref.make(object as Obj.Unknown)),
+          }),
+        );
+
+        yield* session
+          .createRequest({
+            prompt: promptText,
+            system: systemText,
+            toolkit: promptToolkit,
+          })
+          .pipe(
+            Effect.provide(
+              Layer.mergeAll(modelLayer, ToolExecutionService({ feed }), makeToolResolverFromOperations()),
+            ),
+          );
+
+        return yield* Deferred.poll(resultSink).pipe(
+          Effect.flatten,
+          Effect.flatten,
+          Effect.catchTag('NoSuchElementException', () =>
+            Effect.gen(function* () {
+              yield* session
+                .createRequest({
+                  prompt: 'You must signal task completion by calling [completeJob] with the output or failure reason.',
+                  system: systemText,
+                  toolkit: promptToolkit,
+                })
+                .pipe(
+                  Effect.provide(
+                    Layer.mergeAll(modelLayer, ToolExecutionService({ feed }), makeToolResolverFromOperations()),
+                  ),
+                );
+
+              return yield* Deferred.poll(resultSink).pipe(
+                Effect.flatten,
+                Effect.flatten,
+                Effect.catchTag('NoSuchElementException', () =>
+                  Effect.fail(new PromptError('Agent did not signal task completion.', {})),
+                ),
+              );
+            }),
+          ),
+        );
+      },
+      Effect.tapBoth({
+        onSuccess: () => Database.flush(),
+        onFailure: () => Database.flush(),
+      }),
+      Effect.scoped,
+    ),
+  ),
+  Operation.opaqueHandler,
+);
+
+const makePromptAgentToolkit = (options: {
+  output: Schema.Schema.Any;
+  resultSink: Deferred.Deferred<unknown, PromptError>;
+}) => {
+  class PromptAgentToolkit extends Toolkit.make(
+    Tool.make('completeJob', {
+      parameters: {
+        success: Schema.optional(Schema.Any), // TODO(dmaretskyi): Pipe output schema here.
+        failure: Schema.optional(
+          Schema.Struct({
+            message: Schema.String.annotations({
+              description: 'Short message describing the error.',
+            }),
+            description: Schema.optional(Schema.String).annotations({
+              description: 'Optional longer message describing in detail what went wrong',
+            }),
+          }),
+        ),
+      },
+    }),
+  ) {}
+  const layer = PromptAgentToolkit.toLayer({
+    completeJob: Effect.fnUntraced(function* (result) {
+      if (result.failure) {
+        yield* Deferred.fail(
+          options.resultSink,
+          new PromptError(result.failure.message, {
+            description: result.failure.description,
+          }),
+        );
+      } else {
+        yield* Deferred.succeed(options.resultSink, result.success);
+      }
+    }),
+  });
+
+  return OpaqueToolkit.make(PromptAgentToolkit, layer);
+};
+
+interface ToolExecutionServiceOptions {
+  feed: Feed.Feed;
+}
+
+const ToolExecutionService = ({ feed }: ToolExecutionServiceOptions) =>
+  Layer.unwrapEffect(
+    Effect.gen(function* () {
+      const operationInvoker = yield* Operation.Service;
+      return makeToolExecutionService({
+        invoke: (tool, input) =>
+          Effect.gen(function* () {
+            const operationDef = getOperationFromTool(tool).pipe(Option.getOrThrow);
+            log('invoking operation', { operationDef, input });
+            const result = yield* operationInvoker
+              .invoke(operationDef, input, {
+                conversation: Obj.getDXN(feed).toString(),
+              })
+              .pipe(Effect.orDie);
+            log('result', { result });
+            return result;
+          }),
+      });
+    }),
+  );

package/src/functions/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './agent';

package/src/index.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './blueprints';
+export * from './crud';
+export * from './functions';
+export * from './sync';
+export * from './toolkits';
+export * from './types';

package/src/sync/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './sync';

package/src/sync/sync.ts

@@ -0,0 +1,106 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Filter, Obj, Query, Ref } from '@dxos/echo';
+import { Database } from '@dxos/echo';
+import { failedInvariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+/**
+ * Syncs objects to the database.
+ * If there's an object with a matching foreign key in the database, it will be updated.
+ * Otherwise, a new object will be added.
+ * Recursively syncs top-level refs.
+ *
+ * @param opts.foreignKeyId - The key to use for matching objects.
+ */
+export const syncObjects: (
+  objs: Obj.Unknown[],
+  opts: { foreignKeyId: string },
+) => Effect.Effect<Obj.Unknown[], never, Database.Service> = Effect.fn('syncObjects')(function* (
+  objs,
+  { foreignKeyId },
+) {
+  return yield* Effect.forEach(
+    objs,
+    Effect.fnUntraced(function* (obj) {
+      // Sync referenced objects.
+      for (const key of Object.keys(obj)) {
+        if (typeof key !== 'string' || key === 'id') {
+          continue;
+        }
+        if (!Ref.isRef((obj as any)[key])) {
+          continue;
+        }
+        const ref: Ref.Unknown = (obj as any)[key];
+        if (!ref.target) {
+          continue;
+        }
+        if (Obj.getDXN(ref.target).isLocalObjectId()) {
+          // obj not persisted to db.
+          const [target] = yield* syncObjects([ref.target], { foreignKeyId });
+          (obj as any)[key] = Ref.make(target);
+        }
+      }
+
+      const schema = Obj.getSchema(obj) ?? failedInvariant('No schema.');
+      const foreignId = Obj.getKeys(obj, foreignKeyId)[0]?.id ?? failedInvariant('No foreign key.');
+      const [existing] = yield* Database.runQuery(
+        Query.select(Filter.foreignKeys(schema, [{ source: foreignKeyId, id: foreignId }])),
+      );
+      log('sync object', {
+        type: Obj.getTypename(obj),
+        foreignId,
+        existing: existing ? Obj.getDXN(existing) : undefined,
+      });
+      if (!existing) {
+        yield* Database.add(obj);
+        return obj;
+      } else {
+        copyObjectData(existing, obj);
+        return existing;
+      }
+    }),
+    { concurrency: 1 },
+  );
+});
+
+const copyObjectData = (existing: Obj.Unknown, newObj: Obj.Unknown) => {
+  Obj.update(existing, (existing) => {
+    // Copy properties from newObj to existing.
+    for (const key of Object.keys(newObj)) {
+      if (typeof key !== 'string' || key === 'id') {
+        continue;
+      }
+      if (
+        typeof (newObj as any)[key] !== 'string' &&
+        typeof (newObj as any)[key] !== 'number' &&
+        typeof (newObj as any)[key] !== 'boolean' &&
+        !Ref.isRef((newObj as any)[key])
+      ) {
+        continue;
+      }
+      (existing as any)[key] = (newObj as any)[key];
+    }
+
+    // Delete properties that don't exist in newObj.
+    for (const key of Object.keys(existing)) {
+      if (typeof key !== 'string' || key === 'id') {
+        continue;
+      }
+      if (!(key in newObj)) {
+        delete (existing as any)[key];
+      }
+    }
+
+    // Update foreign keys.
+    for (const foreignKey of Obj.getMeta(newObj).keys) {
+      Obj.deleteKeys(existing, foreignKey.source);
+      // TODO(dmaretskyi): Doesn't work: `Obj.getMeta(existing).keys.push(foreignKey);`
+      Obj.getMeta(existing).keys.push({ ...foreignKey });
+    }
+  });
+};

package/src/testing/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from '../blueprints/testing';
+export * from './plugins';

package/src/testing/plugins.tsx

@@ -0,0 +1,79 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import React from 'react';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { Surface } from '@dxos/app-framework/ui';
+import { Format, type Obj, Type } from '@dxos/echo';
+import { Card } from '@dxos/react-ui';
+import { Syntax } from '@dxos/react-ui-syntax-highlighter';
+
+export const MapSchema = Schema.Struct({
+  coordinates: Format.GeoPoint,
+}).pipe(
+  Type.object({
+    typename: 'com.example.type.map',
+    version: '0.1.0',
+  }),
+);
+
+export type MapSchema = Schema.Schema.Type<typeof MapSchema>;
+
+// TODO(burdon): Move to ECHO def.
+export type ArtifactsContext = {
+  items: Obj.Unknown[];
+  getArtifacts: () => Obj.Unknown[];
+  addArtifact: (artifact: Obj.Unknown) => void;
+};
+
+declare global {
+  interface ToolContextExtensions {
+    artifacts?: ArtifactsContext;
+  }
+}
+
+// TODO(dmaretskyi): Removed images from conductor GPT implementation.
+const isImage = (data: any): data is any => false;
+
+export const capabilities: Capability.Any[] = [
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-image',
+      role: 'card--content',
+      filter: (data: any): data is any => isImage(data.value),
+      component: ({ data }) => (
+        <Card.Content>
+          <img
+            className='grow object-cover'
+            src={`data:image/jpeg;base64,${data.value.source.data}`}
+            alt={data.value.prompt ?? `Generated image [id=${data.value.id}]`}
+          />
+        </Card.Content>
+      ),
+    }),
+  ),
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-default',
+      role: 'card--content',
+      position: 'fallback',
+      component: ({ data }) => (
+        <Card.Content>
+          <Syntax.Root data={data}>
+            <Syntax.Content>
+              <Syntax.Filter />
+              <Syntax.Viewport>
+                <Syntax.Code />
+              </Syntax.Viewport>
+            </Syntax.Content>
+          </Syntax.Root>
+        </Card.Content>
+      ),
+    }),
+  ),
+];

package/src/toolkits/WebToolkit.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as AnthropicTool from '@effect/ai-anthropic/AnthropicTool';
+import * as Tool from '@effect/ai/Tool';
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+// TODO(dmaretskyi): Testing only.
+export const WebToolkit = Toolkit.make(
+  AnthropicTool.WebSearch_20250305({}).pipe(
+    // TODO(dmaretskyi): Effect bug -- provider-defined tools don't support annotations.
+    // ToolFormatter.assign({
+    //   debugFormatResult: (result) =>
+    //     Array.isArray(result) ? result.map(Struct.pick('title', 'type', 'url', 'page_age')) : result,
+    // }),
+  ),
+  Tool.make('WebFetch', {
+    parameters: {
+      url: Schema.String,
+    },
+    success: Schema.String,
+  }),
+);
+
+export const layer = WebToolkit.toLayer({
+  WebFetch: Effect.fnUntraced(function* ({ url }) {
+    const response = yield* Effect.promise(() => fetch(url).then((response) => response.text()));
+    return response;
+  }),
+});

package/src/toolkits/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as WebToolkit from './WebToolkit';

package/src/typedefs.d.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+declare module '*?raw' {
+  const content: string;
+  export default content;
+}