@dxos/assistant-toolkit 0.8.4-main.ae835ea

package/src/blueprints/websearch/websearch-blueprint.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { ToolId } from '@dxos/ai';
+import { Blueprint } from '@dxos/blueprints';
+import { Obj, Ref } from '@dxos/echo';
+import { DataType } from '@dxos/schema';
+
+const blueprint: Blueprint.Blueprint = Obj.make(Blueprint.Blueprint, {
+  key: 'dxos.org/blueprint/web-search',
+  name: 'Web Search',
+  description: 'Search the web.',
+  instructions: {
+    source: Ref.make(DataType.makeText('')), // No instructions required.
+  },
+  tools: [ToolId.make('AnthropicWebSearch')],
+});
+
+export default blueprint;

package/src/blueprints/websearch/websearch-toolkit.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as AnthropicTool from '@effect/ai-anthropic/AnthropicTool';
+
+export const WebSearchToolkit = Toolkit.make(AnthropicTool.WebSearch_20250305({}));

package/src/experimental/feed.test.ts

@@ -0,0 +1,108 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { describe, it } from '@effect/vitest';
+import * as Config from 'effect/Config';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Redacted from 'effect/Redacted';
+
+import { AiService } from '@dxos/ai';
+import { AiServiceTestingPreset, EXA_API_KEY } from '@dxos/ai/testing';
+import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
+import { TestHelpers } from '@dxos/effect';
+import { ComputeEventLogger, CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
+import { TestDatabaseLayer, testStoragePath } from '@dxos/functions/testing';
+
+import { Discord, Linear } from '../functions';
+
+const TestLayer = Layer.mergeAll(
+  AiService.model('@anthropic/claude-opus-4-0'),
+  makeToolResolverFromFunctions([], Toolkit.make()),
+  makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
+  ComputeEventLogger.layerFromTracing,
+).pipe(
+  Layer.provideMerge(
+    Layer.mergeAll(
+      AiServiceTestingPreset('direct'),
+      TestDatabaseLayer({
+        indexing: { vector: true },
+        types: [],
+        storagePath: testStoragePath({ name: 'feed-test' }),
+      }),
+      CredentialsService.layerConfig([
+        { service: 'exa.ai', apiKey: Config.succeed(Redacted.make(EXA_API_KEY)) },
+        { service: 'discord.com', apiKey: Config.redacted('DISCORD_TOKEN') },
+        { service: 'linear.app', apiKey: Config.redacted('LINEAR_API_KEY') },
+      ]),
+      FunctionInvocationService.layerTestMocked({ functions: [Linear.sync, Discord.fetch] }).pipe(
+        Layer.provideMerge(ComputeEventLogger.layerFromTracing),
+        Layer.provideMerge(TracingService.layerLogInfo()),
+      ),
+      FetchHttpClient.layer,
+    ),
+  ),
+);
+
+describe('Feed', { timeout: 600_000 }, () => {
+  it.effect(
+    'fetch discord messages',
+    Effect.fnUntraced(
+      function* (_) {
+        // const messages = yield* LocalFunctionExecutionService.invokeFunction(fetchDiscordMessages, {
+        //   serverId: '837138313172353095',
+        //   // channelId: '1404487604761526423',
+        //   after: Date.now() / 1000 - 128 * 3600,
+        // });
+        // for (const message of messages) {
+        //   console.log(message.sender.name, message.blocks.find((block) => block._tag === 'text')?.text);
+        // }
+        // console.log(`Fetched ${messages.length} messages`);
+
+        // const result = yield* AiSession.run({
+        //   history: [
+        //     Obj.make(DataType.Message, {
+        //       created: new Date().toISOString(),
+        //       sender: { role: 'user' },
+        //       blocks: messages
+        //         .map(
+        //           (message) =>
+        //             ({
+        //               _tag: 'text',
+        //               text: message.sender.name + ': ' + message.blocks.find((block) => block._tag === 'text')?.text,
+        //             }) as const,
+        //         )
+        //         .filter((block) => block._tag === 'text' && block.text.trim().length > 0),
+        //     }),
+        //   ],
+        //   prompt: 'Summarize the messages.',
+        //   system: 'Summarize the messages.',
+        // }).pipe(Effect.provide(AiService.model('@anthropic/claude-3-5-haiku-latest')));
+        // console.log(result);
+
+        const linearIssues = yield* FunctionInvocationService.invokeFunction(Linear.sync, {
+          team: '1127c63a-6f77-4725-9229-50f6cd47321c',
+        });
+        console.log(linearIssues);
+
+        // const result = yield* AiSession.run({
+        //   history: [
+        //     Obj.make(DataType.Message, {
+        //       created: new Date().toISOString(),
+        //       sender: { role: 'user' },
+        //       blocks: [{ _tag: 'text', text: JSON.stringify(linearIssues) }],
+        //     }),
+        //   ],
+        //   prompt: 'Summarize the whats new.',
+        //   system: 'Summarize the whats new. Reference specific people by name.',
+        // }).pipe(Effect.provide(AiService.model('@anthropic/claude-sonnet-4-0')));
+        // console.log(result.at(-1)?.blocks.find((block) => block._tag === 'text')?.text);
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.taggedTest('llm'),
+    ),
+  );
+});

package/src/functions/agent/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type FunctionDefinition } from '@dxos/functions';
+
+import { default as prompt$ } from './prompt';
+
+export namespace Agent {
+  export const prompt: FunctionDefinition.Any = prompt$; // TODO(burdon): Temp fix for TS error.
+}

package/src/functions/agent/prompt.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ModelName } from '@dxos/ai';
+import { AiSession, GenerationObserver, createToolkit } from '@dxos/assistant';
+import { Prompt, Template } from '@dxos/blueprints';
+import { Obj, Ref, Type } from '@dxos/echo';
+import { DatabaseService, TracingService, defineFunction } from '@dxos/functions';
+import { log } from '@dxos/log';
+
+const DEFAULT_MODEL: ModelName = '@anthropic/claude-opus-4-0';
+
+export default defineFunction({
+  key: 'dxos.org/function/prompt',
+  name: 'Agent',
+  description: 'Agentic worker that executes a provided prompt using blueprints and tools.',
+  inputSchema: Schema.Struct({
+    prompt: Type.Ref(Prompt.Prompt),
+    systemPrompt: Type.Ref(Prompt.Prompt).pipe(Schema.optional),
+    /**
+     * @default @anthropic/claude-opus-4-0
+     */
+    model: Schema.optional(ModelName),
+    /**
+     * Input object or data.
+     * References get auto-resolved.
+     */
+    input: Schema.Record({ key: Schema.String, value: Schema.Any }),
+  }),
+  outputSchema: Schema.Any,
+  handler: Effect.fnUntraced(function* ({ data }) {
+    log.info('processing input', { input: data.input });
+
+    const input = { ...data.input };
+    for (const key of Object.keys(data.input)) {
+      const value = data.input[key];
+      if (Ref.isRef(value)) {
+        const object = yield* DatabaseService.load(value);
+        input[key] = Obj.toJSON(object);
+      } else {
+        input[key] = JSON.stringify(value);
+      }
+    }
+
+    yield* DatabaseService.flush({ indexes: true });
+    const prompt = yield* DatabaseService.load(data.prompt);
+    const systemPrompt = data.systemPrompt ? yield* DatabaseService.load(data.systemPrompt) : undefined;
+    yield* TracingService.emitStatus({ message: `Running ${prompt.id}` });
+
+    log.info('starting agent', { prompt: prompt.id, input: data.input });
+
+    const blueprints = yield* Function.pipe(
+      prompt.blueprints,
+      Array.appendAll(systemPrompt?.blueprints ?? []),
+      Effect.forEach(DatabaseService.loadOption),
+      Effect.map(Array.filter(Option.isSome)),
+      Effect.map(Array.map((option) => option.value)),
+    );
+    const objects = yield* Function.pipe(
+      prompt.context,
+      Array.appendAll(systemPrompt?.context ?? []),
+      Effect.forEach(DatabaseService.loadOption),
+      Effect.map(Array.filter(Option.isSome)),
+      Effect.map(Array.map((option) => option.value)),
+    );
+    const toolkit = yield* createToolkit({ blueprints });
+
+    const promptInstructions = yield* DatabaseService.load(prompt.instructions.source);
+    const promptText = Template.process(promptInstructions.content, input);
+
+    const systemInstructions = systemPrompt ? yield* DatabaseService.load(systemPrompt.instructions.source) : undefined;
+    const systemText = systemInstructions ? Template.process(systemInstructions.content, {}) : undefined;
+
+    const session = new AiSession();
+    const result = yield* session
+      .run({
+        prompt: promptText,
+        system: systemText,
+        blueprints,
+        objects: objects as Obj.Any[],
+        toolkit,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'agent' })),
+      })
+      .pipe(Effect.provide(AiService.model(data.model ?? DEFAULT_MODEL)));
+    const lastBlock = result
+      .at(-1)
+      ?.blocks.filter((block) => block._tag === 'text')
+      .at(-1);
+
+    return {
+      note: lastBlock?.text,
+    };
+  }),
+});

package/src/functions/discord/fetch-messages.test.ts

@@ -0,0 +1,59 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { describe, it } from '@effect/vitest';
+import * as Config from 'effect/Config';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { AiService } from '@dxos/ai';
+import { AiServiceTestingPreset } from '@dxos/ai/testing';
+import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
+import { TestHelpers } from '@dxos/effect';
+import { ComputeEventLogger, CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
+import { TestDatabaseLayer } from '@dxos/functions/testing';
+
+import { default as fetchMessages } from './fetch-messages';
+
+const TestLayer = Layer.mergeAll(
+  AiService.model('@anthropic/claude-opus-4-0'),
+  makeToolResolverFromFunctions([], Toolkit.make()),
+  makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
+  ComputeEventLogger.layerFromTracing,
+).pipe(
+  Layer.provideMerge(
+    Layer.mergeAll(
+      AiServiceTestingPreset('direct'),
+      TestDatabaseLayer({}),
+      CredentialsService.layerConfig([{ service: 'discord.com', apiKey: Config.redacted('DISCORD_TOKEN') }]),
+      FetchHttpClient.layer,
+      FunctionInvocationService.layerTestMocked({ functions: [fetchMessages] }).pipe(
+        Layer.provideMerge(ComputeEventLogger.layerFromTracing),
+        Layer.provideMerge(TracingService.layerLogInfo()),
+      ),
+    ),
+  ),
+);
+
+const DXOS_SERVER_ID = '837138313172353095';
+
+describe('Feed', { timeout: 600_000 }, () => {
+  it.effect(
+    'fetch discord messages',
+    Effect.fnUntraced(
+      function* (_) {
+        const messages = yield* FunctionInvocationService.invokeFunction(fetchMessages, {
+          serverId: DXOS_SERVER_ID,
+          // channelId: '1404487604761526423',
+          last: '7d',
+        });
+        console.log(messages);
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.taggedTest('sync'),
+    ),
+  );
+});

package/src/functions/discord/fetch-messages.ts

@@ -0,0 +1,251 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { DiscordConfig, DiscordREST, DiscordRESTMemoryLive } from 'dfx';
+import type {
+  GuildChannelResponse,
+  MessageResponse,
+  PrivateChannelResponse,
+  PrivateGroupChannelResponse,
+  ThreadResponse,
+} from 'dfx/types';
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { Obj } from '@dxos/echo';
+import { CredentialsService, TracingService, defineFunction } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
+
+// TODO(dmaretskyi): Extract.
+const TimeRange = class extends Schema.String.pipe(Schema.pattern(/\d+(s|m|h|d)/)).annotations({
+  description: 'Time range. 1d - 1 day, 2h - 2 hours, 30m - 30 minutes, 15s - 15 seconds.',
+  examples: ['1d', '2h', '30m', '15s'],
+}) {
+  static toSeconds(timeRange: Schema.Schema.Type<typeof TimeRange>) {
+    const match = timeRange.match(/(\d+)(s|m|h|d)/);
+    if (!match) {
+      throw new Error(`Invalid time range: ${timeRange}`);
+    }
+    const [_, amount, unit] = match;
+    switch (unit) {
+      case 's':
+        return Number(amount);
+      case 'm':
+        return Number(amount) * 60;
+      case 'h':
+        return Number(amount) * 60 * 60;
+      case 'd':
+        return Number(amount) * 24 * 60 * 60;
+      default:
+        throw new Error(`Invalid time range unit: ${unit}`);
+    }
+  }
+};
+type TimeRange = Schema.Schema.Type<typeof TimeRange>;
+
+const DiscordConfigFromCredential = Layer.unwrapEffect(
+  Effect.gen(function* () {
+    return DiscordConfig.layer({
+      token: yield* CredentialsService.getApiKey({ service: 'discord.com' }),
+      rest: {
+        baseUrl: 'https://api-proxy.dxos.workers.dev/discord.com/api/v10',
+      },
+    });
+  }),
+);
+
+type DiscordChannel = GuildChannelResponse | PrivateChannelResponse | PrivateGroupChannelResponse | ThreadResponse;
+
+const DEFAULT_AFTER = 1704067200; // 2024-01-01
+const DEFAULT_LIMIT = 500;
+const DEFAULT_IGNORE_USERNAMES = ['GitHub', 'Needle'];
+
+// TODO(dmaretskyi): Align with standard thread type.
+type Thread = {
+  discordChannelId: string;
+  name?: string;
+  messages: DataType.Message[];
+};
+
+export default defineFunction({
+  key: 'dxos.org/function/fetch-discord-messages',
+  name: 'Sync Discord messages',
+  inputSchema: Schema.Struct({
+    serverId: Schema.String.annotations({
+      description: 'The ID of the server to fetch messages from.',
+    }),
+    channelId: Schema.optional(Schema.String).annotations({
+      description:
+        'The ID of the channel to fetch messages from. Will crawl all channels from the server if not specified.',
+    }),
+    after: Schema.optional(Schema.Number).annotations({
+      description:
+        'Fetch messages that were sent after a given date. Unix timestamp in seconds. Exclusive with `last`.',
+    }),
+    last: TimeRange.annotations({
+      description:
+        'Time range to fetch most recent messages. Specifies the range in the past, from now. "1d" would fetch messages from the last 24 hours.',
+    }),
+    limit: Schema.optional(Schema.Number).annotations({
+      description: 'The maximum number of messages to fetch.',
+    }),
+    pageSize: Schema.optional(Schema.Number).annotations({
+      description: 'The number of messages to fetch per page.',
+    }),
+    ignoreUsernames: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'Exclude messages from these usernames.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({
+      data: {
+        serverId,
+        channelId,
+        after,
+        last,
+        pageSize = 100,
+        limit = DEFAULT_LIMIT,
+        ignoreUsernames = DEFAULT_IGNORE_USERNAMES,
+      },
+    }) {
+      if (!after && !last) {
+        throw new Error('cannot specify both `after` and `last`');
+      }
+      const afterTs = last ? Date.now() / 1000 - TimeRange.toSeconds(last) : (after ?? DEFAULT_AFTER);
+
+      const rest = yield* DiscordREST;
+
+      let channels: DiscordChannel[] = [];
+      channels.push(...(yield* rest.listGuildChannels(serverId)));
+      const { threads: guildThreads } = yield* rest.getActiveGuildThreads(serverId);
+      channels.push(...guildThreads);
+      if (channelId) {
+        channels = channels.filter((channel) => channel.id === channelId);
+      }
+      if (channels.length === 0) {
+        throw new Error('no channels found');
+      }
+      for (const channel of channels) {
+        console.log(channel.id, 'name' in channel ? channel.name : undefined);
+      }
+
+      yield* TracingService.emitStatus({ message: `Will fetch from channels: ${channels.length}` });
+
+      const threads = yield* Effect.forEach(
+        channels,
+        Effect.fnUntraced(function* (channel) {
+          const allMessages: DataType.Message[] = [];
+
+          let lastMessage: Option.Option<DataType.Message> = Option.none();
+          while (true) {
+            const { id: lastId = undefined } = Function.pipe(
+              lastMessage,
+              Option.map(Obj.getKeys('discord.com')),
+              Option.flatMap(Option.fromIterable),
+              Option.getOrElse(() => ({ id: undefined })),
+            );
+
+            const options = {
+              after: !lastId ? `${generateSnowflake(afterTs)}` : lastId,
+              limit: pageSize,
+            };
+            log.info('fetching messages', {
+              lastId,
+              afterTs,
+              afterSnowflake: options.after,
+              after: parseSnowflake(options.after),
+              limit: options.limit,
+            });
+            const messages = yield* rest.listMessages(channel.id, options).pipe(
+              Effect.map(Array.map(makeMessage)),
+              Effect.map(Array.reverse),
+              Effect.catchTag('ErrorResponse', (err) =>
+                err.cause.code === 50001 ? Effect.succeed([]) : Effect.fail(err),
+              ),
+            );
+            if (messages.length > 0) {
+              lastMessage = Option.fromNullable(messages.at(-1));
+              allMessages.push(...messages);
+            } else {
+              break;
+            }
+            yield* TracingService.emitStatus({ message: `Fetched messages: ${allMessages.length}` });
+            if (allMessages.length >= limit) {
+              break;
+            }
+          }
+
+          return {
+            discordChannelId: channel.id,
+            name: 'name' in channel ? (channel.name ?? undefined) : undefined,
+            messages: allMessages
+              .filter((message) => !message.sender.name || !ignoreUsernames.includes(message.sender.name))
+              .filter((message) =>
+                message.blocks.some((block) => block._tag === 'text' && block.text.trim().length > 0),
+              ),
+          } satisfies Thread;
+        }),
+        { concurrency: 10 },
+      );
+
+      return threads
+        .filter((thread) => thread.messages.length > 0)
+        .map(serializeThread)
+        .join('\n');
+    },
+    Effect.provide(
+      DiscordRESTMemoryLive.pipe(Layer.provideMerge(DiscordConfigFromCredential)).pipe(
+        Layer.provide(FetchHttpClient.layer),
+      ),
+    ),
+    Effect.orDie,
+  ),
+});
+
+/**
+ * @param unixTimestamp in seconds
+ */
+const generateSnowflake = (unixTimestamp: number): bigint => {
+  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
+  return (BigInt(unixTimestamp * 1000) - discordEpoch) << 22n;
+};
+
+const parseSnowflake = (snowflake: string): Date => {
+  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
+  return new Date(Number((BigInt(snowflake) >> 22n) + discordEpoch));
+};
+
+const makeMessage = (message: MessageResponse): DataType.Message =>
+  Obj.make(DataType.Message, {
+    [Obj.Meta]: {
+      keys: [
+        { id: message.id, source: 'discord.com' },
+        { id: message.channel_id, source: 'discord.com/thread' },
+      ],
+    },
+    sender: { name: message.author.username },
+    created: message.timestamp,
+    blocks: [{ _tag: 'text', text: message.content }],
+  });
+
+/**
+ * Standard JSON serialization is to verbose for large amounts of data.
+ */
+const serializeThread = (thread: Thread): string => {
+  return `<thread id=${thread.discordChannelId} name=${thread.name ?? ''}>\n${thread.messages
+    .map(
+      (message) =>
+        `  ${message.sender.name}: ${message.blocks
+          .filter((block) => block._tag === 'text')
+          .map((block) => block.text)
+          .join(' ')}`,
+    )
+    .join('\n')}\n</thread>`;
+};

package/src/functions/discord/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as fetch$ } from './fetch-messages';
+
+export namespace Discord {
+  export const fetch = fetch$;
+}

package/src/functions/document/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as read$ } from './read';
+import { default as update$ } from './update';
+
+export namespace Document {
+  export const read = read$;
+  export const update = update$;
+}

package/src/functions/document/read.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ArtifactId } from '@dxos/assistant';
+import { DatabaseService, defineFunction } from '@dxos/functions';
+import { Markdown } from '@dxos/plugin-markdown/types';
+
+export default defineFunction({
+  key: 'dxos.org/function/markdown/read',
+  name: 'Read markdown document',
+  description: 'Read markdown document.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the document to read.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { id } }) {
+    const doc = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Markdown.Document);
+    const { content } = yield* DatabaseService.load(doc.content);
+    return { content };
+  }),
+});

package/src/functions/document/update.ts

@@ -0,0 +1,30 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ArtifactId } from '@dxos/assistant';
+import { DatabaseService, defineFunction } from '@dxos/functions';
+import { Markdown } from '@dxos/plugin-markdown/types';
+
+export default defineFunction({
+  key: 'dxos.org/function/markdown/update',
+  name: 'Update markdown',
+  description: 'Updates the entire contents of the markdown document.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the document to write.',
+    }),
+    content: Schema.String.annotations({
+      description: 'New content to write to the document.',
+    }),
+  }),
+  outputSchema: Schema.Void,
+  handler: Effect.fn(function* ({ data: { id, content } }) {
+    const doc = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Markdown.Document);
+    const text = yield* DatabaseService.load(doc.content);
+    text.content = content;
+  }),
+});