@dxos/assistant-toolkit 0.8.4-main.1068cf700f

package/src/functions/agent/prompt.ts

@@ -0,0 +1,116 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Match from 'effect/Match';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ModelName } from '@dxos/ai';
+import { AiSession, GenerationObserver, createToolkit } from '@dxos/assistant';
+import { Prompt, Template } from '@dxos/blueprints';
+import { Obj, Ref, Type } from '@dxos/echo';
+import { Database } from '@dxos/echo';
+import { TracingService, defineFunction } from '@dxos/functions';
+import { log } from '@dxos/log';
+
+const DEFAULT_MODEL: ModelName = '@anthropic/claude-opus-4-0';
+
+export default defineFunction({
+  key: 'dxos.org/function/prompt',
+  name: 'Agent',
+  description: 'Agentic worker that executes a provided prompt using blueprints and tools.',
+  inputSchema: Schema.Struct({
+    prompt: Type.Ref(Prompt.Prompt),
+    systemPrompt: Type.Ref(Prompt.Prompt).pipe(Schema.optional),
+    /**
+     * @default @anthropic/claude-opus-4-0
+     */
+    model: Schema.optional(ModelName),
+    /**
+     * Input object or data.
+     * References get auto-resolved.
+     */
+    input: Schema.Any.pipe(Schema.annotations({ title: 'Input' })),
+  }),
+  outputSchema: Schema.Any,
+  handler: Effect.fnUntraced(function* ({ data }) {
+    log.info('processing input', { input: data.input });
+
+    // TODO(wittjosiah): Support templated object as input.
+    //   Currently the object templating only supports direct pass-through or strings.
+    // const input = { ...data.input };
+    // for (const key of Object.keys(data.input)) {
+    //   const value = data.input[key];
+    //   if (Ref.isRef(value)) {
+    //     const object = yield* Database.load(value);
+    //     input[key] = Obj.toJSON(object);
+    //   } else {
+    //     input[key] = JSON.stringify(value);
+    //   }
+    // }
+    const input = yield* Match.value(data.input).pipe(
+      Match.when(
+        (value: any) => Ref.isRef(value),
+        Effect.fnUntraced(function* (ref) {
+          const object = yield* Database.load(ref);
+          return Obj.toJSON(object as Obj.Unknown);
+        }),
+      ),
+      Match.orElse(() => Effect.succeed(data.input)),
+    );
+
+    yield* Database.flush({ indexes: true });
+    const prompt = yield* Database.load(data.prompt);
+    const systemPrompt = data.systemPrompt ? yield* Database.load(data.systemPrompt) : undefined;
+    yield* TracingService.emitStatus({ message: `Running ${prompt.id}` });
+
+    log.info('starting agent', { prompt: prompt.id, input });
+
+    const blueprints = yield* Function.pipe(
+      prompt.blueprints,
+      Array.appendAll(systemPrompt?.blueprints ?? []),
+      Effect.forEach(Database.loadOption),
+      Effect.map(Array.filter(Option.isSome)),
+      Effect.map(Array.map((option) => option.value)),
+    );
+    const toolkit = yield* createToolkit({ blueprints });
+
+    const objects = yield* Function.pipe(
+      prompt.context,
+      Array.appendAll(systemPrompt?.context ?? []),
+      Effect.forEach(Database.loadOption),
+      Effect.map(Array.filter(Option.isSome)),
+      Effect.map(Array.map((option) => option.value)),
+    );
+
+    const promptInstructions = yield* Database.load(prompt.instructions.source);
+    const promptText = Template.process(promptInstructions.content, input);
+
+    const systemInstructions = systemPrompt ? yield* Database.load(systemPrompt.instructions.source) : undefined;
+    const systemText = systemInstructions ? Template.process(systemInstructions.content, {}) : undefined;
+
+    const session = new AiSession();
+    const result = yield* session
+      .run({
+        prompt: promptText,
+        system: systemText,
+        blueprints,
+        objects: objects as Obj.Unknown[],
+        toolkit,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'agent' })),
+      })
+      .pipe(Effect.provide(AiService.model(data.model ?? DEFAULT_MODEL)));
+    const lastBlock = result
+      .at(-1)
+      ?.blocks.filter((block) => block._tag === 'text')
+      .at(-1);
+
+    return {
+      note: lastBlock?.text,
+    };
+  }),
+});

package/src/functions/discord/fetch-messages.test.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { describe, it } from '@effect/vitest';
+import * as Config from 'effect/Config';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { AiService } from '@dxos/ai';
+import { AiServiceTestingPreset } from '@dxos/ai/testing';
+import { GenericToolkit, ToolExecutionServices } from '@dxos/assistant';
+import { TestHelpers } from '@dxos/effect/testing';
+import { CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
+import { FunctionInvocationServiceLayerTestMocked, TestDatabaseLayer } from '@dxos/functions-runtime/testing';
+
+import { default as fetchMessages } from './fetch-messages';
+
+const TestLayer = Layer.mergeAll(AiService.model('@anthropic/claude-opus-4-0'), ToolExecutionServices).pipe(
+  Layer.provideMerge(
+    Layer.mergeAll(
+      GenericToolkit.providerEmpty,
+      AiServiceTestingPreset('direct'),
+      TestDatabaseLayer({}),
+      CredentialsService.layerConfig([{ service: 'discord.com', apiKey: Config.redacted('DISCORD_TOKEN') }]),
+      FetchHttpClient.layer,
+      FunctionInvocationServiceLayerTestMocked({ functions: [fetchMessages] }).pipe(
+        Layer.provideMerge(TracingService.layerNoop),
+      ),
+    ),
+  ),
+);
+
+const DXOS_SERVER_ID = '837138313172353095';
+
+describe('Feed', { timeout: 600_000 }, () => {
+  it.effect(
+    'fetch discord messages',
+    Effect.fnUntraced(
+      function* (_) {
+        const messages = yield* FunctionInvocationService.invokeFunction(fetchMessages, {
+          serverId: DXOS_SERVER_ID,
+          // channelId: '1404487604761526423',
+          last: '7d',
+        });
+        console.log(messages);
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+      TestHelpers.taggedTest('sync'),
+    ),
+  );
+});

package/src/functions/discord/fetch-messages.ts

@@ -0,0 +1,252 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { DiscordConfig, DiscordREST, DiscordRESTMemoryLive } from 'dfx';
+import type {
+  GuildChannelResponse,
+  MessageResponse,
+  PrivateChannelResponse,
+  PrivateGroupChannelResponse,
+  ThreadResponse,
+} from 'dfx/types';
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { Obj } from '@dxos/echo';
+import { CredentialsService, TracingService, defineFunction } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { Message } from '@dxos/types';
+
+// TODO(dmaretskyi): Extract.
+const TimeRange = class extends Schema.String.pipe(Schema.pattern(/\d+(s|m|h|d)/)).annotations({
+  description: 'Time range. 1d - 1 day, 2h - 2 hours, 30m - 30 minutes, 15s - 15 seconds.',
+  examples: ['1d', '2h', '30m', '15s'],
+}) {
+  static toSeconds(timeRange: Schema.Schema.Type<typeof TimeRange>) {
+    const match = timeRange.match(/(\d+)(s|m|h|d)/);
+    if (!match) {
+      throw new Error(`Invalid time range: ${timeRange}`);
+    }
+    const [_, amount, unit] = match;
+    switch (unit) {
+      case 's':
+        return Number(amount);
+      case 'm':
+        return Number(amount) * 60;
+      case 'h':
+        return Number(amount) * 60 * 60;
+      case 'd':
+        return Number(amount) * 24 * 60 * 60;
+      default:
+        throw new Error(`Invalid time range unit: ${unit}`);
+    }
+  }
+};
+type TimeRange = Schema.Schema.Type<typeof TimeRange>;
+
+const DiscordConfigFromCredential = Layer.unwrapEffect(
+  Effect.gen(function* () {
+    return DiscordConfig.layer({
+      token: yield* CredentialsService.getApiKey({ service: 'discord.com' }),
+      rest: {
+        baseUrl: 'https://api-proxy.dxos.workers.dev/discord.com/api/v10',
+      },
+    });
+  }),
+);
+
+type DiscordChannel = GuildChannelResponse | PrivateChannelResponse | PrivateGroupChannelResponse | ThreadResponse;
+
+const DEFAULT_AFTER = 1704067200; // 2024-01-01
+const DEFAULT_LIMIT = 500;
+const DEFAULT_IGNORE_USERNAMES = ['GitHub', 'Needle'];
+
+// TODO(dmaretskyi): Align with standard thread type.
+type Thread = {
+  discordChannelId: string;
+  name?: string;
+  messages: Message.Message[];
+};
+
+export default defineFunction({
+  key: 'dxos.org/function/fetch-discord-messages',
+  name: 'Sync Discord messages',
+  inputSchema: Schema.Struct({
+    serverId: Schema.String.annotations({
+      description: 'The ID of the server to fetch messages from.',
+    }),
+    channelId: Schema.optional(Schema.String).annotations({
+      description:
+        'The ID of the channel to fetch messages from. Will crawl all channels from the server if not specified.',
+    }),
+    after: Schema.optional(Schema.Number).annotations({
+      description:
+        'Fetch messages that were sent after a given date. Unix timestamp in seconds. Exclusive with `last`.',
+    }),
+    last: TimeRange.annotations({
+      description:
+        'Time range to fetch most recent messages. Specifies the range in the past, from now. "1d" would fetch messages from the last 24 hours.',
+    }),
+    limit: Schema.optional(Schema.Number).annotations({
+      description: 'The maximum number of messages to fetch.',
+    }),
+    pageSize: Schema.optional(Schema.Number).annotations({
+      description: 'The number of messages to fetch per page.',
+    }),
+    ignoreUsernames: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'Exclude messages from these usernames.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({
+      data: {
+        serverId,
+        channelId,
+        after,
+        last,
+        pageSize = 100,
+        limit = DEFAULT_LIMIT,
+        ignoreUsernames = DEFAULT_IGNORE_USERNAMES,
+      },
+    }) {
+      if (!after && !last) {
+        throw new Error('cannot specify both `after` and `last`');
+      }
+      const afterTs = last ? Date.now() / 1000 - TimeRange.toSeconds(last) : (after ?? DEFAULT_AFTER);
+
+      const rest = yield* DiscordREST;
+
+      let channels: DiscordChannel[] = [];
+      channels.push(...(yield* rest.listGuildChannels(serverId)));
+      const { threads: guildThreads } = yield* rest.getActiveGuildThreads(serverId);
+      channels.push(...guildThreads);
+      if (channelId) {
+        channels = channels.filter((channel) => channel.id === channelId);
+      }
+      if (channels.length === 0) {
+        throw new Error('no channels found');
+      }
+      for (const channel of channels) {
+        log.info('channel', { id: channel.id, name: 'name' in channel ? channel.name : undefined });
+      }
+
+      yield* TracingService.emitStatus({ message: `Will fetch from channels: ${channels.length}` });
+
+      const threads = yield* Effect.forEach(
+        channels,
+        Effect.fnUntraced(function* (channel) {
+          const allMessages: Message.Message[] = [];
+
+          let lastMessage: Option.Option<Message.Message> = Option.none();
+          while (true) {
+            const { id: lastId = undefined } = Function.pipe(
+              lastMessage,
+              Option.map(Obj.getKeys('discord.com')),
+              Option.flatMap(Option.fromIterable),
+              Option.getOrElse(() => ({ id: undefined })),
+            );
+
+            const options = {
+              after: !lastId ? `${generateSnowflake(afterTs)}` : lastId,
+              limit: pageSize,
+            };
+            log.info('fetching messages', {
+              lastId,
+              afterTs,
+              afterSnowflake: options.after,
+              after: parseSnowflake(options.after),
+              limit: options.limit,
+            });
+            const messages = yield* rest.listMessages(channel.id, options).pipe(
+              Effect.map(Array.map(makeMessage)),
+              Effect.map(Array.reverse),
+              Effect.catchTag('ErrorResponse', (err) =>
+                err.cause.code === 50001 ? Effect.succeed([]) : Effect.fail(err),
+              ),
+            );
+            if (messages.length > 0) {
+              lastMessage = Option.fromNullable(messages.at(-1));
+              allMessages.push(...messages);
+            } else {
+              break;
+            }
+            yield* TracingService.emitStatus({ message: `Fetched messages: ${allMessages.length}` });
+            if (allMessages.length >= limit) {
+              break;
+            }
+          }
+
+          return {
+            discordChannelId: channel.id,
+            name: 'name' in channel ? (channel.name ?? undefined) : undefined,
+            messages: allMessages
+              .filter((message) => !message.sender.name || !ignoreUsernames.includes(message.sender.name))
+              .filter((message) =>
+                message.blocks.some((block: any) => block._tag === 'text' && block.text.trim().length > 0),
+              ),
+          } satisfies Thread;
+        }),
+        { concurrency: 10 },
+      );
+
+      return threads
+        .filter((thread) => thread.messages.length > 0)
+        .map(serializeThread)
+        .join('\n');
+    },
+    Effect.provide(
+      DiscordRESTMemoryLive.pipe(Layer.provideMerge(DiscordConfigFromCredential)).pipe(
+        Layer.provide(FetchHttpClient.layer),
+      ),
+    ),
+    Effect.orDie,
+  ),
+});
+
+/**
+ * @param unixTimestamp in seconds
+ */
+const generateSnowflake = (unixTimestamp: number): bigint => {
+  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
+  return (BigInt(unixTimestamp * 1000) - discordEpoch) << 22n;
+};
+
+const parseSnowflake = (snowflake: string): Date => {
+  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
+  return new Date(Number((BigInt(snowflake) >> 22n) + discordEpoch));
+};
+
+// TODO(burdon): Move to @dxos/types.
+const makeMessage = (message: MessageResponse): Message.Message =>
+  Obj.make(Message.Message, {
+    [Obj.Meta]: {
+      keys: [
+        { id: message.id, source: 'discord.com' },
+        { id: message.channel_id, source: 'discord.com/thread' },
+      ],
+    },
+    sender: { name: message.author.username },
+    created: message.timestamp,
+    blocks: [{ _tag: 'text', text: message.content }],
+  });
+
+/**
+ * Standard JSON serialization is to verbose for large amounts of data.
+ */
+const serializeThread = (thread: Thread): string => {
+  return `<thread id=${thread.discordChannelId} name=${thread.name ?? ''}>\n${thread.messages
+    .map(
+      (message) =>
+        `  ${message.sender.name}: ${message.blocks
+          .filter((block: any) => block._tag === 'text')
+          .map((block: any) => block.text)
+          .join(' ')}`,
+    )
+    .join('\n')}\n</thread>`;
+};

package/src/functions/discord/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as fetch$ } from './fetch-messages';
+
+export namespace Discord {
+  export const fetch = fetch$;
+}

package/src/functions/document/create.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Database, Ref } from '@dxos/echo';
+import { defineFunction } from '@dxos/functions';
+import { Markdown } from '@dxos/plugin-markdown/types';
+
+export default defineFunction({
+  key: 'dxos.org/function/markdown/create',
+  name: 'Create markdown document',
+  description: 'Creates a new markdown document.',
+  inputSchema: Schema.Struct({
+    name: Schema.String.annotations({
+      description: 'Name of the document.',
+    }),
+    content: Schema.String.annotations({
+      description: 'Content of the document.',
+    }),
+  }),
+  outputSchema: Schema.Void,
+  handler: Effect.fn(function* ({ data: { name, content } }) {
+    const doc = yield* Database.add(Markdown.make({ name, content }));
+    return { document: Ref.make(doc) };
+  }),
+});

package/src/functions/document/index.ts

@@ -0,0 +1,13 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as create$ } from './create';
+import { default as read$ } from './read';
+import { default as update$ } from './update';
+
+export namespace Document {
+  export const read = read$;
+  export const update = update$;
+  export const create = create$;
+}

package/src/functions/document/read.ts

@@ -0,0 +1,32 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Database, Type } from '@dxos/echo';
+import { defineFunction } from '@dxos/functions';
+import { Markdown } from '@dxos/plugin-markdown/types';
+
+export default defineFunction({
+  key: 'dxos.org/function/markdown/read',
+  name: 'Read markdown document',
+  description:
+    'Read markdown document. Note that result is a snapshot in time, and might have changed since the document was last read.',
+  inputSchema: Schema.Struct({
+    document: Type.Ref(Markdown.Document).annotations({
+      description: 'The document to read.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { document } }) {
+    const { content } = yield* document.pipe(
+      Database.load,
+      Effect.flatMap((doc) => doc.content.pipe(Database.load)),
+    );
+    return { content };
+  }),
+});

package/src/functions/document/update.ts

@@ -0,0 +1,32 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Database, Obj, Type } from '@dxos/echo';
+import { defineFunction } from '@dxos/functions';
+import { Markdown } from '@dxos/plugin-markdown/types';
+
+export default defineFunction({
+  key: 'dxos.org/function/markdown/update',
+  name: 'Update markdown',
+  description: 'Updates the entire contents of the markdown document.',
+  inputSchema: Schema.Struct({
+    doc: Type.Ref(Markdown.Document).annotations({
+      description: 'The ID of the document to write.',
+    }),
+    content: Schema.String.annotations({
+      description: 'New content to write to the document.',
+    }),
+  }),
+  outputSchema: Schema.Void,
+  handler: Effect.fn(function* ({ data: { doc, content } }) {
+    const document = yield* Database.load(doc);
+    const text = yield* Database.load(document.content);
+    Obj.change(text, (t) => {
+      t.content = content;
+    });
+  }),
+});

package/src/functions/entity-extraction/entity-extraction.conversations.json

@@ -0,0 +1 @@
+{"conversations":[{"parameters":{"model":"@anthropic/claude-sonnet-4-0","stream":true,"tools":[{"name":"graph_writer","description":"Write to the local graph database","inputSchema":{"type":"object","required":[],"properties":{"objects_dxos_org_type_Organization":{"type":"array","items":{"type":"object","required":["id"],"properties":{"name":{"type":"string","title":"Name"},"description":{"type":"string","title":"Description"},"status":{"type":"string","enum":["prospect","qualified","active","commit","reject"],"title":"Status"},"image":{"type":"string","title":"Image"},"website":{"type":"string","title":"Website"},"id":{"type":"string","description":"The id of this object. Come up with a unique id based on your judgement."}},"additionalProperties":false},"description":"The objects of type: dxos.org/type/Organization. An organization."}},"additionalProperties":false}}]},"prompt":{"content":[{"role":"system","content":"Extract the sender's organization from the email. If you are not sure, do nothing.\nThe extracted organization URL must match the sender's email domain.\n,","options":{}},{"role":"user","content":"{\"source\":{\"id\":\"01JGFJJZ00G0WKQSJGMAKCNT87\",\"@type\":\"dxn:type:dxos.org/type/Message:0.2.0\",\"@meta\":{\"keys\":[],\"tags\":[\"important\"]},\"blocks\":[{\"_tag\":\"text\",\"text\":\"Hey team, what's up?\"},{\"_tag\":\"text\",\"text\":\"I'm working on a new algorithm today.\"},{\"_tag\":\"text\",\"text\":\"Anything new from the research team?\"}],\"created\":\"2025-01-01T00:00:00.000Z\",\"sender\":{\"email\":\"john.smith@anthropic.com\",\"name\":\"John Smith\"}},\"contact\":{\"id\":\"01JGFJJZ00G0WKQSJGMAKCNT89\",\"@type\":\"dxn:type:dxos.org/type/Person:0.1.0\",\"@meta\":{\"keys\":[],\"tags\":[\"important\"]},\"emails\":[{\"value\":\"john.smith@anthropic.com\"}],\"fullName\":\"John Smith\"}}","options":{"anthropic":{"cacheControl":{"ttl":"5m","type":"ephemeral"}}}}]},"response":[{"type":"response-metadata","id":"msg_01R1zT7R6NEdWg598pnsyNeA","modelId":"claude-sonnet-4-20250514","timestamp":"1970-01-01T00:00:00.000Z","metadata":{}},{"type":"text-start","id":"0","metadata":{}},{"type":"text-delta","id":"0","delta":"I can","metadata":{}},{"type":"text-delta","id":"0","delta":" see that","metadata":{}},{"type":"text-delta","id":"0","delta":" the","metadata":{}},{"type":"text-delta","id":"0","delta":" sender","metadata":{}},{"type":"text-delta","id":"0","delta":"'s","metadata":{}},{"type":"text-delta","id":"0","delta":" email","metadata":{}},{"type":"text-delta","id":"0","delta":" is from","metadata":{}},{"type":"text-delta","id":"0","delta":" the","metadata":{}},{"type":"text-delta","id":"0","delta":" domain","metadata":{}},{"type":"text-delta","id":"0","delta":" \"anthropic.com\".","metadata":{}},{"type":"text-delta","id":"0","delta":" I","metadata":{}},{"type":"text-delta","id":"0","delta":"'ll extract and","metadata":{}},{"type":"text-delta","id":"0","delta":" create","metadata":{}},{"type":"text-delta","id":"0","delta":" an","metadata":{}},{"type":"text-delta","id":"0","delta":" organization record","metadata":{}},{"type":"text-delta","id":"0","delta":" for","metadata":{}},{"type":"text-delta","id":"0","delta":" Anthropic based on this","metadata":{}},{"type":"text-delta","id":"0","delta":" email","metadata":{}},{"type":"text-delta","id":"0","delta":" domain","metadata":{}},{"type":"text-delta","id":"0","delta":".","metadata":{}},{"type":"text-end","id":"0","metadata":{}},{"type":"tool-params-start","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","name":"graph_writer","providerExecuted":false,"metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"{\"obje","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"cts_dxos","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"_org_","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"type_Org","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"aniza","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"tion\"","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":": ","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"[{\"id\":\"an","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"thropic\",","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"\"nam","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"e\":\"Anthropi","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"c\",","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"\"we","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"bsite\":","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"\"https://a","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"nthrop","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"ic.","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"com\",\"descri","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"ption\":","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"\"A","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"I sa","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"fet","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"y compan","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"y focuse","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"d on d","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"evelop","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"ing s","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"afe, b","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"eneficia","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"l ","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"AI s","metadata":{}},{"type":"tool-params-delta","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","delta":"ystems\"}]}","metadata":{}},{"type":"tool-params-end","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","metadata":{}},{"type":"tool-call","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","name":"graph_writer","params":{"objects_dxos_org_type_Organization":[{"name":"Anthropic","description":"AI safety company focused on developing safe, beneficial AI systems","website":"https://anthropic.com","id":"anthropic"}]},"providerExecuted":false,"metadata":{}},{"type":"finish","reason":"tool-calls","usage":{"inputTokens":853,"outputTokens":154,"totalTokens":1007},"metadata":{"anthropic":{"usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":853,"output_tokens":2,"server_tool_use":null,"service_tier":"standard"}}}}]},{"parameters":{"model":"@anthropic/claude-sonnet-4-0","stream":true,"tools":[{"name":"graph_writer","description":"Write to the local graph database","inputSchema":{"type":"object","required":[],"properties":{"objects_dxos_org_type_Organization":{"type":"array","items":{"type":"object","required":["id"],"properties":{"name":{"type":"string","title":"Name"},"description":{"type":"string","title":"Description"},"status":{"type":"string","enum":["prospect","qualified","active","commit","reject"],"title":"Status"},"image":{"type":"string","title":"Image"},"website":{"type":"string","title":"Website"},"id":{"type":"string","description":"The id of this object. Come up with a unique id based on your judgement."}},"additionalProperties":false},"description":"The objects of type: dxos.org/type/Organization. An organization."}},"additionalProperties":false}}]},"prompt":{"content":[{"role":"system","content":"Extract the sender's organization from the email. If you are not sure, do nothing.\nThe extracted organization URL must match the sender's email domain.\n,","options":{}},{"role":"user","content":"{\"source\":{\"id\":\"01JGFJJZ00G0WKQSJGMAKCNT87\",\"@type\":\"dxn:type:dxos.org/type/Message:0.2.0\",\"@meta\":{\"keys\":[],\"tags\":[\"important\"]},\"blocks\":[{\"_tag\":\"text\",\"text\":\"Hey team, what's up?\"},{\"_tag\":\"text\",\"text\":\"I'm working on a new algorithm today.\"},{\"_tag\":\"text\",\"text\":\"Anything new from the research team?\"}],\"created\":\"2025-01-01T00:00:00.000Z\",\"sender\":{\"email\":\"john.smith@anthropic.com\",\"name\":\"John Smith\"}},\"contact\":{\"id\":\"01JGFJJZ00G0WKQSJGMAKCNT89\",\"@type\":\"dxn:type:dxos.org/type/Person:0.1.0\",\"@meta\":{\"keys\":[],\"tags\":[\"important\"]},\"emails\":[{\"value\":\"john.smith@anthropic.com\"}],\"fullName\":\"John Smith\"}}","options":{}},{"role":"assistant","content":[{"type":"text","text":"I can see that the sender's email is from the domain \"anthropic.com\". I'll extract and create an organization record for Anthropic based on this email domain.","options":{}},{"type":"tool-call","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","name":"graph_writer","params":{"objects_dxos_org_type_Organization":[{"id":"anthropic","name":"Anthropic","website":"https://anthropic.com","description":"AI safety company focused on developing safe, beneficial AI systems"}]},"providerExecuted":false,"options":{}}],"options":{}},{"role":"tool","content":[{"type":"tool-result","id":"toolu_01BMGcZUghBKE9Zj3zs7b2NL","name":"graph_writer","isFailure":false,"result":["dxn:queue:data:BDJXMULPW5FB55UZ7KP5HLDHVTXUTHE2U:01JGFJJZ00G0WKQSJGMAKCNT8B:01JGFJJZ00G0WKQSJGMAKCNT8G"],"providerExecuted":false,"options":{}}],"options":{"anthropic":{"cacheControl":{"ttl":"5m","type":"ephemeral"}}}}]},"response":[{"type":"response-metadata","id":"msg_01EfBN9rt7NxPo6mGTuhB8s2","modelId":"claude-sonnet-4-20250514","timestamp":"1970-01-01T00:00:00.000Z","metadata":{}},{"type":"text-start","id":"0","metadata":{}},{"type":"text-delta","id":"0","delta":"I","metadata":{}},{"type":"text-delta","id":"0","delta":"'ve successfully extracted and","metadata":{}},{"type":"text-delta","id":"0","delta":" created an","metadata":{}},{"type":"text-delta","id":"0","delta":" organization record for Anthropic based","metadata":{}},{"type":"text-delta","id":"0","delta":" on the sender's email domain (","metadata":{}},{"type":"text-delta","id":"0","delta":"anthropic.com). The organization has","metadata":{}},{"type":"text-delta","id":"0","delta":" been added","metadata":{}},{"type":"text-delta","id":"0","delta":" to the graph","metadata":{}},{"type":"text-delta","id":"0","delta":" database","metadata":{}},{"type":"text-delta","id":"0","delta":" with the name","metadata":{}},{"type":"text-delta","id":"0","delta":" \"Anthropic\" and their","metadata":{}},{"type":"text-delta","id":"0","delta":" corresponding","metadata":{}},{"type":"text-delta","id":"0","delta":" website.","metadata":{}},{"type":"text-end","id":"0","metadata":{}},{"type":"finish","reason":"stop","usage":{"inputTokens":6,"outputTokens":51,"totalTokens":57},"metadata":{"anthropic":{"usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":1083,"input_tokens":6,"output_tokens":1,"server_tool_use":null,"service_tier":"standard"}}}}]}]}

package/src/functions/entity-extraction/entity-extraction.test.ts

@@ -0,0 +1,74 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+
+import { MemoizedAiService } from '@dxos/ai/testing';
+import { AssistantTestLayer } from '@dxos/assistant/testing';
+import { Blueprint } from '@dxos/blueprints';
+import { Obj } from '@dxos/echo';
+import { Database } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { FunctionInvocationService } from '@dxos/functions';
+import { ObjectId } from '@dxos/keys';
+import { Message, Organization, Person } from '@dxos/types';
+
+import { ResearchGraph } from '../research';
+
+import { default as entityExtraction } from './entity-extraction';
+
+ObjectId.dangerouslyDisableRandomness();
+
+const TestLayer = AssistantTestLayer({
+  functions: [entityExtraction],
+  types: [Blueprint.Blueprint, Message.Message, Person.Person, Organization.Organization, ResearchGraph],
+});
+
+describe('Entity extraction', () => {
+  it.effect(
+    'call a function to generate a research report',
+    Effect.fnUntraced(
+      function* (_) {
+        const email = yield* Database.add(
+          Obj.make(Message.Message, {
+            [Obj.Meta]: {
+              tags: ['important'],
+            },
+            created: new Date('2025-01-01').toISOString(),
+            sender: {
+              name: 'John Smith',
+              email: 'john.smith@anthropic.com',
+            },
+            blocks: [
+              {
+                _tag: 'text',
+                text: "Hey team, what's up?",
+              },
+              {
+                _tag: 'text',
+                text: "I'm working on a new algorithm today.",
+              },
+              {
+                _tag: 'text',
+                text: 'Anything new from the research team?',
+              },
+            ],
+          }),
+        );
+        yield* Database.flush({ indexes: true });
+        const result = yield* FunctionInvocationService.invokeFunction(entityExtraction, {
+          source: email,
+        });
+        expect(result.entities).toHaveLength(2);
+        for (const entity of result.entities ?? []) {
+          expect(Obj.getMeta(entity)?.tags).toContain('important');
+        }
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+    ),
+    MemoizedAiService.isGenerationEnabled() ? 60_000 : undefined,
+  );
+});