@dxos/assistant-toolkit 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/experimental/feed.test.ts

@@ -1,108 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Toolkit from '@effect/ai/Toolkit';
-import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
-import { describe, it } from '@effect/vitest';
-import * as Config from 'effect/Config';
-import * as Effect from 'effect/Effect';
-import * as Layer from 'effect/Layer';
-import * as Redacted from 'effect/Redacted';
-
-import { AiService } from '@dxos/ai';
-import { AiServiceTestingPreset, EXA_API_KEY } from '@dxos/ai/testing';
-import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
-import { TestHelpers } from '@dxos/effect';
-import { ComputeEventLogger, CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
-import { TestDatabaseLayer, testStoragePath } from '@dxos/functions/testing';
-
-import { Discord, Linear } from '../functions';
-
-const TestLayer = Layer.mergeAll(
-  AiService.model('@anthropic/claude-opus-4-0'),
-  makeToolResolverFromFunctions([], Toolkit.make()),
-  makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
-  ComputeEventLogger.layerFromTracing,
-).pipe(
-  Layer.provideMerge(
-    Layer.mergeAll(
-      AiServiceTestingPreset('direct'),
-      TestDatabaseLayer({
-        indexing: { vector: true },
-        types: [],
-        storagePath: testStoragePath({ name: 'feed-test' }),
-      }),
-      CredentialsService.layerConfig([
-        { service: 'exa.ai', apiKey: Config.succeed(Redacted.make(EXA_API_KEY)) },
-        { service: 'discord.com', apiKey: Config.redacted('DISCORD_TOKEN') },
-        { service: 'linear.app', apiKey: Config.redacted('LINEAR_API_KEY') },
-      ]),
-      FunctionInvocationService.layerTestMocked({ functions: [Linear.sync, Discord.fetch] }).pipe(
-        Layer.provideMerge(ComputeEventLogger.layerFromTracing),
-        Layer.provideMerge(TracingService.layerLogInfo()),
-      ),
-      FetchHttpClient.layer,
-    ),
-  ),
-);
-
-describe('Feed', { timeout: 600_000 }, () => {
-  it.effect(
-    'fetch discord messages',
-    Effect.fnUntraced(
-      function* (_) {
-        // const messages = yield* LocalFunctionExecutionService.invokeFunction(fetchDiscordMessages, {
-        //   serverId: '837138313172353095',
-        //   // channelId: '1404487604761526423',
-        //   after: Date.now() / 1000 - 128 * 3600,
-        // });
-        // for (const message of messages) {
-        //   console.log(message.sender.name, message.blocks.find((block) => block._tag === 'text')?.text);
-        // }
-        // console.log(`Fetched ${messages.length} messages`);
-
-        // const result = yield* AiSession.run({
-        //   history: [
-        //     Obj.make(DataType.Message, {
-        //       created: new Date().toISOString(),
-        //       sender: { role: 'user' },
-        //       blocks: messages
-        //         .map(
-        //           (message) =>
-        //             ({
-        //               _tag: 'text',
-        //               text: message.sender.name + ': ' + message.blocks.find((block) => block._tag === 'text')?.text,
-        //             }) as const,
-        //         )
-        //         .filter((block) => block._tag === 'text' && block.text.trim().length > 0),
-        //     }),
-        //   ],
-        //   prompt: 'Summarize the messages.',
-        //   system: 'Summarize the messages.',
-        // }).pipe(Effect.provide(AiService.model('@anthropic/claude-3-5-haiku-latest')));
-        // console.log(result);
-
-        const linearIssues = yield* FunctionInvocationService.invokeFunction(Linear.sync, {
-          team: '1127c63a-6f77-4725-9229-50f6cd47321c',
-        });
-        console.log(linearIssues);
-
-        // const result = yield* AiSession.run({
-        //   history: [
-        //     Obj.make(DataType.Message, {
-        //       created: new Date().toISOString(),
-        //       sender: { role: 'user' },
-        //       blocks: [{ _tag: 'text', text: JSON.stringify(linearIssues) }],
-        //     }),
-        //   ],
-        //   prompt: 'Summarize the whats new.',
-        //   system: 'Summarize the whats new. Reference specific people by name.',
-        // }).pipe(Effect.provide(AiService.model('@anthropic/claude-sonnet-4-0')));
-        // console.log(result.at(-1)?.blocks.find((block) => block._tag === 'text')?.text);
-      },
-      Effect.provide(TestLayer),
-      TestHelpers.taggedTest('llm'),
-    ),
-  );
-});

package/src/functions/discord/fetch-messages.test.ts

@@ -1,59 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Toolkit from '@effect/ai/Toolkit';
-import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
-import { describe, it } from '@effect/vitest';
-import * as Config from 'effect/Config';
-import * as Effect from 'effect/Effect';
-import * as Layer from 'effect/Layer';
-
-import { AiService } from '@dxos/ai';
-import { AiServiceTestingPreset } from '@dxos/ai/testing';
-import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
-import { TestHelpers } from '@dxos/effect';
-import { ComputeEventLogger, CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
-import { TestDatabaseLayer } from '@dxos/functions/testing';
-
-import { default as fetchMessages } from './fetch-messages';
-
-const TestLayer = Layer.mergeAll(
-  AiService.model('@anthropic/claude-opus-4-0'),
-  makeToolResolverFromFunctions([], Toolkit.make()),
-  makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
-  ComputeEventLogger.layerFromTracing,
-).pipe(
-  Layer.provideMerge(
-    Layer.mergeAll(
-      AiServiceTestingPreset('direct'),
-      TestDatabaseLayer({}),
-      CredentialsService.layerConfig([{ service: 'discord.com', apiKey: Config.redacted('DISCORD_TOKEN') }]),
-      FetchHttpClient.layer,
-      FunctionInvocationService.layerTestMocked({ functions: [fetchMessages] }).pipe(
-        Layer.provideMerge(ComputeEventLogger.layerFromTracing),
-        Layer.provideMerge(TracingService.layerLogInfo()),
-      ),
-    ),
-  ),
-);
-
-const DXOS_SERVER_ID = '837138313172353095';
-
-describe('Feed', { timeout: 600_000 }, () => {
-  it.effect(
-    'fetch discord messages',
-    Effect.fnUntraced(
-      function* (_) {
-        const messages = yield* FunctionInvocationService.invokeFunction(fetchMessages, {
-          serverId: DXOS_SERVER_ID,
-          // channelId: '1404487604761526423',
-          last: '7d',
-        });
-        console.log(messages);
-      },
-      Effect.provide(TestLayer),
-      TestHelpers.taggedTest('sync'),
-    ),
-  );
-});

package/src/functions/discord/fetch-messages.ts

@@ -1,251 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
-import { DiscordConfig, DiscordREST, DiscordRESTMemoryLive } from 'dfx';
-import type {
-  GuildChannelResponse,
-  MessageResponse,
-  PrivateChannelResponse,
-  PrivateGroupChannelResponse,
-  ThreadResponse,
-} from 'dfx/types';
-import * as Array from 'effect/Array';
-import * as Effect from 'effect/Effect';
-import * as Function from 'effect/Function';
-import * as Layer from 'effect/Layer';
-import * as Option from 'effect/Option';
-import * as Schema from 'effect/Schema';
-
-import { Obj } from '@dxos/echo';
-import { CredentialsService, TracingService, defineFunction } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-
-// TODO(dmaretskyi): Extract.
-const TimeRange = class extends Schema.String.pipe(Schema.pattern(/\d+(s|m|h|d)/)).annotations({
-  description: 'Time range. 1d - 1 day, 2h - 2 hours, 30m - 30 minutes, 15s - 15 seconds.',
-  examples: ['1d', '2h', '30m', '15s'],
-}) {
-  static toSeconds(timeRange: Schema.Schema.Type<typeof TimeRange>) {
-    const match = timeRange.match(/(\d+)(s|m|h|d)/);
-    if (!match) {
-      throw new Error(`Invalid time range: ${timeRange}`);
-    }
-    const [_, amount, unit] = match;
-    switch (unit) {
-      case 's':
-        return Number(amount);
-      case 'm':
-        return Number(amount) * 60;
-      case 'h':
-        return Number(amount) * 60 * 60;
-      case 'd':
-        return Number(amount) * 24 * 60 * 60;
-      default:
-        throw new Error(`Invalid time range unit: ${unit}`);
-    }
-  }
-};
-type TimeRange = Schema.Schema.Type<typeof TimeRange>;
-
-const DiscordConfigFromCredential = Layer.unwrapEffect(
-  Effect.gen(function* () {
-    return DiscordConfig.layer({
-      token: yield* CredentialsService.getApiKey({ service: 'discord.com' }),
-      rest: {
-        baseUrl: 'https://api-proxy.dxos.workers.dev/discord.com/api/v10',
-      },
-    });
-  }),
-);
-
-type DiscordChannel = GuildChannelResponse | PrivateChannelResponse | PrivateGroupChannelResponse | ThreadResponse;
-
-const DEFAULT_AFTER = 1704067200; // 2024-01-01
-const DEFAULT_LIMIT = 500;
-const DEFAULT_IGNORE_USERNAMES = ['GitHub', 'Needle'];
-
-// TODO(dmaretskyi): Align with standard thread type.
-type Thread = {
-  discordChannelId: string;
-  name?: string;
-  messages: DataType.Message[];
-};
-
-export default defineFunction({
-  key: 'dxos.org/function/fetch-discord-messages',
-  name: 'Sync Discord messages',
-  inputSchema: Schema.Struct({
-    serverId: Schema.String.annotations({
-      description: 'The ID of the server to fetch messages from.',
-    }),
-    channelId: Schema.optional(Schema.String).annotations({
-      description:
-        'The ID of the channel to fetch messages from. Will crawl all channels from the server if not specified.',
-    }),
-    after: Schema.optional(Schema.Number).annotations({
-      description:
-        'Fetch messages that were sent after a given date. Unix timestamp in seconds. Exclusive with `last`.',
-    }),
-    last: TimeRange.annotations({
-      description:
-        'Time range to fetch most recent messages. Specifies the range in the past, from now. "1d" would fetch messages from the last 24 hours.',
-    }),
-    limit: Schema.optional(Schema.Number).annotations({
-      description: 'The maximum number of messages to fetch.',
-    }),
-    pageSize: Schema.optional(Schema.Number).annotations({
-      description: 'The number of messages to fetch per page.',
-    }),
-    ignoreUsernames: Schema.optional(Schema.Array(Schema.String)).annotations({
-      description: 'Exclude messages from these usernames.',
-    }),
-  }),
-  handler: Effect.fnUntraced(
-    function* ({
-      data: {
-        serverId,
-        channelId,
-        after,
-        last,
-        pageSize = 100,
-        limit = DEFAULT_LIMIT,
-        ignoreUsernames = DEFAULT_IGNORE_USERNAMES,
-      },
-    }) {
-      if (!after && !last) {
-        throw new Error('cannot specify both `after` and `last`');
-      }
-      const afterTs = last ? Date.now() / 1000 - TimeRange.toSeconds(last) : (after ?? DEFAULT_AFTER);
-
-      const rest = yield* DiscordREST;
-
-      let channels: DiscordChannel[] = [];
-      channels.push(...(yield* rest.listGuildChannels(serverId)));
-      const { threads: guildThreads } = yield* rest.getActiveGuildThreads(serverId);
-      channels.push(...guildThreads);
-      if (channelId) {
-        channels = channels.filter((channel) => channel.id === channelId);
-      }
-      if (channels.length === 0) {
-        throw new Error('no channels found');
-      }
-      for (const channel of channels) {
-        console.log(channel.id, 'name' in channel ? channel.name : undefined);
-      }
-
-      yield* TracingService.emitStatus({ message: `Will fetch from channels: ${channels.length}` });
-
-      const threads = yield* Effect.forEach(
-        channels,
-        Effect.fnUntraced(function* (channel) {
-          const allMessages: DataType.Message[] = [];
-
-          let lastMessage: Option.Option<DataType.Message> = Option.none();
-          while (true) {
-            const { id: lastId = undefined } = Function.pipe(
-              lastMessage,
-              Option.map(Obj.getKeys('discord.com')),
-              Option.flatMap(Option.fromIterable),
-              Option.getOrElse(() => ({ id: undefined })),
-            );
-
-            const options = {
-              after: !lastId ? `${generateSnowflake(afterTs)}` : lastId,
-              limit: pageSize,
-            };
-            log.info('fetching messages', {
-              lastId,
-              afterTs,
-              afterSnowflake: options.after,
-              after: parseSnowflake(options.after),
-              limit: options.limit,
-            });
-            const messages = yield* rest.listMessages(channel.id, options).pipe(
-              Effect.map(Array.map(makeMessage)),
-              Effect.map(Array.reverse),
-              Effect.catchTag('ErrorResponse', (err) =>
-                err.cause.code === 50001 ? Effect.succeed([]) : Effect.fail(err),
-              ),
-            );
-            if (messages.length > 0) {
-              lastMessage = Option.fromNullable(messages.at(-1));
-              allMessages.push(...messages);
-            } else {
-              break;
-            }
-            yield* TracingService.emitStatus({ message: `Fetched messages: ${allMessages.length}` });
-            if (allMessages.length >= limit) {
-              break;
-            }
-          }
-
-          return {
-            discordChannelId: channel.id,
-            name: 'name' in channel ? (channel.name ?? undefined) : undefined,
-            messages: allMessages
-              .filter((message) => !message.sender.name || !ignoreUsernames.includes(message.sender.name))
-              .filter((message) =>
-                message.blocks.some((block) => block._tag === 'text' && block.text.trim().length > 0),
-              ),
-          } satisfies Thread;
-        }),
-        { concurrency: 10 },
-      );
-
-      return threads
-        .filter((thread) => thread.messages.length > 0)
-        .map(serializeThread)
-        .join('\n');
-    },
-    Effect.provide(
-      DiscordRESTMemoryLive.pipe(Layer.provideMerge(DiscordConfigFromCredential)).pipe(
-        Layer.provide(FetchHttpClient.layer),
-      ),
-    ),
-    Effect.orDie,
-  ),
-});
-
-/**
- * @param unixTimestamp in seconds
- */
-const generateSnowflake = (unixTimestamp: number): bigint => {
-  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
-  return (BigInt(unixTimestamp * 1000) - discordEpoch) << 22n;
-};
-
-const parseSnowflake = (snowflake: string): Date => {
-  const discordEpoch = 1420070400000n; // Discord Epoch (ms)
-  return new Date(Number((BigInt(snowflake) >> 22n) + discordEpoch));
-};
-
-const makeMessage = (message: MessageResponse): DataType.Message =>
-  Obj.make(DataType.Message, {
-    [Obj.Meta]: {
-      keys: [
-        { id: message.id, source: 'discord.com' },
-        { id: message.channel_id, source: 'discord.com/thread' },
-      ],
-    },
-    sender: { name: message.author.username },
-    created: message.timestamp,
-    blocks: [{ _tag: 'text', text: message.content }],
-  });
-
-/**
- * Standard JSON serialization is to verbose for large amounts of data.
- */
-const serializeThread = (thread: Thread): string => {
-  return `<thread id=${thread.discordChannelId} name=${thread.name ?? ''}>\n${thread.messages
-    .map(
-      (message) =>
-        `  ${message.sender.name}: ${message.blocks
-          .filter((block) => block._tag === 'text')
-          .map((block) => block.text)
-          .join(' ')}`,
-    )
-    .join('\n')}\n</thread>`;
-};

package/src/functions/discord/index.ts

@@ -1,9 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { default as fetch$ } from './fetch-messages';
-
-export namespace Discord {
-  export const fetch = fetch$;
-}

package/src/functions/document/index.ts

@@ -1,11 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { default as read$ } from './read';
-import { default as update$ } from './update';
-
-export namespace Document {
-  export const read = read$;
-  export const update = update$;
-}

package/src/functions/document/read.ts

@@ -1,29 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { ArtifactId } from '@dxos/assistant';
-import { DatabaseService, defineFunction } from '@dxos/functions';
-import { Markdown } from '@dxos/plugin-markdown/types';
-
-export default defineFunction({
-  key: 'dxos.org/function/markdown/read',
-  name: 'Read markdown document',
-  description: 'Read markdown document.',
-  inputSchema: Schema.Struct({
-    id: ArtifactId.annotations({
-      description: 'The ID of the document to read.',
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    content: Schema.String,
-  }),
-  handler: Effect.fn(function* ({ data: { id } }) {
-    const doc = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Markdown.Document);
-    const { content } = yield* DatabaseService.load(doc.content);
-    return { content };
-  }),
-});

package/src/functions/document/update.ts

@@ -1,30 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { ArtifactId } from '@dxos/assistant';
-import { DatabaseService, defineFunction } from '@dxos/functions';
-import { Markdown } from '@dxos/plugin-markdown/types';
-
-export default defineFunction({
-  key: 'dxos.org/function/markdown/update',
-  name: 'Update markdown',
-  description: 'Updates the entire contents of the markdown document.',
-  inputSchema: Schema.Struct({
-    id: ArtifactId.annotations({
-      description: 'The ID of the document to write.',
-    }),
-    content: Schema.String.annotations({
-      description: 'New content to write to the document.',
-    }),
-  }),
-  outputSchema: Schema.Void,
-  handler: Effect.fn(function* ({ data: { id, content } }) {
-    const doc = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Markdown.Document);
-    const text = yield* DatabaseService.load(doc.content);
-    text.content = content;
-  }),
-});