@dxos/assistant-toolkit 0.8.4-main.ae835ea

package/src/functions/entity-extraction/entity-extraction.ts

@@ -0,0 +1,163 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Predicate from 'effect/Predicate';
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+import { AiSession, makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
+import { Filter, Obj, Ref } from '@dxos/echo';
+import { DatabaseService, FunctionInvocationService, defineFunction } from '@dxos/functions';
+import { type DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
+import { trim } from '@dxos/util';
+
+import { contextQueueLayerFromResearchGraph, makeGraphWriterHandler, makeGraphWriterToolkit } from '../research';
+
+export default defineFunction({
+  key: 'dxos.org/functions/entity-extraction',
+  name: 'Entity Extraction',
+  description: 'Extracts entities from emails and transcripts.',
+  inputSchema: Schema.Struct({
+    source: DataType.Message.annotations({ description: 'Email or transcript to extract entities from.' }),
+
+    // TODO(dmaretskyi): Consider making this an array of blueprints instead.
+    instructions: Schema.optional(Schema.String).annotations({ description: 'Instructions extraction process.' }),
+  }),
+  outputSchema: Schema.Struct({
+    entities: Schema.optional(
+      Schema.Array(Obj.Any).annotations({
+        description: 'Extracted entities.',
+      }),
+    ),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({ data: { source, instructions } }) {
+      const contact = yield* extractContact(source);
+      let organization: DataType.Organization | null = null;
+
+      if (contact && !contact.organization) {
+        const created: DXN[] = [];
+        const GraphWriterToolkit = makeGraphWriterToolkit({ schema: [DataType.LegacyOrganization] }).pipe();
+        const GraphWriterHandler = makeGraphWriterHandler(GraphWriterToolkit, {
+          onAppend: (dxns) => created.push(...dxns),
+        });
+        const toolkit = yield* GraphWriterToolkit.pipe(
+          Effect.provide(GraphWriterHandler.pipe(Layer.provide(contextQueueLayerFromResearchGraph))),
+        );
+
+        yield* new AiSession().run({
+          system: trim`
+            Extract the sender's organization from the email. If you are not sure, do nothing.
+            The extracted organization URL must match the sender's email domain.
+            ${instructions ? '<user_intructions>' + instructions + '</user_intructions>' : ''},
+          `,
+          prompt: JSON.stringify({ source, contact }),
+          toolkit,
+        });
+
+        if (created.length > 1) {
+          throw new Error('Multiple organizations created');
+        } else if (created.length === 1) {
+          organization = yield* DatabaseService.resolve(created[0], DataType.Organization);
+          Obj.getMeta(organization).tags ??= [];
+          Obj.getMeta(organization).tags!.push(...(Obj.getMeta(source)?.tags ?? []));
+          contact.organization = Ref.make(organization);
+        }
+      }
+
+      return {
+        entities: [contact, organization].filter(Predicate.isNotNullable),
+      };
+    },
+    Effect.provide(
+      Layer.mergeAll(
+        AiService.model('@anthropic/claude-sonnet-4-0'), // TODO(dmaretskyi): Extract.
+        makeToolResolverFromFunctions([], Toolkit.make()),
+        makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
+      ).pipe(
+        Layer.provide(
+          // TODO(dmaretskyi): This should be provided by environment.
+          Layer.mergeAll(FunctionInvocationService.layerTest()),
+        ),
+      ),
+    ),
+  ),
+});
+
+const extractContact = Effect.fn('extractContact')(function* (message: DataType.Message) {
+  const name = message.sender.name;
+  const email = message.sender.email;
+  if (!email) {
+    log.warn('email is required for contact extraction', { sender: message.sender });
+    return undefined;
+  }
+
+  const { objects: existingContacts } = yield* DatabaseService.runQuery(Filter.type(DataType.Person));
+
+  // Check for existing contact
+  // TODO(dmaretskyi): Query filter DSL - https://linear.app/dxos/issue/DX-541/filtercontains-should-work-with-partial-objects
+  const existingContact = existingContacts.find((contact) =>
+    contact.emails?.some((contactEmail) => contactEmail.value === email),
+  );
+
+  if (existingContact) {
+    log.info('Contact already exists', { email, existingContact });
+    return existingContact;
+  }
+
+  const newContact = Obj.make(DataType.Person, {
+    [Obj.Meta]: {
+      tags: Obj.getMeta(message)?.tags,
+    },
+    emails: [{ value: email }],
+  });
+  yield* DatabaseService.add(newContact);
+
+  if (name) {
+    newContact.fullName = name;
+  }
+
+  const emailDomain = email.split('@')[1]?.toLowerCase();
+  if (!emailDomain) {
+    log.warn('Invalid email format, cannot extract domain', { email });
+    return newContact;
+  }
+
+  log.info('extracted email domain', { emailDomain });
+
+  const { objects: existingOrganisations } = yield* DatabaseService.runQuery(Filter.type(DataType.Organization));
+  const matchingOrg = existingOrganisations.find((org) => {
+    if (org.website) {
+      try {
+        const websiteUrl =
+          org.website.startsWith('http://') || org.website.startsWith('https://')
+            ? org.website
+            : `https://${org.website}`;
+
+        const websiteDomain = new URL(websiteUrl).hostname.toLowerCase();
+        return (
+          websiteDomain === emailDomain ||
+          websiteDomain.endsWith(`.${emailDomain}`) ||
+          emailDomain.endsWith(`.${websiteDomain}`)
+        );
+      } catch (e) {
+        log.warn('Error parsing website URL', { website: org.website, error: e });
+        return false;
+      }
+    }
+    return false;
+  });
+
+  if (matchingOrg) {
+    log.info('found matching organization', { organization: matchingOrg });
+    newContact.organization = Ref.make(matchingOrg);
+  }
+
+  return newContact;
+});

package/src/functions/entity-extraction/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as extract$ } from './entity-extraction';
+
+export namespace EntityExtraction {
+  export const extract = extract$;
+}

package/src/functions/exa/exa.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+import Exa from 'exa-js';
+
+import { CredentialsService, defineFunction } from '@dxos/functions';
+
+export default defineFunction({
+  key: 'dxos.org/function/exa',
+  name: 'Exa',
+  description: 'Search the web for information',
+  inputSchema: Schema.Struct({
+    query: Schema.String.annotations({
+      description: 'The query to search for.',
+    }),
+  }),
+  outputSchema: Schema.Unknown,
+  handler: Effect.fnUntraced(function* ({ data: { query } }) {
+    const credential = yield* CredentialsService.getCredential({ service: 'exa.ai' });
+    const exa = new Exa(credential.apiKey);
+
+    const context = yield* Effect.promise(async () =>
+      exa.searchAndContents(query, {
+        type: 'auto',
+        text: {
+          maxCharacters: 3_000,
+        },
+        livecrawl: 'always',
+      }),
+    );
+
+    return context;
+  }),
+});

package/src/functions/exa/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export { default as exaFunction } from './exa';
+export { default as exaMockFunction } from './mock';

package/src/functions/exa/mock.ts

@@ -0,0 +1,71 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { defineFunction } from '@dxos/functions';
+
+import { SEARCH_RESULTS } from '../../testing';
+
+export default defineFunction({
+  key: 'dxos.org/function/exa-mock',
+  name: 'Exa mock',
+  description: 'Search the web for information',
+  inputSchema: Schema.Struct({
+    query: Schema.String.annotations({
+      description: 'The query to search for.',
+    }),
+  }),
+  outputSchema: Schema.Unknown,
+  handler: Effect.fnUntraced(function* ({ data: { query } }) {
+    const result = SEARCH_RESULTS.reduce(
+      (closest, current) => {
+        if (!current.autopromptString) {
+          return closest;
+        }
+        if (!closest) {
+          return current;
+        }
+
+        // Calculate Levenshtein distance
+        const dist1 = levenshteinDistance(query, current.autopromptString);
+        const dist2 = levenshteinDistance(query, closest.autopromptString || '');
+
+        // Weight by length of the longer string to normalize
+        const weight1 = dist1 / Math.max(query.length, current.autopromptString.length);
+        const weight2 = dist2 / Math.max(query.length, closest.autopromptString?.length || 0);
+
+        return weight1 < weight2 ? current : closest;
+      },
+      null as (typeof SEARCH_RESULTS)[0] | null,
+    );
+
+    return result;
+  }),
+});
+
+const levenshteinDistance = (str1: string, str2: string): number => {
+  const m = str1.length;
+  const n = str2.length;
+  const dp: number[][] = Array(m + 1)
+    .fill(null)
+    .map(() => Array(n + 1).fill(0));
+
+  for (let i = 0; i <= m; i++) {
+    dp[i][0] = i;
+  }
+  for (let j = 0; j <= n; j++) {
+    dp[0][j] = j;
+  }
+
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      dp[i][j] =
+        str1[i - 1] === str2[j - 1] ? dp[i - 1][j - 1] : Math.min(dp[i - 1][j - 1], dp[i - 1][j], dp[i][j - 1]) + 1;
+    }
+  }
+
+  return dp[m][n];
+};

package/src/functions/github/fetch-prs.ts

@@ -0,0 +1,30 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { defineFunction, withAuthorization } from '@dxos/functions';
+
+export default defineFunction({
+  key: 'dxos.org/function/github/fetch-prs',
+  name: 'Fetch PRs',
+  description: 'Fetches PRs from GitHub.',
+  inputSchema: Schema.Struct({
+    owner: Schema.String.annotations({
+      description: 'GitHub owner.',
+    }),
+    repo: Schema.String.annotations({
+      description: 'GitHub repository.',
+    }),
+  }),
+  handler: Effect.fnUntraced(function* ({ data }) {
+    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization({ service: 'github.com' })));
+
+    const response = yield* client.get(`https://api.github.com/repos/${data.owner}/${data.repo}/pulls`);
+    const json: any = yield* response.json;
+    return json;
+  }),
+});

package/src/functions/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './agent';
+export * from './discord';
+export * from './document';
+export * from './entity-extraction';
+export * from './linear';
+export * from './research';
+export * from './tasks';

package/src/functions/linear/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { default as sync$ } from './sync-issues';
+
+export namespace Linear {
+  export const sync = sync$;
+}

package/src/functions/linear/linear.test.ts

@@ -0,0 +1,86 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import { describe, it } from '@effect/vitest';
+import * as Config from 'effect/Config';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { AiService } from '@dxos/ai';
+import { AiServiceTestingPreset } from '@dxos/ai/testing';
+import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
+import { Obj, Query } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect';
+import {
+  ComputeEventLogger,
+  CredentialsService,
+  DatabaseService,
+  FunctionInvocationService,
+  TracingService,
+} from '@dxos/functions';
+import { TestDatabaseLayer, testStoragePath } from '@dxos/functions/testing';
+import { DataType } from '@dxos/schema';
+
+import { LINEAR_ID_KEY, default as fetchLinearIssues } from './sync-issues';
+
+const TestLayer = Layer.mergeAll(
+  AiService.model('@anthropic/claude-opus-4-0'),
+  makeToolResolverFromFunctions([], Toolkit.make()),
+  makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
+  ComputeEventLogger.layerFromTracing,
+).pipe(
+  Layer.provideMerge(
+    Layer.mergeAll(
+      AiServiceTestingPreset('direct'),
+      TestDatabaseLayer({
+        // indexing: { vector: true },
+        types: [DataType.Task, DataType.Person, DataType.Project],
+        storagePath: testStoragePath({ name: 'feed-test-13' }),
+      }),
+      CredentialsService.layerConfig([{ service: 'linear.app', apiKey: Config.redacted('LINEAR_API_KEY') }]),
+      FunctionInvocationService.layerTestMocked({ functions: [fetchLinearIssues] }).pipe(
+        Layer.provideMerge(ComputeEventLogger.layerFromTracing),
+        Layer.provideMerge(TracingService.layerLogInfo()),
+      ),
+      FetchHttpClient.layer,
+    ),
+  ),
+);
+
+describe('Linear', { timeout: 600_000 }, () => {
+  it.effect(
+    'sync',
+    Effect.fnUntraced(
+      function* (_) {
+        yield* DatabaseService.flush({ indexes: true });
+
+        yield* FunctionInvocationService.invokeFunction(fetchLinearIssues, {
+          team: '1127c63a-6f77-4725-9229-50f6cd47321c',
+        });
+
+        const { objects: persons } = yield* DatabaseService.runQuery(Query.type(DataType.Person));
+        console.log('people', {
+          count: persons.length,
+          people: persons.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
+        });
+        const { objects: projects } = yield* DatabaseService.runQuery(Query.type(DataType.Project));
+        console.log('projects', {
+          count: projects.length,
+          projects: projects.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
+        });
+        const { objects: tasks } = yield* DatabaseService.runQuery(Query.type(DataType.Task));
+        console.log('tasks', {
+          count: tasks.length,
+          tasks: tasks.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
+        });
+
+        yield* DatabaseService.flush({ indexes: true });
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.taggedTest('sync'),
+    ),
+  );
+});

package/src/functions/linear/sync-issues.ts

@@ -0,0 +1,189 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Schema from 'effect/Schema';
+
+import { Filter, Obj, Query, Ref, type Type } from '@dxos/echo';
+import { DatabaseService, defineFunction, withAuthorization } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
+
+import { syncObjects } from '../../sync';
+import { graphqlRequestBody } from '../../util';
+
+const queryIssues = `
+query Issues($teamId: String!, $after: DateTimeOrDuration!) {
+  team(id: $teamId) {
+    id
+    name
+
+
+   issues(last: 150, orderBy: updatedAt, filter: {
+    updatedAt: { gt: $after }
+   }) {
+    edges {
+        node {
+            id
+            title
+            createdAt
+            updatedAt
+            description
+            assignee { id, name }
+            state { 
+                name
+            }
+            project {
+                id
+                name
+            }
+        }
+        cursor
+    }
+    pageInfo {
+        hasNextPage
+        endCursor
+    }
+    }
+  }
+}
+`;
+
+type LinearIssue = {
+  id: string;
+  title: string;
+  createdAt: string;
+  updatedAt: string;
+  description: string;
+  assignee: LinearPerson;
+  state: { name: string };
+  project: { id: string; name: string };
+};
+
+type LinearPerson = {
+  id: string;
+  name: string;
+};
+
+export const LINEAR_ID_KEY = 'linear.app/id';
+export const LINEAR_TEAM_ID_KEY = 'linear.app/teamId';
+export const LINEAR_UPDATED_AT_KEY = 'linear.app/updatedAt';
+
+export default defineFunction({
+  key: 'dxos.org/function/linear/sync-issues',
+  name: 'Linear',
+  description: 'Sync issues from Linear.',
+  inputSchema: Schema.Struct({
+    team: Schema.String.annotations({
+      description: 'Linear team id.',
+    }),
+  }),
+  handler: Effect.fnUntraced(function* ({ data }) {
+    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization({ service: 'linear.app' })));
+
+    // Get the timestamp that was previosly synced.
+    const after = yield* getLatestUpdateTimestamp(data.team, DataType.Task);
+    log.info('will fetch', { after });
+
+    // Fetch the issues that have changed since the last sync.
+    const response = yield* client.post('https://api.linear.app/graphql', {
+      body: yield* graphqlRequestBody(queryIssues, {
+        teamId: data.team,
+        after,
+      }),
+    });
+    const json: any = yield* response.json;
+    const tasks = (json.data.team.issues.edges as any[]).map((edge: any) =>
+      mapLinearIssue(edge.node as LinearIssue, { teamId: data.team }),
+    );
+    log.info('Fetched tasks', { count: tasks.length });
+
+    // Synchronize new objects with ECHO.
+    return {
+      objects: yield* syncObjects(tasks, { foreignKeyId: LINEAR_ID_KEY }),
+      syncComplete: tasks.length < 150,
+    };
+  }, Effect.provide(FetchHttpClient.layer)),
+});
+
+const getLatestUpdateTimestamp: (
+  teamId: string,
+  dataType: Type.Obj.Any,
+) => Effect.Effect<string, never, DatabaseService> = Effect.fnUntraced(function* (teamId, dataType) {
+  const { objects: existingTasks } = yield* DatabaseService.runQuery(
+    Query.type(dataType).select(Filter.foreignKeys(dataType, [{ source: LINEAR_TEAM_ID_KEY, id: teamId }])),
+  );
+  return Function.pipe(
+    existingTasks,
+    Array.map((task) => Obj.getKeys(task, LINEAR_UPDATED_AT_KEY).at(0)?.id),
+    Array.filter((x) => x !== undefined),
+    Array.reduce('2025-01-01T00:00:00.000Z', (acc: string, x: string) => (x > acc ? x : acc)),
+  );
+});
+
+const mapLinearPerson = (person: LinearPerson, { teamId }: { teamId: string }): DataType.Person =>
+  Obj.make(DataType.Person, {
+    [Obj.Meta]: {
+      keys: [
+        {
+          id: person.id,
+          source: LINEAR_ID_KEY,
+        },
+        {
+          id: teamId,
+          source: LINEAR_TEAM_ID_KEY,
+        },
+      ],
+    },
+    nickname: person.name,
+  });
+
+const mapLinearIssue = (issue: LinearIssue, { teamId }: { teamId: string }): DataType.Task =>
+  Obj.make(DataType.Task, {
+    [Obj.Meta]: {
+      keys: [
+        {
+          id: issue.id,
+          source: LINEAR_ID_KEY,
+        },
+        {
+          id: issue.updatedAt,
+          source: LINEAR_UPDATED_AT_KEY,
+        },
+        {
+          id: teamId,
+          source: LINEAR_TEAM_ID_KEY,
+        },
+      ],
+    },
+    title: issue.title ?? undefined,
+    description: issue.description ?? undefined,
+    assigned: !issue.assignee ? undefined : Ref.make(mapLinearPerson(issue.assignee, { teamId })),
+    // TODO(dmaretskyi): Sync those (+ linear team as org?).
+    // state: issue.state.name,
+
+    project: !issue.project
+      ? undefined
+      : Ref.make(
+          DataType.makeProject({
+            [Obj.Meta]: {
+              keys: [
+                {
+                  id: issue.project.id,
+                  source: LINEAR_ID_KEY,
+                },
+                {
+                  id: teamId,
+                  source: LINEAR_TEAM_ID_KEY,
+                },
+              ],
+            },
+            name: issue.project.name,
+          }),
+        ),
+  });

package/src/functions/research/create-document.ts

@@ -0,0 +1,69 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Relation } from '@dxos/echo';
+import { DatabaseService, TracingService, defineFunction } from '@dxos/functions';
+import { invariant } from '@dxos/invariant';
+import { DXN, ObjectId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Markdown } from '@dxos/plugin-markdown/types';
+import { DataType } from '@dxos/schema';
+import { trim } from '@dxos/util';
+
+export default defineFunction({
+  key: 'dxos.org/function/research/create-document',
+  name: 'Create research document',
+  description: 'Creates a note summarizing the research.',
+  inputSchema: Schema.Struct({
+    name: Schema.String.annotations({
+      description: 'Name of the note.',
+    }),
+
+    content: Schema.String.annotations({
+      description: trim`
+        Content of the note. 
+        Supports (and are prefered) references to research objects using @ syntax and <object> tags (refer to research blueprint instructions).
+      `,
+    }),
+
+    // TODO(dmaretskyi): Use a specialized type for this (e.g., ArtifactId renamed as RefFromLLM).
+    target: Schema.String.annotations({
+      description: trim`
+        Id of the object (organization, contact, etc.) for which the research was performed. 
+        This must be a ulid.
+      `,
+    }),
+  }),
+  outputSchema: Schema.Struct({}), // TODO(burdon): Schema.Void?
+  handler: Effect.fnUntraced(function* ({ data: { target, name, content } }) {
+    log.info('Creating research document', { target, name, content });
+
+    yield* DatabaseService.flush({ indexes: true });
+    yield* TracingService.emitStatus({ message: 'Creating research document...' });
+    invariant(ObjectId.isValid(target));
+
+    const targetObj = yield* DatabaseService.resolve(DXN.fromLocalObjectId(target));
+
+    const doc = yield* DatabaseService.add(
+      Markdown.makeDocument({
+        name,
+        content,
+      }),
+    );
+    yield* DatabaseService.add(
+      Relation.make(DataType.HasSubject, {
+        [Relation.Source]: doc,
+        [Relation.Target]: targetObj as any,
+        completedAt: new Date().toISOString(),
+      }),
+    );
+    yield* DatabaseService.flush({ indexes: true });
+
+    log.info('Created research document', { target, name, content });
+    return {};
+  }),
+});