@dxos/assistant-toolkit 0.8.4-main.72ec0f3 → 0.8.4-main.937b3ca

package/src/functions/entity-extraction/entity-extraction.test.ts

@@ -6,13 +6,14 @@ import { describe, expect, it } from '@effect/vitest';
 import * as Effect from 'effect/Effect';
 import * as Layer from 'effect/Layer';
 
-import { AiService, MemoizedAiService } from '@dxos/ai';
-import { TestAiService } from '@dxos/ai/testing';
+import { AiService } from '@dxos/ai';
+import { MemoizedAiService, TestAiService } from '@dxos/ai/testing';
 import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
 import { Blueprint } from '@dxos/blueprints';
 import { Obj } from '@dxos/echo';
-import { TestHelpers } from '@dxos/effect';
-import { CredentialsService, DatabaseService, FunctionInvocationService, TracingService } from '@dxos/functions';
+import { Database } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { CredentialsService, FunctionInvocationService, TracingService } from '@dxos/functions';
 import { FunctionInvocationServiceLayerTest, TestDatabaseLayer } from '@dxos/functions-runtime/testing';
 import { ObjectId } from '@dxos/keys';
 import { Message, Organization, Person } from '@dxos/types';
@@ -49,7 +50,7 @@ describe('Entity extraction', () => {
     'call a function to generate a research report',
     Effect.fnUntraced(
       function* (_) {
-        const email = yield* DatabaseService.add(
+        const email = yield* Database.Service.add(
           Obj.make(Message.Message, {
             [Obj.Meta]: {
               tags: ['important'],
@@ -75,7 +76,7 @@ describe('Entity extraction', () => {
             ],
           }),
         );
-        yield* DatabaseService.flush({ indexes: true });
+        yield* Database.Service.flush({ indexes: true });
         const result = yield* FunctionInvocationService.invokeFunction(entityExtraction, {
           source: email,
         });

package/src/functions/entity-extraction/entity-extraction.ts

@@ -10,12 +10,13 @@ import * as Schema from 'effect/Schema';
 
 import { AiService } from '@dxos/ai';
 import { AiSession, makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
-import { Filter, Obj, Ref } from '@dxos/echo';
-import { DatabaseService, defineFunction } from '@dxos/functions';
+import { Filter, Obj, Ref, Type } from '@dxos/echo';
+import { Database } from '@dxos/echo';
+import { defineFunction } from '@dxos/functions';
 import { FunctionInvocationServiceLayerTest } from '@dxos/functions-runtime/testing';
 import { type DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { LegacyOrganization, Message, Organization, Person } from '@dxos/types';
+import { type Actor, LegacyOrganization, Message, Organization, Person } from '@dxos/types';
 import { trim } from '@dxos/util';
 
 import { makeGraphWriterHandler, makeGraphWriterToolkit } from '../../crud';
@@ -26,26 +27,33 @@ export default defineFunction({
   name: 'Entity Extraction',
   description: 'Extracts entities from emails and transcripts.',
   inputSchema: Schema.Struct({
-    source: Message.Message.annotations({ description: 'Email or transcript to extract entities from.' }),
+    source: Message.Message.annotations({
+      description: 'Email or transcript to extract entities from.',
+    }),
 
     // TODO(dmaretskyi): Consider making this an array of blueprints instead.
-    instructions: Schema.optional(Schema.String).annotations({ description: 'Instructions extraction process.' }),
+    instructions: Schema.optional(Schema.String).annotations({
+      description: 'Instructions extraction process.',
+    }),
   }),
   outputSchema: Schema.Struct({
     entities: Schema.optional(
-      Schema.Array(Obj.Any).annotations({
+      Schema.Array(Type.Obj).annotations({
         description: 'Extracted entities.',
       }),
     ),
   }),
   handler: Effect.fnUntraced(
-    function* ({ data: { source, instructions } }) {
-      const contact = yield* extractContact(source);
+    function* ({ data: { source: message, instructions } }) {
+      const tags = Obj.getMeta(message)?.tags;
+      const contact = yield* extractContact(message.sender, tags);
       let organization: Organization.Organization | null = null;
 
       if (contact && !contact.organization) {
         const created: DXN[] = [];
-        const GraphWriterToolkit = makeGraphWriterToolkit({ schema: [LegacyOrganization] }).pipe();
+        const GraphWriterToolkit = makeGraphWriterToolkit({
+          schema: [LegacyOrganization],
+        }).pipe();
         const GraphWriterHandler = makeGraphWriterHandler(GraphWriterToolkit, {
           onAppend: (dxns) => created.push(...dxns),
         });
@@ -59,17 +67,22 @@ export default defineFunction({
             The extracted organization URL must match the sender's email domain.
             ${instructions ? '<user_intructions>' + instructions + '</user_intructions>' : ''},
           `,
-          prompt: JSON.stringify({ source, contact }),
+          prompt: JSON.stringify({ source: message, contact }),
           toolkit,
         });
 
         if (created.length > 1) {
           throw new Error('Multiple organizations created');
         } else if (created.length === 1) {
-          organization = yield* DatabaseService.resolve(created[0], Organization.Organization);
-          Obj.getMeta(organization).tags ??= [];
-          Obj.getMeta(organization).tags!.push(...(Obj.getMeta(source)?.tags ?? []));
-          contact.organization = Ref.make(organization);
+          organization = yield* Database.Service.resolve(created[0], Organization.Organization);
+          Obj.change(organization, (org) => {
+            const meta = Obj.getMeta(org);
+            meta.tags ??= [];
+            meta.tags.push(...(tags ?? []));
+          });
+          Obj.change(contact, (c) => {
+            c.organization = Ref.make(organization!);
+          });
         }
       }
 
@@ -92,20 +105,20 @@ export default defineFunction({
   ),
 });
 
-const extractContact = Effect.fn('extractContact')(function* (message: Message.Message) {
-  const name = message.sender.name;
-  const email = message.sender.email;
+const extractContact = Effect.fn('extractContact')(function* (actor: Actor.Actor, tags?: readonly string[]) {
+  const name = actor.name;
+  const email = actor.email;
   if (!email) {
-    log.warn('email is required for contact extraction', { sender: message.sender });
+    log.warn('email is required for contact extraction', { actor });
     return undefined;
   }
 
-  const { objects: existingContacts } = yield* DatabaseService.runQuery(Filter.type(Person.Person));
+  const existingContacts = yield* Database.Service.runQuery(Filter.type(Person.Person));
 
   // Check for existing contact
   // TODO(dmaretskyi): Query filter DSL - https://linear.app/dxos/issue/DX-541/filtercontains-should-work-with-partial-objects
   const existingContact = existingContacts.find((contact) =>
-    contact.emails?.some((contactEmail: any) => contactEmail.value === email),
+    contact.emails?.some((contactEmail) => contactEmail.value === email),
   );
 
   if (existingContact) {
@@ -114,15 +127,15 @@ const extractContact = Effect.fn('extractContact')(function* (message: Message.M
   }
 
   const newContact = Obj.make(Person.Person, {
-    [Obj.Meta]: {
-      tags: Obj.getMeta(message)?.tags,
-    },
+    ...(tags ? { [Obj.Meta]: { tags: [...tags] } } : {}),
     emails: [{ value: email }],
   });
-  yield* DatabaseService.add(newContact);
+  yield* Database.Service.add(newContact);
 
   if (name) {
-    newContact.fullName = name;
+    Obj.change(newContact, (c) => {
+      c.fullName = name;
+    });
   }
 
   const emailDomain = email.split('@')[1]?.toLowerCase();
@@ -133,7 +146,7 @@ const extractContact = Effect.fn('extractContact')(function* (message: Message.M
 
   log.info('extracted email domain', { emailDomain });
 
-  const { objects: existingOrganisations } = yield* DatabaseService.runQuery(Filter.type(Organization.Organization));
+  const existingOrganisations = yield* Database.Service.runQuery(Filter.type(Organization.Organization));
   const matchingOrg = existingOrganisations.find((org) => {
     if (org.website) {
       try {
@@ -149,7 +162,10 @@ const extractContact = Effect.fn('extractContact')(function* (message: Message.M
           emailDomain.endsWith(`.${websiteDomain}`)
         );
       } catch (e) {
-        log.warn('Error parsing website URL', { website: org.website, error: e });
+        log.warn('Error parsing website URL', {
+          website: org.website,
+          error: e,
+        });
         return false;
       }
     }
@@ -158,7 +174,9 @@ const extractContact = Effect.fn('extractContact')(function* (message: Message.M
 
   if (matchingOrg) {
     log.info('found matching organization', { organization: matchingOrg });
-    newContact.organization = Ref.make(matchingOrg);
+    Obj.change(newContact, (c) => {
+      c.organization = Ref.make(matchingOrg);
+    });
   }
 
   return newContact;

package/src/functions/exa/mock.ts

@@ -7,7 +7,7 @@ import * as Schema from 'effect/Schema';
 
 import { defineFunction } from '@dxos/functions';
 
-import { SEARCH_RESULTS } from '../../testing';
+import { SEARCH_RESULTS } from './data';
 
 export default defineFunction({
   key: 'dxos.org/function/exa-mock',

package/src/functions/github/fetch-prs.ts

@@ -6,7 +6,7 @@ import * as HttpClient from '@effect/platform/HttpClient';
 import * as Effect from 'effect/Effect';
 import * as Schema from 'effect/Schema';
 
-import { defineFunction, withAuthorization } from '@dxos/functions';
+import { CredentialsService, defineFunction, withAuthorization } from '@dxos/functions';
 
 export default defineFunction({
   key: 'dxos.org/function/github/fetch-prs',
@@ -21,7 +21,8 @@ export default defineFunction({
     }),
   }),
   handler: Effect.fnUntraced(function* ({ data }) {
-    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization({ service: 'github.com' })));
+    const credential = yield* CredentialsService.getCredential({ service: 'github.com' });
+    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization(credential.apiKey!)));
 
     const response = yield* client.get(`https://api.github.com/repos/${data.owner}/${data.repo}/pulls`);
     const json: any = yield* response.json;

package/src/functions/linear/linear.test.ts

@@ -13,8 +13,9 @@ import { AiService } from '@dxos/ai';
 import { AiServiceTestingPreset } from '@dxos/ai/testing';
 import { makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
 import { Obj, Query } from '@dxos/echo';
-import { TestHelpers } from '@dxos/effect';
-import { CredentialsService, DatabaseService, FunctionInvocationService } from '@dxos/functions';
+import { Database } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { CredentialsService, FunctionInvocationService } from '@dxos/functions';
 import { TracingServiceExt } from '@dxos/functions-runtime';
 import {
   FunctionInvocationServiceLayerTestMocked,
@@ -39,9 +40,9 @@ const TestLayer = Layer.mergeAll(
         storagePath: testStoragePath({ name: 'feed-test-13' }),
       }),
       CredentialsService.layerConfig([{ service: 'linear.app', apiKey: Config.redacted('LINEAR_API_KEY') }]),
-      FunctionInvocationServiceLayerTestMocked({ functions: [fetchLinearIssues] }).pipe(
-        Layer.provideMerge(TracingServiceExt.layerLogInfo()),
-      ),
+      FunctionInvocationServiceLayerTestMocked({
+        functions: [fetchLinearIssues],
+      }).pipe(Layer.provideMerge(TracingServiceExt.layerLogInfo())),
       FetchHttpClient.layer,
     ),
   ),
@@ -52,29 +53,29 @@ describe('Linear', { timeout: 600_000 }, () => {
     'sync',
     Effect.fnUntraced(
       function* (_) {
-        yield* DatabaseService.flush({ indexes: true });
+        yield* Database.Service.flush({ indexes: true });
 
         yield* FunctionInvocationService.invokeFunction(fetchLinearIssues, {
           team: '1127c63a-6f77-4725-9229-50f6cd47321c',
         });
 
-        const { objects: persons } = yield* DatabaseService.runQuery(Query.type(Person.Person));
+        const persons = yield* Database.Service.runQuery(Query.type(Person.Person));
         console.log('people', {
           count: persons.length,
           people: persons.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
         });
-        const { objects: projects } = yield* DatabaseService.runQuery(Query.type(Project.Project));
+        const projects = yield* Database.Service.runQuery(Query.type(Project.Project));
         console.log('projects', {
           count: projects.length,
           projects: projects.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
         });
-        const { objects: tasks } = yield* DatabaseService.runQuery(Query.type(Task.Task));
+        const tasks = yield* Database.Service.runQuery(Query.type(Task.Task));
         console.log('tasks', {
           count: tasks.length,
           tasks: tasks.map((_) => `(${_.id}) ${Obj.getLabel(_)} [${Obj.getKeys(_, LINEAR_ID_KEY)[0]?.id}]`),
         });
 
-        yield* DatabaseService.flush({ indexes: true });
+        yield* Database.Service.flush({ indexes: true });
       },
       Effect.provide(TestLayer),
       TestHelpers.taggedTest('sync'),

package/src/functions/linear/sync-issues.ts

@@ -10,7 +10,8 @@ import * as Function from 'effect/Function';
 import * as Schema from 'effect/Schema';
 
 import { Filter, Obj, Query, Ref, type Type } from '@dxos/echo';
-import { DatabaseService, defineFunction, withAuthorization } from '@dxos/functions';
+import { Database } from '@dxos/echo';
+import { CredentialsService, defineFunction, withAuthorization } from '@dxos/functions';
 import { log } from '@dxos/log';
 import { Person, Project, Task } from '@dxos/types';
 
@@ -35,7 +36,7 @@ query Issues($teamId: String!, $after: DateTimeOrDuration!) {
             updatedAt
             description
             assignee { id, name }
-            state { 
+            state {
                 name
             }
             project {
@@ -84,7 +85,8 @@ export default defineFunction({
     }),
   }),
   handler: Effect.fnUntraced(function* ({ data }) {
-    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization({ service: 'linear.app' })));
+    const credential = yield* CredentialsService.getCredential({ service: 'linear.app' });
+    const client = yield* HttpClient.HttpClient.pipe(Effect.map(withAuthorization(credential.apiKey!)));
 
     // Get the timestamp that was previosly synced.
     const after = yield* getLatestUpdateTimestamp(data.team, Task.Task);
@@ -114,8 +116,8 @@ export default defineFunction({
 const getLatestUpdateTimestamp: (
   teamId: string,
   dataType: Type.Obj.Any,
-) => Effect.Effect<string, never, DatabaseService> = Effect.fnUntraced(function* (teamId, dataType) {
-  const { objects: existingTasks } = yield* DatabaseService.runQuery(
+) => Effect.Effect<string, never, Database.Service> = Effect.fnUntraced(function* (teamId, dataType) {
+  const existingTasks = yield* Database.Service.runQuery(
     Query.type(dataType).select(Filter.foreignKeys(dataType, [{ source: LINEAR_TEAM_ID_KEY, id: teamId }])),
   );
   return Function.pipe(

package/src/functions/research/document-create.ts

@@ -7,7 +7,8 @@ import * as Schema from 'effect/Schema';
 
 import { ArtifactId } from '@dxos/assistant';
 import { Obj, Relation } from '@dxos/echo';
-import { DatabaseService, TracingService, defineFunction } from '@dxos/functions';
+import { Database } from '@dxos/echo';
+import { TracingService, defineFunction } from '@dxos/functions';
 import { log } from '@dxos/log';
 import { Markdown } from '@dxos/plugin-markdown/types';
 import { HasSubject } from '@dxos/types';
@@ -42,14 +43,14 @@ export default defineFunction({
     log.info('Creating research document', { subject, name, content });
 
     // TODO(burdon): Auto flush before and after calling function?
-    yield* DatabaseService.flush({ indexes: true });
+    yield* Database.Service.flush({ indexes: true });
     yield* TracingService.emitStatus({ message: 'Creating research document...' });
 
     // TODO(burdon): Type check.
-    const target = (yield* DatabaseService.resolve(ArtifactId.toDXN(subject))) as Obj.Any;
+    const target = (yield* Database.Service.resolve(ArtifactId.toDXN(subject))) as Obj.Unknown;
 
     // Create document.
-    const object = yield* DatabaseService.add(
+    const object = yield* Database.Service.add(
       Markdown.make({
         name,
         content,
@@ -57,7 +58,7 @@ export default defineFunction({
     );
 
     // Create relation.
-    yield* DatabaseService.add(
+    yield* Database.Service.add(
       Relation.make(HasSubject.HasSubject, {
         [Relation.Source]: object,
         [Relation.Target]: target,
@@ -65,7 +66,7 @@ export default defineFunction({
       }),
     );
 
-    yield* DatabaseService.flush({ indexes: true });
+    yield* Database.Service.flush({ indexes: true });
     log.info('Created research document', { subject, object });
 
     return {

package/src/functions/research/research-graph.ts

@@ -7,8 +7,10 @@ import * as Layer from 'effect/Layer';
 import * as Schema from 'effect/Schema';
 
 import { Obj, Query, Ref, Type } from '@dxos/echo';
+import { Database } from '@dxos/echo';
+import { SystemTypeAnnotation } from '@dxos/echo/internal';
 import { Queue } from '@dxos/echo-db';
-import { ContextQueueService, DatabaseService, QueueService } from '@dxos/functions';
+import { ContextQueueService, QueueService } from '@dxos/functions';
 
 /**
  * Container for a set of ephemeral research results.
@@ -16,32 +18,32 @@ import { ContextQueueService, DatabaseService, QueueService } from '@dxos/functi
 export const ResearchGraph = Schema.Struct({
   queue: Type.Ref(Queue),
 }).pipe(
-  Type.Obj({
+  Type.object({
     typename: 'dxos.org/type/ResearchGraph',
     version: '0.1.0',
   }),
+  SystemTypeAnnotation.set(true),
 );
 
 export interface ResearchGraph extends Schema.Schema.Type<typeof ResearchGraph> {}
 
-export const queryResearchGraph: () => Effect.Effect<ResearchGraph | undefined, never, DatabaseService> = Effect.fn(
+export const queryResearchGraph: () => Effect.Effect<ResearchGraph | undefined, never, Database.Service> = Effect.fn(
   'queryResearchGraph',
 )(function* () {
-  const { objects } = yield* DatabaseService.runQuery(Query.type(ResearchGraph));
+  const objects = yield* Database.Service.runQuery(Query.type(ResearchGraph));
   return objects.at(0);
 });
 
-export const createResearchGraph: () => Effect.Effect<ResearchGraph, never, DatabaseService | QueueService> = Effect.fn(
-  'createResearchGraph',
-)(function* () {
-  const queue = yield* QueueService.createQueue();
-  return yield* DatabaseService.add(Obj.make(ResearchGraph, { queue: Ref.fromDXN(queue.dxn) }));
-});
+export const createResearchGraph: () => Effect.Effect<ResearchGraph, never, Database.Service | QueueService> =
+  Effect.fn('createResearchGraph')(function* () {
+    const queue = yield* QueueService.createQueue();
+    return yield* Database.Service.add(Obj.make(ResearchGraph, { queue: Ref.fromDXN(queue.dxn) }));
+  });
 
 export const contextQueueLayerFromResearchGraph = Layer.unwrapEffect(
   Effect.gen(function* () {
     const researchGraph = (yield* queryResearchGraph()) ?? (yield* createResearchGraph());
-    const researchQueue = yield* DatabaseService.load(researchGraph.queue);
+    const researchQueue = yield* Database.Service.load(researchGraph.queue);
     return ContextQueueService.layer(researchQueue);
   }),
 );