@dxos/assistant 0.7.4-df37807

package/src/testing/logger.ts

@@ -0,0 +1,128 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type { ImageSource, MessageContentBlock } from '../ai-service';
+import type { ConversationEvent } from '../conversation';
+
+export const createLogger = ({
+  stream,
+  filter,
+  onImage,
+}: {
+  stream?: boolean;
+  filter?: (event: ConversationEvent) => boolean;
+  onImage?: (img: { id: string; description?: string; source: ImageSource }) => void;
+} = {}) => {
+  const images: Record<string, any> = {};
+  let currentText = '';
+
+  return (event: ConversationEvent) => {
+    if (typeof filter === 'function' && !filter(event)) {
+      return;
+    }
+
+    const printContentBlock = (content: MessageContentBlock) => {
+      switch (content.type) {
+        case 'text':
+          process.stdout.write(content.text);
+          break;
+        case 'tool_use':
+          process.stdout.write(`⚙️ [Tool Use] ${content.name}\n`);
+          break;
+        case 'image': {
+          if (content.id && content.source) {
+            images[content.id] = content.source;
+          }
+          process.stdout.write(`[Image id=${content.id} media_type=${content.source?.media_type}]`);
+          break;
+        }
+      }
+    };
+
+    const onTextBlockPrinted = (text: string) => {
+      const imageMatch = text.match(/<image id="([^"]+)"(?:\s+prompt="([^"]+)")?\s*\/>/);
+      if (imageMatch && onImage && images[imageMatch[1]]) {
+        onImage({
+          id: imageMatch[1],
+          description: imageMatch[2], // Extract prompt as description if present
+          source: images[imageMatch[1]],
+        });
+      }
+    };
+
+    if (stream) {
+      switch (event.type) {
+        case 'message_start': {
+          process.stdout.write(`${event.message.role.toUpperCase()}\n\n`);
+          for (const content of event.message.content) {
+            printContentBlock(content);
+          }
+          break;
+        }
+        case 'content_block_start': {
+          currentText = '';
+          printContentBlock(event.content);
+          break;
+        }
+        case 'content_block_delta': {
+          switch (event.delta.type) {
+            case 'text_delta': {
+              currentText += event.delta.text;
+              process.stdout.write(event.delta.text);
+              break;
+            }
+            case 'input_json_delta': {
+              process.stdout.write(event.delta.partial_json);
+              break;
+            }
+          }
+          break;
+        }
+        case 'content_block_stop': {
+          process.stdout.write('\n');
+          onTextBlockPrinted(currentText);
+          currentText = '';
+          break;
+        }
+        case 'message_delta': {
+          break;
+        }
+        case 'message_stop': {
+          process.stdout.write('\n\n');
+          break;
+        }
+      }
+    }
+
+    switch (event.type) {
+      case 'message': {
+        if (!stream || event.message.role !== 'assistant') {
+          process.stdout.write(`${event.message.role.toUpperCase()}\n\n`);
+          for (const block of event.message.content) {
+            switch (block.type) {
+              case 'text':
+                process.stdout.write(block.text + '\n');
+                onTextBlockPrinted(block.text);
+                break;
+              case 'tool_use':
+                process.stdout.write(`⚙️ [Tool Use] ${block.name}\n`);
+                process.stdout.write(`  ${JSON.stringify(block.input)}\n`);
+                break;
+              case 'tool_result':
+                if (block.isError) {
+                  process.stdout.write('❌ [Tool Error]\n');
+                  process.stdout.write(block.content + '\n');
+                } else {
+                  process.stdout.write('✅ [Tool Success]\n');
+                  process.stdout.write(block.content + '\n');
+                }
+            }
+            process.stdout.write('\n\n');
+          }
+        }
+        break;
+      }
+    }
+  };
+};

package/src/testing/object-extraction.test.ts

@@ -0,0 +1,71 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// import { AnthropicBackend } from '../conversation';
+// import { runLLM } from '../conversation';
+// import { createUserMessage } from '../conversation';
+
+// test('extract objects from the DXOS explainer', async () => {
+//   const result = await runLLM({
+//     model: '@anthropic/claude-3-5-haiku-20241022',
+//     messages: [
+//       createUserMessage(`
+// # README
+
+// > Welcome to DXOS Composer.
+
+// Here, you can:
+// - Create and organize things
+// - Share and collaborate with others in real-time
+// - Work offline
+
+// Composer is:
+// - Free. Software built with DXOS requires no expensive server infrastructure to operate.
+// - Private. Your data never travels to or lives on any servers. Connections with collaborators are peer-to-peer.
+// - Local-first. Your data is stored entirely in your browser.
+// - Extensible. Add features by building plugins.
+// - Open source.
+
+// Your feedback is most welcome: feature requests, bug reports, questions: [Discord](https://discord.gg/uTYyx6srAW).
+
+// See also:
+// - [DXOS Documentation](https://docs.dxos.org) where you can build your own local-first, collaborative software.
+// `),
+//     ],
+//     system: `
+//       <instructions>
+//        Extract a list of objects that appear in the text.
+//        Objects can include names, places, technologies, people, concepts, etc.
+//        Extracts objects that are interesting or important in the context of the text.
+//       </instructions>
+
+//       <formatting>
+//         Output a list of object names from text, each surrounded with an object <object> tag.
+//       </formatting>
+
+//       <example>
+//         <input>
+//           Czerniaków was born on 30 November 1880 in Warsaw, Poland (then part of the Congress Poland). He studied engineering in Warsaw and Dresden and taught in the Jewish community's vocational school in Warsaw.
+//         </input>
+//         <output>
+//           <object>Czerniaków</object>
+//           <object>30 November 1880</object>
+//           <object>Warsaw</object>
+//           <object>Poland</object>
+//           <object>Congress Poland</object>
+//           <object>engineering</object>
+//           <object>Dresden</object>
+//         </output>
+//       </example>
+//     `,
+//     tools: [],
+//     backend,
+//   });
+
+//   log.info('result', { result });
+// });
+
+// const backend = new AnthropicBackend({
+//   apiKey: process.env.ANTHROPIC_API_KEY!,
+// });

package/src/testing/query-promts.ts

@@ -0,0 +1,75 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Schema as S } from '@effect/schema';
+
+import { toJsonSchema } from '@dxos/echo-schema';
+import { log } from '@dxos/log';
+
+import { defineTool, LLMToolResult } from '../conversation';
+import { executeQuery, formatJsonSchemaForLLM, type DataSource } from '../cypher';
+import { trim } from '../util';
+
+export const createCypherTool = (dataSource: DataSource, schemaTypes: S.Schema.Any[] = []) =>
+  defineTool({
+    name: 'graphQuery',
+    description:
+      'Query the ECHO graph database in cypher query language. Returns data from the database.' +
+      (schemaTypes.length > 0 ? `\n\n${createSystemPrompt(schemaTypes)}` : ''),
+    schema: S.Struct({
+      query: S.String.annotations({
+        description: `
+      A valid cypher query string to execute.
+      Query must have one MATCH clause.
+      Match clause can have multiple patterns but they must all be connected via a relationship chain.
+      Query might have zero or one WHERE clauses.
+      Query must have one RETURN clause.
+      RETURN clause can have multiple expressions separated by commas.
+      DISTINCT keyword is not allowed.
+
+      <example>
+      MATCH (n:Person) RETURN n.name
+      </example>
+
+      <example>
+      MATCH (s:Studio)-[:PRODUCED]->(m:Movie)-[:PRODUCED_IN]->(c:City {name: "Las Vegas"}) RETURN s
+      </example>
+      
+      <example>
+      MATCH (a:Actor)-[:STARRED_IN]->(m:Movie)<-[:PRODUCED]-(s:Studio {name: "Cinema Pictures"}) WHERE m.name = "Once upon a time" RETURN m.name, a.name
+      </example>
+      `,
+      }),
+    }),
+    execute: async ({ query }) => {
+      try {
+        log('cypher query', { query });
+        const results = await executeQuery(dataSource, query);
+        log('query complete', { results });
+        return LLMToolResult.Success(results);
+      } catch (e: any) {
+        log.catch(e);
+        return LLMToolResult.Error(e.message);
+      }
+    },
+  });
+
+export const createSystemPrompt = (schemaTypes: S.Schema.Any[]) => trim`
+  You have access to ECHO graph database. You can query the database to get data to satisfy user prompts.
+
+  Database schema is defined in pseud-cypher syntax. The schema is:
+
+  <schema>
+    ${formatJsonSchemaForLLM(schemaTypes.map((schema) => toJsonSchema(schema)))}
+  <schema>
+
+  Before replying always think step-by-step on how to proceed.
+  Print your thoughts inside <cot> tags.
+  
+  <example>
+    <cot>To answer the question I need to ...</cot>
+  </example>
+
+  Before answering the user's question, decide what tools you need to use to answer the question.
+`;

package/src/testing/query.test.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { test } from 'vitest';
+
+import { SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { EchoTestBuilder } from '@dxos/echo-db/testing';
+
+import { createLogger } from './logger';
+import { createCypherTool, createSystemPrompt } from './query-promts';
+import { createTestData, seedTestData } from './test-data';
+import { Contact, Org, Project, Task } from './test-schema';
+import { AIServiceClientImpl, ObjectId } from '../ai-service';
+import { runLLM, createUserMessage } from '../conversation';
+import { EchoDataSource } from '../cypher/echo-data-source';
+
+const ENDPOINT = 'http://localhost:8787';
+
+const client = new AIServiceClientImpl({
+  endpoint: ENDPOINT,
+});
+
+test.skip('cypher query', async () => {
+  const dataSource = createTestData();
+  const schemaTypes = [Org, Project, Task, Contact];
+
+  const cypherTool = createCypherTool(dataSource);
+
+  const spaceId = SpaceId.random();
+  const threadId = ObjectId.random();
+
+  await client.insertMessages([
+    createUserMessage(
+      spaceId,
+      threadId,
+      'Query the database and give me all employees from DXOS organization that work on Composer and what their tasks are.',
+    ),
+  ]);
+
+  const result = await runLLM({
+    model: '@anthropic/claude-3-5-sonnet-20241022',
+    tools: [cypherTool],
+    spaceId,
+    threadId,
+    system: createSystemPrompt(schemaTypes),
+    client,
+    logger: createLogger({ stream: false }),
+  });
+
+  log.info('DONE', { result: result.result });
+});
+
+test.only('query ECHO', async () => {
+  await using builder = await new EchoTestBuilder().open();
+  const { db, graph } = await builder.createDatabase();
+
+  const schemaTypes = [Org, Project, Task, Contact];
+  graph.schemaRegistry.addSchema(schemaTypes);
+
+  seedTestData(db);
+  await db.flush({ indexes: true });
+
+  const dataSource = new EchoDataSource(db);
+  const cypherTool = createCypherTool(dataSource);
+
+  const spaceId = SpaceId.random();
+  const threadId = ObjectId.random();
+
+  await client.insertMessages([
+    createUserMessage(
+      spaceId,
+      threadId,
+      'Query the database and give me all employees from DXOS organization that work on Composer and what their tasks are.',
+    ),
+  ]);
+
+  const result = await runLLM({
+    model: '@anthropic/claude-3-5-sonnet-20241022',
+    tools: [cypherTool],
+    spaceId,
+    threadId,
+    system: createSystemPrompt(schemaTypes),
+    client,
+    logger: createLogger({ stream: false }),
+  });
+
+  log.info('DONE', { result: result.result });
+});