@dxos/assistant 0.7.4-df37807

package/src/conversation/types.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Schema as S } from '@effect/schema';
+
+import { toJsonSchema } from '@dxos/echo-schema';
+import { JsonSchemaType } from '@dxos/echo-schema';
+import type { SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { ObjectId, type Message } from '../ai-service';
+
+export const createUserMessage = (spaceId: SpaceId, threadId: ObjectId, text: string): Message => ({
+  id: ObjectId.random(),
+  spaceId,
+  threadId,
+  role: 'user',
+  content: [{ type: 'text', text }],
+});
+
+export const LLMToolDefinition = S.Struct({
+  name: S.String,
+  description: S.String,
+  parameters: JsonSchemaType,
+  execute: S.Any,
+});
+
+export type LLMToolDefinition = {
+  name: string;
+  description: string;
+  parameters: JsonSchemaType;
+  execute: (params: unknown, context: ToolExecutionContext) => Promise<LLMToolResult>;
+};
+
+export type ToolExecutionContext = {};
+
+export type LLMToolResult =
+  | { kind: 'success'; result: unknown }
+  | { kind: 'error'; message: string }
+  | { kind: 'break'; result: unknown };
+
+export const LLMToolResult = Object.freeze({
+  /**
+   * The tool execution was successful.
+   * Gives the result back to the LLM.
+   */
+  Success: (result: unknown): LLMToolResult => ({ kind: 'success', result }),
+
+  Error: (message: string): LLMToolResult => ({ kind: 'error', message }),
+  /**
+   * Stop the conversation and return the result.
+   */
+  Break: (result: unknown): LLMToolResult => ({ kind: 'break', result }),
+});
+
+export type DefineToolParams<Params extends S.Schema.AnyNoContext> = {
+  name: string;
+  description: string;
+  schema: Params;
+  execute: (params: S.Schema.Type<Params>, context: ToolExecutionContext) => Promise<LLMToolResult>;
+};
+
+export const defineTool = <Params extends S.Schema.AnyNoContext>({
+  name,
+  description,
+  schema,
+  execute,
+}: DefineToolParams<Params>): LLMToolDefinition => {
+  return {
+    name,
+    description,
+    parameters: toFunctionParameterSchema(schema),
+    execute: (params, context) => {
+      const sanitized = S.decodeSync(schema)(params);
+      return execute(sanitized, context);
+    },
+  };
+};
+
+const toFunctionParameterSchema = (schema: S.Schema.All) => {
+  const jsonSchema = toJsonSchema(schema);
+  log('tool schema', { jsonSchema });
+  for (const key in jsonSchema.properties) {
+    if (typeof jsonSchema.properties![key].description !== 'string') {
+      throw new Error(`Property missing description: ${key}`);
+    }
+  }
+  return jsonSchema;
+};

package/src/cypher/ast.ts

@@ -0,0 +1,75 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// TODO(burdon): Why interface not type?
+
+export interface AstNode<Kind extends string> {
+  astKind: Kind;
+}
+
+export interface Identifier extends AstNode<'Identifier'> {
+  name: string;
+}
+
+export interface Property {
+  key: Identifier;
+  value: StringLiteral;
+}
+
+export interface Properties extends AstNode<'Properties'> {
+  properties: Property[];
+}
+
+export interface NodePattern extends AstNode<'NodePattern'> {
+  variable: Identifier | null;
+  label: Identifier;
+  properties: Properties | null;
+}
+
+export interface RelationshipPattern extends AstNode<'RelationshipPattern'> {
+  variable: Identifier | null;
+  label: Identifier;
+  properties: Properties | null;
+}
+
+export interface GraphPatternConnector extends AstNode<'GraphPatternConnector'> {
+  direction: '-' | '->' | '<-';
+}
+
+export interface GraphPattern extends AstNode<'GraphPattern'> {
+  segments: (NodePattern | RelationshipPattern)[] & {
+    separators: GraphPatternConnector[];
+  };
+}
+
+export interface MatchClause extends AstNode<'MatchClause'> {
+  pattern: GraphPattern;
+}
+
+export interface StringLiteral extends AstNode<'StringLiteral'> {
+  value: string;
+}
+
+export interface MemberExpression extends AstNode<'MemberExpression'> {
+  path: Identifier[];
+}
+
+export interface EqualsPredicate extends AstNode<'EqualsPredicate'> {
+  left: MemberExpression;
+  right: StringLiteral;
+}
+
+export interface WhereClause extends AstNode<'WhereClause'> {
+  predicates: EqualsPredicate[];
+}
+
+export interface ReturnClause extends AstNode<'ReturnClause'> {
+  fields: MemberExpression[];
+}
+
+export interface CypherQuery extends AstNode<'CypherQuery'> {
+  matchClause: MatchClause;
+  whereClause: WhereClause | null;
+  returnClause: ReturnClause;
+}

package/src/cypher/combinators.test.ts

@@ -0,0 +1,137 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+/* eslint-disable unused-imports/no-unused-vars */
+/* eslint-disable @typescript-eslint/consistent-type-imports */
+
+import { createRequire } from 'node:module';
+import type { Parjser } from 'parjs';
+import { describe, test } from 'vitest';
+
+import { log } from '@dxos/log';
+
+import {
+  cypherQuery,
+  identifier,
+  matchClause,
+  nodePattern,
+  properties,
+  returnClause,
+  whereClause,
+} from './combinators';
+
+describe('Cypher Combinators', () => {
+  // TODO(dmaretskyi): Must be like this or it breaks vitest.
+  const require = createRequire(import.meta.url);
+  const { string, regexp, whitespace } = require('parjs') as typeof import('parjs');
+  const { between, many, manySepBy, or, map, then, recover } =
+    require('parjs/combinators') as typeof import('parjs/combinators');
+
+  test('string', ({ expect }) => {
+    expect(string('hello').parse('hello')).toMatchObject({ isOk: true });
+  });
+
+  test('string with whitespace', ({ expect }) => {
+    const ws = whitespace(); // Parses whitespace
+    // many;
+    // const optionalWs = many(100)(ws);
+
+    expect(string('hello').parse('hello')).toMatchObject({ isOk: true });
+  });
+
+  test('optional vairable tag', ({ expect }) => {
+    const node = identifier.pipe(
+      then(':'),
+      recover(() => ({ kind: 'Soft' })),
+      or(''),
+      then(identifier),
+      between('(', ')'),
+    );
+    const relation = identifier.pipe(
+      then(':'),
+      recover(() => ({ kind: 'Soft' })),
+      or(''),
+      then(identifier),
+      between('[', ']'),
+    );
+    const parser = or(node)(relation);
+    const input = '[a:b]';
+    const result = parser.parse(input);
+    log.info('', { result });
+  });
+
+  test('properties', ({ expect }) => {
+    const input = "{name: 'John'}";
+    const result = properties.parse(input);
+    log.info('', { result });
+    expect(result).toMatchObject({ isOk: true });
+  });
+
+  test('node pattern', ({ expect }) => {
+    testParser(nodePattern, '(:Person)');
+    testParser(nodePattern, '(n:Person)');
+    testParser(nodePattern, "(n:Person {name: 'John'})");
+  });
+
+  test('match clause', ({ expect }) => {
+    testParser(matchClause, 'MATCH (n:Person)');
+    testParser(matchClause, 'MATCH (n:Person)-[r:KNOWS]->(m:Person)');
+    testParser(matchClause, 'MATCH (o:Org)-[:HAS_EMPLOYEE]->(e:Employee)');
+  });
+
+  test('where clause', ({ expect }) => {
+    testParser(whereClause, 'WHERE n.name = "John"');
+    testParser(whereClause, 'WHERE o.name = "DXOS" AND p.name = "Composer"');
+  });
+
+  test('return clause', ({ expect }) => {
+    testParser(returnClause, 'RETURN n, m');
+    testParser(returnClause, 'RETURN m.name AS employee');
+
+    const cases = ['RETURN n, m'];
+    for (const input of cases) {
+      const result = returnClause.parse(input);
+      log.info('', { result });
+      expect(result).toMatchObject({ isOk: true });
+    }
+  });
+
+  test('query', async ({ expect }) => {
+    testParser(cypherQuery, 'MATCH (n:Person) RETURN n');
+    testParser(
+      cypherQuery,
+      `
+      MATCH (n:Person {name: 'John'})-[r:KNOWS]->(m:Person)
+      RETURN n, m
+    `,
+    );
+    testParser(
+      cypherQuery,
+      `
+      MATCH (n:Person)-[r:KNOWS]->(m:Person)
+      WHERE n.name = "John"
+      RETURN n, m
+    `,
+    );
+    testParser(
+      cypherQuery,
+      'MATCH (o:Org)-[:HAS_EMPLOYEE]->(e:Employee)-[:WORKS_ON]->(p:Project) WHERE o.name = "DXOS" AND p.name = "Composer" RETURN e.name, e.id, p.name, p.id',
+    );
+    testParser(
+      cypherQuery,
+      'MATCH (n:Org {name: "DXOS"})-[:HAS_EMPLOYEE]->(m:Employee)<-[:WORKS_ON]-(p:Project {name: "Composer"}) RETURN m.name AS employee',
+    );
+    testParser(
+      cypherQuery,
+      "MATCH (org:Org {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(t:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
+    );
+  });
+});
+
+const testParser = (parser: Parjser<any>, input: string) => {
+  const result = parser.parse(input);
+  if (!result.isOk) {
+    throw new Error(`Failed to parse: ${result}`);
+  }
+};

package/src/cypher/combinators.ts

@@ -0,0 +1,149 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+/* eslint-disable unused-imports/no-unused-vars */
+/* eslint-disable @typescript-eslint/consistent-type-imports */
+
+import type {
+  CypherQuery,
+  EqualsPredicate,
+  GraphPattern,
+  GraphPatternConnector,
+  Identifier,
+  MatchClause,
+  MemberExpression,
+  NodePattern,
+  Properties,
+  Property,
+  RelationshipPattern,
+  ReturnClause,
+  StringLiteral,
+  WhereClause,
+} from './ast';
+
+import { Parjser, parjs, parjsCombinators } from './parjs';
+
+// Helper parsers
+const ws = parjs.whitespace(); // Parses whitespace
+
+const nothing = parjs.string('').pipe(parjsCombinators.map(() => null)); // Parses nothing
+
+const optional = <T>(parser: Parjser<T>) =>
+  parser.pipe(
+    parjsCombinators.recover(() => ({ kind: 'Soft' })),
+    parjsCombinators.or(nothing),
+  ); // Makes a parser optional
+
+const keyword = (word: string) => parjs.string(word).pipe(parjsCombinators.map(() => word)); // Parses specific keywords
+
+// Identifiers (node labels, relationship types, variables, etc.)
+export const identifier = parjs.regexp(/[a-zA-Z_][a-zA-Z0-9_]*/).pipe(
+  parjsCombinators.map((match): Identifier => ({ astKind: 'Identifier', name: match[0] })), // Extract the matched string
+);
+
+// Literals (numbers and strings)
+const numberLiteral = parjs.regexp(/-?\d+(\.\d+)?/).pipe(parjsCombinators.map(Number)); // Numbers
+const stringLiteral = parjs.regexp(/"([^"]*)"|'([^']*)'/).pipe(
+  parjsCombinators.map((match): StringLiteral => ({ astKind: 'StringLiteral', value: match[1] || match[2] })), // Extract content without quotes
+);
+
+// Node pattern (e.g., "(n:Person {name: 'John'})"
+const variableTag = identifier.pipe(parjsCombinators.map((variable) => variable)).expects('variable tag');
+export const label = identifier.pipe(parjsCombinators.map((name) => name));
+export const property = identifier.pipe(
+  parjsCombinators.then(parjs.string(':'), ws, stringLiteral),
+  parjsCombinators.map(([key, , , value]): Property => ({ key, value })),
+);
+export const properties = property.pipe(
+  parjsCombinators.manySepBy(','),
+  parjsCombinators.between('{', '}'),
+  parjsCombinators.map((properties): Properties => ({ astKind: 'Properties', properties })),
+);
+export const nodePattern = optional(variableTag).pipe(
+  parjsCombinators.then(':'),
+  parjsCombinators.then(ws, label, ws),
+  parjsCombinators.then(optional(properties)),
+  parjsCombinators.between('(', ')'),
+  parjsCombinators.map(
+    ([[[variable], _, label], properties]): NodePattern => ({ astKind: 'NodePattern', variable, label, properties }),
+  ),
+);
+
+export const relationshipPattern = optional(variableTag).pipe(
+  parjsCombinators.then(':'),
+  parjsCombinators.then(ws, label, ws),
+  parjsCombinators.then(optional(properties)),
+  parjsCombinators.between('[', ']'),
+  parjsCombinators.map(
+    ([[[variable], _, label], properties]): RelationshipPattern => ({
+      astKind: 'RelationshipPattern',
+      variable,
+      label,
+      properties,
+    }),
+  ),
+);
+
+const graphPatternConnector = parjs.string('->').pipe(
+  parjsCombinators.or('<-' as const, '-' as const),
+  parjsCombinators.map((x): GraphPatternConnector => ({ astKind: 'GraphPatternConnector', direction: x })),
+);
+
+export const graphPattern = nodePattern.pipe(
+  parjsCombinators.or(relationshipPattern),
+  parjsCombinators.manySepBy(graphPatternConnector),
+  parjsCombinators.map((segments): GraphPattern => ({ astKind: 'GraphPattern', segments })),
+);
+
+export const matchClause = keyword('MATCH').pipe(
+  parjsCombinators.then(ws),
+  parjsCombinators.then(graphPattern),
+  parjsCombinators.map(([, pattern]): MatchClause => ({ astKind: 'MatchClause', pattern })),
+);
+
+const memberExpression = identifier.pipe(
+  parjsCombinators.manySepBy('.'),
+  parjsCombinators.map((path): MemberExpression => ({ astKind: 'MemberExpression', path })),
+);
+
+const equalsPredicate = memberExpression.pipe(
+  parjsCombinators.then(ws, parjs.string('='), ws, stringLiteral),
+  parjsCombinators.map(([left, , , , right]): EqualsPredicate => ({ astKind: 'EqualsPredicate', left, right })),
+);
+
+export const whereClause = keyword('WHERE').pipe(
+  parjsCombinators.then(
+    ws,
+    equalsPredicate.pipe(parjsCombinators.between(ws, ws), parjsCombinators.manySepBy(parjs.string('AND'))),
+    ws,
+  ),
+  parjsCombinators.map(([, , predicates]): WhereClause => ({ astKind: 'WhereClause', predicates })),
+);
+
+export const returnClause = keyword('RETURN').pipe(
+  parjsCombinators.then(
+    ws,
+    memberExpression.pipe(
+      parjsCombinators.then(
+        optional(parjs.string('AS').pipe(parjsCombinators.between(ws), parjsCombinators.then(identifier))),
+        ws,
+      ),
+      parjsCombinators.manySepBy(parjs.string(',').pipe(parjsCombinators.then(ws))),
+    ),
+    ws,
+  ),
+  parjsCombinators.map(([, , fields]): ReturnClause => ({ astKind: 'ReturnClause', fields: fields.map(([f]) => f) })),
+);
+
+export const cypherQuery = ws.pipe(
+  parjsCombinators.then(matchClause, ws, optional(whereClause), ws, returnClause, ws),
+  parjsCombinators.map(
+    ([, matchClause, , whereClause, , returnClause]): CypherQuery => ({
+      astKind: 'CypherQuery',
+      matchClause,
+      whereClause,
+      returnClause,
+    }),
+  ),
+);

package/src/cypher/echo-data-source.ts

@@ -0,0 +1,105 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { raise } from '@dxos/debug';
+import { type EchoDatabase, Filter, type ReactiveEchoObject } from '@dxos/echo-db';
+import { type S, getSchemaTypename, StoredSchema, toJsonSchema } from '@dxos/echo-schema';
+import { getSchema } from '@dxos/live-object';
+import { log } from '@dxos/log';
+
+import type { DataSource, Node, Relationship } from './query-executor';
+import { formatInferredRelationshipLabel, formatNodeLabel, isReference } from './schema';
+
+export class EchoDataSource implements DataSource {
+  constructor(private readonly _db: EchoDatabase) {}
+
+  @log.method()
+  async getNodes({ label }: { label?: string }): Promise<Node[]> {
+    if (!label) {
+      const { objects } = await this._db.query().run();
+      return objects.map(this._objectToNode);
+    }
+
+    const schema = (await this._getAllSchema()).find((s) => getSchemaTypename(s)?.endsWith(label));
+
+    if (!schema) {
+      return [];
+    }
+
+    const { objects } = await this._db.query(Filter.schema(schema)).run();
+    return objects.map(this._objectToNode);
+  }
+
+  @log.method()
+  async getRelationships({ label }: { label?: string }): Promise<Relationship[]> {
+    const syntheticRefRelationships = (await this._getAllSchema())
+      .filter((schema) => getSchemaTypename(schema) !== StoredSchema.typename)
+      .flatMap((schema) => {
+        const jsonSchema = toJsonSchema(schema);
+        const relationships = Object.entries(jsonSchema.properties!).filter(([key, prop]) => isReference(prop));
+
+        return relationships.map(([key, value]) => {
+          const relationshipLabel = formatInferredRelationshipLabel(
+            jsonSchema.typename ?? raise(new Error('Schema must have typename')),
+            key,
+          );
+
+          return {
+            label: relationshipLabel,
+            schema,
+            property: key,
+          };
+        });
+      });
+    const relationship = syntheticRefRelationships.find((r) => r.label === label);
+    if (!relationship) {
+      return []; // TODO(dmaretskyi): Handle real relationships.
+    }
+
+    const { objects } = await this._db.query(Filter.schema(relationship.schema)).run();
+    return (
+      await Promise.all(objects.map((object) => this._projectRefRelationship(object, relationship.property)))
+    ).flat();
+  }
+
+  private async _getAllSchema(): Promise<S.Schema.AnyNoContext[]> {
+    return [
+      ...(await this._db.schemaRegistry.query().run()),
+      // TODO(dmaretskyi): Remove once we can serialize recursive schema.
+      ...this._db.graph.schemaRegistry.schemas.filter((s) => getSchemaTypename(s)?.startsWith('example.org')),
+    ].filter((schema) => getSchemaTypename(schema) !== StoredSchema.typename);
+  }
+
+  private _objectToNode(object: ReactiveEchoObject<any>): Node {
+    const { id, ...properties } = object;
+    return {
+      id,
+      kind: 'node',
+      label: formatNodeLabel(getSchemaTypename(getSchema(object)!)!),
+      properties,
+    };
+  }
+
+  private async _projectRefRelationship(object: ReactiveEchoObject<any>, prop: string): Promise<Relationship[]> {
+    if (!object[prop]) {
+      return [];
+    }
+
+    const value = Array.isArray(object[prop]) ? object[prop] : [object[prop]];
+
+    return Promise.all(
+      value.map(async (ref): Promise<Relationship> => {
+        const target = await ref.load();
+        return {
+          id: `${object.id}-${prop}-${target.id}`,
+          kind: 'relationship',
+          label: formatInferredRelationshipLabel(getSchemaTypename(getSchema(object)!)!, prop),
+          properties: {},
+          source: this._objectToNode(object),
+          target: this._objectToNode(target),
+        };
+      }),
+    );
+  }
+}

package/src/cypher/execution-plan.test.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { log } from '@dxos/log';
+
+import { createExecutionPlan } from './execution-plan';
+import { parseCypherQuery } from './parser';
+
+describe('Execution Plan', () => {
+  test('trivial', ({ expect }) => {
+    const plan = createExecutionPlan(parseCypherQuery('MATCH (n:Person) RETURN n'));
+    log.info('', { plan });
+    expect(plan.steps.map((step) => step.type)).toEqual(['NodeScan', 'ProduceResults']);
+  });
+
+  test('complex query', ({ expect }) => {
+    const plan = createExecutionPlan(
+      parseCypherQuery(
+        "MATCH (org:Org {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
+      ),
+    );
+    log.info('', { plan });
+    for (const step of plan.steps) {
+      log.info('', { step });
+    }
+    expect(plan.steps.map((step) => step.type)).toEqual([
+      'NodeScan',
+      'FilterByProperty',
+      'RelationshipScan',
+      'FilterByLabel',
+      'RelationshipScan',
+      'FilterByLabel',
+      'RelationshipScan',
+      'FilterByLabel',
+      'FilterByProperty',
+      'ProduceResults',
+    ]);
+  });
+});