@dxos/functions 0.5.3-main.cb47aab → 0.5.3-main.d3c5e1f

package/src/trigger/trigger-registry.test.ts

@@ -11,7 +11,7 @@ import { type Space } from '@dxos/client/echo';
 import { TestBuilder } from '@dxos/client/testing';
 import { Context } from '@dxos/context';
 import { Filter } from '@dxos/echo-db';
-import { create } from '@dxos/echo-schema';
+import { create, splitMeta } from '@dxos/echo-schema';
 import { describe, test } from '@dxos/test';
 import { range } from '@dxos/util';
 
@@ -19,9 +19,17 @@ import { TriggerRegistry } from './trigger-registry';
 import { createInitializedClients, TestType, triggerWebhook } from '../testing';
 import { type FunctionManifest, FunctionTrigger } from '../types';
 
-const testManifest: FunctionManifest = {
+const manifest: FunctionManifest = {
   triggers: [
     {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-1',
+          },
+        ],
+      },
       function: 'example.com/function/webhook-test',
       spec: {
         type: 'webhook',
@@ -29,10 +37,22 @@ const testManifest: FunctionManifest = {
       },
     },
     {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-2',
+          },
+        ],
+      },
       function: 'example.com/function/subscription-test',
       spec: {
         type: 'subscription',
-        filter: [{ type: TestType.typename }],
+        filter: [
+          {
+            type: TestType.typename,
+          },
+        ],
       },
     },
   ],
@@ -57,11 +77,12 @@ describe('trigger registry', () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const registry = createRegistry(client);
       const space = await client.spaces.create();
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       const { objects } = await space.db.query(Filter.schema(FunctionTrigger)).run();
-      expect(objects.length).to.eq(testManifest.triggers?.length);
-      const expected = testManifest.triggers?.map((t) => t.function).sort();
-      expect(objects.map((o: FunctionTrigger) => o.function).sort()).to.deep.eq(expected);
+      expect(objects.length).to.eq(manifest.triggers?.length);
+
+      const expected = manifest.triggers?.map((trigger) => trigger.function).sort();
+      expect(objects.map((object: FunctionTrigger) => object.function).sort()).to.deep.eq(expected);
     });
   });
 
@@ -70,13 +91,13 @@ describe('trigger registry', () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const space = await client.spaces.create();
       const registry = createRegistry(client);
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       await registry.open(ctx);
-      await waitHasInactiveTriggers(registry, space);
+      await waitForInactiveTriggers(registry, space);
 
       const callbackInvoked = new Trigger();
       const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
-      const webhookTrigger = allTriggers.find((t: FunctionTrigger) => t.spec.type === 'webhook')!;
+      const webhookTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'webhook')!;
       await registry.activate({ space }, webhookTrigger, async () => {
         callbackInvoked.wake();
         return 200;
@@ -90,15 +111,16 @@ describe('trigger registry', () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const space = await client.spaces.create();
       const registry = createRegistry(client);
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       await registry.open(ctx);
-      await waitHasInactiveTriggers(registry, space);
+      await waitForInactiveTriggers(registry, space);
 
       const inactiveTrigger = registry.getInactiveTriggers(space)[0];
       await registry.activate({ space }, inactiveTrigger, async () => 200);
 
       const updatedInactiveList = registry.getInactiveTriggers(space);
-      expect(updatedInactiveList.find((t: FunctionTrigger) => t.function === inactiveTrigger.function)).to.be.undefined;
+      expect(updatedInactiveList.find((trigger: FunctionTrigger) => trigger.function === inactiveTrigger.function)).to
+        .be.undefined;
     });
   });
 
@@ -107,12 +129,12 @@ describe('trigger registry', () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const space = await client.spaces.create();
       const registry = createRegistry(client);
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       await registry.open(ctx);
-      await waitHasInactiveTriggers(registry, space);
+      await waitForInactiveTriggers(registry, space);
 
       const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
-      const echoTrigger = allTriggers.find((t: FunctionTrigger) => t.spec.type === 'subscription')!;
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
       let count = 0;
       await registry.activate({ space }, echoTrigger, async () => {
         count++;
@@ -124,7 +146,6 @@ describe('trigger registry', () => {
       expect(count).to.eq(1);
 
       space.db.remove(echoTrigger);
-
       space.db.add(create(TestType, { title: '2' }));
       await sleep(20);
       expect(count).to.eq(1);
@@ -134,12 +155,12 @@ describe('trigger registry', () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const space = await client.spaces.create();
       const registry = createRegistry(client);
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       await registry.open(ctx);
-      await waitHasInactiveTriggers(registry, space);
+      await waitForInactiveTriggers(registry, space);
 
       const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
-      const echoTrigger = allTriggers.find((t: FunctionTrigger) => t.spec.type === 'subscription')!;
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
       let count = 0;
       await registry.activate({ space }, echoTrigger, async () => {
         count++;
@@ -155,19 +176,20 @@ describe('trigger registry', () => {
   });
 
   describe('trigger events', () => {
-    test('event fired when all registered when opened', async () => {
+    test.only('event fired when all registered when opened', async () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const registry = createRegistry(client);
-      const triggers = createTriggersInSpace(client.spaces.default, 3);
+      const triggers = createTriggers(client.spaces.default, 3);
 
       const triggersRegistered = new Trigger<FunctionTrigger[]>();
       registry.registered.on((fn) => {
         expect(fn.space.key.toHex()).to.eq(client.spaces.default.key.toHex());
         triggersRegistered.wake(fn.triggers);
       });
+
       void registry.open(ctx);
       const functions = await triggersRegistered.wait();
-      const expected = triggers.map((obj) => obj.id).sort();
+      const expected = triggers.map((object) => object.id).sort();
       expect(functions.map((fn) => fn.id).sort()).to.deep.eq(expected);
     });
 
@@ -182,16 +204,16 @@ describe('trigger registry', () => {
         triggerRegistered.wake(fn.triggers[0]);
       });
       await registry.open(ctx);
-      await registry.register(space, { triggers: testManifest?.triggers?.slice(0, 1) });
+      await registry.register(space, { triggers: manifest?.triggers?.slice(0, 1) });
       const registered = await triggerRegistered.wait();
-      expect(registered.function).to.eq(testManifest.triggers![0].function);
+      expect(registered.function).to.eq(manifest.triggers![0].function);
     });
 
     test('event fired when a new trigger is removed', async () => {
       const client = (await createInitializedClients(testBuilder))[0];
       const registry = createRegistry(client);
       const space = await client.spaces.create();
-      const triggers = createTriggersInSpace(space, 3);
+      const triggers = createTriggers(space, 3);
 
       const triggerLoaded = new Trigger();
       registry.registered.on((fn) => triggerLoaded.wake());
@@ -201,7 +223,7 @@ describe('trigger registry', () => {
         expect(fn.triggers.length).to.eq(1);
         triggerRemoved.wake(fn.triggers[0]);
       });
-      await registry.register(space, testManifest);
+      await registry.register(space, manifest);
       await registry.open(ctx);
       await triggerLoaded.wait();
 
@@ -211,19 +233,23 @@ describe('trigger registry', () => {
     });
   });
 
-  const waitHasInactiveTriggers = async (registry: TriggerRegistry, space: Space) => {
-    await waitForCondition({ condition: () => registry.getInactiveTriggers(space).length > 0 });
-  };
-
   const createRegistry = (client: Client) => {
     const registry = new TriggerRegistry(client);
     ctx.onDispose(() => registry.close());
     return registry;
   };
 
-  const createTriggersInSpace = (space: Space, count: number) => {
-    const triggers = range(count, () => create(FunctionTrigger, { ...testManifest.triggers![0] }));
-    triggers.forEach((def) => space.db.add(def));
+  const createTriggers = (space: Space, count: number) => {
+    const triggers = range(count, () => {
+      const { meta, object } = splitMeta(manifest.triggers![0]);
+      return create(FunctionTrigger, object, meta);
+    });
+
+    triggers.forEach((trigger) => space.db.add(trigger));
     return triggers;
   };
+
+  const waitForInactiveTriggers = async (registry: TriggerRegistry, space: Space) => {
+    await waitForCondition({ condition: () => registry.getInactiveTriggers(space).length > 0 });
+  };
 });

package/src/trigger/trigger-registry.ts

@@ -4,8 +4,9 @@
 
 import { Event } from '@dxos/async';
 import { type Client } from '@dxos/client';
-import { create, Filter, type Space } from '@dxos/client/echo';
+import { create, Filter, getMeta, type Space } from '@dxos/client/echo';
 import { Context, Resource } from '@dxos/context';
+import { ECHO_ATTR_META, foreignKey, foreignKeyEquals, splitMeta } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -13,6 +14,7 @@ import { ComplexMap } from '@dxos/util';
 
 import { createSubscriptionTrigger, createTimerTrigger, createWebhookTrigger, createWebsocketTrigger } from './type';
 import { type FunctionManifest, FunctionTrigger, type FunctionTriggerType, type TriggerSpec } from '../types';
+import { diff, intersection } from '../util';
 
 type ResponseCode = number;
 
@@ -100,10 +102,21 @@ export class TriggerRegistry extends Resource {
       space.db.graph.runtimeSchemaRegistry.registerSchema(FunctionTrigger);
     }
 
-    const reactiveObjects = manifest.triggers.map((template: Omit<FunctionTrigger, 'id'>) =>
-      create(FunctionTrigger, { ...template }),
-    );
-    reactiveObjects.forEach((obj) => space.db.add(obj));
+    // Sync triggers.
+    const { objects: existing } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+    const { added, removed } = diff(existing, manifest.triggers, (a, b) => {
+      // Create FK to enable syncing if none are set.
+      // TODO(burdon): Warn if not unique.
+      const keys = b[ECHO_ATTR_META]?.keys ?? [foreignKey('manifest', [b.function, b.spec.type].join('-'))];
+      return intersection(getMeta(a)?.keys ?? [], keys, foreignKeyEquals).length > 0;
+    });
+
+    added.forEach((trigger) => {
+      const { meta, object } = splitMeta(trigger);
+      space.db.add(create(FunctionTrigger, object, meta));
+    });
+    // TODO(burdon): Update existing triggers.
+    removed.forEach((trigger) => space.db.remove(trigger));
   }
 
   protected override async _open(): Promise<void> {

package/src/trigger/type/subscription-trigger.ts

@@ -3,7 +3,7 @@
 //
 
 import { TextV0Type } from '@braneframe/types';
-import { debounce, DeferredTask } from '@dxos/async';
+import { debounce, UpdateScheduler } from '@dxos/async';
 import { type Context } from '@dxos/context';
 import { createSubscription, Filter, getAutomergeObjectCore, type Query } from '@dxos/echo-db';
 import { log } from '@dxos/log';
@@ -18,26 +18,33 @@ export const createSubscriptionTrigger: TriggerFactory<SubscriptionTrigger> = as
   callback: TriggerCallback,
 ) => {
   const objectIds = new Set<string>();
-  const task = new DeferredTask(ctx, async () => {
-    if (objectIds.size > 0) {
-      await callback({ objects: Array.from(objectIds) });
-      objectIds.clear();
-    }
-  });
+  const task = new UpdateScheduler(
+    ctx,
+    async () => {
+      if (objectIds.size > 0) {
+        const objects = Array.from(objectIds);
+        objectIds.clear();
+        await callback({ objects });
+      }
+    },
+    { maxFrequency: 4 },
+  );
 
   // TODO(burdon): Don't fire initially?
   // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
   const subscriptions: (() => void)[] = [];
   const subscription = createSubscription(({ added, updated }) => {
-    log.info('updated', { added: added.length, updated: updated.length });
+    const sizeBefore = objectIds.size;
     for (const object of added) {
       objectIds.add(object.id);
     }
     for (const object of updated) {
       objectIds.add(object.id);
     }
-
-    task.schedule();
+    if (objectIds.size > sizeBefore) {
+      log.info('updated', { added: added.length, updated: updated.length });
+      task.trigger();
+    }
   });
 
   subscriptions.push(() => subscription.unsubscribe());

package/src/types.ts

@@ -2,10 +2,7 @@
 // Copyright 2023 DXOS.org
 //
 
-import { AST, S, TypedObject } from '@dxos/echo-schema';
-
-// TODO(burdon): Factor out.
-const omitEchoId = <T>(schema: S.Schema<T>): S.Schema<Omit<T, 'id'>> => S.make(AST.omit(schema.ast, ['id']));
+import { RawObject, S, TypedObject } from '@dxos/echo-schema';
 
 /**
  * Type discriminator for TriggerSpec.
@@ -17,7 +14,7 @@ export type FunctionTriggerType = 'subscription' | 'timer' | 'webhook' | 'websoc
 
 const SubscriptionTriggerSchema = S.struct({
   type: S.literal('subscription'),
-  // TODO(burdon): Define query DSL.
+  // TODO(burdon): Define query DSL (from ECHO).
   filter: S.array(
     S.struct({
       type: S.string,
@@ -33,12 +30,14 @@ const SubscriptionTriggerSchema = S.struct({
     }),
   ),
 });
+
 export type SubscriptionTrigger = S.Schema.Type<typeof SubscriptionTriggerSchema>;
 
 const TimerTriggerSchema = S.struct({
   type: S.literal('timer'),
   cron: S.string,
 });
+
 export type TimerTrigger = S.Schema.Type<typeof TimerTriggerSchema>;
 
 const WebhookTriggerSchema = S.mutable(
@@ -49,6 +48,7 @@ const WebhookTriggerSchema = S.mutable(
     port: S.optional(S.number),
   }),
 );
+
 export type WebhookTrigger = S.Schema.Type<typeof WebhookTriggerSchema>;
 
 const WebsocketTriggerSchema = S.struct({
@@ -56,6 +56,7 @@ const WebsocketTriggerSchema = S.struct({
   url: S.string,
   init: S.optional(S.record(S.string, S.any)),
 });
+
 export type WebsocketTrigger = S.Schema.Type<typeof WebsocketTriggerSchema>;
 
 const TriggerSpecSchema = S.union(
@@ -64,6 +65,7 @@ const TriggerSpecSchema = S.union(
   WebsocketTriggerSchema,
   SubscriptionTriggerSchema,
 );
+
 export type TriggerSpec = TimerTrigger | WebhookTrigger | WebsocketTrigger | SubscriptionTrigger;
 
 /**
@@ -84,9 +86,9 @@ export class FunctionTrigger extends TypedObject({
   typename: 'dxos.org/type/FunctionTrigger',
   version: '0.1.0',
 })({
-  function: S.string.pipe(S.description('Function ID/URI.')),
-  // Context passed to a function.
-  meta: S.optional(S.record(S.string, S.any)),
+  function: S.string.pipe(S.description('Function URI.')),
+  // Context is merged into the event data passed to the function.
+  meta: S.optional(S.object),
   spec: TriggerSpecSchema,
 }) {}
 
@@ -94,8 +96,8 @@ export class FunctionTrigger extends TypedObject({
  * Function manifest file.
  */
 export const FunctionManifestSchema = S.struct({
-  functions: S.optional(S.mutable(S.array(omitEchoId(FunctionDef)))),
-  triggers: S.optional(S.mutable(S.array(omitEchoId(FunctionTrigger)))),
+  functions: S.optional(S.mutable(S.array(RawObject(FunctionDef)))),
+  triggers: S.optional(S.mutable(S.array(RawObject(FunctionTrigger)))),
 });
 
 export type FunctionManifest = S.Schema.Type<typeof FunctionManifestSchema>;

package/src/util.test.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { describe, test } from '@dxos/test';
+
+import { diff, intersection } from './util';
+
+describe('diff', () => {
+  test('returns the difference between two sets', () => {
+    {
+      const { added, updated, removed } = diff<number>([], [], (a, b) => a === b);
+      expect(added).to.deep.eq([]);
+      expect(updated).to.deep.eq([]);
+      expect(removed).to.deep.eq([]);
+    }
+    {
+      const previous = [1, 2, 3];
+      const next = [2, 3, 4];
+      const { added, updated, removed } = diff(previous, next, (a, b) => a === b);
+      expect(added).to.deep.eq([4]);
+      expect(updated).to.deep.eq([2, 3]);
+      expect(removed).to.deep.eq([1]);
+    }
+    {
+      const previous = [{ x: 1 }, { x: 2 }, { x: 3 }];
+      const next = [{ x: 2 }, { x: 3 }, { x: 4 }];
+      const { added, updated, removed } = diff(previous, next, (a, b) => a.x === b.x);
+      expect(added).to.deep.eq([{ x: 4 }]);
+      expect(updated).to.deep.eq([{ x: 2 }, { x: 3 }]);
+      expect(removed).to.deep.eq([{ x: 1 }]);
+    }
+  });
+
+  test('intersection', () => {
+    expect(intersection([1, 2, 3], [2, 3, 4], (a, b) => a === b)).to.deep.eq([2, 3]);
+    expect(
+      intersection([{ x: 1 }, { x: 2 }, { x: 3 }], [{ x: 2 }, { x: 3 }, { x: 4 }], (a, b) => a.x === b.x),
+    ).to.deep.eq([{ x: 2 }, { x: 3 }]);
+  });
+});

package/src/util.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export type Comparator<A, B = A> = (a: A, b: B) => boolean;
+
+export type DiffResult<A, B = A> = {
+  added: B[];
+  updated: A[];
+  removed: A[];
+};
+
+/**
+ *
+ * @param previous
+ * @param next
+ * @param comparator
+ */
+// TODO(burdon): Factor out.
+export const diff = <A, B = A>(
+  previous: readonly A[],
+  next: readonly B[],
+  comparator: Comparator<A, B>,
+): DiffResult<A, B> => {
+  const remaining = [...previous];
+  const result: DiffResult<A, B> = {
+    added: [],
+    updated: [],
+    removed: remaining,
+  };
+
+  // TODO(burdon): Mark and sweep.
+  for (const object of next) {
+    const index = remaining.findIndex((item) => comparator(item, object));
+    if (index === -1) {
+      result.added.push(object);
+    } else {
+      result.updated.push(remaining[index]);
+      remaining.splice(index, 1);
+    }
+  }
+
+  return result;
+};
+
+// TODO(burdon): Factor out.
+export const intersection = <A, B = A>(a: A[], b: B[], comparator: Comparator<A, B>): A[] =>
+  a.filter((a) => b.find((b) => comparator(a, b)) !== undefined);

package/dist/types/src/registry/function-registry.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"function-registry.d.ts","sourceRoot":"","sources":["../../../../src/registry/function-registry.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAkB,KAAK,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,KAAK,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAIvD,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE9D,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,WAAW,EAAE,CAAC;CAC7B,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,QAAQ;IAKhC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJpC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAA4D;IAEhG,SAAgB,qBAAqB,kCAAyC;gBAEjD,OAAO,EAAE,MAAM;IAIrC,YAAY,CAAC,KAAK,EAAE,KAAK,GAAG,WAAW,EAAE;IAIhD;;;OAGG;IAEU,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;cAcrD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cA0BtB,MAAM,CAAC,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAG3D"}

package/dist/types/src/registry/function-registry.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"function-registry.test.d.ts","sourceRoot":"","sources":["../../../../src/registry/function-registry.test.ts"],"names":[],"mappings":""}

package/dist/types/src/registry/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/registry/index.ts"],"names":[],"mappings":"AAIA,cAAc,qBAAqB,CAAC"}