@synnaxlabs/client 0.20.0 → 0.22.0

package/src/ontology/client.ts

@@ -7,83 +7,108 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { type UnaryClient } from "@synnaxlabs/freighter";
-import { type AsyncTermSearcher } from "@synnaxlabs/x";
+import { sendRequired, type UnaryClient } from "@synnaxlabs/freighter";
+import { type AsyncTermSearcher } from "@synnaxlabs/x/search";
 
-import { type Client as FrameClient } from "@/framer/client";
+import { QueryError } from "@/errors";
+import { framer } from "@/framer";
 import { group } from "@/ontology/group";
-import { type ID, type Resource } from "@/ontology/payload";
-import { Retriever } from "@/ontology/retriever";
+import {
+  ID,
+  IDPayload,
+  RelationshipChange,
+  ResourceChange,
+  idZ,
+  parseRelationship,
+  resourceSchemaZ,
+  type Resource,
+} from "@/ontology/payload";
 import { Writer } from "@/ontology/writer";
+import { z } from "zod";
+import { observe, toArray } from "@synnaxlabs/x";
+import { Frame } from "@/framer/frame";
 
-import { QueryError } from "@/errors";
+const RETRIEVE_ENDPOINT = "/ontology/retrieve";
+
+const retrieveReqZ = z.object({
+  ids: idZ.array().optional(),
+  children: z.boolean().optional(),
+  parents: z.boolean().optional(),
+  includeSchema: z.boolean().optional(),
+  excludeFieldData: z.boolean().optional(),
+  term: z.string().optional(),
+  limit: z.number().optional(),
+  offset: z.number().optional(),
+});
+
+type RetrieveRequest = z.infer<typeof retrieveReqZ>;
+
+export type RetrieveOptions = Pick<
+  RetrieveRequest,
+  "includeSchema" | "excludeFieldData"
+>;
 
-import { ChangeTracker } from "@/ontology/signals";
+const retrieveResZ = z.object({
+  resources: resourceSchemaZ.array(),
+});
+
+const parseIDs = (ids: ID | ID[] | string | string[]): IDPayload[] =>
+  toArray(ids).map((id) => new ID(id).payload);
 
 /** The core client class for executing queries against a Synnax cluster ontology */
 export class Client implements AsyncTermSearcher<string, string, Resource> {
+  readonly type: string = "ontology";
   groups: group.Client;
-  retriever: Retriever;
+  private readonly client: UnaryClient;
   private readonly writer: Writer;
-  private readonly framer: FrameClient;
+  private readonly framer: framer.Client;
 
-  constructor(unary: UnaryClient, framer: FrameClient) {
-    this.retriever = new Retriever(unary);
+  constructor(unary: UnaryClient, framer: framer.Client) {
+    this.client = unary;
     this.writer = new Writer(unary);
     this.groups = new group.Client(unary);
     this.framer = framer;
   }
 
-  async search(term: string): Promise<Resource[]> {
-    return await this.retriever.search(term);
+  async search(term: string, options?: RetrieveOptions): Promise<Resource[]> {
+    return await this.execRetrieve({ term, ...options });
   }
 
-  async retrieve(
-    id: ID | string,
-    includeSchema?: boolean,
-    includeFieldData?: boolean,
-  ): Promise<Resource>;
+  async retrieve(id: ID | string, options?: RetrieveOptions): Promise<Resource>;
 
-  async retrieve(
-    ids: ID[] | string[],
-    includeSchema?: boolean,
-    includeFieldData?: boolean,
-  ): Promise<Resource[]>;
+  async retrieve(ids: ID[] | string[], options?: RetrieveOptions): Promise<Resource[]>;
 
   async retrieve(
     ids: ID | ID[] | string | string[],
-    includeSchema?: boolean,
-    includeFieldData?: boolean,
+    options?: RetrieveOptions,
   ): Promise<Resource | Resource[]> {
-    const resources = await this.retriever.retrieve(
-      ids,
-      includeSchema,
-      includeFieldData,
-    );
+    const resources = await this.execRetrieve({ ids: parseIDs(ids), ...options });
     if (Array.isArray(ids)) return resources;
     if (resources.length === 0)
       throw new QueryError(`No resource found with ID ${ids.toString()}`);
     return resources[0];
   }
 
-  async page(offset: number, limit: number): Promise<Resource[]> {
-    return [];
+  async page(
+    offset: number,
+    limit: number,
+    options?: RetrieveOptions,
+  ): Promise<Resource[]> {
+    return await this.execRetrieve({ offset, limit, ...options });
   }
 
   async retrieveChildren(
-    IDs: ID | ID[],
-    includeSchema?: boolean,
-    includeFieldData?: boolean,
+    ids: ID | ID[],
+    options?: RetrieveOptions,
   ): Promise<Resource[]> {
-    return await this.retriever.retrieveChildren(IDs, includeSchema, includeFieldData);
+    return await this.execRetrieve({ ids: parseIDs(ids), children: true, ...options });
   }
 
   async retrieveParents(
-    IDs: ID | ID[],
-    includeSchema?: boolean,
-    includeFieldData?: boolean,
+    ids: ID | ID[],
+    options?: RetrieveOptions,
   ): Promise<Resource[]> {
-    return await this.retriever.retrieveParents(IDs, includeSchema, includeFieldData);
+    return await this.execRetrieve({ ids: parseIDs(ids), parents: true, ...options });
   }
 
   async addChildren(id: ID, ...children: ID[]): Promise<void> {
@@ -99,6 +124,135 @@ export class Client implements AsyncTermSearcher<string, string, Resource> {
   }
 
   async openChangeTracker(): Promise<ChangeTracker> {
-    return await ChangeTracker.open(this.framer, this.retriever);
+    return await ChangeTracker.open(this.framer, this);
+  }
+
+  newSearcherWithOptions(
+    options: RetrieveOptions,
+  ): AsyncTermSearcher<string, string, Resource> {
+    return {
+      type: this.type,
+      search: (term: string) => this.search(term, options),
+      retrieve: (ids: string[]) => this.retrieve(ids, options),
+      page: (offset: number, limit: number) => this.page(offset, limit, options),
+    };
+  }
+
+  private async execRetrieve(request: RetrieveRequest): Promise<Resource[]> {
+    const { resources } = await sendRequired(
+      this.client,
+      RETRIEVE_ENDPOINT,
+      request,
+      retrieveReqZ,
+      retrieveResZ,
+    );
+    return resources;
+  }
+}
+
+const RESOURCE_SET_NAME = "sy_ontology_resource_set";
+const RESOURCE_DELETE_NAME = "sy_ontology_resource_delete";
+const RELATIONSHIP_SET_NAME = "sy_ontology_relationship_set";
+const RELATIONSHIP_DELETE_NAME = "sy_ontology_relationship_delete";
+
+export class ChangeTracker {
+  private readonly resourceObs: observe.Observer<ResourceChange[]>;
+  private readonly relationshipObs: observe.Observer<RelationshipChange[]>;
+
+  readonly relationships: observe.Observable<RelationshipChange[]>;
+  readonly resources: observe.Observable<ResourceChange[]>;
+
+  private readonly streamer: framer.Streamer;
+  private readonly client: Client;
+  private readonly closePromise: Promise<void>;
+
+  constructor(streamer: framer.Streamer, client: Client) {
+    this.relationshipObs = new observe.Observer<RelationshipChange[]>();
+    this.relationships = this.relationshipObs;
+    this.resourceObs = new observe.Observer<ResourceChange[]>();
+    this.resources = this.resourceObs;
+    this.client = client;
+    this.streamer = streamer;
+    this.closePromise = this.start();
+  }
+
+  async close(): Promise<void> {
+    this.streamer.close();
+    await this.closePromise;
+  }
+
+  private async start(): Promise<void> {
+    for await (const frame of this.streamer) {
+      await this.update(frame);
+    }
+  }
+
+  private async update(frame: Frame): Promise<void> {
+    const resSets = await this.parseResourceSets(frame);
+    const resDeletes = this.parseResourceDeletes(frame);
+    const allResources = resSets.concat(resDeletes);
+    if (allResources.length > 0) this.resourceObs.notify(resSets.concat(resDeletes));
+    const relSets = this.parseRelationshipSets(frame);
+    const relDeletes = this.parseRelationshipDeletes(frame);
+    const allRelationships = relSets.concat(relDeletes);
+    if (allRelationships.length > 0)
+      this.relationshipObs.notify(relSets.concat(relDeletes));
+  }
+
+  private parseRelationshipSets(frame: Frame): RelationshipChange[] {
+    const relationships = frame.get(RELATIONSHIP_SET_NAME);
+    if (relationships.length === 0) return [];
+    return Array.from(relationships.as("string")).map((rel) => ({
+      variant: "set",
+      key: parseRelationship(rel),
+      value: undefined,
+    }));
+  }
+
+  private parseRelationshipDeletes(frame: Frame): RelationshipChange[] {
+    const relationships = frame.get(RELATIONSHIP_DELETE_NAME);
+    if (relationships.length === 0) return [];
+    return Array.from(relationships.as("string")).map((rel) => ({
+      variant: "delete",
+      key: parseRelationship(rel),
+    }));
+  }
+
+  private async parseResourceSets(frame: Frame): Promise<ResourceChange[]> {
+    const sets = frame.get(RESOURCE_SET_NAME);
+    if (sets.length === 0) return [];
+    // We should only ever get one series of sets
+    const ids = Array.from(sets.as("string")).map((id: string) => new ID(id));
+    try {
+      const resources = await this.client.retrieve(ids);
+      return resources.map((resource) => ({
+        variant: "set",
+        key: resource.id,
+        value: resource,
+      }));
+    } catch (e) {
+      if (e instanceof QueryError) return [];
+      throw e;
+    }
+  }
+
+  private parseResourceDeletes(frame: Frame): ResourceChange[] {
+    const deletes = frame.get(RESOURCE_DELETE_NAME);
+    if (deletes.length === 0) return [];
+    // We should only ever get one series of deletes
+    return Array.from(deletes.as("string")).map((str) => ({
+      variant: "delete",
+      key: new ID(str),
+    }));
+  }
+
+  static async open(client: framer.Client, retriever: Client): Promise<ChangeTracker> {
+    const streamer = await client.openStreamer([
+      RESOURCE_SET_NAME,
+      RESOURCE_DELETE_NAME,
+      RELATIONSHIP_SET_NAME,
+      RELATIONSHIP_DELETE_NAME,
+    ]);
+    return new ChangeTracker(streamer, retriever);
   }
 }

package/src/ontology/external.ts

@@ -9,4 +9,3 @@
 
 export * from "@/ontology/client";
 export * from "@/ontology/payload";
-export * from "@/ontology/signals";

package/src/ontology/group/client.ts

@@ -9,11 +9,10 @@
 
 import { type UnaryClient } from "@synnaxlabs/freighter";
 
-import { type ID } from "@/ontology/payload";
-
 import { Group } from "@/ontology/group/group";
 import { type Payload } from "@/ontology/group/payload";
 import { Writer } from "@/ontology/group/writer";
+import { type ID } from "@/ontology/payload";
 
 export class Client {
   private readonly creator: Writer;

package/src/ontology/group/payload.ts

@@ -7,7 +7,7 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { toArray } from "@synnaxlabs/x";
+import { toArray } from "@synnaxlabs/x/toArray";
 import { z } from "zod";
 
 export const keyZ = z.string().uuid();

package/src/ontology/ontology.spec.ts

@@ -42,6 +42,22 @@ describe("Ontology", () => {
       expect(parents.length).toEqual(1);
       expect(parents[0].name).toEqual(name);
     });
+  });
+  describe("page", () => {
+    it("should return a page of resources", async () => {
+      for (let i = 0; i < 10; i++)
+        await client.ontology.groups.create(ontology.Root, randomName());
+      const page = await client.ontology.page(0, 5);
+      expect(page.length).toEqual(5);
+      const page2 = await client.ontology.page(5, 5);
+      expect(page2.length).toEqual(5);
+      const page1Keys = page.map((r) => r.key);
+      const page2Keys = page2.map((r) => r.key);
+      const intersection = page1Keys.filter((key) => page2Keys.includes(key));
+      expect(intersection.length).toEqual(0);
+    });
+  });
+  describe("write", () => {
     test("add children", async () => {
       const name = randomName();
       const g = await client.ontology.groups.create(ontology.Root, name);

package/src/ontology/payload.ts

@@ -7,8 +7,16 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
+import { change } from "@synnaxlabs/x";
 import { z } from "zod";
 
+export type ResourceChange = change.Change<ID, Resource>;
+export type ResourceSet = change.Set<ID, Resource>;
+export type ResourceDelete = change.Delete<ID, Resource>;
+export type RelationshipChange = change.Change<Relationship, undefined>;
+export type RelationshipSet = change.Set<Relationship, undefined>;
+export type RelationshipDelete = change.Delete<Relationship, undefined>;
+
 const resourceTypeZ = z.union([
   z.literal("label"),
   z.literal("builtin"),
@@ -20,24 +28,22 @@ const resourceTypeZ = z.union([
   z.literal("range-alias"),
   z.literal("user"),
   z.literal("workspace"),
-  z.literal("pid"),
+  z.literal("schematic"),
   z.literal("lineplot"),
-  z.literal("rack")
+  z.literal("rack"),
+  z.literal("device"),
+  z.literal("task"),
 ]);
 
 export type ResourceType = z.infer<typeof resourceTypeZ>;
 
-export const idZ = z.object({
-  type: resourceTypeZ,
-  key: z.string(),
-});
+export const idZ = z.object({ type: resourceTypeZ, key: z.string() });
+
+export type IDPayload = z.infer<typeof idZ>;
 
 export const stringIDZ = z.string().transform((v) => {
   const [type, key] = v.split(":");
-  return {
-    type: type as ResourceType,
-    key,
-  };
+  return { type: type as ResourceType, key };
 });
 
 export const crudeIDZ = z.union([stringIDZ, idZ]);
@@ -93,8 +99,8 @@ export const resourceSchemaZ = z
   .object({
     id: ID.z,
     name: z.string(),
-    schema: schemaZ.optional(),
-    data: z.record(z.unknown()).optional(),
+    schema: schemaZ.optional().nullable(),
+    data: z.record(z.unknown()).optional().nullable(),
   })
   .transform((resource) => {
     return {
@@ -103,7 +109,10 @@ export const resourceSchemaZ = z
     };
   });
 
-export type Resource = z.infer<typeof resourceSchemaZ>;
+export type Resource<T extends {} = {}> = Omit<
+  z.output<typeof resourceSchemaZ>,
+  "data"
+> & { data?: T | null };
 
 export const relationshipSchemaZ = z.object({
   from: ID.z,

package/src/ranger/active.ts

@@ -43,9 +43,9 @@ export class Active {
   async setActive(range: Key): Promise<void> {
     await sendRequired<typeof setActiveReqZ, typeof setActiveResZ>(
       this.client,
-      SET_ENDPOINT, 
-      { range }, 
-      setActiveReqZ, 
+      SET_ENDPOINT,
+      { range },
+      setActiveReqZ,
       setActiveResZ,
     );
   }
@@ -65,8 +65,8 @@ export class Active {
   async clearActive(range: Key): Promise<void> {
     await sendRequired<typeof clearActiveReqZ, typeof clearActiveResZ>(
       this.client,
-      CLEAR_ENDPOINT, 
-      { range }, 
+      CLEAR_ENDPOINT,
+      { range },
       clearActiveReqZ,
       clearActiveResZ,
     );

package/src/ranger/alias.ts

@@ -8,7 +8,7 @@
 // included in the file licenses/APL.txt.
 
 import { sendRequired, type UnaryClient } from "@synnaxlabs/freighter";
-import { type change } from "@synnaxlabs/x";
+import { type change } from "@synnaxlabs/x/change";
 import { z } from "zod";
 
 import { type channel } from "@/channel";
@@ -133,7 +133,7 @@ export class Aliaser {
   }
 
   async openChangeTracker(): Promise<signals.Observable<string, Alias>> {
-    return await signals.Observable.open<string, Alias>(
+    return await signals.openObservable<string, Alias>(
       this.frameClient,
       ALIAS_SET_NAME,
       ALIAS_DELETE_NAME,

package/src/ranger/client.ts

@@ -7,11 +7,12 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { type UnaryClient } from "@synnaxlabs/freighter";
-import { type AsyncTermSearcher, toArray } from "@synnaxlabs/x";
+import { sendRequired, type UnaryClient } from "@synnaxlabs/freighter";
+import { type AsyncTermSearcher } from "@synnaxlabs/x/search";
+import { toArray } from "@synnaxlabs/x/toArray";
 
 import { type Retriever as ChannelRetriever } from "@/channel/retriever";
-import { QueryError } from "@/errors";
+import { MultipleFoundError, NotFoundError, QueryError } from "@/errors";
 import { type framer } from "@/framer";
 import { type label } from "@/label";
 import { Active } from "@/ranger/active";
@@ -26,14 +27,34 @@ import {
   type Params,
   type Payload,
   analyzeParams,
+  payloadZ,
+  keyZ,
 } from "@/ranger/payload";
 import { Range } from "@/ranger/range";
-import { type Retriever } from "@/ranger/retriever";
 import { type Writer } from "@/ranger/writer";
+import { signals } from "@/signals";
+import { z } from "zod";
+import { CrudeTimeRange, TimeRange } from "@synnaxlabs/x";
+import { nullableArrayZ } from "@/util/zod";
+
+const retrieveReqZ = z.object({
+  keys: keyZ.array().optional(),
+  names: z.array(z.string()).optional(),
+  term: z.string().optional(),
+  overlapsWith: TimeRange.z.optional(),
+});
+
+export type RetrieveRequest = z.infer<typeof retrieveReqZ>;
+
+const RETRIEVE_ENDPOINT = "/range/retrieve";
+
+const retrieveResZ = z.object({
+  ranges: nullableArrayZ(payloadZ),
+});
 
 export class Client implements AsyncTermSearcher<string, Key, Range> {
+  readonly type: string = "range";
   private readonly frameClient: framer.Client;
-  private readonly retriever: Retriever;
   private readonly writer: Writer;
   private readonly unaryClient: UnaryClient;
   private readonly channels: ChannelRetriever;
@@ -42,14 +63,12 @@ export class Client implements AsyncTermSearcher<string, Key, Range> {
 
   constructor(
     frameClient: framer.Client,
-    retriever: Retriever,
     writer: Writer,
     unary: UnaryClient,
     channels: ChannelRetriever,
     labelClient: label.Client,
   ) {
     this.frameClient = frameClient;
-    this.retriever = retriever;
     this.writer = writer;
     this.unaryClient = unary;
     this.channels = channels;
@@ -76,25 +95,42 @@ export class Client implements AsyncTermSearcher<string, Key, Range> {
   }
 
   async search(term: string): Promise<Range[]> {
-    return this.sugar(await this.retriever.search(term));
+    return this.sugar(await this.execRetrieve({ term }));
   }
 
   async page(offset: number, limit: number): Promise<Range[]> {
     return [];
   }
 
+  async retrieve(range: CrudeTimeRange): Promise<Range[]>;
+
   async retrieve(range: Key | Name): Promise<Range>;
 
-  async retrieve(params: Keys | Names): Promise<Range[]>;
+  async retrieve(range: Keys | Names): Promise<Range[]>;
+
+  async retrieve(params: Params | CrudeTimeRange): Promise<Range | Range[]> {
+    if (typeof params === "object" && "start" in params)
+      return await this.execRetrieve({ overlapsWith: new TimeRange(params) });
+    const { single, actual, variant, normalized } = analyzeParams(params);
+    const ranges = await this.execRetrieve({ [variant]: normalized });
+    if (!single) return ranges;
+    if (ranges.length === 0)
+      throw new NotFoundError(`range matching ${actual} not found`);
+    if (ranges.length > 1)
+      throw new MultipleFoundError(`multiple ranges matching ${actual} found`);
+    return ranges[0];
+  }
 
-  async retrieve(params: Params): Promise<Range | Range[]> {
-    const { single, actual } = analyzeParams(params);
-    const res = this.sugar(await this.retriever.retrieve(params));
-    if (!single) return res;
-    if (res.length === 0) throw new QueryError(`range matching ${actual} not found`);
-    if (res.length > 1)
-      throw new QueryError(`multiple ranges matching ${actual} found`);
-    return res[0];
+  private async execRetrieve(req: RetrieveRequest): Promise<Range[]> {
+    return this.sugar(
+      await sendRequired<typeof retrieveReqZ, typeof retrieveResZ>(
+        this.unaryClient,
+        RETRIEVE_ENDPOINT,
+        req,
+        retrieveReqZ,
+        retrieveResZ,
+      ).then((res) => res.ranges),
+    );
   }
 
   async setActive(range: Key): Promise<void> {
@@ -117,6 +153,7 @@ export class Client implements AsyncTermSearcher<string, Key, Range> {
         payload.name,
         payload.timeRange,
         payload.key,
+        payload.color,
         this.frameClient,
         new KV(payload.key, this.unaryClient),
         new Aliaser(payload.key, this.frameClient, this.unaryClient),
@@ -125,4 +162,18 @@ export class Client implements AsyncTermSearcher<string, Key, Range> {
       );
     });
   }
+
+  async openTracker(): Promise<signals.Observable<string, Range>> {
+    return await signals.openObservable<string, Range>(
+      this.frameClient,
+      "sy_range_set",
+      "sy_range_delete",
+      (variant, data) => {
+        if (variant === "delete")
+          return data.toStrings().map((k) => ({ variant, key: k, value: undefined }));
+        const sugared = this.sugar(data.parseJSON(payloadZ));
+        return sugared.map((r) => ({ variant, key: r.key, value: r }));
+      },
+    );
+  }
 }

package/src/ranger/external.ts

@@ -11,5 +11,4 @@ export * from "@/ranger/client";
 export * from "@/ranger/writer";
 export * from "@/ranger/payload";
 export * from "@/ranger/range";
-export * from "@/ranger/retriever";
 export type { Alias, AliasChange } from "@/ranger/alias";

package/src/ranger/kv.ts

@@ -8,7 +8,8 @@
 // included in the file licenses/APL.txt.
 
 import { type UnaryClient, sendRequired } from "@synnaxlabs/freighter";
-import { isObject, toArray } from "@synnaxlabs/x";
+import { isObject } from "@synnaxlabs/x/identity";
+import { toArray } from "@synnaxlabs/x/toArray";
 import { z } from "zod";
 
 import { type Key, keyZ } from "@/ranger/payload";
@@ -65,6 +66,10 @@ export class KV {
     return Array.isArray(keys) ? res.pairs : res.pairs[keys];
   }
 
+  async list(): Promise<Record<string, string>> {
+    return this.get([]);
+  }
+
   async set(key: string, value: string): Promise<void>;
 
   async set(kv: Record<string, string>): Promise<void>;

package/src/ranger/payload.ts

@@ -7,7 +7,8 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { TimeRange, toArray } from "@synnaxlabs/x";
+import { TimeRange } from "@synnaxlabs/x/telem";
+import { toArray } from "@synnaxlabs/x/toArray";
 import { z } from "zod";
 
 export const keyZ = z.string().uuid();
@@ -21,6 +22,7 @@ export const payloadZ = z.object({
   key: keyZ,
   name: z.string().min(1),
   timeRange: TimeRange.z,
+  color: z.string().optional(),
 });
 export type Payload = z.infer<typeof payloadZ>;
 
@@ -29,7 +31,7 @@ export const newPayloadZ = payloadZ.extend({
 });
 export type NewPayload = z.infer<typeof newPayloadZ>;
 
-export type ParamAnalsysisResult =
+export type ParamAnalysisResult =
   | {
       single: true;
       variant: "keys";
@@ -55,7 +57,7 @@ export type ParamAnalsysisResult =
       actual: Names;
     };
 
-export const analyzeParams = (params: Params): ParamAnalsysisResult => {
+export const analyzeParams = (params: Params): ParamAnalysisResult => {
   const normal = toArray(params) as Keys | Names;
   if (normal.length === 0) {
     throw new Error("Range params must not be empty");
@@ -66,5 +68,5 @@ export const analyzeParams = (params: Params): ParamAnalsysisResult => {
     variant: isKey ? "keys" : "names",
     normalized: normal,
     actual: params,
-  } as const as ParamAnalsysisResult;
+  } as const as ParamAnalysisResult;
 };