@synnaxlabs/client 0.2.0 → 0.13.5

package/src/framer/writer.ts

@@ -0,0 +1,236 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+/* eslint-disable @typescript-eslint/no-throw-literal */
+import type { Stream, StreamClient } from "@synnaxlabs/freighter";
+import { decodeError, errorZ } from "@synnaxlabs/freighter";
+import {
+  type NativeTypedArray,
+  Series,
+  TimeStamp,
+  type CrudeTimeStamp,
+  toArray,
+} from "@synnaxlabs/x";
+import { z } from "zod";
+
+import { type Key, type KeyOrName, type Params } from "@/channel/payload";
+import { type Retriever } from "@/channel/retriever";
+import { Authority } from "@/control/authority";
+import {
+  subjectZ as controlSubjectZ,
+  type Subject as ControlSubject,
+} from "@/control/state";
+import { ForwardFrameAdapter } from "@/framer/adapter";
+import { type CrudeFrame, Frame, frameZ } from "@/framer/frame";
+import { StreamProxy } from "@/framer/streamProxy";
+
+enum Command {
+  Open = 0,
+  Write = 1,
+  Commit = 2,
+  Error = 3,
+  SetAuthority = 4,
+}
+
+const netConfigZ = z.object({
+  start: TimeStamp.z.optional(),
+  controlSubject: controlSubjectZ.optional(),
+  keys: z.number().array(),
+  authorities: Authority.z.array().optional(),
+});
+
+const reqZ = z.object({
+  command: z.nativeEnum(Command),
+  config: netConfigZ.optional(),
+  frame: frameZ.optional(),
+});
+
+type Request = z.infer<typeof reqZ>;
+
+const resZ = z.object({
+  ack: z.boolean(),
+  command: z.nativeEnum(Command),
+  error: errorZ.optional().nullable(),
+});
+
+type Response = z.infer<typeof resZ>;
+
+export interface WriterConfig {
+  start: CrudeTimeStamp;
+  channels: Params;
+  controlSubject?: ControlSubject;
+  authorities?: Authority | Authority[];
+}
+
+/**
+ * Writer is used to write telemetry to a set of channels in time order.
+ * It should not be instantiated directly, and should instead be instantited via the
+ * FramerClient {@link FrameClient#openWriter}.
+ *
+ * The writer is a streaming protocol that is heavily optimized for prerformance. This
+ * comes at the cost of icnreased complexity, and should only be used directly when
+ * writing large volumes of data (such as recording telemetry from a sensor or ingsting
+ * data froma file). Simpler methods (such as the frame client's write method) should
+ * be used for most use cases.
+ *
+ * The protocol is as follows:
+ *
+ * 1. The writer is opened with a starting timestamp and a list of channel keys. The
+ * writer will fail to open if the starting timstamp overlaps with any existing telemetry
+ * for any channels specified. If the writer opens successfuly, the caller is then
+ * free to write frames to the writer.
+ *
+ * 2. To write a frame, the caller can use the write method and follow the validation
+ * rules described in its method's documentation. This process is asynchronous, meaning
+ * that write calls may return before teh frame has been written to the cluster. This
+ * also means that the writer can accumulate an error after write is called. If the writer
+ * accumulates an erorr, all subsequent write and commit calls will return False. The
+ * caller can check for errors by calling the error mehtod, which returns the accumulated
+ * error and resets the writer for future use. The caller can also check for errors by
+ * closing the writer, which will throw any accumulated error.
+ *
+ * 3. To commit the written frames to the cluster, the caller can call the commit method.
+ * Unlike write, commit is synchronous, meaning that it will not return until the frames
+ * have been written to the cluster. If the writer has accumulated an erorr, commit will
+ * return false. After the caller acknowledges the erorr, they can attempt to commit again.
+ * Commit can be called several times throughout a writer's lifetime, and will only
+ * commit the frames that have been written since the last commit.
+ *
+ * 4. A writer MUST be closed after use in order to prevent resource leaks. Close should
+ * typically be called in a 'finally' block. If the writer has accumulated an error,
+ * close will throw the error.
+ */
+export class Writer {
+  private static readonly ENDPOINT = "/frame/write";
+  private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
+  private readonly adapter: ForwardFrameAdapter;
+
+  private constructor(
+    stream: Stream<typeof reqZ, typeof resZ>,
+    adapter: ForwardFrameAdapter,
+  ) {
+    this.stream = new StreamProxy("Writer", stream);
+    this.adapter = adapter;
+  }
+
+  static async _open(
+    retriever: Retriever,
+    client: StreamClient,
+    {
+      channels,
+      authorities = Authority.ABSOLUTE,
+      controlSubject: subject,
+      start,
+    }: WriterConfig,
+  ): Promise<Writer> {
+    const adapter = await ForwardFrameAdapter.open(retriever, channels);
+    const stream = await client.stream(Writer.ENDPOINT, reqZ, resZ);
+    const writer = new Writer(stream, adapter);
+    await writer.execute({
+      command: Command.Open,
+      config: {
+        start: new TimeStamp(start),
+        keys: adapter.keys,
+        controlSubject: subject,
+        authorities: toArray(authorities),
+      },
+    });
+    return writer;
+  }
+
+  async write(channel: KeyOrName, data: NativeTypedArray): Promise<boolean>;
+
+  async write(frame: CrudeFrame): Promise<boolean>;
+
+  /**
+   * Writes the given frame to the database.
+   *
+   * @param frame - The frame to write to the database. The frame must:
+   *
+   *    1. Have exactly one array for each key in the list of keys provided to the
+   *    writer's open method.
+   *    2. Have equal length arrays for each key.
+   *    3. When writing to an index (i.e. TimeStamp) channel, the values must be
+   *    monotonically increasing.
+   *
+   * @returns false if the writer has accumulated an error. In this case, the caller
+   * should acknowledge the error by calling the error method or closing the writer.
+   */
+  async write(
+    frame: CrudeFrame | KeyOrName,
+    data?: NativeTypedArray,
+  ): Promise<boolean> {
+    const isKeyOrName = ["string", "number"].includes(typeof frame);
+    if (isKeyOrName) {
+      frame = new Frame(frame, new Series(data as NativeTypedArray));
+    }
+    frame = this.adapter.adapt(new Frame(frame));
+    // @ts-expect-error
+    this.stream.send({ command: Command.Write, frame: frame.toPayload() });
+    return true;
+  }
+
+  async setAuthority(value: Record<Key, Authority>): Promise<boolean> {
+    const res = await this.execute({
+      command: Command.SetAuthority,
+      config: {
+        keys: Object.keys(value).map((k) => Number(k)),
+        authorities: Object.values(value),
+      },
+    });
+    return res.ack;
+  }
+
+  /**
+   * Commits the written frames to the database. Commit is synchronous, meaning that it
+   * will not return until all frames have been commited to the database.
+   *
+   * @returns false if the commit failed due to an error. In this case, the caller
+   * should acknowledge the error by calling the error method or closing the writer.
+   * After the caller acknowledges the error, they can attempt to commit again.
+   */
+  async commit(): Promise<boolean> {
+    if (this.errorAccumulated) return false;
+    const res = await this.execute({ command: Command.Commit });
+    return res.ack;
+  }
+
+  /**
+   * @returns  The accumulated error, if any. This method will clear the writer's error
+   * state, allowing the writer to be used again.
+   */
+  async error(): Promise<Error | null> {
+    this.stream.send({ command: Command.Error });
+    const res = await this.execute({ command: Command.Error });
+    return res.error != null ? decodeError(res.error) : null;
+  }
+
+  /**
+   * Closes the writer, raising any accumulated error encountered during operation.
+   * A writer MUST be closed after use, and this method should probably be placed
+   * in a 'finally' block.
+   */
+  async close(): Promise<void> {
+    await this.stream.closeAndAck();
+  }
+
+  async execute(req: Request): Promise<Response> {
+    // @ts-expect-error
+    this.stream.send(req);
+    while (true) {
+      const res = await this.stream.receive();
+      if (res.command === req.command) return res;
+      console.warn("writer received unexpected response", res);
+    }
+  }
+
+  private get errorAccumulated(): boolean {
+    return this.stream.received();
+  }
+}

package/src/index.ts

@@ -1,5 +1,17 @@
-export { default as Synnax } from './lib/client';
-export * from './lib/telem';
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+export * from "@/channel";
+export { default as Synnax, synnaxPropsZ } from "@/client";
+export type { SynnaxProps } from "@/client";
+export * from "@/connection";
+export { Channel } from "@/channel/client";
 export {
   AuthError,
   ContiguityError,
@@ -9,5 +21,32 @@ export {
   RouteError,
   UnexpectedError,
   ValidationError,
-} from './lib/errors';
-export { Channel } from './lib/channel/client';
+} from "@/errors";
+export { framer } from "@/framer";
+export { Frame } from "@/framer/frame";
+export { ontology } from "@/ontology";
+export { control } from "@/control";
+export { Authority } from "@/control/authority";
+export {
+  DataType,
+  Density,
+  Rate,
+  Series,
+  TimeRange,
+  TimeSpan,
+  TimeStamp,
+} from "@synnaxlabs/x";
+export type {
+  NativeTypedArray,
+  CrudeDataType,
+  CrudeDensity,
+  CrudeRate,
+  CrudeSize,
+  CrudeTimeSpan,
+  CrudeTimeStamp,
+  SampleValue,
+  TimeStampStringFormat,
+  TZInfo,
+} from "@synnaxlabs/x";
+export { workspace } from "@/workspace";
+export { ranger } from "@/ranger";

package/src/ontology/cdc.ts

@@ -0,0 +1,135 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { observe, type change } from "@synnaxlabs/x";
+
+import { QueryError } from "@/errors";
+import { type Client as FrameClient } from "@/framer/client";
+import { type Frame } from "@/framer/frame";
+import { type Streamer as FrameStreamer } from "@/framer/streamer";
+import {
+  ID,
+  parseRelationship,
+  type Relationship,
+  type Resource,
+} from "@/ontology/payload";
+import { type Retriever } from "@/ontology/retriever";
+
+export type ResourceChange = change.Change<ID, Resource>;
+export type RelationshipChange = change.Change<Relationship, undefined>;
+
+const RESOURCE_SET_NAME = "sy_ontology_resource_set";
+const RESOURCE_DELETE_NAME = "sy_ontology_resource_delete";
+const RELATIONSHIP_SET_NAME = "sy_ontology_relationship_set";
+const RELATIONSHIP_DELETE_NAME = "sy_ontology_relationship_delete";
+
+export class ChangeTracker {
+  private readonly resourceObs: observe.Observer<ResourceChange[]>;
+  private readonly relationshipObs: observe.Observer<RelationshipChange[]>;
+
+  readonly relationships: observe.Observable<RelationshipChange[]>;
+  readonly resources: observe.Observable<ResourceChange[]>;
+
+  private readonly streamer: FrameStreamer;
+  private readonly retriever: Retriever;
+  private readonly closePromise: Promise<void>;
+
+  constructor(streamer: FrameStreamer, retriever: Retriever) {
+    this.relationshipObs = new observe.Observer<RelationshipChange[]>();
+    this.relationships = this.relationshipObs;
+    this.resourceObs = new observe.Observer<ResourceChange[]>();
+    this.resources = this.resourceObs;
+    this.retriever = retriever;
+    this.streamer = streamer;
+    this.closePromise = this.start();
+  }
+
+  async close(): Promise<void> {
+    this.streamer.close();
+    await this.closePromise;
+  }
+
+  private async start(): Promise<void> {
+    for await (const frame of this.streamer) {
+      await this.update(frame);
+    }
+  }
+
+  private async update(frame: Frame): Promise<void> {
+    const resSets = await this.parseResourceSets(frame);
+    const resDeletes = this.parseResourceDeletes(frame);
+    const allResources = resSets.concat(resDeletes);
+    if (allResources.length > 0) this.resourceObs.notify(resSets.concat(resDeletes));
+    const relSets = this.parseRelationshipSets(frame);
+    const relDeletes = this.parseRelationshipDeletes(frame);
+    const allRels = relSets.concat(relDeletes);
+    if (allRels.length > 0) this.relationshipObs.notify(relSets.concat(relDeletes));
+  }
+
+  private parseRelationshipSets(frame: Frame): RelationshipChange[] {
+    const relationships = frame.get(RELATIONSHIP_SET_NAME);
+    if (relationships.length === 0) return [];
+    // We should only ever get one series of relationships
+    return relationships[0].toStrings().map((rel) => ({
+      variant: "set",
+      key: parseRelationship(rel),
+      value: undefined,
+    }));
+  }
+
+  private parseRelationshipDeletes(frame: Frame): RelationshipChange[] {
+    const relationships = frame.get(RELATIONSHIP_DELETE_NAME);
+    if (relationships.length === 0) return [];
+    // We should only ever get one series of relationships
+    return relationships[0].toStrings().map((rel) => ({
+      variant: "delete",
+      key: parseRelationship(rel),
+    }));
+  }
+
+  private async parseResourceSets(frame: Frame): Promise<ResourceChange[]> {
+    const sets = frame.get(RESOURCE_SET_NAME);
+    if (sets.length === 0) return [];
+    // We should only ever get one series of sets
+    const ids = sets[0].toStrings().map((id) => new ID(id));
+    try {
+      const resources = await this.retriever.retrieve(ids);
+      return resources.map((resource) => ({
+        variant: "set",
+        key: resource.id,
+        value: resource,
+      }));
+    } catch (e) {
+      if (e instanceof QueryError) {
+        console.warn(e);
+        return [];
+      }
+      throw e;
+    }
+  }
+
+  private parseResourceDeletes(frame: Frame): ResourceChange[] {
+    const deletes = frame.get(RESOURCE_DELETE_NAME);
+    if (deletes.length === 0) return [];
+    // We should only ever get one series of deletes
+    return deletes[0]
+      .toStrings()
+      .map((str) => ({ variant: "delete", key: new ID(str) }));
+  }
+
+  static async open(client: FrameClient, retriever: Retriever): Promise<ChangeTracker> {
+    const streamer = await client.newStreamer([
+      RESOURCE_SET_NAME,
+      RESOURCE_DELETE_NAME,
+      RELATIONSHIP_SET_NAME,
+      RELATIONSHIP_DELETE_NAME,
+    ]);
+    return new ChangeTracker(streamer, retriever);
+  }
+}

package/src/ontology/client.ts

@@ -0,0 +1,103 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { type UnaryClient } from "@synnaxlabs/freighter";
+import { type AsyncTermSearcher } from "@synnaxlabs/x";
+
+import { type Client as FrameClient } from "@/framer/client";
+import { ChangeTracker } from "@/ontology/cdc";
+import { group } from "@/ontology/group";
+import { type ID, type Resource } from "@/ontology/payload";
+import { Retriever } from "@/ontology/retriever";
+import { Writer } from "@/ontology/writer";
+
+import { QueryError } from "..";
+
+/** The core client class for executing queries against a Synnax cluster ontology */
+export class Client implements AsyncTermSearcher<string, string, Resource> {
+  groups: group.Client;
+  retriever: Retriever;
+  private readonly writer: Writer;
+  private readonly framer: FrameClient;
+
+  constructor(unary: UnaryClient, framer: FrameClient) {
+    this.retriever = new Retriever(unary);
+    this.writer = new Writer(unary);
+    this.groups = new group.Client(unary);
+    this.framer = framer;
+  }
+
+  async search(term: string): Promise<Resource[]> {
+    return await this.retriever.search(term);
+  }
+
+  async retrieve(
+    id: ID | string,
+    includeSchema?: boolean,
+    includeFieldData?: boolean,
+  ): Promise<Resource>;
+
+  async retrieve(
+    ids: ID[] | string[],
+    includeSchema?: boolean,
+    includeFieldData?: boolean,
+  ): Promise<Resource[]>;
+
+  async retrieve(
+    ids: ID | ID[] | string | string[],
+    includeSchema?: boolean,
+    includeFieldData?: boolean,
+  ): Promise<Resource | Resource[]> {
+    const resources = await this.retriever.retrieve(
+      ids,
+      includeSchema,
+      includeFieldData,
+    );
+    if (Array.isArray(ids)) return resources;
+    if (resources.length === 0)
+      throw new QueryError(`No resource found with ID ${ids.toString()}`);
+    return resources[0];
+  }
+
+  async page(offset: number, limit: number): Promise<Resource[]> {
+    return [];
+  }
+
+  async retrieveChildren(
+    IDs: ID | ID[],
+    includeSchema?: boolean,
+    includeFieldData?: boolean,
+  ): Promise<Resource[]> {
+    return await this.retriever.retrieveChildren(IDs, includeSchema, includeFieldData);
+  }
+
+  async retrieveParents(
+    IDs: ID | ID[],
+    includeSchema?: boolean,
+    includeFieldData?: boolean,
+  ): Promise<Resource[]> {
+    return await this.retriever.retrieveParents(IDs, includeSchema, includeFieldData);
+  }
+
+  async addChildren(id: ID, ...children: ID[]): Promise<void> {
+    return await this.writer.addChildren(id, ...children);
+  }
+
+  async removeChildren(id: ID, ...children: ID[]): Promise<void> {
+    return await this.writer.removeChildren(id, ...children);
+  }
+
+  async moveChildren(from: ID, to: ID, ...children: ID[]): Promise<void> {
+    return await this.writer.moveChildren(from, to, ...children);
+  }
+
+  async openChangeTracker(): Promise<ChangeTracker> {
+    return await ChangeTracker.open(this.framer, this.retriever);
+  }
+}

package/src/ontology/external.ts

@@ -0,0 +1,12 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+export * from "@/ontology/client";
+export * from "@/ontology/payload";
+export * from "@/ontology/cdc";

package/src/ontology/group/client.ts

@@ -0,0 +1,40 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { type UnaryClient } from "@synnaxlabs/freighter";
+
+import { type ID } from "@/ontology/payload";
+
+import { Group } from "./group";
+import { type Payload } from "./payload";
+import { Writer } from "./writer";
+
+export class Client {
+  private readonly creator: Writer;
+
+  constructor(unary: UnaryClient) {
+    this.creator = new Writer(unary);
+  }
+
+  async create(parent: ID, name: string): Promise<Group> {
+    return this.sugar(await this.creator.create(parent, name));
+  }
+
+  async rename(key: string, name: string): Promise<void> {
+    return await this.creator.rename(key, name);
+  }
+
+  async delete(...keys: string[]): Promise<void> {
+    return await this.creator.delete(keys);
+  }
+
+  private sugar(payload: Payload): Group {
+    return new Group(payload.name, payload.key);
+  }
+}

package/src/ontology/group/external.ts

@@ -0,0 +1,11 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+export * from "@/ontology/group/group";
+export * from "@/ontology/group/client";

package/src/ontology/group/group.spec.ts

@@ -0,0 +1,46 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { describe, it, expect } from "vitest";
+
+import { QueryError } from "@/errors";
+import { ontology } from "@/ontology";
+import { newClient } from "@/setupspecs";
+
+const client = newClient();
+
+describe("Group", () => {
+  describe("create", () => {
+    it("should correctly create a group", async () => {
+      const name = `group-${Math.random()}`;
+      const g = await client.ontology.groups.create(ontology.Root, name);
+      expect(g.name).toEqual(name);
+    });
+  });
+  describe("rename", () => {
+    it("should correctly rename a group", async () => {
+      const name = `group-${Math.random()}`;
+      const g = await client.ontology.groups.create(ontology.Root, name);
+      const newName = `group-${Math.random()}`;
+      await client.ontology.groups.rename(g.key, newName);
+      const g2 = await client.ontology.retrieve(g.ontologyID);
+      expect(g2.name).toEqual(newName);
+    });
+  });
+  describe("delete", () => {
+    it("should correctly delete the group", async () => {
+      const name = `group-${Math.random()}`;
+      const g = await client.ontology.groups.create(ontology.Root, name);
+      await client.ontology.groups.delete(g.key);
+      await expect(
+        async () => await client.ontology.retrieve(g.ontologyID),
+      ).rejects.toThrow(QueryError);
+    });
+  });
+});

package/src/ontology/group/group.ts

@@ -0,0 +1,27 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { ontology } from "@/ontology";
+
+export class Group {
+  key: string;
+  name: string;
+
+  constructor(name: string, key: string) {
+    this.key = key;
+    this.name = name;
+  }
+
+  get ontologyID(): ontology.ID {
+    return new ontology.ID({
+      type: "group",
+      key: this.key,
+    });
+  }
+}

package/src/ontology/group/index.ts

@@ -0,0 +1,10 @@
+// Copyright 2023 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+export * as group from "@/ontology/group/external";