@milaboratories/pl-tree 1.3.6

package/src/sync.test.ts

@@ -0,0 +1,101 @@
+import { field, TestHelpers } from '@milaboratories/pl-client';
+import { PlTreeState } from './state';
+import { constructTreeLoadingRequest, loadTreeState } from './sync';
+import { Computable } from '@milaboratories/computable';
+import { TestStructuralResourceType1 } from './test_utils';
+
+test('load resources', async () => {
+  await TestHelpers.withTempRoot(async (cl) => {
+    const r1 = await cl.withWriteTx(
+      'CreatingStructure1',
+      async (tx) => {
+        const rr1 = tx.createStruct(TestStructuralResourceType1);
+        const ff1 = field(tx.clientRoot, 'f1');
+        tx.createField(ff1, 'Dynamic');
+        tx.setField(ff1, rr1);
+        await tx.commit();
+        return await rr1.globalId;
+      },
+      { sync: true }
+    );
+
+    const treeState = new PlTreeState(r1);
+
+    const theComputable = Computable.make((c) =>
+      c.accessor(treeState.entry()).node().traverse('a', 'b')?.getDataAsString()
+    );
+
+    const refreshState = async (): Promise<void> => {
+      const req = constructTreeLoadingRequest(treeState);
+      const states = await cl.withReadTx('loadingTree', (tx) => loadTreeState(tx, req));
+      treeState.updateFromResourceData(states);
+    };
+
+    await refreshState();
+
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    const r2 = await cl.withWriteTx(
+      'CreatingStructure2',
+      async (tx) => {
+        const rr2 = tx.createStruct(TestStructuralResourceType1);
+        const ff2 = field(r1, 'a');
+        tx.createField(ff2, 'Input');
+        tx.setField(ff2, rr2);
+        await tx.commit();
+        return await rr2.globalId;
+      },
+      { sync: true }
+    );
+
+    await refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    const r3 = await cl.withWriteTx(
+      'CreatingStructure3',
+      async (tx) => {
+        const rr3 = tx.createValue(TestStructuralResourceType1, 'hi!');
+        const ff3 = field(r2, 'b');
+        tx.createField(ff3, 'Input');
+        tx.setField(ff3, rr3);
+        await tx.commit();
+        return await rr3.globalId;
+      },
+      { sync: true }
+    );
+
+    await refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: true,
+      value: 'hi!'
+    });
+
+    await cl.withWriteTx(
+      'CreatingStructure3',
+      async (tx) => {
+        tx.lock(r1);
+        tx.lock(r2);
+        await tx.commit();
+      },
+      { sync: true }
+    );
+
+    await refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: true,
+      value: 'hi!'
+    });
+  });
+});

package/src/sync.ts

@@ -0,0 +1,129 @@
+import {
+  FieldData,
+  isNullResourceId,
+  OptionalResourceId,
+  PlTransaction,
+  ResourceId
+} from '@milaboratories/pl-client';
+import Denque from 'denque';
+import { ExtendedResourceData, PlTreeState } from './state';
+
+/** Applied to list of fields in resource data. */
+export type PruningFunction = (resource: ExtendedResourceData) => FieldData[];
+
+export interface TreeLoadingRequest {
+  /** Resource to prime the traversal algorithm. It is ok, if some of them
+   * doesn't exist anymore. Should not contain elements from final resource
+   * set. */
+  readonly seedResources: ResourceId[];
+
+  /** Resource ids for which state is already known and not expected to change.
+   * Algorithm will not continue traversal over those ids, and states will not
+   * be retrieved for them. */
+  readonly finalResources: Set<ResourceId>;
+
+  /** This function is applied to each resource data field list, before
+   * using it continue traversal. This modification also is applied to
+   * output data to make result self-consistent in terms that it will contain
+   * all referenced resources, this is required to be able to pass it to tree
+   * to update the state. */
+  readonly pruningFunction?: PruningFunction;
+}
+
+/** Given the current tree state, build the request object to pass to
+ * {@link loadTreeState} to load updated state. */
+export function constructTreeLoadingRequest(
+  tree: PlTreeState,
+  pruningFunction?: PruningFunction
+): TreeLoadingRequest {
+  const seedResources: ResourceId[] = [];
+  const finalResources = new Set<ResourceId>();
+  tree.forEachResource((res) => {
+    if (res.final) finalResources.add(res.id);
+    else seedResources.push(res.id);
+  });
+
+  // if tree is empty, seeding tree reconstruction from the specified root
+  if (seedResources.length === 0 && finalResources.size === 0) seedResources.push(tree.root);
+
+  return { seedResources, finalResources, pruningFunction };
+}
+
+/** Given the transaction (preferably read-only) and loading request, executes
+ * the tree traversal algorithm, and collects fresh states of resources
+ * to update the tree state. */
+export async function loadTreeState(
+  tx: PlTransaction,
+  loadingRequest: TreeLoadingRequest
+): Promise<ExtendedResourceData[]> {
+  const { seedResources, finalResources, pruningFunction } = loadingRequest;
+
+  // Main idea of using a queue here is that responses will arrive in the same order as they were
+  // sent, so we can only wait for the earliest sent unprocessed response promise at any given moment.
+  // In such a way logic become linear without recursion, and at the same time deal with data
+  // as soon as it arrives.
+
+  const pending = new Denque<Promise<ExtendedResourceData | undefined>>();
+
+  // tracking resources we already requested
+  const requested = new Set<ResourceId>();
+  const requestState = (rid: OptionalResourceId) => {
+    if (isNullResourceId(rid) || requested.has(rid) || finalResources.has(rid)) return;
+
+    // adding the id, so we will not request it's state again if somebody else
+    // references the same resource
+    requested.add(rid);
+
+    // requesting resource and all kv records
+    const resourceData = tx.getResourceDataIfExists(rid, true);
+    const kvData = tx.listKeyValuesIfResourceExists(rid);
+
+    // pushing combined promise
+    pending.push(
+      (async () => {
+        const resource = await resourceData;
+        const kv = await kvData;
+
+        if (resource === undefined) return undefined;
+
+        if (kv === undefined) throw new Error('Inconsistent replies');
+
+        return { ...resource, kv };
+      })()
+    );
+  };
+
+  // sending seed requests
+  seedResources.forEach((rid) => requestState(rid));
+
+  const result: ExtendedResourceData[] = [];
+  while (true) {
+    // taking next pending request
+    const nextResourcePromise = pending.shift();
+    if (nextResourcePromise === undefined)
+      // this means we have no pending requests and traversal is over
+      break;
+
+    // at this point we pause and wait for the nest requested resource state to arrive
+    let nextResource = await nextResourcePromise;
+    if (nextResource === undefined)
+      // ignoring resources that were not found (this may happen for seed resource ids)
+      continue;
+
+    // apply field pruning, if requested
+    if (pruningFunction !== undefined)
+      nextResource = { ...nextResource, fields: pruningFunction(nextResource) };
+
+    // continue traversal over the referenced resource
+    requestState(nextResource.error);
+    for (const field of nextResource.fields) {
+      requestState(field.value);
+      requestState(field.error);
+    }
+
+    // aggregating the state
+    result.push(nextResource);
+  }
+
+  return result;
+}

package/src/synchronized_tree.test.ts

@@ -0,0 +1,210 @@
+import { field, TestHelpers } from '@milaboratories/pl-client';
+import { TestStructuralResourceType1 } from './test_utils';
+import { Computable } from '@milaboratories/computable';
+import { SynchronizedTreeState } from './synchronized_tree';
+
+test('simple synchronized tree test', async () => {
+  await TestHelpers.withTempRoot(async (pl) => {
+    const r1 = await pl.withWriteTx(
+      'CreatingStructure1',
+      async (tx) => {
+        const rr1 = tx.createStruct(TestStructuralResourceType1);
+        const ff1 = field(tx.clientRoot, 'f1');
+        tx.createField(ff1, 'Dynamic');
+        tx.setField(ff1, rr1);
+        await tx.commit();
+        return await rr1.globalId;
+      },
+      { sync: true }
+    );
+
+    const treeState = await SynchronizedTreeState.init(pl, r1, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+
+    const theComputable = Computable.make((c) =>
+      c.accessor(treeState.entry()).node().traverse('a', 'b')?.getDataAsString()
+    );
+
+    await theComputable.refreshState();
+
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    const r2 = await pl.withWriteTx(
+      'CreatingStructure2',
+      async (tx) => {
+        const rr2 = tx.createStruct(TestStructuralResourceType1);
+        const ff2 = field(r1, 'a');
+        tx.createField(ff2, 'Input');
+        tx.setField(ff2, rr2);
+        await tx.commit();
+        return await rr2.globalId;
+      },
+      { sync: true }
+    );
+
+    await theComputable.refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    const r3 = await pl.withWriteTx(
+      'CreatingStructure3',
+      async (tx) => {
+        const rr3 = tx.createValue(TestStructuralResourceType1, 'hi!');
+        const ff3 = field(r2, 'b');
+        tx.createField(ff3, 'Input');
+        tx.setField(ff3, rr3);
+        await tx.commit();
+        return await rr3.globalId;
+      },
+      { sync: true }
+    );
+
+    await theComputable.refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: true,
+      value: 'hi!'
+    });
+
+    await pl.withWriteTx(
+      'CreatingStructure3',
+      async (tx) => {
+        tx.lock(r1);
+        tx.lock(r2);
+        await tx.commit();
+      },
+      { sync: true }
+    );
+
+    await theComputable.refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: true,
+      value: 'hi!'
+    });
+
+    await treeState.awaitSyncLoopTermination();
+  });
+});
+
+test('synchronized tree test with KV', async () => {
+  await TestHelpers.withTempRoot(async (pl) => {
+    const r1 = await pl.withWriteTx(
+      'CreatingStructure1',
+      async (tx) => {
+        const rr1 = tx.createStruct(TestStructuralResourceType1);
+        const ff1 = field(tx.clientRoot, 'f1');
+        tx.createField(ff1, 'Dynamic');
+        tx.setField(ff1, rr1);
+        await tx.commit();
+        return await rr1.globalId;
+      },
+      { sync: true }
+    );
+
+    const treeState = await SynchronizedTreeState.init(pl, r1, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+
+    const theComputable = Computable.make((c) =>
+      c.accessor(treeState.entry()).node().traverse('a')?.getKeyValueAsString('b', true)
+    );
+
+    await theComputable.refreshState();
+
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    const r2 = await pl.withWriteTx(
+      'CreatingStructure2',
+      async (tx) => {
+        const rr2 = tx.createStruct(TestStructuralResourceType1);
+        const ff2 = field(r1, 'a');
+        tx.createField(ff2, 'Input');
+        tx.setField(ff2, rr2);
+        await tx.commit();
+        return await rr2.globalId;
+      },
+      { sync: true }
+    );
+
+    await theComputable.refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    await pl.withWriteTx('AssignKeyValue', async (tx) => {
+      tx.setKValue(r2, 'b', 'hi!');
+      await tx.commit();
+    });
+
+    await theComputable.refreshState();
+
+    expect(theComputable.isChanged()).toBe(true);
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: true,
+      value: 'hi!'
+    });
+
+    await treeState.awaitSyncLoopTermination();
+  });
+});
+
+test('termination test', async () => {
+  await TestHelpers.withTempRoot(async (pl) => {
+    const r1 = await pl.withWriteTx(
+      'CreatingStructure1',
+      async (tx) => {
+        const rr1 = tx.createStruct(TestStructuralResourceType1);
+        const ff1 = field(tx.clientRoot, 'f1');
+        tx.createField(ff1, 'Dynamic');
+        tx.setField(ff1, rr1);
+        await tx.commit();
+        return await rr1.globalId;
+      },
+      { sync: true }
+    );
+
+    const treeState = await SynchronizedTreeState.init(pl, r1, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+
+    const entry = treeState.entry();
+    const theComputable = Computable.make((c) =>
+      c.accessor(entry).node().traverse('a')?.getKeyValueAsString('b', true)
+    );
+
+    await theComputable.refreshState();
+
+    expect(await theComputable.getValueOrError()).toMatchObject({
+      stable: false,
+      value: undefined
+    });
+
+    await treeState.terminate();
+
+    const resultAfterTermination = await theComputable.getValueOrError();
+    expect(resultAfterTermination).toMatchObject({
+      type: 'error'
+    });
+    expect((resultAfterTermination as any).errors[0].message).toMatch('terminated');
+  });
+});

package/src/synchronized_tree.ts

@@ -0,0 +1,189 @@
+import { PollingComputableHooks } from '@milaboratories/computable';
+import { PlTreeEntry } from './accessors';
+import { isTimeoutOrCancelError, PlClient, ResourceId } from '@milaboratories/pl-client';
+import { FinalPredicate, PlTreeState, TreeStateUpdateError } from './state';
+import { constructTreeLoadingRequest, loadTreeState, PruningFunction } from './sync';
+import * as tp from 'node:timers/promises';
+import { MiLogger } from '@milaboratories/ts-helpers';
+
+export type SynchronizedTreeOps = {
+  finalPredicate?: FinalPredicate;
+  pruning?: PruningFunction;
+
+  /** Interval after last sync to sleep before the next one */
+  pollingInterval: number;
+  /** For how long to continue polling after the last derived value access */
+  stopPollingDelay: number;
+};
+
+type ScheduledRefresh = {
+  resolve: () => void;
+  reject: (err: any) => void;
+};
+
+export class SynchronizedTreeState {
+  private readonly finalPredicate: FinalPredicate | undefined;
+  private state: PlTreeState;
+  private readonly pollingInterval: number;
+  private readonly pruning?: PruningFunction;
+  private readonly hooks: PollingComputableHooks;
+  private readonly abortController = new AbortController();
+
+  private constructor(
+    private readonly pl: PlClient,
+    private readonly root: ResourceId,
+    ops: SynchronizedTreeOps,
+    private readonly logger?: MiLogger
+  ) {
+    const { finalPredicate, pruning, pollingInterval, stopPollingDelay } = ops;
+    this.pruning = pruning;
+    this.pollingInterval = pollingInterval;
+    this.finalPredicate = finalPredicate;
+    this.state = new PlTreeState(root, finalPredicate);
+    this.hooks = new PollingComputableHooks(
+      () => this.startUpdating(),
+      () => this.stopUpdating(),
+      { stopDebounce: stopPollingDelay },
+      (resolve, reject) => this.scheduleOnNextState(resolve, reject)
+    );
+  }
+
+  /** @deprecated use "entry" instead */
+  public accessor(rid: ResourceId = this.root): PlTreeEntry {
+    if (this.terminated) throw new Error('tree synchronization is terminated');
+    return this.entry(rid);
+  }
+
+  public entry(rid: ResourceId = this.root): PlTreeEntry {
+    if (this.terminated) throw new Error('tree synchronization is terminated');
+    return new PlTreeEntry({ treeProvider: () => this.state, hooks: this.hooks }, rid);
+  }
+
+  /** Can be used to externally kick off the synchronization polling loop, and
+   * await for the first synchronization to happen. */
+  public async refreshState(): Promise<void> {
+    if (this.terminated) throw new Error('tree synchronization is terminated');
+    await this.hooks.refreshState();
+  }
+
+  private scheduledOnNextState: ScheduledRefresh[] = [];
+
+  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {
+    if (this.terminated) reject(new Error('tree synchronization is terminated'));
+    else this.scheduledOnNextState.push({ resolve, reject });
+  }
+
+  /** Called from observer */
+  private startUpdating(): void {
+    if (this.terminated) return;
+    this.keepRunning = true;
+    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();
+  }
+
+  /** Called from observer */
+  private stopUpdating(): void {
+    this.keepRunning = false;
+  }
+
+  /** If true, main loop will continue polling pl state. */
+  private keepRunning = false;
+  /** Actual state of main loop. */
+  private currentLoop: Promise<void> | undefined = undefined;
+
+  /** Executed from the main loop, and initialization procedure. */
+  private async refresh(): Promise<void> {
+    if (this.terminated) throw new Error('tree synchronization is terminated');
+    const request = constructTreeLoadingRequest(this.state, this.pruning);
+    const data = await this.pl.withReadTx('ReadingTree', async (tx) => {
+      return await loadTreeState(tx, request);
+    });
+    this.state.updateFromResourceData(data, true);
+  }
+
+  /** If true this tree state is permanently terminaed. */
+  private terminated = false;
+
+  private async mainLoop() {
+    while (true) {
+      if (!this.keepRunning) break;
+
+      // saving those who want to be notified about new state here
+      // because those who will be added during the tree retrieval
+      // should be notified only on the next round
+      let toNotify: ScheduledRefresh[] | undefined = undefined;
+      if (this.scheduledOnNextState.length > 0) {
+        toNotify = this.scheduledOnNextState;
+        this.scheduledOnNextState = [];
+      }
+
+      try {
+        // actual tree synchronization
+        await this.refresh();
+
+        // notifying that we got new state
+        if (toNotify !== undefined) for (const n of toNotify) n.resolve();
+      } catch (e: any) {
+        // notifying that we failed to refresh the state
+        if (toNotify !== undefined) for (const n of toNotify) n.reject(e);
+
+        // catching tree update errors, as they may leave our tree in inconsistent state
+        if (e instanceof TreeStateUpdateError) {
+          // important error logging, this should never happen
+          this.logger?.error(e);
+
+          // marking everybody who used previous state as changed
+          this.state.invalidateTree('stat update error');
+          // creating new tree
+          this.state = new PlTreeState(this.root, this.finalPredicate);
+
+          // scheduling state update without delay
+          continue;
+
+          // unfortunately external observer may still see tree in its default
+          // empty state, though this is best we can do in this exceptional
+          // situation, and hope on caching layers inside computables to present
+          // some stale state until we reconstruct the tree again
+        } else this.logger?.warn(e);
+      }
+
+      if (!this.keepRunning || this.terminated) break;
+
+      try {
+        await tp.setTimeout(this.pollingInterval, this.abortController.signal);
+      } catch (e: unknown) {
+        if (!isTimeoutOrCancelError(e)) throw new Error('Unexpected error', { cause: e });
+        break;
+      }
+    }
+
+    // reset only as a very last line
+    this.currentLoop = undefined;
+  }
+
+  /**
+   * Terminates the internal loop, and permanently destoys all internal state, so
+   * all computables using this state will resolve to errors.
+   * */
+  public async terminate(): Promise<void> {
+    this.keepRunning = false;
+    this.terminated = true;
+    this.abortController.abort();
+
+    if (this.currentLoop === undefined) return;
+    await this.currentLoop;
+
+    this.state.invalidateTree('synchronization terminated for the tree');
+  }
+
+  /** @deprecated */
+  public async awaitSyncLoopTermination(): Promise<void> {
+    if (this.currentLoop === undefined) return;
+    await this.currentLoop;
+  }
+
+  public static async init(pl: PlClient, root: ResourceId, ops: SynchronizedTreeOps) {
+    const tree = new SynchronizedTreeState(pl, root, ops);
+    await tree.refresh();
+    return tree;
+  }
+}