@milaboratories/pl-client 3.17.6 → 3.18.0

package/src/core/finality.test.ts

@@ -0,0 +1,298 @@
+import { describe, expect, test, vi } from "vitest";
+import { ResourceTypeName, ResourceTypePrefix } from "@milaboratories/pl-model-common";
+import {
+  CacheFinality,
+  DefaultFinalResourceDataPredicate,
+  StrictFinality,
+  TreeFinality,
+} from "./final";
+import { FinalityTable } from "./finality";
+import type { FinalResourceDataPredicate } from "./finality";
+import type { FieldData, ResourceData } from "./types";
+import { createSignedResourceId, NullSignedResourceId } from "./types";
+
+function resource(typeName: string, resourceReady = true): ResourceData {
+  return {
+    id: createSignedResourceId(1n),
+    originalResourceId: NullSignedResourceId,
+    kind: "Structural",
+    type: { name: typeName, version: "1" },
+    data: undefined,
+    error: NullSignedResourceId,
+    inputsLocked: resourceReady,
+    outputsLocked: resourceReady,
+    resourceReady,
+    final: false,
+    fields: [],
+  };
+}
+
+const base = FinalityTable.of("base", [
+  { match: { name: "A" }, rule: "readyOrDuplicateOrError", why: "test" },
+  { match: { name: "B" }, rule: "never", why: "test" },
+]);
+
+test("a child layer is its parent plus its additions", () => {
+  const child = base.extend("child", [
+    { match: { name: "B" }, rule: "always", why: "test" },
+    { match: { prefix: "C/" }, rule: "always", why: "test" },
+  ]);
+  expect(child.isFinal(resource("A"))).toBe(true);
+  expect(child.isFinal(resource("A", false))).toBe(false);
+  expect(child.isFinal(resource("B"))).toBe(true);
+  expect(child.isFinal(resource("C/x"))).toBe(true);
+  expect(base.isFinal(resource("B"))).toBe(false);
+  expect(base.isFinal(resource("C/x"))).toBe(false);
+});
+
+test("an addition cannot make its parent's final resource non-final", () => {
+  const child = base.extend("child", [{ match: { name: "A" }, rule: "never", why: "test" }]);
+  expect(child.isFinal(resource("A"))).toBe(true);
+});
+
+test("a type no layer covers is never final", () => {
+  expect(base.extend("child", []).isFinal(resource("Unknown"))).toBe(false);
+});
+
+test("a layer lists each match once", () => {
+  expect(() =>
+    FinalityTable.of("dup", [
+      { match: { name: "A" }, rule: "always", why: "test" },
+      { match: { name: "A" }, rule: "never", why: "test" },
+    ]),
+  ).toThrow(/listed twice/);
+});
+
+test("entries list the base first", () => {
+  const child = base.extend("child", [{ match: { prefix: "C/" }, rule: "always", why: "test" }]);
+  expect(child.entries.map((e) => ("name" in e.match ? e.match.name : e.match.prefix))).toEqual([
+    "A",
+    "B",
+    "C/",
+  ]);
+});
+
+describe("the built-in tables", () => {
+  type Snapshot = Parameters<FinalResourceDataPredicate>[0];
+  const id = createSignedResourceId(101n);
+  const target = createSignedResourceId(102n);
+  const error = createSignedResourceId(103n);
+
+  function field(patch: Partial<FieldData> = {}): FieldData {
+    return {
+      name: "out",
+      type: "Output",
+      status: "Resolved",
+      value: target,
+      error: NullSignedResourceId,
+      valueIsFinal: true,
+      ...patch,
+    };
+  }
+
+  function resource(name: string, patch: Partial<Snapshot> = {}): Snapshot {
+    return {
+      id,
+      type: { name, version: "1" },
+      kind: "Structural",
+      data: undefined,
+      originalResourceId: NullSignedResourceId,
+      error: NullSignedResourceId,
+      inputsLocked: true,
+      outputsLocked: true,
+      resourceReady: true,
+      final: false,
+      fields: [field()],
+      ...patch,
+    };
+  }
+
+  // The expected answers, listed independently of the table's entries.
+  const alwaysFinal = new Set([
+    "Blob",
+    "WorkingDirectory",
+    "json/object",
+    "json-gz/object",
+    "json/string",
+    "json/array",
+    "json/number",
+    "json/bool",
+    "json/null",
+    "json/errorTrace",
+    "BContextEnd",
+    "Frontend/FromUrl",
+    "Frontend/FromFolder",
+    "Frontend/FromLocalTgz",
+    "BObjectSpec",
+    "Null",
+    "binary",
+  ]);
+  const finalWhenTerminal = new Set([
+    "Dummy",
+    "StdMap",
+    "std/map",
+    "EphStdMap",
+    "PFrame",
+    "ParquetChunk",
+    "BContext",
+    "BlockPackCustom",
+    "BinaryMap",
+    "BinaryValue",
+    "BlobMap",
+    "BResolveSingleNoResult",
+    "BQueryResult",
+    "TengoTemplate",
+    "TengoLib",
+    "SoftwareInfo",
+  ]);
+  const neverFinal = new Set([
+    "UserProject",
+    "Projects",
+    "ClientRoot",
+    "SharingOutbox",
+    "SharingState",
+    "SharedEnvelope",
+    "BResolveSingle",
+    "BResolveChoice",
+    "LSProvider",
+  ]);
+  function terminal(r: Snapshot): boolean {
+    return (
+      r.resourceReady ||
+      r.originalResourceId !== NullSignedResourceId ||
+      r.error !== NullSignedResourceId
+    );
+  }
+  function filled(r: Snapshot): boolean {
+    return (
+      terminal(r) &&
+      r.outputsLocked &&
+      (r.fields === undefined ||
+        r.fields.every(
+          (f) =>
+            f.error !== NullSignedResourceId ||
+            (f.value !== NullSignedResourceId && f.valueIsFinal),
+        ))
+    );
+  }
+  function expectedTree(r: Snapshot): boolean {
+    const name = r.type.name;
+    if (name === "BResolveSingle" || name === "BResolveChoice" || name.startsWith("BlobCopy/"))
+      return filled(r);
+    if (name === "StreamManager") {
+      if (!terminal(r)) return false;
+      if (r.fields === undefined || r.error !== NullSignedResourceId) return true;
+      const stream = r.fields.find((f) => f.name === "stream");
+      const downloadable = r.fields.find((f) => f.name === "downloadable");
+      return (
+        stream !== undefined &&
+        downloadable !== undefined &&
+        stream.value !== NullSignedResourceId &&
+        stream.value === downloadable.value
+      );
+    }
+    if (alwaysFinal.has(name)) return true;
+    if (finalWhenTerminal.has(name)) return terminal(r);
+    if (neverFinal.has(name)) return false;
+    if (name === "json/resourceError") return r.type.version === "1";
+    if (
+      ["Blob/", "LS/", "WorkingDirectory/", "StorageSpaceAllocation/"].some((p) =>
+        name.startsWith(p),
+      )
+    )
+      return true;
+    if (["BlobUpload/", "BlobIndex/"].some((p) => name.startsWith(p))) return filled(r);
+    if (["PColumnData/", "StreamWorkdir/"].some((p) => name.startsWith(p))) return terminal(r);
+    return false;
+  }
+
+  const names = [
+    ...new Set([
+      ...Object.values(ResourceTypeName),
+      ...Object.values(ResourceTypePrefix).flatMap((p) => [
+        p,
+        p + "fs",
+        p + "BlobUpload/fs",
+        p.slice(0, -1),
+        "x" + p,
+        p.replace("/", "X/"),
+      ]),
+      "Blob/BlobCopy/fs",
+      "BlobUpload",
+      "BlobIndex",
+      "BlobCopy",
+      "BlobUploadX/fs",
+      "LSProvider/x",
+      "StdMap/child",
+      "std/map/child",
+      "json/resourceError/child",
+    ]),
+  ];
+  const fieldStates: Array<Snapshot["fields"]> = [
+    undefined,
+    [],
+    [field()],
+    [field({ valueIsFinal: false })],
+    [field({ value: NullSignedResourceId, valueIsFinal: false, status: "Empty" })],
+    [field({ value: NullSignedResourceId, valueIsFinal: false, error })],
+    [field(), field({ name: "service", type: "Service", valueIsFinal: false })],
+    [field({ name: "stream" }), field({ name: "downloadable" })],
+    [field({ name: "stream" }), field({ name: "downloadable", value: error })],
+  ];
+
+  test("every built-in name and prefix, at every version and state, gets the answer of the rules it is listed with", () => {
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+    const mismatches: string[] = [];
+    try {
+      for (const name of names)
+        for (const version of ["", "1", "2", "01"]) {
+          for (let bits = 0; bits < 16; bits++)
+            for (const fields of fieldStates) {
+              const r = resource(name, {
+                type: { name, version },
+                resourceReady: !!(bits & 1),
+                outputsLocked: !!(bits & 2),
+                originalResourceId: bits & 4 ? target : NullSignedResourceId,
+                error: bits & 8 ? error : NullSignedResourceId,
+                fields,
+              });
+              const tree = expectedTree(r);
+              const cache = name !== "Blob" && name !== "StreamManager" && tree;
+              if (
+                TreeFinality.isFinal(r) !== tree ||
+                DefaultFinalResourceDataPredicate(r) !== tree ||
+                CacheFinality.isFinal(r) !== cache
+              )
+                mismatches.push(
+                  JSON.stringify(r, (_, v) => (typeof v === "bigint" ? String(v) : v)),
+                );
+            }
+        }
+      expect(mismatches).toEqual([]);
+    } finally {
+      log.mockRestore();
+    }
+  });
+
+  test("a built-in type is never logged as unknown; an unknown one is logged once, whatever its version or layer", () => {
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+    try {
+      for (const name of [
+        ...Object.values(ResourceTypeName),
+        ...Object.values(ResourceTypePrefix).map((p) => p + "probe"),
+      ]) {
+        for (const layer of [StrictFinality, CacheFinality, TreeFinality])
+          layer.isFinal(resource(name));
+      }
+      expect(log).not.toHaveBeenCalled();
+      const name = "unknown-log-probe/x";
+      for (const version of ["1", "2", "1"])
+        for (const layer of [StrictFinality, CacheFinality, TreeFinality]) {
+          expect(layer.isFinal(resource(name, { type: { name, version } }))).toBe(false);
+        }
+      expect(log.mock.calls).toEqual([["UNKNOWN RESOURCE TYPE: " + name]]);
+    } finally {
+      log.mockRestore();
+    }
+  });
+});

package/src/core/finality.ts

@@ -0,0 +1,162 @@
+import type { Optional } from "utility-types";
+import type { BasicResourceData, ResourceData } from "./types";
+import { isNotNullSignedResourceId, isNullSignedResourceId } from "./types";
+
+/**
+ * Tells whether a resource state is final for a consumer: whether it will not change in any way
+ * that consumer observes, under the write exceptions the predicate documents. If the data carries
+ * no fields (`fields` undefined), the answer is about the basic part of the resource data only.
+ */
+export type FinalResourceDataPredicate = (
+  resourceData: Optional<ResourceData, "fields">,
+) => boolean;
+
+/** Ready for calculation, a duplicate of an original, or carrying a resource error. */
+export function readyOrDuplicateOrError(r: ResourceData | BasicResourceData): boolean {
+  return (
+    r.resourceReady ||
+    isNotNullSignedResourceId(r.originalResourceId) ||
+    isNotNullSignedResourceId(r.error)
+  );
+}
+
+/** {@link readyOrDuplicateOrError}, outputs locked, and every supplied field holding a final
+ * non-null value or an error. Without fields the answer is about the basic part only. */
+export function readyAndAllOutputsFilled(r: Optional<ResourceData, "fields">): boolean {
+  if (!readyOrDuplicateOrError(r)) return false;
+  if (!r.outputsLocked) return false;
+  if (r.fields === undefined) return true;
+  for (const f of r.fields)
+    if (isNullSignedResourceId(f.error) && (isNullSignedResourceId(f.value) || !f.valueIsFinal))
+      return false;
+  return true;
+}
+
+/** A condition with a traversal stop-rule translation. */
+export type TranslatableFinalityRule =
+  | "always"
+  | "readyOrDuplicateOrError"
+  | "readyAndAllOutputsFilled";
+
+/** When a resource of a type counts as final. */
+export type FinalityRule =
+  | TranslatableFinalityRule
+  | "never"
+  | { readonly custom: FinalResourceDataPredicate };
+
+/**
+ * How an entry is expressed as a traversal stop rule: `"exact"` translates its rule;
+ * `approx` stops on the given condition, which differs from the rule for the stated reason;
+ * `none` contributes no clause for this entry.
+ */
+export type FinalityStopRule =
+  | "exact"
+  | { readonly approx: TranslatableFinalityRule; readonly reason: string }
+  | { readonly none: string };
+
+/** The resource types an entry covers: one type name, or every name with a prefix. */
+export type FinalityMatch = { readonly name: string } | { readonly prefix: string };
+
+type EntryBase = {
+  readonly match: FinalityMatch;
+  /** What the backend writes after the rule holds, and why that is safe for this layer. */
+  readonly why: string;
+  /** The field pruning the entry relies on in trees that prune; the project tree must apply it. */
+  readonly requiresPruning?: { readonly type: string; readonly fields: "all" | readonly string[] };
+};
+
+/** One row of a finality table: the types it matches, when they count as final, why, and how the
+ * rule becomes a stop rule. A custom rule must declare a non-exact stop rule. */
+export type FinalityEntry = EntryBase &
+  (
+    | {
+        readonly rule: TranslatableFinalityRule | "never";
+        readonly stopRule?: Exclude<FinalityStopRule, "exact"> | "exact";
+      }
+    | {
+        readonly rule: { readonly custom: FinalResourceDataPredicate };
+        readonly stopRule: Exclude<FinalityStopRule, "exact">;
+      }
+  );
+
+function matches(match: FinalityMatch, typeName: string): boolean {
+  return "name" in match ? typeName === match.name : typeName.startsWith(match.prefix);
+}
+
+function matchKey(match: FinalityMatch): string {
+  return "name" in match ? `name:${match.name}` : `prefix:${match.prefix}`;
+}
+
+function holds(rule: FinalityRule, r: Optional<ResourceData, "fields">): boolean {
+  switch (rule) {
+    case "always":
+      return true;
+    case "never":
+      return false;
+    case "readyOrDuplicateOrError":
+      return readyOrDuplicateOrError(r);
+    case "readyAndAllOutputsFilled":
+      return readyAndAllOutputsFilled(r);
+    default:
+      return rule.custom(r);
+  }
+}
+
+// Unknown type names are logged once, across every table and predicate.
+const unknownResourceTypeNames = new Set<string>();
+
+/**
+ * A layer of the finality table. Each layer is its parent plus its own entries: a resource is
+ * final in a layer if it is final in the parent or by an entry of the layer, so a layer never
+ * calls final less than its parent does. A type no entry of any layer covers is never final.
+ */
+export class FinalityTable {
+  /** The entries of this layer and its ancestors, the base first. */
+  public readonly entries: readonly FinalityEntry[];
+
+  private constructor(
+    /** Names the layer in messages and tests. */
+    public readonly name: string,
+    own: readonly FinalityEntry[],
+    parent: FinalityTable | undefined,
+  ) {
+    const seen = new Set<string>();
+    for (const e of own) {
+      const key = matchKey(e.match);
+      if (seen.has(key)) throw new Error(`finality table ${name}: ${key} is listed twice`);
+      seen.add(key);
+    }
+    this.entries = parent === undefined ? own : [...parent.entries, ...own];
+  }
+
+  /** A base layer. Throws if `entries` lists a match twice. */
+  static of(name: string, entries: readonly FinalityEntry[]): FinalityTable {
+    return new FinalityTable(name, entries, undefined);
+  }
+
+  /** A child layer: this one plus `additions`. Throws if `additions` lists a match twice. */
+  extend(name: string, additions: readonly FinalityEntry[]): FinalityTable {
+    return new FinalityTable(name, additions, this);
+  }
+
+  /** True when any entry of the layer or its ancestors matching `r`'s type holds for `r`. A type
+   * no entry matches is false, and its name is logged once. */
+  isFinal(r: Optional<ResourceData, "fields">): boolean {
+    let covered = false;
+    for (const e of this.entries) {
+      if (!matches(e.match, r.type.name)) continue;
+      covered = true;
+      if (holds(e.rule, r)) return true;
+    }
+    if (!covered && !unknownResourceTypeNames.has(r.type.name)) {
+      unknownResourceTypeNames.add(r.type.name);
+      console.log("UNKNOWN RESOURCE TYPE: " + r.type.name);
+    }
+    return false;
+  }
+
+  /** {@link isFinal} as a standalone function. */
+  predicate(): FinalResourceDataPredicate {
+    return (r) => this.isFinal(r);
+  }
+}

package/src/core/transaction.ts

@@ -278,7 +278,8 @@ export class PlTransaction {
     public readonly name: string,
     public readonly writable: boolean,
     private readonly _clientRoot: OptionalSignedResourceId,
-    private readonly finalPredicate: FinalResourceDataPredicate,
+    /** What {@link sharedResourceDataCache} may keep. */
+    private readonly cachePredicate: FinalResourceDataPredicate,
     private readonly sharedResourceDataCache: LRUCache<SignedResourceId, ResourceDataCacheRecord>,
     private readonly enableFormattedErrors: boolean = false,
   ) {
@@ -767,7 +768,7 @@ export class PlTransaction {
 
       // we will cache only final resource data states
       // caching result even if we were ignore the cache
-      if (isResourceId(rId) && this.finalPredicate(result)) {
+      if (isResourceId(rId) && this.cachePredicate(result)) {
         deepFreeze(result);
         const fromCache = this.sharedResourceDataCache.get(rId);
         if (fromCache) {

package/src/index.ts

@@ -10,6 +10,7 @@ export * from "./core/unauth_client";
 export * from "./core/auth";
 export * from "./core/capabilities";
 export * from "./core/final";
+export * from "./core/finality";
 export * from "./core/tree_filter";
 export * from "./core/user_resources";
 export * from "./core/wire";