@milaboratories/pl-client 3.17.7 → 3.18.0

package/src/core/final.test.ts

@@ -1,5 +1,10 @@
 import { expect, test } from "vitest";
-import { DefaultFinalResourceDataPredicate } from "./final";
+import {
+  CacheFinality,
+  DefaultFinalResourceDataPredicate,
+  StrictFinality,
+  TreeFinality,
+} from "./final";
 import type { FieldData, ResourceData } from "./types";
 import { createSignedResourceId, NullSignedResourceId } from "./types";
 
@@ -71,3 +76,100 @@ test("a StreamManager with both fields still empty is not final", () => {
     ),
   ).toBe(false);
 });
+
+function resource(typeName: string, patch: Partial<ResourceData> = {}): ResourceData {
+  return streamManager({ type: { name: typeName, version: "1" }, ...patch });
+}
+
+const notReady: Partial<ResourceData> = {
+  resourceReady: false,
+  inputsLocked: false,
+  outputsLocked: false,
+};
+
+const filledOutput = { fields: [field("out", A)] };
+const unfilledOutput = {
+  fields: [
+    {
+      ...field("out", A),
+      value: NullSignedResourceId,
+      status: "Empty" as const,
+      valueIsFinal: false,
+    },
+  ],
+};
+
+test("a ready resolver with locked outputs is final only once every supplied field is settled", () => {
+  for (const name of ["BResolveSingle", "BResolveChoice"]) {
+    expect(DefaultFinalResourceDataPredicate(resource(name, filledOutput))).toBe(true);
+    expect(DefaultFinalResourceDataPredicate(resource(name, unfilledOutput))).toBe(false);
+    expect(
+      DefaultFinalResourceDataPredicate(resource(name, { ...notReady, ...filledOutput })),
+    ).toBe(false);
+  }
+});
+
+test("a ready blob copy with locked outputs is final only once every supplied field is settled", () => {
+  expect(DefaultFinalResourceDataPredicate(resource("BlobCopy/aToB", filledOutput))).toBe(true);
+  expect(DefaultFinalResourceDataPredicate(resource("BlobCopy/aToB", unfilledOutput))).toBe(false);
+});
+
+test("BResolveSingleNoResult is final at ready", () => {
+  expect(DefaultFinalResourceDataPredicate(resource("BResolveSingleNoResult"))).toBe(true);
+  expect(DefaultFinalResourceDataPredicate(resource("BResolveSingleNoResult", notReady))).toBe(
+    false,
+  );
+});
+
+test("LSProvider is never final", () => {
+  expect(DefaultFinalResourceDataPredicate(resource("LSProvider"))).toBe(false);
+});
+
+test("frontend archives and JSON scalar and error-trace snapshots are always final", () => {
+  for (const name of ["Frontend/FromLocalTgz", "json/bool", "json/null", "json/errorTrace"])
+    expect(StrictFinality.isFinal(resource(name, notReady))).toBe(true);
+});
+
+test("the strict layer keeps blobs, transfer handles, working directories and stream managers non-final", () => {
+  for (const name of ["Blob", "Blob/fs", "BlobIndex/fs", "BlobUpload/fs", "BlobCopy/aToB"])
+    expect(StrictFinality.isFinal(resource(name, filledOutput))).toBe(false);
+  expect(StrictFinality.isFinal(resource("WorkingDirectory", notReady))).toBe(false);
+  expect(StrictFinality.isFinal(streamManager({ error: A, fields: [] }))).toBe(false);
+});
+
+test("the cache adds the types whose only later writes are KV", () => {
+  for (const name of ["Blob/fs", "BlobIndex/fs", "BlobUpload/fs", "BlobCopy/aToB"])
+    expect(CacheFinality.isFinal(resource(name, filledOutput))).toBe(true);
+  expect(CacheFinality.isFinal(resource("WorkingDirectory", notReady))).toBe(true);
+});
+
+test("the cache keeps a Blob and a StreamManager out: their fields change later", () => {
+  expect(CacheFinality.isFinal(resource("Blob", notReady))).toBe(false);
+  expect(CacheFinality.isFinal(streamManager({ error: A, fields: [] }))).toBe(false);
+  expect(TreeFinality.isFinal(resource("Blob", notReady))).toBe(true);
+  expect(TreeFinality.isFinal(streamManager({ error: A, fields: [] }))).toBe(true);
+});
+
+test("each layer calls final everything its parent does", () => {
+  const states: Partial<ResourceData>[] = [
+    {},
+    notReady,
+    filledOutput,
+    unfilledOutput,
+    { error: A },
+    { ...notReady, originalResourceId: B },
+    { fields: undefined },
+  ];
+  const typeNames = new Set<string>();
+  for (const e of TreeFinality.entries)
+    typeNames.add("name" in e.match ? e.match.name : e.match.prefix + "x");
+  for (const name of typeNames)
+    for (const state of states) {
+      const r = resource(name, state);
+      const strict = StrictFinality.isFinal(r);
+      const cache = CacheFinality.isFinal(r);
+      const tree = TreeFinality.isFinal(r);
+      expect(!strict || cache, `${name} ${JSON.stringify(state)}: strict ⊆ cache`).toBe(true);
+      expect(!cache || tree, `${name} ${JSON.stringify(state)}: cache ⊆ tree`).toBe(true);
+    }
+});

package/src/core/final.ts

@@ -1,132 +1,273 @@
-import type { Optional } from "utility-types";
-import type { BasicResourceData, ResourceData } from "./types";
-import { isNotNullSignedResourceId, isNullSignedResourceId } from "./types";
 import { ResourceTypeName, ResourceTypePrefix } from "@milaboratories/pl-model-common";
+import type { FinalityEntry, FinalityStopRule, FinalResourceDataPredicate } from "./finality";
+import { FinalityTable, readyOrDuplicateOrError } from "./finality";
+import { isNotNullSignedResourceId, isNullSignedResourceId } from "./types";
 export { ResourceTypeName, ResourceTypePrefix };
+export type { FinalResourceDataPredicate } from "./finality";
+export { readyOrDuplicateOrError } from "./finality";
 
-/**
- * Function is used to guide multiple layers of caching in pl-client and derived pl-tree.
- *
- * This function defines expected resource-specific state mutation behaviour,
- * if it returns true, system will expect that this data will never change as long as resource exist.
- *
- * If resource data contain information about fields, if should be taken into account, fields are undefined,
- * "final" state should be calculated for "basic" part of resource data only.
- */
-export type FinalResourceDataPredicate = (
-  resourceData: Optional<ResourceData, "fields">,
-) => boolean;
+// The finality tables, one layer per consumer. Every layer accepts two backend writes after
+// final: a data-loss error cascade, lifting the error of a stored blob found lost or corrupt to
+// the resources built from it, and the first duplicate's `hasOriginalListeners` flag on its
+// original, backend-internal and never on the wire. An entry's `why` names any other write
+// it accepts. A layer adds only the types whose later writes its consumer does not observe.
 
-export function readyOrDuplicateOrError(r: ResourceData | BasicResourceData): boolean {
-  return (
-    r.resourceReady ||
-    isNotNullSignedResourceId(r.originalResourceId) ||
-    isNotNullSignedResourceId(r.error)
-  );
-}
+// Where the backend's filter stops earlier than the predicate. An early stop is safe: the
+// backend sends a body-less stop marker, and the streaming loader fetches every stopped resource
+// it does not hold as final again without stop rules (field filtering and pruning still apply),
+// so the predicate decides on the fetched body.
+const earlyOnFieldErrors: FinalityStopRule = {
+  approx: "readyOrDuplicateOrError",
+  reason:
+    "HAS_ERRORS is set by an error on any field, the predicate reads the resource error only; " +
+    "an early stop is followed up with a plain fetch",
+};
+const earlyOnFieldErrorsWithOutputs: FinalityStopRule = {
+  approx: "readyAndAllOutputsFilled",
+  reason:
+    "HAS_ERRORS is set by an error on any field, the predicate reads the resource error only; " +
+    "ALL_OUTPUTS_FINAL, like the predicate, checks every retained field; an early stop is " +
+    "followed up with a plain fetch",
+};
+
+const value = (name: string): FinalityEntry => ({
+  match: { name },
+  rule: "always",
+  why: "a value, written once at creation",
+});
 
-function readyAndHasAllOutputsFilled(r: Optional<ResourceData, "fields">): boolean {
+const settledAtReady = (name: string, why: string): FinalityEntry => ({
+  match: { name },
+  rule: "readyOrDuplicateOrError",
+  why,
+  stopRule: earlyOnFieldErrors,
+});
+
+const never = (match: FinalityEntry["match"], why: string): FinalityEntry => ({
+  match,
+  rule: "never",
+  why,
+});
+
+/** The base layer: nothing observable (state, fields, KV) changes after the rule holds, whatever
+ * the consumer prunes. Exceptions: the data-loss error cascade, the `hasOriginalListeners` flag,
+ * and the writes its entries name. */
+export const StrictFinality: FinalityTable = FinalityTable.of("strict", [
+  value(ResourceTypeName.JsonObject),
+  value(ResourceTypeName.JsonGzObject),
+  value(ResourceTypeName.JsonString),
+  value(ResourceTypeName.JsonArray),
+  value(ResourceTypeName.JsonNumber),
+  value(ResourceTypeName.JsonBool),
+  value(ResourceTypeName.JsonNull),
+  {
+    match: { name: ResourceTypeName.JsonErrorTrace },
+    rule: "always",
+    why: "an error-trace snapshot, populated and locked in its creating transaction",
+  },
+  value(ResourceTypeName.BContextEnd),
+  value(ResourceTypeName.FrontendFromUrl),
+  value(ResourceTypeName.FrontendFromFolder),
+  value(ResourceTypeName.FrontendFromLocalTgz),
+  value(ResourceTypeName.BObjectSpec),
+  value(ResourceTypeName.Null),
+  value(ResourceTypeName.Binary),
+  {
+    match: { name: ResourceTypeName.JsonResourceError },
+    rule: { custom: (r) => r.type.version === "1" },
+    why: "version 1 is a value, written once at creation",
+    stopRule: { approx: "always", reason: "a stop rule cannot see the type version" },
+  },
+  { match: { prefix: ResourceTypePrefix.LS }, rule: "always", why: "no write after creation" },
+  {
+    match: { prefix: ResourceTypePrefix.WorkingDirectory },
+    rule: "always",
+    why: "no write after creation",
+  },
+  {
+    match: { prefix: ResourceTypePrefix.StorageSpaceAllocation },
+    rule: "always",
+    why: "no write after creation",
+  },
+  ...[
+    ResourceTypeName.StdMap,
+    ResourceTypeName.StdMapSlash,
+    ResourceTypeName.EphStdMap,
+    ResourceTypeName.PFrame,
+    ResourceTypeName.ParquetChunk,
+    ResourceTypeName.BContext,
+    ResourceTypeName.BinaryMap,
+    ResourceTypeName.BinaryValue,
+    ResourceTypeName.BlobMap,
+    ResourceTypeName.BResolveSingleNoResult,
+    ResourceTypeName.BQueryResult,
+    ResourceTypeName.TengoLib,
+  ].map((name) =>
+    settledAtReady(name, "its inputs are final at ready, and no controller fills it afterwards"),
+  ),
+  ...[
+    ResourceTypeName.TengoTemplate,
+    ResourceTypeName.SoftwareInfo,
+    ResourceTypeName.BlockPackCustom,
+  ].map((name) =>
+    settledAtReady(
+      name,
+      "its only later write is the `hasOriginalListeners` flag its first duplicate sets",
+    ),
+  ),
+  settledAtReady(
+    ResourceTypeName.Dummy,
+    "its only later write is a data-loss error from the Blob whose incarnation field holds it",
+  ),
+  {
+    match: { prefix: ResourceTypePrefix.PColumnData },
+    rule: "readyOrDuplicateOrError",
+    why: "its inputs are final at ready, and no controller fills it afterwards",
+    stopRule: earlyOnFieldErrors,
+  },
+  {
+    match: { prefix: ResourceTypePrefix.StreamWorkdir },
+    rule: "readyOrDuplicateOrError",
+    why: "no write after ready",
+    stopRule: {
+      approx: "readyOrDuplicateOrError",
+      reason:
+        "HAS_ERRORS is set by an error on any field; on ResourceTree walks the project field " +
+        "filter also removes every field, `resourceError` included, before decoding, so the " +
+        "predicate cannot see an error the filter stops on; an early stop is followed up with " +
+        "a plain fetch",
+    },
+  },
+  {
+    match: { name: ResourceTypeName.BResolveSingle },
+    rule: "readyAndAllOutputsFilled",
+    why:
+      "outputs are locked at creation and assigned after ready, never overwritten; one " +
+      "exception: a later context with more than one match sets an error on a resolver that " +
+      "already succeeded, and a reader holding it final misses that error",
+    stopRule: earlyOnFieldErrorsWithOutputs,
+  },
+  {
+    match: { name: ResourceTypeName.BResolveChoice },
+    rule: "readyAndAllOutputsFilled",
+    why:
+      "outputs are locked at creation and point at result maps that are completed and locked " +
+      "as resolution finishes",
+    stopRule: earlyOnFieldErrorsWithOutputs,
+  },
+  never({ name: ResourceTypeName.UserProject }, "blocks and fields come and go over its life"),
+  never({ name: ResourceTypeName.Projects }, "projects are added and removed"),
+  never({ name: ResourceTypeName.ClientRoot }, "its fields are added and removed"),
+  ...[
+    ResourceTypeName.SharingOutbox,
+    ResourceTypeName.SharingState,
+    ResourceTypeName.SharedEnvelope,
+  ].map((name) => never({ name }, "gains and loses dynamic child fields over its life")),
+  never(
+    { name: ResourceTypeName.LSProvider },
+    "its storage fields are rewritten on a controller restart with a changed storage config",
+  ),
+  never({ name: ResourceTypeName.Blob }, "its incarnation field is attached or replaced later"),
+  never({ name: ResourceTypeName.StreamManager }, "`stream` is reset after an error"),
+  never({ prefix: ResourceTypePrefix.Blob }, "its `ctl/file/blobInfo` KV is re-pointed later"),
+  never(
+    { prefix: ResourceTypePrefix.BlobIndex },
+    "a controller restart writes its `ctl/ctlsdk/bootstrapDone` KV",
+  ),
+  never(
+    { prefix: ResourceTypePrefix.BlobUpload },
+    "the Reset RPC deletes its `ctl/file/storage/uploadState` KV",
+  ),
+  never(
+    { prefix: ResourceTypePrefix.BlobCopy },
+    "a controller bootstrap records its `ctl/ctlsdk/bootstrapDone` KV in a separate transaction, which may run after the copy settled",
+  ),
+  never(
+    { name: ResourceTypeName.WorkingDirectory },
+    "lock acquisition can write its `internal/locks/lockedBy` KV after creation",
+  ),
+]);
+
+/** What the transaction resource cache may keep while an entry stays in its LRU. The cache holds
+ * state and fields, never KV, so it adds the types whose only later writes are KV. */
+export const CacheFinality: FinalityTable = StrictFinality.extend("cache", [
+  {
+    match: { prefix: ResourceTypePrefix.Blob },
+    rule: "always",
+    why:
+      "`ctl/file/blobInfo` is re-pointed for the same content: the cache holds no KV, and a " +
+      "tree reads only its `sizeBytes`, invariant for the same content",
+  },
+  {
+    match: { prefix: ResourceTypePrefix.BlobIndex },
+    rule: "readyAndAllOutputsFilled",
+    why: "`ctl/ctlsdk/bootstrapDone` is backend bookkeeping KV; no client reads it",
+    stopRule: earlyOnFieldErrorsWithOutputs,
+  },
+  {
+    match: { prefix: ResourceTypePrefix.BlobUpload },
+    rule: "readyAndAllOutputsFilled",
+    why: "`ctl/file/storage/uploadState` is backend bookkeeping KV; no client reads it",
+    stopRule: earlyOnFieldErrorsWithOutputs,
+  },
+  {
+    match: { prefix: ResourceTypePrefix.BlobCopy },
+    rule: "readyAndAllOutputsFilled",
+    why:
+      "outputs are locked at creation; the only later write, the bootstrap's " +
+      "`ctl/ctlsdk/bootstrapDone`, is backend bookkeeping KV; no client reads it",
+    stopRule: earlyOnFieldErrorsWithOutputs,
+  },
+  {
+    match: { name: ResourceTypeName.WorkingDirectory },
+    rule: "always",
+    why: "the lock KV is backend bookkeeping; no client reads it",
+  },
+]);
+
+function streamSwitchedOrErrored(r: Parameters<FinalResourceDataPredicate>[0]): boolean {
   if (!readyOrDuplicateOrError(r)) return false;
-  if (!r.outputsLocked) return false;
-  if (r.fields === undefined) return true; // if fields are not provided basic resource state is not expected to change in the future
-  for (const f of r.fields)
-    if (isNullSignedResourceId(f.error) && (isNullSignedResourceId(f.value) || !f.valueIsFinal))
-      return false;
-  return true;
+  if (r.fields === undefined) return true;
+  if (isNotNullSignedResourceId(r.error)) return true;
+  // Fields can be pruned away by the reader: without them nothing proves the switch. The
+  // predicate must not throw, since a throw inside a tree update rebuilds the whole tree.
+  const downloadable = r.fields.find((f) => f.name === "downloadable");
+  const stream = r.fields.find((f) => f.name === "stream");
+  if (downloadable === undefined || stream === undefined || isNullSignedResourceId(stream.value))
+    return false;
+  // No backend flag marks the switch: equal non-null fields mean the controller's success branch
+  // has switched `stream` to the downloadable blob.
+  return stream.value === downloadable.value;
 }
 
-// solely for logging
-const unknownResourceTypeNames = new Set<string>();
+/**
+ * The finality of trees, and the default of `PlClient.finalPredicate`: a resource it calls final
+ * leaves the refresh seeds, later bodies for it are ignored, and its mutable-state change sources
+ * are retired. It adds Blob and StreamManager under reader assumptions stated at each entry; a
+ * consumer that reads what they exclude passes its own predicate.
+ */
+export const TreeFinality: FinalityTable = CacheFinality.extend("tree", [
+  {
+    match: { name: ResourceTypeName.Blob },
+    rule: "always",
+    why:
+      "the incarnation field is replaced later; the project tree prunes Blob's fields, and the " +
+      "drivers read a blob by id, never through its fields",
+    requiresPruning: { type: ResourceTypeName.Blob, fields: "all" },
+  },
+  {
+    match: { name: ResourceTypeName.StreamManager },
+    rule: { custom: streamSwitchedOrErrored },
+    why:
+      "after an error `stream` is reset in a later transaction; default traversal raises the " +
+      "error before exposing the fields, so only an `ignoreError` read sees `stream`",
+    stopRule: {
+      approx: "readyOrDuplicateOrError",
+      reason: "a stop rule cannot compare two fields, so it stops before `stream` switches",
+    },
+  },
+]);
 
-/** Default implementation, defining behaviour for built-in resource types. */
-export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r): boolean => {
-  switch (r.type.name) {
-    case ResourceTypeName.StreamManager: {
-      if (!readyOrDuplicateOrError(r)) return false;
-      if (r.fields === undefined) return true; // if fields are not provided basic resource state is not expected to change in the future
-      if (isNotNullSignedResourceId(r.error)) return true;
-      // Fields can be pruned away by the reader: without them nothing proves the switch, so
-      // the resource is not final. The predicate must not throw, since a throw inside the
-      // tree's update invalidates and rebuilds the whole tree.
-      const downloadable = r.fields.find((f) => f.name === "downloadable");
-      const stream = r.fields.find((f) => f.name === "stream");
-      // Both still empty is not a switch either.
-      if (
-        downloadable === undefined ||
-        stream === undefined ||
-        isNullSignedResourceId(stream.value)
-      )
-        return false;
-      return stream.value === downloadable.value; // it's equal to the resource is marked as final on backend side
-    }
-    case ResourceTypeName.StdMap:
-    case ResourceTypeName.StdMapSlash:
-    case ResourceTypeName.EphStdMap:
-    case ResourceTypeName.PFrame:
-    case ResourceTypeName.ParquetChunk:
-    case ResourceTypeName.BContext:
-    case ResourceTypeName.BlockPackCustom:
-    case ResourceTypeName.BinaryMap:
-    case ResourceTypeName.BinaryValue:
-    case ResourceTypeName.BlobMap:
-    case ResourceTypeName.BResolveSingle:
-    case ResourceTypeName.BResolveSingleNoResult:
-    case ResourceTypeName.BQueryResult:
-    case ResourceTypeName.TengoTemplate:
-    case ResourceTypeName.TengoLib:
-    case ResourceTypeName.SoftwareInfo:
-    case ResourceTypeName.Dummy:
-      return readyOrDuplicateOrError(r);
-    case ResourceTypeName.JsonResourceError:
-      return r.type.version === "1";
-    case ResourceTypeName.JsonObject:
-    case ResourceTypeName.JsonGzObject:
-    case ResourceTypeName.JsonString:
-    case ResourceTypeName.JsonArray:
-    case ResourceTypeName.JsonNumber:
-    case ResourceTypeName.BContextEnd:
-    case ResourceTypeName.FrontendFromUrl:
-    case ResourceTypeName.FrontendFromFolder:
-    case ResourceTypeName.BObjectSpec:
-    case ResourceTypeName.Blob:
-    case ResourceTypeName.Null:
-    case ResourceTypeName.Binary:
-    case ResourceTypeName.LSProvider:
-    case ResourceTypeName.WorkingDirectory:
-      return true;
-    case ResourceTypeName.UserProject:
-    case ResourceTypeName.Projects:
-    case ResourceTypeName.ClientRoot:
-    // Never final — these sharing resources gain and lose dynamic child fields over their lifetime.
-    case ResourceTypeName.SharingOutbox:
-    case ResourceTypeName.SharingState:
-    case ResourceTypeName.SharedEnvelope:
-      return false;
-    default:
-      if (
-        r.type.name.startsWith(ResourceTypePrefix.Blob) ||
-        r.type.name.startsWith(ResourceTypePrefix.LS) ||
-        r.type.name.startsWith(ResourceTypePrefix.WorkingDirectory) ||
-        r.type.name.startsWith(ResourceTypePrefix.StorageSpaceAllocation)
-      ) {
-        return true;
-      } else if (
-        r.type.name.startsWith(ResourceTypePrefix.BlobUpload) ||
-        r.type.name.startsWith(ResourceTypePrefix.BlobIndex)
-      ) {
-        return readyAndHasAllOutputsFilled(r);
-      } else if (r.type.name.startsWith(ResourceTypePrefix.PColumnData)) {
-        return readyOrDuplicateOrError(r);
-      } else if (r.type.name.startsWith(ResourceTypePrefix.StreamWorkdir)) {
-        return readyOrDuplicateOrError(r);
-      } else {
-        // Unknown resource type detected
-        // Set used to log this message only once
-        if (!unknownResourceTypeNames.has(r.type.name)) {
-          console.log("UNKNOWN RESOURCE TYPE: " + r.type.name);
-          unknownResourceTypeNames.add(r.type.name);
-        }
-      }
-  }
-  return false;
-};
+/** @deprecated use {@link TreeFinality} */
+export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate =
+  TreeFinality.predicate();