@milaboratories/pl-middle-layer 1.58.0 → 1.58.2

package/dist/middle_layer/driver_kit.cjs

@@ -3,8 +3,8 @@ const require_driver = require("../pool/driver.cjs");
 require("../pool/index.cjs");
 const require_ops = require("./ops.cjs");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
-let _milaboratories_helpers = require("@milaboratories/helpers");
 let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
+let _milaboratories_helpers = require("@milaboratories/helpers");
 //#region src/middle_layer/driver_kit.ts
 async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
 	const ops = {

package/dist/middle_layer/driver_kit.d.ts

@@ -2,8 +2,8 @@ import { InternalPFrameDriver } from "../pool/driver.js";
 import { DriverKitOpsConstructor } from "./ops.js";
 import { Signer } from "@milaboratories/ts-helpers";
 import { PlClient } from "@milaboratories/pl-client";
-import * as Sdk from "@milaboratories/pl-model-common";
 import { DownloadBlobToURLDriver, DownloadDriver, DownloadUrlDriver, InternalLsDriver, LogsDriver, UploadDriver } from "@milaboratories/pl-drivers";
+import * as Sdk from "@milaboratories/pl-model-common";
 
 //#region src/middle_layer/driver_kit.d.ts
 /**

package/dist/middle_layer/driver_kit.js

@@ -2,8 +2,8 @@ import { createPFrameDriver } from "../pool/driver.js";
 import "../pool/index.js";
 import { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from "./ops.js";
 import { HmacSha256Signer } from "@milaboratories/ts-helpers";
-import { isAsyncDisposable } from "@milaboratories/helpers";
 import { DownloadBlobToURLDriver, DownloadDriver, DownloadUrlDriver, LogsDriver, LogsStreamDriver, LsDriver, UploadDriver, createDownloadClient, createLogsClient, createUploadBlobClient, createUploadProgressClient } from "@milaboratories/pl-drivers";
+import { isAsyncDisposable } from "@milaboratories/helpers";
 //#region src/middle_layer/driver_kit.ts
 async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
 	const ops = {

package/dist/middle_layer/middle_layer.d.ts

@@ -12,9 +12,9 @@ import { Dispatcher } from "undici";
 import { BlockEventDispatcher, MiLogger, Signer } from "@milaboratories/ts-helpers";
 import { PlClient, ResourceId } from "@milaboratories/pl-client";
 import { ComputableStableDefined } from "@milaboratories/computable";
+import { DownloadUrlDriver } from "@milaboratories/pl-drivers";
 import { QuickJSWASMModule } from "quickjs-emscripten";
 import { ModelServiceRegistry } from "@milaboratories/pl-model-common";
-import { DownloadUrlDriver } from "@milaboratories/pl-drivers";
 
 //#region src/middle_layer/middle_layer.d.ts
 interface MiddleLayerEnvironment {

package/dist/pool/data.cjs

@@ -1,10 +1,40 @@
 const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
 let _platforma_sdk_model = require("@platforma-sdk/model");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
+let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 let node_crypto = require("node:crypto");
 let canonicalize = require("canonicalize");
 canonicalize = require_runtime.__toESM(canonicalize);
+let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
 //#region src/pool/data.ts
+/**
+* Tree-independent reference to a blob resource used by the PFrame data flow.
+*
+* The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;
+* resolution then went back through the originating tree, so when that tree dropped
+* the resource (e.g., a project recalculated with new settings), shared pool entries
+* — held alive by other projects — would start failing with "resource not found in
+* the tree" even though the underlying blob was still valid backend-side.
+*
+* BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed
+* to resolve. The pools then key by rid (`resourceInfo.id`) and call into the
+* download driver with the snapshot directly — independent of any specific tree.
+*/
+var BlobResourceRef = class {
+	constructor(resourceInfo, onDemandSnapshot) {
+		this.resourceInfo = resourceInfo;
+		this.onDemandSnapshot = onDemandSnapshot;
+	}
+	toJSON() {
+		return (0, _milaboratories_pl_client.resourceIdToString)(this.resourceInfo.id);
+	}
+};
+function makeLocalBlobRef(accessor) {
+	return new BlobResourceRef(accessor.resourceInfo, void 0);
+}
+function makeRemoteBlobRef(accessor) {
+	return new BlobResourceRef(accessor.resourceInfo, (0, _milaboratories_pl_tree.makeResourceSnapshot)(accessor, _milaboratories_pl_drivers.OnDemandBlobResourceSnapshot));
+}
 const PColumnDataJsonPartitioned = (0, _milaboratories_pl_client.resourceType)("PColumnData/JsonPartitioned", "1");
 const PColumnDataJsonSuperPartitioned = (0, _milaboratories_pl_client.resourceType)("PColumnData/Partitioned/JsonPartitioned", "1");
 const PColumnDataBinaryPartitioned = (0, _milaboratories_pl_client.resourceType)("PColumnData/BinaryPartitioned", "1");
@@ -28,10 +58,10 @@ function parseDataInfoResource(data) {
 		};
 	} else if ((0, _milaboratories_pl_client.resourceTypesEqual)(data.resourceType, PColumnDataJsonPartitioned)) {
 		const meta = resourceData;
-		const parts = Object.fromEntries(data.listInputFields().map((field) => [field, data.traverse({
+		const parts = Object.fromEntries(data.listInputFields().map((field) => [field, makeLocalBlobRef(data.traverse({
 			field,
 			errorIfFieldNotSet: true
-		}).persist()]));
+		}))]));
 		return {
 			type: "JsonPartitioned",
 			partitionKeyLength: meta.partitionKeyLength,
@@ -49,10 +79,10 @@ function parseDataInfoResource(data) {
 			if (keys === void 0) throw new _platforma_sdk_model.PFrameDriverError(`no partition keys for super key ${superKey}`);
 			for (const key of keys) {
 				const partKey = JSON.stringify([...JSON.parse(superKey), ...JSON.parse(key)]);
-				parts[partKey] = superPart.traverse({
+				parts[partKey] = makeLocalBlobRef(superPart.traverse({
 					field: key,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			}
 		}
 		return {
@@ -70,10 +100,10 @@ function parseDataInfoResource(data) {
 				part = {};
 				parts[partKey] = part;
 			}
-			part.index = data.traverse({
+			part.index = makeLocalBlobRef(data.traverse({
 				field,
 				errorIfFieldNotSet: true
-			}).persist();
+			}));
 		} else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
 			const partKey = field.slice(0, -7);
 			let part = parts[partKey];
@@ -81,10 +111,10 @@ function parseDataInfoResource(data) {
 				part = {};
 				parts[partKey] = part;
 			}
-			part.values = data.traverse({
+			part.values = makeLocalBlobRef(data.traverse({
 				field,
 				errorIfFieldNotSet: true
-			}).persist();
+			}));
 		} else throw new _platforma_sdk_model.PFrameDriverError(`unrecognized part field name: ${field}`);
 		for (const [key, part] of Object.entries(parts)) {
 			if (part.index === void 0) throw new _platforma_sdk_model.PFrameDriverError(`no index for part ${key}`);
@@ -113,10 +143,10 @@ function parseDataInfoResource(data) {
 					part = {};
 					parts[partKey] = part;
 				}
-				parts[partKey].index = superData.traverse({
+				parts[partKey].index = makeLocalBlobRef(superData.traverse({
 					field,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			} else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
 				const key = field.slice(0, -7);
 				const partKey = JSON.stringify([...JSON.parse(superKey), ...JSON.parse(key)]);
@@ -125,10 +155,10 @@ function parseDataInfoResource(data) {
 					part = {};
 					parts[partKey] = part;
 				}
-				parts[partKey].values = superData.traverse({
+				parts[partKey].values = makeLocalBlobRef(superData.traverse({
 					field,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			} else throw new _platforma_sdk_model.PFrameDriverError(`unrecognized part field name: ${field}`);
 		}
 		return {
@@ -177,11 +207,11 @@ function parseDataInfoResource(data) {
 }
 function traverseParquetChunkResource(resource) {
 	if (!(0, _milaboratories_pl_client.resourceTypesEqual)(resource.resourceType, ParquetChunkResourceType)) throw new _platforma_sdk_model.PFrameDriverError(`unknown resource type: ${(0, _milaboratories_pl_client.resourceTypeToString)(resource.resourceType)}, expected: ${(0, _milaboratories_pl_client.resourceTypeToString)(ParquetChunkResourceType)}`);
-	const blob = resource.traverse({
+	const blob = makeRemoteBlobRef(resource.traverse({
 		field: "blob",
 		assertFieldType: "Service",
 		errorIfFieldNotSet: true
-	}).persist();
+	}));
 	const partInfo = resource.getDataAsJson();
 	const mapping = resource.traverse({
 		field: "mapping",
@@ -217,6 +247,7 @@ function deriveLocalPObjectId(resolvePath, outputName) {
 exports.deriveGlobalPObjectId = deriveGlobalPObjectId;
 exports.deriveLegacyPObjectId = deriveLegacyPObjectId;
 exports.deriveLocalPObjectId = deriveLocalPObjectId;
+exports.makeLocalBlobRef = makeLocalBlobRef;
 exports.parseDataInfoResource = parseDataInfoResource;
 exports.traverseParquetChunkResource = traverseParquetChunkResource;
 

package/dist/pool/data.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"data.cjs","names":["PFrameDriverError"],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport type { PlTreeEntry, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<PlTreeEntry> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).persist()]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, PlTreeEntry> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).persist();\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<PlTreeEntry> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = resource\n    .traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .persist();\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;AAuBA,MAAa,8BAAA,GAAA,0BAAA,cAA0C,+BAA+B,IAAI;AAC1F,MAAa,mCAAA,GAAA,0BAAA,cACX,2CACA,IACD;AACD,MAAa,gCAAA,GAAA,0BAAA,cAA4C,iCAAiC,IAAI;AAC9F,MAAa,qCAAA,GAAA,0BAAA,cACX,6CACA,IACD;AACD,MAAa,iCAAA,GAAA,0BAAA,cAA6C,kCAAkC,IAAI;AAChG,MAAa,sCAAA,GAAA,0BAAA,cACX,8CACA,IACD;AACD,MAAa,mBAAA,GAAA,0BAAA,cAA+B,oBAAoB,IAAI;AAEpE,MAAa,4BAAA,GAAA,0BAAA,cAAwC,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MACsC;AACtC,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAIA,qBAAAA,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAIA,qBAAAA,kBAAkB,mDAAmD;AAEjF,MAAA,GAAA,0BAAA,oBAAuB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;8DAC2B,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CAAC,OAAO,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CACzF;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAqC,EAAE;AAC7C,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAAC,SAAS;;;AAI3F,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;8DAC2B,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAG7E,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;aAChE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;QACrE,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;8DAC2B,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAC7E,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,UACpB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;cACH,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,UACrB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;SACP,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;8DAC2B,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAIA,qBAAAA,kBACR,+BAAA,GAAA,0BAAA,sBAAmD,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC2B;AAC3B,KAAI,EAAA,GAAA,0BAAA,oBAAoB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAIA,qBAAAA,kBACR,2BAAA,GAAA,0BAAA,sBAA+C,SAAS,aAAa,CAAC,eAAA,GAAA,0BAAA,sBAClC,yBAAyB,GAC9D;CAGH,MAAM,OAAO,SACV,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACjF,SAAS;CACZ,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,QAAA,GAAA,YAAA,YAAkB,SAAS;AACjC,MAAK,QAAA,GAAA,aAAA,SAAoB,KAAK,CAAE;AAChC,MAAK,OAAO,OAAO,EAAA,GAAA,0BAAA,kBAAkB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK,GAAG,CAAC;AACnF,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,SAAA,GAAA,aAAA,SAAoB;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,SAAA,GAAA,aAAA,SAAoB;EAAE;EAAa,MAAM;EAAY,CAAC"}
+{"version":3,"file":"data.cjs","names":["OnDemandBlobResourceSnapshot","PFrameDriverError"],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceIdToString,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n  type ResourceId,\n  type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n  constructor(\n    public readonly resourceInfo: { readonly id: ResourceId; readonly type: ResourceType },\n    /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n    public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n  ) {}\n\n  toJSON(): string {\n    return resourceIdToString(this.resourceInfo.id);\n  }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n  return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n  return new BlobResourceRef(\n    accessor.resourceInfo,\n    makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n  );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<BlobResourceRef> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [\n          field,\n          makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n        ]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, BlobResourceRef> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = makeLocalBlobRef(\n          superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n        );\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = makeLocalBlobRef(\n            superData.traverse({ field, errorIfFieldNotSet: true }),\n          );\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = makeLocalBlobRef(\n            superData.traverse({ field, errorIfFieldNotSet: true }),\n          );\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = makeRemoteBlobRef(\n    resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n  );\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,UAAA,GAAA,0BAAA,oBAA0B,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,eAAA,GAAA,wBAAA,sBACY,UAAUA,2BAAAA,6BAA6B,CAC7D;;AAGH,MAAa,8BAAA,GAAA,0BAAA,cAA0C,+BAA+B,IAAI;AAC1F,MAAa,mCAAA,GAAA,0BAAA,cACX,2CACA,IACD;AACD,MAAa,gCAAA,GAAA,0BAAA,cAA4C,iCAAiC,IAAI;AAC9F,MAAa,qCAAA,GAAA,0BAAA,cACX,6CACA,IACD;AACD,MAAa,iCAAA,GAAA,0BAAA,cAA6C,kCAAkC,IAAI;AAChG,MAAa,sCAAA,GAAA,0BAAA,cACX,8CACA,IACD;AACD,MAAa,mBAAA,GAAA,0BAAA,cAA+B,oBAAoB,IAAI;AAEpE,MAAa,4BAAA,GAAA,0BAAA,cAAwC,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MAC0C;AAC1C,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAIC,qBAAAA,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAIA,qBAAAA,kBAAkB,mDAAmD;AAEjF,MAAA,GAAA,0BAAA,oBAAuB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;8DAC2B,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;8DAC2B,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;8DAC2B,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;8DAC2B,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAIA,qBAAAA,kBACR,+BAAA,GAAA,0BAAA,sBAAmD,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,EAAA,GAAA,0BAAA,oBAAoB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAIA,qBAAAA,kBACR,2BAAA,GAAA,0BAAA,sBAA+C,SAAS,aAAa,CAAC,eAAA,GAAA,0BAAA,sBAClC,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,QAAA,GAAA,YAAA,YAAkB,SAAS;AACjC,MAAK,QAAA,GAAA,aAAA,SAAoB,KAAK,CAAE;AAChC,MAAK,OAAO,OAAO,EAAA,GAAA,0BAAA,kBAAkB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK,GAAG,CAAC;AACnF,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,SAAA,GAAA,aAAA,SAAoB;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,SAAA,GAAA,aAAA,SAAoB;EAAE;EAAa,MAAM;EAAY,CAAC"}

package/dist/pool/data.d.ts

@@ -1,6 +1,8 @@
 import { PObjectId } from "@platforma-sdk/model";
 import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
+import { ResourceId, ResourceType } from "@milaboratories/pl-client";
 import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import { OnDemandBlobResourceSnapshot } from "@milaboratories/pl-drivers";
 
 //#region src/pool/data.d.ts
 declare function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId;

package/dist/pool/data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data.d.ts","names":[],"sources":["../../src/pool/data.ts"],"mappings":";;;;;iBAmSgB,qBAAA,CAAsB,OAAA,UAAiB,UAAA,WAAqB,SAAA;AAAA,iBAI5D,oBAAA,CAAqB,WAAA,YAAuB,UAAA,WAAqB,SAAA"}
+{"version":3,"file":"data.d.ts","names":[],"sources":["../../src/pool/data.ts"],"mappings":";;;;;;;iBA0UgB,qBAAA,CAAsB,OAAA,UAAiB,UAAA,WAAqB,SAAA;AAAA,iBAI5D,oBAAA,CAAqB,WAAA,YAAuB,UAAA,WAAqB,SAAA"}

package/dist/pool/data.js

@@ -1,8 +1,38 @@
 import { PFrameDriverError } from "@platforma-sdk/model";
-import { isNullResourceId, resourceType, resourceTypeToString, resourceTypesEqual } from "@milaboratories/pl-client";
+import { isNullResourceId, resourceIdToString, resourceType, resourceTypeToString, resourceTypesEqual } from "@milaboratories/pl-client";
+import { makeResourceSnapshot } from "@milaboratories/pl-tree";
 import { createHash } from "node:crypto";
 import canonicalize from "canonicalize";
+import { OnDemandBlobResourceSnapshot } from "@milaboratories/pl-drivers";
 //#region src/pool/data.ts
+/**
+* Tree-independent reference to a blob resource used by the PFrame data flow.
+*
+* The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;
+* resolution then went back through the originating tree, so when that tree dropped
+* the resource (e.g., a project recalculated with new settings), shared pool entries
+* — held alive by other projects — would start failing with "resource not found in
+* the tree" even though the underlying blob was still valid backend-side.
+*
+* BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed
+* to resolve. The pools then key by rid (`resourceInfo.id`) and call into the
+* download driver with the snapshot directly — independent of any specific tree.
+*/
+var BlobResourceRef = class {
+	constructor(resourceInfo, onDemandSnapshot) {
+		this.resourceInfo = resourceInfo;
+		this.onDemandSnapshot = onDemandSnapshot;
+	}
+	toJSON() {
+		return resourceIdToString(this.resourceInfo.id);
+	}
+};
+function makeLocalBlobRef(accessor) {
+	return new BlobResourceRef(accessor.resourceInfo, void 0);
+}
+function makeRemoteBlobRef(accessor) {
+	return new BlobResourceRef(accessor.resourceInfo, makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot));
+}
 const PColumnDataJsonPartitioned = resourceType("PColumnData/JsonPartitioned", "1");
 const PColumnDataJsonSuperPartitioned = resourceType("PColumnData/Partitioned/JsonPartitioned", "1");
 const PColumnDataBinaryPartitioned = resourceType("PColumnData/BinaryPartitioned", "1");
@@ -26,10 +56,10 @@ function parseDataInfoResource(data) {
 		};
 	} else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {
 		const meta = resourceData;
-		const parts = Object.fromEntries(data.listInputFields().map((field) => [field, data.traverse({
+		const parts = Object.fromEntries(data.listInputFields().map((field) => [field, makeLocalBlobRef(data.traverse({
 			field,
 			errorIfFieldNotSet: true
-		}).persist()]));
+		}))]));
 		return {
 			type: "JsonPartitioned",
 			partitionKeyLength: meta.partitionKeyLength,
@@ -47,10 +77,10 @@ function parseDataInfoResource(data) {
 			if (keys === void 0) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);
 			for (const key of keys) {
 				const partKey = JSON.stringify([...JSON.parse(superKey), ...JSON.parse(key)]);
-				parts[partKey] = superPart.traverse({
+				parts[partKey] = makeLocalBlobRef(superPart.traverse({
 					field: key,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			}
 		}
 		return {
@@ -68,10 +98,10 @@ function parseDataInfoResource(data) {
 				part = {};
 				parts[partKey] = part;
 			}
-			part.index = data.traverse({
+			part.index = makeLocalBlobRef(data.traverse({
 				field,
 				errorIfFieldNotSet: true
-			}).persist();
+			}));
 		} else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
 			const partKey = field.slice(0, -7);
 			let part = parts[partKey];
@@ -79,10 +109,10 @@ function parseDataInfoResource(data) {
 				part = {};
 				parts[partKey] = part;
 			}
-			part.values = data.traverse({
+			part.values = makeLocalBlobRef(data.traverse({
 				field,
 				errorIfFieldNotSet: true
-			}).persist();
+			}));
 		} else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
 		for (const [key, part] of Object.entries(parts)) {
 			if (part.index === void 0) throw new PFrameDriverError(`no index for part ${key}`);
@@ -111,10 +141,10 @@ function parseDataInfoResource(data) {
 					part = {};
 					parts[partKey] = part;
 				}
-				parts[partKey].index = superData.traverse({
+				parts[partKey].index = makeLocalBlobRef(superData.traverse({
 					field,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			} else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
 				const key = field.slice(0, -7);
 				const partKey = JSON.stringify([...JSON.parse(superKey), ...JSON.parse(key)]);
@@ -123,10 +153,10 @@ function parseDataInfoResource(data) {
 					part = {};
 					parts[partKey] = part;
 				}
-				parts[partKey].values = superData.traverse({
+				parts[partKey].values = makeLocalBlobRef(superData.traverse({
 					field,
 					errorIfFieldNotSet: true
-				}).persist();
+				}));
 			} else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
 		}
 		return {
@@ -175,11 +205,11 @@ function parseDataInfoResource(data) {
 }
 function traverseParquetChunkResource(resource) {
 	if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) throw new PFrameDriverError(`unknown resource type: ${resourceTypeToString(resource.resourceType)}, expected: ${resourceTypeToString(ParquetChunkResourceType)}`);
-	const blob = resource.traverse({
+	const blob = makeRemoteBlobRef(resource.traverse({
 		field: "blob",
 		assertFieldType: "Service",
 		errorIfFieldNotSet: true
-	}).persist();
+	}));
 	const partInfo = resource.getDataAsJson();
 	const mapping = resource.traverse({
 		field: "mapping",
@@ -212,6 +242,6 @@ function deriveLocalPObjectId(resolvePath, outputName) {
 	});
 }
 //#endregion
-export { deriveGlobalPObjectId, deriveLegacyPObjectId, deriveLocalPObjectId, parseDataInfoResource, traverseParquetChunkResource };
+export { deriveGlobalPObjectId, deriveLegacyPObjectId, deriveLocalPObjectId, makeLocalBlobRef, parseDataInfoResource, traverseParquetChunkResource };
 
 //# sourceMappingURL=data.js.map

package/dist/pool/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","names":[],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport type { PlTreeEntry, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<PlTreeEntry> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).persist()]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, PlTreeEntry> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).persist();\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<PlTreeEntry> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = resource\n    .traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .persist();\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;AAuBA,MAAa,6BAA6B,aAAa,+BAA+B,IAAI;AAC1F,MAAa,kCAAkC,aAC7C,2CACA,IACD;AACD,MAAa,+BAA+B,aAAa,iCAAiC,IAAI;AAC9F,MAAa,oCAAoC,aAC/C,6CACA,IACD;AACD,MAAa,gCAAgC,aAAa,kCAAkC,IAAI;AAChG,MAAa,qCAAqC,aAChD,8CACA,IACD;AACD,MAAa,kBAAkB,aAAa,oBAAoB,IAAI;AAEpE,MAAa,2BAA2B,aAAa,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MACsC;AACtC,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAI,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAI,kBAAkB,mDAAmD;AAEjF,KAAI,mBAAmB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;YACQ,mBAAmB,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CAAC,OAAO,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CACzF;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAqC,EAAE;AAC7C,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAAC,SAAS;;;AAI3F,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;YACQ,mBAAmB,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAG7E,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;aAChE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;QACrE,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAI,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAI,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;YACQ,mBAAmB,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAC7E,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,UACpB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;cACH,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,UACrB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;SACP,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;YACQ,mBAAmB,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAI,kBACR,8BAA8B,qBAAqB,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC2B;AAC3B,KAAI,CAAC,mBAAmB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAI,kBACR,0BAA0B,qBAAqB,SAAS,aAAa,CAAC,cACvD,qBAAqB,yBAAyB,GAC9D;CAGH,MAAM,OAAO,SACV,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACjF,SAAS;CACZ,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,OAAO,WAAW,SAAS;AACjC,MAAK,OAAO,aAAa,KAAK,CAAE;AAChC,MAAK,OAAO,OAAO,CAAC,iBAAiB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK,GAAG,CAAC;AACnF,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,QAAO,aAAa;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,QAAO,aAAa;EAAE;EAAa,MAAM;EAAY,CAAC"}
+{"version":3,"file":"data.js","names":[],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceIdToString,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n  type ResourceId,\n  type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n  constructor(\n    public readonly resourceInfo: { readonly id: ResourceId; readonly type: ResourceType },\n    /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n    public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n  ) {}\n\n  toJSON(): string {\n    return resourceIdToString(this.resourceInfo.id);\n  }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n  return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n  return new BlobResourceRef(\n    accessor.resourceInfo,\n    makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n  );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<BlobResourceRef> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [\n          field,\n          makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n        ]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, BlobResourceRef> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = makeLocalBlobRef(\n          superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n        );\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = makeLocalBlobRef(\n            superData.traverse({ field, errorIfFieldNotSet: true }),\n          );\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = makeLocalBlobRef(\n            superData.traverse({ field, errorIfFieldNotSet: true }),\n          );\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = makeRemoteBlobRef(\n    resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n  );\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,SAAO,mBAAmB,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,cACT,qBAAqB,UAAU,6BAA6B,CAC7D;;AAGH,MAAa,6BAA6B,aAAa,+BAA+B,IAAI;AAC1F,MAAa,kCAAkC,aAC7C,2CACA,IACD;AACD,MAAa,+BAA+B,aAAa,iCAAiC,IAAI;AAC9F,MAAa,oCAAoC,aAC/C,6CACA,IACD;AACD,MAAa,gCAAgC,aAAa,kCAAkC,IAAI;AAChG,MAAa,qCAAqC,aAChD,8CACA,IACD;AACD,MAAa,kBAAkB,aAAa,oBAAoB,IAAI;AAEpE,MAAa,2BAA2B,aAAa,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MAC0C;AAC1C,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAI,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAI,kBAAkB,mDAAmD;AAEjF,KAAI,mBAAmB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;YACQ,mBAAmB,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;YACQ,mBAAmB,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAI,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAI,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;YACQ,mBAAmB,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;YACQ,mBAAmB,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAI,kBACR,8BAA8B,qBAAqB,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,CAAC,mBAAmB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAI,kBACR,0BAA0B,qBAAqB,SAAS,aAAa,CAAC,cACvD,qBAAqB,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,OAAO,WAAW,SAAS;AACjC,MAAK,OAAO,aAAa,KAAK,CAAE;AAChC,MAAK,OAAO,OAAO,CAAC,iBAAiB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK,GAAG,CAAC;AACnF,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,QAAO,aAAa;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,QAAO,aAAa;EAAE;EAAa,MAAM;EAAY,CAAC"}

package/dist/pool/driver.cjs

@@ -6,14 +6,14 @@ let node_path = require("node:path");
 node_path = require_runtime.__toESM(node_path);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
+let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
 let _milaboratories_helpers = require("@milaboratories/helpers");
 let _milaboratories_pf_driver = require("@milaboratories/pf-driver");
 let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
 let node_stream = require("node:stream");
-let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
 //#region src/pool/driver.ts
 function makeBlobId(res) {
-	return String(res.rid);
+	return String(res.resourceInfo.id);
 }
 var LocalBlobProviderImpl = class extends _milaboratories_helpers.RefCountPoolBase {
 	constructor(blobDriver, logger) {
@@ -25,7 +25,7 @@ var LocalBlobProviderImpl = class extends _milaboratories_helpers.RefCountPoolBa
 		return makeBlobId(params);
 	}
 	createNewResource(params, _key) {
-		return this.blobDriver.getDownloadedBlob(params);
+		return this.blobDriver.getDownloadedBlob(params.resourceInfo);
 	}
 	getByKey(blobId) {
 		const resource = super.tryGetByKey(blobId);
@@ -58,7 +58,8 @@ var RemoteBlobPool = class extends _milaboratories_helpers.RefCountPoolBase {
 		return makeBlobId(params);
 	}
 	createNewResource(params, _key) {
-		return this.blobDriver.getOnDemandBlob(params);
+		if (params.onDemandSnapshot === void 0) throw new _platforma_sdk_model.PFrameDriverError(`BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; remote (parquet) blobs must be captured via makeRemoteBlobRef.`);
+		return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);
 	}
 	getByKey(blobId) {
 		const resource = super.tryGetByKey(blobId);
@@ -173,7 +174,7 @@ async function createPFrameDriver(params) {
 	const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
 	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
 	const resolveDataInfo = (spec, data) => {
-		return (0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data) ? require_data.parseDataInfoResource(data) : (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => a.persist()) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);
+		return (0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data) ? require_data.parseDataInfoResource(data) : (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.makeLocalBlobRef(a)) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);
 	};
 	return new _milaboratories_pf_driver.AbstractPFrameDriver({
 		logger,

package/dist/pool/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  PlTreeEntry,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;AAsCA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,kBAA4B,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,kBAA4B,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,UAAA,GAAA,wBAAA,sBAA4B,KAAK,GAC7BC,aAAAA,sBAAsB,KAAK,IAAA,GAAA,qBAAA,YAChB,KAAK,GACd,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAM,EAAE,SAAS,CAAC,IAAA,GAAA,0BAAA,kBACtB,MAAM,KAAK;;AAGpC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  return String(res.resourceInfo.id) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;AAuCA,SAAS,WAAW,KAAmD;AACrE,QAAO,OAAO,IAAI,aAAa,GAAG;;AAIpC,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,UAAA,GAAA,wBAAA,sBAA4B,KAAK,GAC7BC,aAAAA,sBAAsB,KAAK,IAAA,GAAA,qBAAA,YAChB,KAAK,GACd,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGpC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/dist/pool/driver.d.ts

@@ -1,7 +1,7 @@
 import { PColumnDataUniversal } from "@platforma-sdk/model";
 import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
-import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
 import { DownloadDriver } from "@milaboratories/pl-drivers";
+import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
 
 //#region src/pool/driver.d.ts
 interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {}

package/dist/pool/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;UA6QiB,oBAAA,SAA6B,4BAAA,CAC5C,oBAAA,CAAqB,kBAAA;AAAA,KAGX,eAAA,GAAkB,uBAAA;EAJQ,0CAMpC,iBAAA;AAAA"}
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;UA0RiB,oBAAA,SAA6B,4BAAA,CAC5C,oBAAA,CAAqB,kBAAA;AAAA,KAGX,eAAA,GAAkB,uBAAA;EAJQ,0CAMpC,iBAAA;AAAA"}

package/dist/pool/driver.js

@@ -1,17 +1,17 @@
-import { parseDataInfoResource, traverseParquetChunkResource } from "./data.js";
+import { makeLocalBlobRef, parseDataInfoResource, traverseParquetChunkResource } from "./data.js";
 import { PFrameDriverError, ensureError, isAbortError, isDataInfo, mapDataInfo } from "@platforma-sdk/model";
 import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
 import path from "node:path";
 import { emptyDir } from "@milaboratories/ts-helpers";
 import { isPlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import { isDownloadNetworkError400 } from "@milaboratories/pl-drivers";
 import { RefCountPoolBase } from "@milaboratories/helpers";
 import { AbstractPFrameDriver, AbstractPFrameDriverOpsDefaults, makeJsonDataInfo } from "@milaboratories/pf-driver";
 import { HttpHelpers } from "@milaboratories/pframes-rs-node";
 import { Readable } from "node:stream";
-import { isDownloadNetworkError400 } from "@milaboratories/pl-drivers";
 //#region src/pool/driver.ts
 function makeBlobId(res) {
-	return String(res.rid);
+	return String(res.resourceInfo.id);
 }
 var LocalBlobProviderImpl = class extends RefCountPoolBase {
 	constructor(blobDriver, logger) {
@@ -23,7 +23,7 @@ var LocalBlobProviderImpl = class extends RefCountPoolBase {
 		return makeBlobId(params);
 	}
 	createNewResource(params, _key) {
-		return this.blobDriver.getDownloadedBlob(params);
+		return this.blobDriver.getDownloadedBlob(params.resourceInfo);
 	}
 	getByKey(blobId) {
 		const resource = super.tryGetByKey(blobId);
@@ -56,7 +56,8 @@ var RemoteBlobPool = class extends RefCountPoolBase {
 		return makeBlobId(params);
 	}
 	createNewResource(params, _key) {
-		return this.blobDriver.getOnDemandBlob(params);
+		if (params.onDemandSnapshot === void 0) throw new PFrameDriverError(`BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; remote (parquet) blobs must be captured via makeRemoteBlobRef.`);
+		return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);
 	}
 	getByKey(blobId) {
 		const resource = super.tryGetByKey(blobId);
@@ -171,7 +172,7 @@ async function createPFrameDriver(params) {
 	const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
 	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
 	const resolveDataInfo = (spec, data) => {
-		return isPlTreeNodeAccessor(data) ? parseDataInfoResource(data) : isDataInfo(data) ? data.type === "ParquetPartitioned" ? mapDataInfo(data, (a) => traverseParquetChunkResource(a)) : mapDataInfo(data, (a) => a.persist()) : makeJsonDataInfo(spec, data);
+		return isPlTreeNodeAccessor(data) ? parseDataInfoResource(data) : isDataInfo(data) ? data.type === "ParquetPartitioned" ? mapDataInfo(data, (a) => traverseParquetChunkResource(a)) : mapDataInfo(data, (a) => makeLocalBlobRef(a)) : makeJsonDataInfo(spec, data);
 	};
 	return new AbstractPFrameDriver({
 		logger,

package/dist/pool/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  PlTreeEntry,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;AAsCA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,kBAA4B,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,kBAA4B,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,SAAO,qBAAqB,KAAK,GAC7B,sBAAsB,KAAK,GAC3B,WAAW,KAAK,GACd,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,EAAE,SAAS,CAAC,GACvC,iBAAiB,MAAM,KAAK;;AAGpC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  return String(res.resourceInfo.id) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;AAuCA,SAAS,WAAW,KAAmD;AACrE,QAAO,OAAO,IAAI,aAAa,GAAG;;AAIpC,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,SAAO,qBAAqB,KAAK,GAC7B,sBAAsB,KAAK,GAC3B,WAAW,KAAK,GACd,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGpC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-middle-layer",
-  "version": "1.58.0",
+  "version": "1.58.2",
   "description": "Pl Middle Layer",
   "keywords": [],
   "license": "UNLICENSED",
@@ -31,23 +31,23 @@
     "yaml": "^2.8.0",
     "zod": "~3.25.76",
     "@milaboratories/computable": "2.9.3",
+    "@milaboratories/pf-spec-driver": "1.3.9",
     "@milaboratories/helpers": "1.14.1",
     "@milaboratories/pf-driver": "1.4.5",
-    "@milaboratories/pf-spec-driver": "1.3.9",
-    "@milaboratories/pl-client": "3.2.5",
     "@milaboratories/pl-deployments": "2.17.12",
     "@milaboratories/pl-drivers": "1.13.1",
-    "@milaboratories/pl-errors": "1.3.13",
-    "@milaboratories/pl-model-backend": "1.2.21",
     "@milaboratories/pl-http": "1.2.4",
-    "@milaboratories/pl-model-middle-layer": "1.18.10",
-    "@milaboratories/pl-tree": "1.9.23",
+    "@milaboratories/pl-client": "3.2.5",
+    "@milaboratories/pl-model-backend": "1.2.21",
     "@milaboratories/pl-model-common": "1.39.0",
+    "@milaboratories/pl-errors": "1.3.13",
+    "@milaboratories/pl-model-middle-layer": "1.18.10",
     "@milaboratories/resolve-helper": "1.1.3",
+    "@milaboratories/pl-tree": "1.9.23",
     "@milaboratories/ts-helpers": "1.8.1",
-    "@platforma-sdk/model": "1.72.0",
-    "@platforma-sdk/workflow-tengo": "5.20.0",
-    "@platforma-sdk/block-tools": "2.7.19"
+    "@platforma-sdk/model": "1.73.0",
+    "@platforma-sdk/block-tools": "2.7.19",
+    "@platforma-sdk/workflow-tengo": "5.20.1"
   },
   "devDependencies": {
     "@types/node": "~24.5.2",
@@ -56,8 +56,8 @@
     "semver": "^7.7.2",
     "typescript": "~5.9.3",
     "vitest": "^4.1.3",
-    "@milaboratories/build-configs": "2.0.0",
     "@milaboratories/ts-builder": "1.3.2",
+    "@milaboratories/build-configs": "2.0.0",
     "@milaboratories/ts-configs": "1.2.3"
   },
   "engines": {

package/src/pool/data.ts

@@ -9,17 +9,57 @@ import {
   type PObjectId,
   type PObjectSpec,
 } from "@platforma-sdk/model";
-import type { PlTreeEntry, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import { makeResourceSnapshot, type PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 import canonicalize from "canonicalize";
 import {
   isNullResourceId,
+  resourceIdToString,
   resourceType,
   resourceTypeToString,
   resourceTypesEqual,
+  type ResourceId,
+  type ResourceType,
 } from "@milaboratories/pl-client";
 import type { Writable } from "utility-types";
 import { createHash } from "node:crypto";
 import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
+import { OnDemandBlobResourceSnapshot } from "@milaboratories/pl-drivers";
+
+/**
+ * Tree-independent reference to a blob resource used by the PFrame data flow.
+ *
+ * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;
+ * resolution then went back through the originating tree, so when that tree dropped
+ * the resource (e.g., a project recalculated with new settings), shared pool entries
+ * — held alive by other projects — would start failing with "resource not found in
+ * the tree" even though the underlying blob was still valid backend-side.
+ *
+ * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed
+ * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the
+ * download driver with the snapshot directly — independent of any specific tree.
+ */
+export class BlobResourceRef {
+  constructor(
+    public readonly resourceInfo: { readonly id: ResourceId; readonly type: ResourceType },
+    /** Present only for on-demand (remote) blobs; needed for size and signed handle. */
+    public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,
+  ) {}
+
+  toJSON(): string {
+    return resourceIdToString(this.resourceInfo.id);
+  }
+}
+
+export function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {
+  return new BlobResourceRef(accessor.resourceInfo, undefined);
+}
+
+function makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {
+  return new BlobResourceRef(
+    accessor.resourceInfo,
+    makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),
+  );
+}
 
 export const PColumnDataJsonPartitioned = resourceType("PColumnData/JsonPartitioned", "1");
 export const PColumnDataJsonSuperPartitioned = resourceType(
@@ -59,7 +99,7 @@ const BinaryPartitionedValuesFieldSuffix = ".values";
 
 export function parseDataInfoResource(
   data: PlTreeNodeAccessor,
-): PFrameInternal.DataInfo<PlTreeEntry> {
+): PFrameInternal.DataInfo<BlobResourceRef> {
   if (!data.getIsReadyOrError()) throw new PFrameDriverError("Data not ready.");
 
   const resourceData = data.getDataAsJson();
@@ -80,7 +120,10 @@ export function parseDataInfoResource(
     const parts = Object.fromEntries(
       data
         .listInputFields()
-        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).persist()]),
+        .map((field) => [
+          field,
+          makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),
+        ]),
     );
 
     return {
@@ -91,7 +134,7 @@ export function parseDataInfoResource(
   } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {
     const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
 
-    const parts: Record<string, PlTreeEntry> = {};
+    const parts: Record<string, BlobResourceRef> = {};
     for (const superKey of data.listInputFields()) {
       const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });
       const keys = superPart.listInputFields();
@@ -103,7 +146,9 @@ export function parseDataInfoResource(
           ...(JSON.parse(superKey) as PColumnValue[]),
           ...(JSON.parse(key) as PColumnValue[]),
         ]);
-        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).persist();
+        parts[partKey] = makeLocalBlobRef(
+          superPart.traverse({ field: key, errorIfFieldNotSet: true }),
+        );
       }
     }
 
@@ -115,7 +160,7 @@ export function parseDataInfoResource(
   } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {
     const meta = resourceData as PColumnDataPartitionedResourceValue;
 
-    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};
+    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};
 
     // parsing the structure
     for (const field of data.listInputFields()) {
@@ -126,7 +171,7 @@ export function parseDataInfoResource(
           part = {};
           parts[partKey] = part;
         }
-        part.index = data.traverse({ field, errorIfFieldNotSet: true }).persist();
+        part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));
       } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
         const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);
         let part = parts[partKey];
@@ -134,7 +179,7 @@ export function parseDataInfoResource(
           part = {};
           parts[partKey] = part;
         }
-        part.values = data.traverse({ field, errorIfFieldNotSet: true }).persist();
+        part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));
       } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
     }
 
@@ -147,12 +192,12 @@ export function parseDataInfoResource(
     return {
       type: "BinaryPartitioned",
       partitionKeyLength: meta.partitionKeyLength,
-      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,
+      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,
     };
   } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {
     const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
 
-    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};
+    const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};
     for (const superKey of data.listInputFields()) {
       const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });
       const keys = superData.listInputFields();
@@ -172,12 +217,9 @@ export function parseDataInfoResource(
             part = {};
             parts[partKey] = part;
           }
-          parts[partKey].index = superData
-            .traverse({
-              field,
-              errorIfFieldNotSet: true,
-            })
-            .persist();
+          parts[partKey].index = makeLocalBlobRef(
+            superData.traverse({ field, errorIfFieldNotSet: true }),
+          );
         } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {
           const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);
 
@@ -190,12 +232,9 @@ export function parseDataInfoResource(
             part = {};
             parts[partKey] = part;
           }
-          parts[partKey].values = superData
-            .traverse({
-              field,
-              errorIfFieldNotSet: true,
-            })
-            .persist();
+          parts[partKey].values = makeLocalBlobRef(
+            superData.traverse({ field, errorIfFieldNotSet: true }),
+          );
         } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);
       }
     }
@@ -203,12 +242,12 @@ export function parseDataInfoResource(
     return {
       type: "BinaryPartitioned",
       partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
-      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,
+      parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,
     };
   } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {
     const meta = resourceData as PColumnDataPartitionedResourceValue;
 
-    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};
+    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};
     for (const key of data.listInputFields()) {
       const resource = data.traverse({
         field: key,
@@ -227,7 +266,7 @@ export function parseDataInfoResource(
   } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {
     const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
 
-    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};
+    const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};
     for (const superKey of data.listInputFields()) {
       const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });
       const keys = superPart.listInputFields();
@@ -259,7 +298,7 @@ export function parseDataInfoResource(
 
 export function traverseParquetChunkResource(
   resource: PlTreeNodeAccessor,
-): ParquetChunk<PlTreeEntry> {
+): ParquetChunk<BlobResourceRef> {
   if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {
     throw new PFrameDriverError(
       `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +
@@ -267,9 +306,9 @@ export function traverseParquetChunkResource(
     );
   }
 
-  const blob = resource
-    .traverse({ field: "blob", assertFieldType: "Service", errorIfFieldNotSet: true })
-    .persist();
+  const blob = makeRemoteBlobRef(
+    resource.traverse({ field: "blob", assertFieldType: "Service", errorIfFieldNotSet: true }),
+  );
   const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;
   const mapping = resource
     .traverse({ field: "mapping", assertFieldType: "Service", errorIfFieldNotSet: true })

package/src/pool/driver.ts

@@ -15,11 +15,7 @@ import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
 import { emptyDir } from "@milaboratories/ts-helpers";
 import { RefCountPoolBase, type PoolEntry, type MiLogger } from "@milaboratories/helpers";
 import type { DownloadDriver } from "@milaboratories/pl-drivers";
-import {
-  isPlTreeNodeAccessor,
-  type PlTreeEntry,
-  type PlTreeNodeAccessor,
-} from "@milaboratories/pl-tree";
+import { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 import type { Computable, ComputableStableDefined } from "@milaboratories/computable";
 import {
   makeJsonDataInfo,
@@ -33,17 +29,22 @@ import {
 import { HttpHelpers } from "@milaboratories/pframes-rs-node";
 import path from "node:path";
 import { Readable } from "node:stream";
-import { parseDataInfoResource, traverseParquetChunkResource } from "./data";
+import {
+  BlobResourceRef,
+  makeLocalBlobRef,
+  parseDataInfoResource,
+  traverseParquetChunkResource,
+} from "./data";
 import { isDownloadNetworkError400 } from "@milaboratories/pl-drivers";
 
-function makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {
-  return String(res.rid) as PFrameInternal.PFrameBlobId;
+function makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {
+  return String(res.resourceInfo.id) as PFrameInternal.PFrameBlobId;
 }
 
 type LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;
 class LocalBlobProviderImpl
-  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>
-  implements LocalBlobProvider<PlTreeEntry>
+  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>
+  implements LocalBlobProvider<BlobResourceRef>
 {
   constructor(
     private readonly blobDriver: DownloadDriver,
@@ -52,12 +53,15 @@ class LocalBlobProviderImpl
     super();
   }
 
-  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {
+  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {
     return makeBlobId(params);
   }
 
-  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {
-    return this.blobDriver.getDownloadedBlob(params);
+  protected createNewResource(
+    params: BlobResourceRef,
+    _key: PFrameInternal.PFrameBlobId,
+  ): LocalBlob {
+    return this.blobDriver.getDownloadedBlob(params.resourceInfo);
   }
 
   public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {
@@ -90,7 +94,7 @@ class LocalBlobProviderImpl
 
 type RemoteBlob = Computable<RemoteBlobHandleAndSize>;
 class RemoteBlobPool extends RefCountPoolBase<
-  PlTreeEntry,
+  BlobResourceRef,
   PFrameInternal.PFrameBlobId,
   RemoteBlob
 > {
@@ -101,12 +105,21 @@ class RemoteBlobPool extends RefCountPoolBase<
     super();
   }
 
-  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {
+  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {
     return makeBlobId(params);
   }
 
-  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {
-    return this.blobDriver.getOnDemandBlob(params);
+  protected createNewResource(
+    params: BlobResourceRef,
+    _key: PFrameInternal.PFrameBlobId,
+  ): RemoteBlob {
+    if (params.onDemandSnapshot === undefined) {
+      throw new PFrameDriverError(
+        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +
+          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,
+      );
+    }
+    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);
   }
 
   public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {
@@ -232,7 +245,7 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
   }
 }
 
-class RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {
+class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
   constructor(
     private readonly pool: RemoteBlobPool,
     private readonly server: PFrameInternal.HttpServer,
@@ -253,7 +266,7 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {
     return new RemoteBlobProviderImpl(pool, server);
   }
 
-  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {
+  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {
     return this.pool.acquire(params);
   }
 
@@ -302,7 +315,7 @@ export async function createPFrameDriver(params: {
       : isDataInfo(data)
         ? data.type === "ParquetPartitioned"
           ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))
-          : mapDataInfo(data, (a) => a.persist())
+          : mapDataInfo(data, (a) => makeLocalBlobRef(a))
         : makeJsonDataInfo(spec, data);
   };