@milaboratories/pl-middle-layer 1.60.4 → 1.60.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/pool/data.cjs +1 -1
  2. package/dist/pool/data.cjs.map +1 -1
  3. package/dist/pool/data.js +1 -1
  4. package/dist/pool/data.js.map +1 -1
  5. package/dist/pool/driver.cjs +2 -1
  6. package/dist/pool/driver.cjs.map +1 -1
  7. package/dist/pool/driver.js +2 -1
  8. package/dist/pool/driver.js.map +1 -1
  9. package/package.json +8 -8
  10. package/src/pool/data.ts +2 -2
  11. package/src/pool/driver.ts +9 -7
package/dist/pool/data.cjs CHANGED
@@ -46,7 +46,7 @@ const ParquetChunkResourceType = (0, _milaboratories_pl_client.resourceType)("Pa
46
46
  const BinaryPartitionedIndexFieldSuffix = ".index";
47
47
  const BinaryPartitionedValuesFieldSuffix = ".values";
48
48
  function parseDataInfoResource(data) {
49
- if (!data.getIsReadyOrError()) throw new _platforma_sdk_model.PFrameDriverError("Data not ready.");
49
+ if (!data.getIsReadyOrError()) return void 0;
50
50
  const resourceData = data.getDataAsJson();
51
51
  if (resourceData === void 0) throw new _platforma_sdk_model.PFrameDriverError("unexpected data info structure, no resource data");
52
52
  if ((0, _milaboratories_pl_client.resourceTypesEqual)(data.resourceType, PColumnDataJson)) {
package/dist/pool/data.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data.cjs","names":["OnDemandBlobResourceSnapshot","PFrameDriverError"],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n PFrameDriverError,\n type BinaryChunk,\n type ParquetChunk,\n type ParquetChunkMapping,\n type ParquetChunkMetadata,\n type PColumnValue,\n type PlRef,\n type PObjectId,\n type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n anyResourceIdToBigint,\n isNullSignedResourceId,\n resourceIdToString,\n resourceType,\n resourceTypeToString,\n resourceTypesEqual,\n type SignedResourceId,\n type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n constructor(\n public readonly resourceInfo: { readonly id: SignedResourceId; readonly type: ResourceType },\n /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n ) {}\n\n toJSON(): string {\n return resourceIdToString(this.resourceInfo.id);\n }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(\n accessor.resourceInfo,\n makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/JsonPartitioned\",\n \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n \"PColumnData/Partitioned/BinaryPartitioned\",\n \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/ParquetPartitioned\",\n \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n keyLength: number;\n data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n superPartitionKeyLength: number;\n partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<BlobResourceRef> {\n if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n const resourceData = data.getDataAsJson();\n if (resourceData === undefined)\n throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n return {\n type: \"Json\",\n keyLength: dataContent.keyLength,\n data: dataContent.data,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts = Object.fromEntries(\n data\n .listInputFields()\n .map((field) => [\n field,\n makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n ]),\n );\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, BlobResourceRef> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = makeLocalBlobRef(\n superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n );\n }\n }\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n // parsing the structure\n for (const field of data.listInputFields()) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n\n // structure validation\n for (const [key, part] of Object.entries(parts)) {\n if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n for (const superKey of data.listInputFields()) {\n const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superData.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const field of keys) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].index = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].values = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const key of data.listInputFields()) {\n const resource = data.traverse({\n field: key,\n assertFieldType: \"Input\",\n errorIfFieldNotSet: true,\n });\n\n parts[key] = traverseParquetChunkResource(resource);\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = traverseParquetChunkResource(resource);\n }\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n }\n\n throw new PFrameDriverError(\n `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n );\n}\n\nexport function traverseParquetChunkResource(\n resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n throw new PFrameDriverError(\n `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n );\n }\n\n const blob = makeRemoteBlobRef(\n resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n );\n const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n const mapping = resource\n .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n .getDataAsJson() as ParquetChunkMapping;\n\n return {\n data: blob,\n ...partInfo,\n ...mapping,\n };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n const hash = createHash(\"sha256\");\n hash.update(canonicalize(spec)!);\n const rid = !isNullSignedResourceId(data.originalId) ? data.originalId : data.id;\n hash.update(String(anyResourceIdToBigint(rid)));\n return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAyCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,UAAA,GAAA,0BAAA,oBAA0B,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,eAAA,GAAA,wBAAA,sBACY,UAAUA,2BAAAA,6BAA6B,CAC7D;;AAGH,MAAa,8BAAA,GAAA,0BAAA,cAA0C,+BAA+B,IAAI;AAC1F,MAAa,mCAAA,GAAA,0BAAA,cACX,2CACA,IACD;AACD,MAAa,gCAAA,GAAA,0BAAA,cAA4C,iCAAiC,IAAI;AAC9F,MAAa,qCAAA,GAAA,0BAAA,cACX,6CACA,IACD;AACD,MAAa,iCAAA,GAAA,0BAAA,cAA6C,kCAAkC,IAAI;AAChG,MAAa,sCAAA,GAAA,0BAAA,cACX,8CACA,IACD;AACD,MAAa,mBAAA,GAAA,0BAAA,cAA+B,oBAAoB,IAAI;AAEpE,MAAa,4BAAA,GAAA,0BAAA,cAAwC,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MAC0C;AAC1C,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAIC,qBAAAA,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAIA,qBAAAA,kBAAkB,mDAAmD;AAEjF,MAAA,GAAA,0BAAA,oBAAuB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;8DAC2B,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;8DAC2B,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;8DAC2B,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;8DAC2B,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAIA,qBAAAA,kBACR,+BAAA,GAAA,0BAAA,sBAAmD,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,EAAA,GAAA,0BAAA,oBAAoB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAIA,qBAAAA,kBACR,2BAAA,GAAA,0BAAA,sBAA+C,SAAS,aAAa,CAAC,eAAA,GAAA,0BAAA,sBAClC,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,QAAA,GAAA,YAAA,YAAkB,SAAS;AACjC,MAAK,QAAA,GAAA,aAAA,SAAoB,KAAK,CAAE;CAChC,MAAM,MAAM,EAAA,GAAA,0BAAA,wBAAwB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK;AAC9E,MAAK,OAAO,QAAA,GAAA,0BAAA,uBAA6B,IAAI,CAAC,CAAC;AAC/C,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,SAAA,GAAA,aAAA,SAAoB;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,SAAA,GAAA,aAAA,SAAoB;EAAE;EAAa,MAAM;EAAY,CAAC"}
1
+ {"version":3,"file":"data.cjs","names":["OnDemandBlobResourceSnapshot","PFrameDriverError"],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n PFrameDriverError,\n type BinaryChunk,\n type ParquetChunk,\n type ParquetChunkMapping,\n type ParquetChunkMetadata,\n type PColumnValue,\n type PlRef,\n type PObjectId,\n type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n anyResourceIdToBigint,\n isNullSignedResourceId,\n resourceIdToString,\n resourceType,\n resourceTypeToString,\n resourceTypesEqual,\n type SignedResourceId,\n type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n constructor(\n public readonly resourceInfo: { readonly id: SignedResourceId; readonly type: ResourceType },\n /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n ) {}\n\n toJSON(): string {\n return resourceIdToString(this.resourceInfo.id);\n }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(\n accessor.resourceInfo,\n makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/JsonPartitioned\",\n \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n \"PColumnData/Partitioned/BinaryPartitioned\",\n \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/ParquetPartitioned\",\n \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n keyLength: number;\n data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n superPartitionKeyLength: number;\n partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n data: PlTreeNodeAccessor,\n): undefined | PFrameInternal.DataInfo<BlobResourceRef> {\n if (!data.getIsReadyOrError()) return undefined;\n\n const resourceData = data.getDataAsJson();\n if (resourceData === undefined)\n throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n return {\n type: \"Json\",\n keyLength: dataContent.keyLength,\n data: dataContent.data,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts = Object.fromEntries(\n data\n .listInputFields()\n .map((field) => [\n field,\n makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n ]),\n );\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, BlobResourceRef> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = makeLocalBlobRef(\n superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n );\n }\n }\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n // parsing the structure\n for (const field of data.listInputFields()) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n\n // structure validation\n for (const [key, part] of Object.entries(parts)) {\n if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n for (const superKey of data.listInputFields()) {\n const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superData.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const field of keys) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].index = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].values = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const key of data.listInputFields()) {\n const resource = data.traverse({\n field: key,\n assertFieldType: \"Input\",\n errorIfFieldNotSet: true,\n });\n\n parts[key] = traverseParquetChunkResource(resource);\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = traverseParquetChunkResource(resource);\n }\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n }\n\n throw new PFrameDriverError(\n `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n );\n}\n\nexport function traverseParquetChunkResource(\n resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n throw new PFrameDriverError(\n `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n );\n }\n\n const blob = makeRemoteBlobRef(\n resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n );\n const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n const mapping = resource\n .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n .getDataAsJson() as ParquetChunkMapping;\n\n return {\n data: blob,\n ...partInfo,\n ...mapping,\n };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n const hash = createHash(\"sha256\");\n hash.update(canonicalize(spec)!);\n const rid = !isNullSignedResourceId(data.originalId) ? data.originalId : data.id;\n hash.update(String(anyResourceIdToBigint(rid)));\n return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAyCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,UAAA,GAAA,0BAAA,oBAA0B,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,eAAA,GAAA,wBAAA,sBACY,UAAUA,2BAAAA,6BAA6B,CAC7D;;AAGH,MAAa,8BAAA,GAAA,0BAAA,cAA0C,+BAA+B,IAAI;AAC1F,MAAa,mCAAA,GAAA,0BAAA,cACX,2CACA,IACD;AACD,MAAa,gCAAA,GAAA,0BAAA,cAA4C,iCAAiC,IAAI;AAC9F,MAAa,qCAAA,GAAA,0BAAA,cACX,6CACA,IACD;AACD,MAAa,iCAAA,GAAA,0BAAA,cAA6C,kCAAkC,IAAI;AAChG,MAAa,sCAAA,GAAA,0BAAA,cACX,8CACA,IACD;AACD,MAAa,mBAAA,GAAA,0BAAA,cAA+B,oBAAoB,IAAI;AAEpE,MAAa,4BAAA,GAAA,0BAAA,cAAwC,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MACsD;AACtD,KAAI,CAAC,KAAK,mBAAmB,CAAE,QAAO,KAAA;CAEtC,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAIC,qBAAAA,kBAAkB,mDAAmD;AAEjF,MAAA,GAAA,0BAAA,oBAAuB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;8DAC2B,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;8DAC2B,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAIA,qBAAAA,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;8DAC2B,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAIA,qBAAAA,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;8DAC2B,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;8DAC2B,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAIA,qBAAAA,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAIA,qBAAAA,kBACR,+BAAA,GAAA,0BAAA,sBAAmD,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,EAAA,GAAA,0BAAA,oBAAoB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAIA,qBAAAA,kBACR,2BAAA,GAAA,0BAAA,sBAA+C,SAAS,aAAa,CAAC,eAAA,GAAA,0BAAA,sBAClC,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,QAAA,GAAA,YAAA,YAAkB,SAAS;AACjC,MAAK,QAAA,GAAA,aAAA,SAAoB,KAAK,CAAE;CAChC,MAAM,MAAM,EAAA,GAAA,0BAAA,wBAAwB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK;AAC9E,MAAK,OAAO,QAAA,GAAA,0BAAA,uBAA6B,IAAI,CAAC,CAAC;AAC/C,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,SAAA,GAAA,aAAA,SAAoB;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,SAAA,GAAA,aAAA,SAAoB;EAAE;EAAa,MAAM;EAAY,CAAC"}
package/dist/pool/data.js CHANGED
@@ -44,7 +44,7 @@ const ParquetChunkResourceType = resourceType("ParquetChunk", "1");
44
44
  const BinaryPartitionedIndexFieldSuffix = ".index";
45
45
  const BinaryPartitionedValuesFieldSuffix = ".values";
46
46
  function parseDataInfoResource(data) {
47
- if (!data.getIsReadyOrError()) throw new PFrameDriverError("Data not ready.");
47
+ if (!data.getIsReadyOrError()) return void 0;
48
48
  const resourceData = data.getDataAsJson();
49
49
  if (resourceData === void 0) throw new PFrameDriverError("unexpected data info structure, no resource data");
50
50
  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {
package/dist/pool/data.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data.js","names":[],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n PFrameDriverError,\n type BinaryChunk,\n type ParquetChunk,\n type ParquetChunkMapping,\n type ParquetChunkMetadata,\n type PColumnValue,\n type PlRef,\n type PObjectId,\n type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n anyResourceIdToBigint,\n isNullSignedResourceId,\n resourceIdToString,\n resourceType,\n resourceTypeToString,\n resourceTypesEqual,\n type SignedResourceId,\n type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n constructor(\n public readonly resourceInfo: { readonly id: SignedResourceId; readonly type: ResourceType },\n /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n ) {}\n\n toJSON(): string {\n return resourceIdToString(this.resourceInfo.id);\n }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(\n accessor.resourceInfo,\n makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/JsonPartitioned\",\n \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n \"PColumnData/Partitioned/BinaryPartitioned\",\n \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/ParquetPartitioned\",\n \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n keyLength: number;\n data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n superPartitionKeyLength: number;\n partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<BlobResourceRef> {\n if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n const resourceData = data.getDataAsJson();\n if (resourceData === undefined)\n throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n return {\n type: \"Json\",\n keyLength: dataContent.keyLength,\n data: dataContent.data,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts = Object.fromEntries(\n data\n .listInputFields()\n .map((field) => [\n field,\n makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n ]),\n );\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, BlobResourceRef> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = makeLocalBlobRef(\n superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n );\n }\n }\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n // parsing the structure\n for (const field of data.listInputFields()) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n\n // structure validation\n for (const [key, part] of Object.entries(parts)) {\n if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n for (const superKey of data.listInputFields()) {\n const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superData.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const field of keys) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].index = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].values = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const key of data.listInputFields()) {\n const resource = data.traverse({\n field: key,\n assertFieldType: \"Input\",\n errorIfFieldNotSet: true,\n });\n\n parts[key] = traverseParquetChunkResource(resource);\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = traverseParquetChunkResource(resource);\n }\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n }\n\n throw new PFrameDriverError(\n `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n );\n}\n\nexport function traverseParquetChunkResource(\n resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n throw new PFrameDriverError(\n `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n );\n }\n\n const blob = makeRemoteBlobRef(\n resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n );\n const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n const mapping = resource\n .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n .getDataAsJson() as ParquetChunkMapping;\n\n return {\n data: blob,\n ...partInfo,\n ...mapping,\n };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n const hash = createHash(\"sha256\");\n hash.update(canonicalize(spec)!);\n const rid = !isNullSignedResourceId(data.originalId) ? data.originalId : data.id;\n hash.update(String(anyResourceIdToBigint(rid)));\n return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,SAAO,mBAAmB,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,cACT,qBAAqB,UAAU,6BAA6B,CAC7D;;AAGH,MAAa,6BAA6B,aAAa,+BAA+B,IAAI;AAC1F,MAAa,kCAAkC,aAC7C,2CACA,IACD;AACD,MAAa,+BAA+B,aAAa,iCAAiC,IAAI;AAC9F,MAAa,oCAAoC,aAC/C,6CACA,IACD;AACD,MAAa,gCAAgC,aAAa,kCAAkC,IAAI;AAChG,MAAa,qCAAqC,aAChD,8CACA,IACD;AACD,MAAa,kBAAkB,aAAa,oBAAoB,IAAI;AAEpE,MAAa,2BAA2B,aAAa,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MAC0C;AAC1C,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAI,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAI,kBAAkB,mDAAmD;AAEjF,KAAI,mBAAmB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;YACQ,mBAAmB,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;YACQ,mBAAmB,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAI,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAI,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;YACQ,mBAAmB,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;YACQ,mBAAmB,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAI,kBACR,8BAA8B,qBAAqB,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,CAAC,mBAAmB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAI,kBACR,0BAA0B,qBAAqB,SAAS,aAAa,CAAC,cACvD,qBAAqB,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,OAAO,WAAW,SAAS;AACjC,MAAK,OAAO,aAAa,KAAK,CAAE;CAChC,MAAM,MAAM,CAAC,uBAAuB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK;AAC9E,MAAK,OAAO,OAAO,sBAAsB,IAAI,CAAC,CAAC;AAC/C,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,QAAO,aAAa;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,QAAO,aAAa;EAAE;EAAa,MAAM;EAAY,CAAC"}
1
+ {"version":3,"file":"data.js","names":[],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n PFrameDriverError,\n type BinaryChunk,\n type ParquetChunk,\n type ParquetChunkMapping,\n type ParquetChunkMetadata,\n type PColumnValue,\n type PlRef,\n type PObjectId,\n type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport { makeResourceSnapshot, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n anyResourceIdToBigint,\n isNullSignedResourceId,\n resourceIdToString,\n resourceType,\n resourceTypeToString,\n resourceTypesEqual,\n type SignedResourceId,\n type ResourceType,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { OnDemandBlobResourceSnapshot } from \"@milaboratories/pl-drivers\";\n\n/**\n * Tree-independent reference to a blob resource used by the PFrame data flow.\n *\n * The earlier design carried a {@link PlTreeEntry} all the way into the blob pools;\n * resolution then went back through the originating tree, so when that tree dropped\n * the resource (e.g., a project recalculated with new settings), shared pool entries\n * — held alive by other projects — would start failing with \"resource not found in\n * the tree\" even though the underlying blob was still valid backend-side.\n *\n * BlobResourceRef captures the snapshot at parse time, where the tree is guaranteed\n * to resolve. The pools then key by rid (`resourceInfo.id`) and call into the\n * download driver with the snapshot directly — independent of any specific tree.\n */\nexport class BlobResourceRef {\n constructor(\n public readonly resourceInfo: { readonly id: SignedResourceId; readonly type: ResourceType },\n /** Present only for on-demand (remote) blobs; needed for size and signed handle. */\n public readonly onDemandSnapshot: OnDemandBlobResourceSnapshot | undefined,\n ) {}\n\n toJSON(): string {\n return resourceIdToString(this.resourceInfo.id);\n }\n}\n\nexport function makeLocalBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(accessor.resourceInfo, undefined);\n}\n\nfunction makeRemoteBlobRef(accessor: PlTreeNodeAccessor): BlobResourceRef {\n return new BlobResourceRef(\n accessor.resourceInfo,\n makeResourceSnapshot(accessor, OnDemandBlobResourceSnapshot),\n );\n}\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/JsonPartitioned\",\n \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n \"PColumnData/Partitioned/BinaryPartitioned\",\n \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n \"PColumnData/Partitioned/ParquetPartitioned\",\n \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n keyLength: number;\n data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n superPartitionKeyLength: number;\n partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n data: PlTreeNodeAccessor,\n): undefined | PFrameInternal.DataInfo<BlobResourceRef> {\n if (!data.getIsReadyOrError()) return undefined;\n\n const resourceData = data.getDataAsJson();\n if (resourceData === undefined)\n throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n return {\n type: \"Json\",\n keyLength: dataContent.keyLength,\n data: dataContent.data,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts = Object.fromEntries(\n data\n .listInputFields()\n .map((field) => [\n field,\n makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true })),\n ]),\n );\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, BlobResourceRef> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = makeLocalBlobRef(\n superPart.traverse({ field: key, errorIfFieldNotSet: true }),\n );\n }\n }\n\n return {\n type: \"JsonPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n\n // parsing the structure\n for (const field of data.listInputFields()) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.index = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n part.values = makeLocalBlobRef(data.traverse({ field, errorIfFieldNotSet: true }));\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n\n // structure validation\n for (const [key, part] of Object.entries(parts)) {\n if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, Partial<Writable<BinaryChunk<BlobResourceRef>>>> = {};\n for (const superKey of data.listInputFields()) {\n const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superData.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const field of keys) {\n if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].index = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n let part = parts[partKey];\n if (part === undefined) {\n part = {};\n parts[partKey] = part;\n }\n parts[partKey].values = makeLocalBlobRef(\n superData.traverse({ field, errorIfFieldNotSet: true }),\n );\n } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n }\n }\n\n return {\n type: \"BinaryPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts: parts as Record<string, BinaryChunk<BlobResourceRef>>,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const key of data.listInputFields()) {\n const resource = data.traverse({\n field: key,\n assertFieldType: \"Input\",\n errorIfFieldNotSet: true,\n });\n\n parts[key] = traverseParquetChunkResource(resource);\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.partitionKeyLength,\n parts,\n };\n } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n const parts: Record<string, ParquetChunk<BlobResourceRef>> = {};\n for (const superKey of data.listInputFields()) {\n const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n const keys = superPart.listInputFields();\n if (keys === undefined)\n throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n for (const key of keys) {\n const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n const partKey = JSON.stringify([\n ...(JSON.parse(superKey) as PColumnValue[]),\n ...(JSON.parse(key) as PColumnValue[]),\n ]);\n parts[partKey] = traverseParquetChunkResource(resource);\n }\n }\n\n return {\n type: \"ParquetPartitioned\",\n partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n parts,\n };\n }\n\n throw new PFrameDriverError(\n `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n );\n}\n\nexport function traverseParquetChunkResource(\n resource: PlTreeNodeAccessor,\n): ParquetChunk<BlobResourceRef> {\n if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n throw new PFrameDriverError(\n `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n );\n }\n\n const blob = makeRemoteBlobRef(\n resource.traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true }),\n );\n const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n const mapping = resource\n .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n .getDataAsJson() as ParquetChunkMapping;\n\n return {\n data: blob,\n ...partInfo,\n ...mapping,\n };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n const hash = createHash(\"sha256\");\n hash.update(canonicalize(spec)!);\n const rid = !isNullSignedResourceId(data.originalId) ? data.originalId : data.id;\n hash.update(String(anyResourceIdToBigint(rid)));\n return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,IAAa,kBAAb,MAA6B;CAC3B,YACE,cAEA,kBACA;AAHgB,OAAA,eAAA;AAEA,OAAA,mBAAA;;CAGlB,SAAiB;AACf,SAAO,mBAAmB,KAAK,aAAa,GAAG;;;AAInD,SAAgB,iBAAiB,UAA+C;AAC9E,QAAO,IAAI,gBAAgB,SAAS,cAAc,KAAA,EAAU;;AAG9D,SAAS,kBAAkB,UAA+C;AACxE,QAAO,IAAI,gBACT,SAAS,cACT,qBAAqB,UAAU,6BAA6B,CAC7D;;AAGH,MAAa,6BAA6B,aAAa,+BAA+B,IAAI;AAC1F,MAAa,kCAAkC,aAC7C,2CACA,IACD;AACD,MAAa,+BAA+B,aAAa,iCAAiC,IAAI;AAC9F,MAAa,oCAAoC,aAC/C,6CACA,IACD;AACD,MAAa,gCAAgC,aAAa,kCAAkC,IAAI;AAChG,MAAa,qCAAqC,aAChD,8CACA,IACD;AACD,MAAa,kBAAkB,aAAa,oBAAoB,IAAI;AAEpE,MAAa,2BAA2B,aAAa,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MACsD;AACtD,KAAI,CAAC,KAAK,mBAAmB,CAAE,QAAO,KAAA;CAEtC,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,KAAA,EACnB,OAAM,IAAI,kBAAkB,mDAAmD;AAEjF,KAAI,mBAAmB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;YACQ,mBAAmB,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CACd,OACA,iBAAiB,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,CACrE,CAAC,CACL;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAyC,EAAE;AACjD,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,iBACf,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAC7D;;;AAIL,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;YACQ,mBAAmB,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AAGjF,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;aACxE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,KAAA,GAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,iBAAiB,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC;QAC7E,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,KAAA,EAAW,OAAM,IAAI,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,KAAA,EAAW,OAAM,IAAI,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;YACQ,mBAAmB,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAyE,EAAE;AACjF,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,iBACrB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;cACQ,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,KAAA,GAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,iBACtB,UAAU,SAAS;KAAE;KAAO,oBAAoB;KAAM,CAAC,CACxD;SACI,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;YACQ,mBAAmB,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAuD,EAAE;AAC/D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,KAAA,EACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAI,kBACR,8BAA8B,qBAAqB,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC+B;AAC/B,KAAI,CAAC,mBAAmB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAI,kBACR,0BAA0B,qBAAqB,SAAS,aAAa,CAAC,cACvD,qBAAqB,yBAAyB,GAC9D;CAGH,MAAM,OAAO,kBACX,SAAS,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CAC3F;CACD,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,OAAO,WAAW,SAAS;AACjC,MAAK,OAAO,aAAa,KAAK,CAAE;CAChC,MAAM,MAAM,CAAC,uBAAuB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK;AAC9E,MAAK,OAAO,OAAO,sBAAsB,IAAI,CAAC,CAAC;AAC/C,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,QAAO,aAAa;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,QAAO,aAAa;EAAE;EAAa,MAAM;EAAY,CAAC"}
package/dist/pool/driver.cjs CHANGED
@@ -176,7 +176,8 @@ async function createPFrameDriver(params) {
176
176
  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
177
177
  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
178
178
  const resolveDataInfo = (spec, data) => {
179
- return (0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data) ? require_data.parseDataInfoResource(data) : (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.makeLocalBlobRef(a)) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);
179
+ if ((0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data)) return require_data.parseDataInfoResource(data) ?? (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, []);
180
+ return (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.makeLocalBlobRef(a)) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);
180
181
  };
181
182
  return new _milaboratories_pf_driver.AbstractPFrameDriver({
182
183
  logger,
package/dist/pool/driver.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,UAAA,GAAA,wBAAA,sBAA4B,KAAK,GAC7BC,aAAAA,sBAAsB,KAAK,IAAA,GAAA,qBAAA,YAChB,KAAK,GACd,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGpC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
1
+ {"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
package/dist/pool/driver.js CHANGED
@@ -174,7 +174,8 @@ async function createPFrameDriver(params) {
174
174
  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
175
175
  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
176
176
  const resolveDataInfo = (spec, data) => {
177
- return isPlTreeNodeAccessor(data) ? parseDataInfoResource(data) : isDataInfo(data) ? data.type === "ParquetPartitioned" ? mapDataInfo(data, (a) => traverseParquetChunkResource(a)) : mapDataInfo(data, (a) => makeLocalBlobRef(a)) : makeJsonDataInfo(spec, data);
177
+ if (isPlTreeNodeAccessor(data)) return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);
178
+ return isDataInfo(data) ? data.type === "ParquetPartitioned" ? mapDataInfo(data, (a) => traverseParquetChunkResource(a)) : mapDataInfo(data, (a) => makeLocalBlobRef(a)) : makeJsonDataInfo(spec, data);
178
179
  };
179
180
  return new AbstractPFrameDriver({
180
181
  logger,
package/dist/pool/driver.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,SAAO,qBAAqB,KAAK,GAC7B,sBAAsB,KAAK,GAC3B,WAAW,KAAK,GACd,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGpC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
1
+ {"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-middle-layer",
3
- "version": "1.60.4",
3
+ "version": "1.60.5",
4
4
  "description": "Pl Middle Layer",
5
5
  "keywords": [],
6
6
  "license": "UNLICENSED",
@@ -32,21 +32,21 @@
32
32
  "zod": "~3.25.76",
33
33
  "@milaboratories/computable": "2.9.4",
34
34
  "@milaboratories/pf-spec-driver": "1.3.16",
35
- "@milaboratories/helpers": "1.14.2",
36
- "@milaboratories/pf-driver": "1.4.11",
37
35
  "@milaboratories/pl-drivers": "1.14.8",
38
- "@milaboratories/pl-client": "3.6.0",
39
36
  "@milaboratories/pl-deployments": "2.17.18",
37
+ "@milaboratories/pf-driver": "1.4.11",
38
+ "@milaboratories/helpers": "1.14.2",
39
+ "@milaboratories/pl-client": "3.6.0",
40
40
  "@milaboratories/pl-errors": "1.4.8",
41
41
  "@milaboratories/pl-http": "1.2.4",
42
42
  "@milaboratories/pl-model-common": "1.42.0",
43
- "@milaboratories/pl-tree": "1.11.1",
44
- "@milaboratories/pl-model-backend": "1.2.30",
45
43
  "@milaboratories/pl-model-middle-layer": "1.19.4",
46
44
  "@milaboratories/resolve-helper": "1.1.3",
47
- "@platforma-sdk/workflow-tengo": "5.24.0",
45
+ "@milaboratories/pl-model-backend": "1.2.30",
46
+ "@milaboratories/pl-tree": "1.11.1",
48
47
  "@milaboratories/ts-helpers": "1.8.2",
49
48
  "@platforma-sdk/block-tools": "2.7.25",
49
+ "@platforma-sdk/workflow-tengo": "5.24.0",
50
50
  "@platforma-sdk/model": "1.77.0"
51
51
  },
52
52
  "devDependencies": {
@@ -56,8 +56,8 @@
56
56
  "semver": "^7.7.2",
57
57
  "typescript": "~5.9.3",
58
58
  "vitest": "^4.1.3",
59
- "@milaboratories/ts-builder": "1.5.0",
60
59
  "@milaboratories/ts-configs": "1.2.3",
60
+ "@milaboratories/ts-builder": "1.5.0",
61
61
  "@milaboratories/build-configs": "2.0.0"
62
62
  },
63
63
  "engines": {
package/src/pool/data.ts CHANGED
@@ -100,8 +100,8 @@ const BinaryPartitionedValuesFieldSuffix = ".values";
100
100
 
101
101
  export function parseDataInfoResource(
102
102
  data: PlTreeNodeAccessor,
103
- ): PFrameInternal.DataInfo<BlobResourceRef> {
104
- if (!data.getIsReadyOrError()) throw new PFrameDriverError("Data not ready.");
103
+ ): undefined | PFrameInternal.DataInfo<BlobResourceRef> {
104
+ if (!data.getIsReadyOrError()) return undefined;
105
105
 
106
106
  const resourceData = data.getDataAsJson();
107
107
  if (resourceData === undefined)
package/src/pool/driver.ts CHANGED
@@ -312,13 +312,15 @@ export async function createPFrameDriver(params: {
312
312
  });
313
313
 
314
314
  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {
315
- return isPlTreeNodeAccessor(data)
316
- ? parseDataInfoResource(data)
317
- : isDataInfo(data)
318
- ? data.type === "ParquetPartitioned"
319
- ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))
320
- : mapDataInfo(data, (a) => makeLocalBlobRef(a))
321
- : makeJsonDataInfo(spec, data);
315
+ if (isPlTreeNodeAccessor(data)) {
316
+ return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);
317
+ }
318
+
319
+ return isDataInfo(data)
320
+ ? data.type === "ParquetPartitioned"
321
+ ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))
322
+ : mapDataInfo(data, (a) => makeLocalBlobRef(a))
323
+ : makeJsonDataInfo(spec, data);
322
324
  };
323
325
 
324
326
  return new AbstractPFrameDriver({