@milaboratories/pl-middle-layer 1.48.11 → 1.48.13

package/dist/pool/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport type { PlTreeEntry, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<PlTreeEntry> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).persist()]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, PlTreeEntry> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).persist();\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<PlTreeEntry> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = resource\n    .traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .persist();\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"names":[],"mappings":";;;;;AAuBO,MAAM,0BAA0B,GAAG,YAAY,CAAC,6BAA6B,EAAE,GAAG;AAClF,MAAM,+BAA+B,GAAG,YAAY,CACzD,yCAAyC,EACzC,GAAG;AAEE,MAAM,4BAA4B,GAAG,YAAY,CAAC,+BAA+B,EAAE,GAAG;AACtF,MAAM,iCAAiC,GAAG,YAAY,CAC3D,2CAA2C,EAC3C,GAAG;AAEE,MAAM,6BAA6B,GAAG,YAAY,CAAC,gCAAgC,EAAE,GAAG;AACxF,MAAM,kCAAkC,GAAG,YAAY,CAC5D,4CAA4C,EAC5C,GAAG;AAEE,MAAM,eAAe,GAAG,YAAY,CAAC,kBAAkB,EAAE,GAAG;AAE5D,MAAM,wBAAwB,GAAG,YAAY,CAAC,cAAc,EAAE,GAAG;AAgBxE,MAAM,iCAAiC,GAAG,QAAQ;AAClD,MAAM,kCAAkC,GAAG,SAAS;AAE9C,SAAU,qBAAqB,CACnC,IAAwB,EAAA;AAExB,IAAA,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;AAAE,QAAA,MAAM,IAAI,iBAAiB,CAAC,iBAAiB,CAAC;AAE7E,IAAA,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE;IACzC,IAAI,YAAY,KAAK,SAAS;AAC5B,QAAA,MAAM,IAAI,iBAAiB,CAAC,kDAAkD,CAAC;IAEjF,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,EAAE;QAC1D,MAAM,WAAW,GAAG,YAA4C;QAEhE,OAAO;AACL,YAAA,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,IAAI,EAAE,WAAW,CAAC,IAAI;SACvB;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,0BAA0B,CAAC,EAAE;QAC5E,MAAM,IAAI,GAAG,YAAmD;AAEhE,QAAA,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAC9B;AACG,aAAA,eAAe;aACf,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CACzF;QAED,OAAO;AACL,YAAA,IAAI,EAAE,iBAAiB;YACvB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,+BAA+B,CAAC,EAAE;QACjF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAAgC,EAAE;QAC7C,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AACpB,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAE5E,YAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;AACtB,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,oBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAoB;AAC3C,oBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB;AACvC,iBAAA,CAAC;gBACF,KAAK,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;YACzF;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,iBAAiB;AACvB,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;YAC1E,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,4BAA4B,CAAC,EAAE;QAC9E,MAAM,IAAI,GAAG,YAAmD;QAEhE,MAAM,KAAK,GAAgE,EAAE;;QAG7E,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC1C,YAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EAAE;AACrD,gBAAA,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,iCAAiC,CAAC,MAAM,CAAC;AACzE,gBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,gBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;oBACtB,IAAI,GAAG,EAAE;AACT,oBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;gBACvB;AACA,gBAAA,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;YAC3E;AAAO,iBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE;AAC7D,gBAAA,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,kCAAkC,CAAC,MAAM,CAAC;AAC1E,gBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,gBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;oBACtB,IAAI,GAAG,EAAE;AACT,oBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;gBACvB;AACA,gBAAA,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;YAC5E;;AAAO,gBAAA,MAAM,IAAI,iBAAiB,CAAC,iCAAiC,KAAK,CAAA,CAAE,CAAC;QAC9E;;AAGA,QAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AAC/C,YAAA,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,qBAAqB,GAAG,CAAA,CAAE,CAAC;AACrF,YAAA,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,sBAAsB,GAAG,CAAA,CAAE,CAAC;QACzF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,mBAAmB;YACzB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;AAC3C,YAAA,KAAK,EAAE,KAAiD;SACzD;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,iCAAiC,CAAC,EAAE;QACnF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAAgE,EAAE;QAC7E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AACpB,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAE5E,YAAA,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE;AACxB,gBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EAAE;AACrD,oBAAA,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,iCAAiC,CAAC,MAAM,CAAC;AAErE,oBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,wBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAoB;AAC3C,wBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB;AACvC,qBAAA,CAAC;AACF,oBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,oBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;wBACtB,IAAI,GAAG,EAAE;AACT,wBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;oBACvB;AACA,oBAAA,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,GAAG;AACpB,yBAAA,QAAQ,CAAC;wBACR,KAAK;AACL,wBAAA,kBAAkB,EAAE,IAAI;qBACzB;AACA,yBAAA,OAAO,EAAE;gBACd;AAAO,qBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE;AAC7D,oBAAA,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,kCAAkC,CAAC,MAAM,CAAC;AAEtE,oBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,wBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAoB;AAC3C,wBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB;AACvC,qBAAA,CAAC;AACF,oBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,oBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;wBACtB,IAAI,GAAG,EAAE;AACT,wBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;oBACvB;AACA,oBAAA,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG;AACrB,yBAAA,QAAQ,CAAC;wBACR,KAAK;AACL,wBAAA,kBAAkB,EAAE,IAAI;qBACzB;AACA,yBAAA,OAAO,EAAE;gBACd;;AAAO,oBAAA,MAAM,IAAI,iBAAiB,CAAC,iCAAiC,KAAK,CAAA,CAAE,CAAC;YAC9E;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,mBAAmB;AACzB,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;AAC1E,YAAA,KAAK,EAAE,KAAiD;SACzD;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,6BAA6B,CAAC,EAAE;QAC/E,MAAM,IAAI,GAAG,YAAmD;QAEhE,MAAM,KAAK,GAA8C,EAAE;QAC3D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AACxC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;AAC7B,gBAAA,KAAK,EAAE,GAAG;AACV,gBAAA,eAAe,EAAE,OAAO;AACxB,gBAAA,kBAAkB,EAAE,IAAI;AACzB,aAAA,CAAC;YAEF,KAAK,CAAC,GAAG,CAAC,GAAG,4BAA4B,CAAC,QAAQ,CAAC;QACrD;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,oBAAoB;YAC1B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,kCAAkC,CAAC,EAAE;QACpF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAA8C,EAAE;QAC3D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AACpB,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAE5E,YAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;AACtB,gBAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAExE,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,oBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAoB;AAC3C,oBAAA,GAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB;AACvC,iBAAA,CAAC;gBACF,KAAK,CAAC,OAAO,CAAC,GAAG,4BAA4B,CAAC,QAAQ,CAAC;YACzD;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,oBAAoB;AAC1B,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;YAC1E,KAAK;SACN;IACH;AAEA,IAAA,MAAM,IAAI,iBAAiB,CACzB,CAAA,2BAAA,EAA8B,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA,CAAE,CACxE;AACH;AAEM,SAAU,4BAA4B,CAC1C,QAA4B,EAAA;IAE5B,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,YAAY,EAAE,wBAAwB,CAAC,EAAE;QACxE,MAAM,IAAI,iBAAiB,CACzB,CAAA,uBAAA,EAA0B,oBAAoB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA,EAAA,CAAI;AACvE,YAAA,CAAA,UAAA,EAAa,oBAAoB,CAAC,wBAAwB,CAAC,CAAA,CAAE,CAChE;IACH;IAEA,MAAM,IAAI,GAAG;AACV,SAAA,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,IAAI,EAAE;AAChF,SAAA,OAAO,EAAE;AACZ,IAAA,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,EAA0B;IACjE,MAAM,OAAO,GAAG;AACb,SAAA,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,IAAI,EAAE;AACnF,SAAA,aAAa,EAAyB;IAEzC,OAAO;AACL,QAAA,IAAI,EAAE,IAAI;AACV,QAAA,GAAG,QAAQ;AACX,QAAA,GAAG,OAAO;KACX;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAiB,EAAE,IAAwB,EAAA;AAC/E,IAAA,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAE,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAc;AACnD;AAEM,SAAU,qBAAqB,CAAC,OAAe,EAAE,UAAkB,EAAA;AACvE,IAAA,OAAO,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAkB,CAAe;AACjG;AAEM,SAAU,oBAAoB,CAAC,WAAqB,EAAE,UAAkB,EAAA;IAC5E,OAAO,YAAY,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,CAAe;AACtE;;;;"}
+{"version":3,"file":"data.js","names":[],"sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnValue,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from \"@platforma-sdk/model\";\nimport type { PlTreeEntry, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport canonicalize from \"canonicalize\";\nimport {\n  isNullResourceId,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n} from \"@milaboratories/pl-client\";\nimport type { Writable } from \"utility-types\";\nimport { createHash } from \"node:crypto\";\nimport type { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\n\nexport const PColumnDataJsonPartitioned = resourceType(\"PColumnData/JsonPartitioned\", \"1\");\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/JsonPartitioned\",\n  \"1\",\n);\nexport const PColumnDataBinaryPartitioned = resourceType(\"PColumnData/BinaryPartitioned\", \"1\");\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/BinaryPartitioned\",\n  \"1\",\n);\nexport const PColumnDataParquetPartitioned = resourceType(\"PColumnData/ParquetPartitioned\", \"1\");\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  \"PColumnData/Partitioned/ParquetPartitioned\",\n  \"1\",\n);\nexport const PColumnDataJson = resourceType(\"PColumnData/Json\", \"1\");\n\nexport const ParquetChunkResourceType = resourceType(\"ParquetChunk\", \"1\");\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nconst BinaryPartitionedIndexFieldSuffix = \".index\";\nconst BinaryPartitionedValuesFieldSuffix = \".values\";\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<PlTreeEntry> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError(\"Data not ready.\");\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError(\"unexpected data info structure, no resource data\");\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: \"Json\",\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).persist()]),\n    );\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, PlTreeEntry> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).persist();\n      }\n    }\n\n    return {\n      type: \"JsonPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n        const partKey = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = data.traverse({ field, errorIfFieldNotSet: true }).persist();\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, Partial<Writable<BinaryChunk<PlTreeEntry>>>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith(BinaryPartitionedIndexFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedIndexFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else if (field.endsWith(BinaryPartitionedValuesFieldSuffix)) {\n          const key = field.slice(0, -BinaryPartitionedValuesFieldSuffix.length);\n\n          const partKey = JSON.stringify([\n            ...(JSON.parse(superKey) as PColumnValue[]),\n            ...(JSON.parse(key) as PColumnValue[]),\n          ]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = superData\n            .traverse({\n              field,\n              errorIfFieldNotSet: true,\n            })\n            .persist();\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: \"BinaryPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<PlTreeEntry>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({\n        field: key,\n        assertFieldType: \"Input\",\n        errorIfFieldNotSet: true,\n      });\n\n      parts[key] = traverseParquetChunkResource(resource);\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<PlTreeEntry>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined)\n        throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...(JSON.parse(superKey) as PColumnValue[]),\n          ...(JSON.parse(key) as PColumnValue[]),\n        ]);\n        parts[partKey] = traverseParquetChunkResource(resource);\n      }\n    }\n\n    return {\n      type: \"ParquetPartitioned\",\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(\n    `unsupported resource type: ${resourceTypeToString(data.resourceType)}`,\n  );\n}\n\nexport function traverseParquetChunkResource(\n  resource: PlTreeNodeAccessor,\n): ParquetChunk<PlTreeEntry> {\n  if (!resourceTypesEqual(resource.resourceType, ParquetChunkResourceType)) {\n    throw new PFrameDriverError(\n      `unknown resource type: ${resourceTypeToString(resource.resourceType)}, ` +\n        `expected: ${resourceTypeToString(ParquetChunkResourceType)}`,\n    );\n  }\n\n  const blob = resource\n    .traverse({ field: \"blob\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .persist();\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource\n    .traverse({ field: \"mapping\", assertFieldType: \"Service\", errorIfFieldNotSet: true })\n    .getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash(\"sha256\");\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString(\"hex\") as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"mappings":";;;;;;AAuBA,MAAa,6BAA6B,aAAa,+BAA+B,IAAI;AAC1F,MAAa,kCAAkC,aAC7C,2CACA,IACD;AACD,MAAa,+BAA+B,aAAa,iCAAiC,IAAI;AAC9F,MAAa,oCAAoC,aAC/C,6CACA,IACD;AACD,MAAa,gCAAgC,aAAa,kCAAkC,IAAI;AAChG,MAAa,qCAAqC,aAChD,8CACA,IACD;AACD,MAAa,kBAAkB,aAAa,oBAAoB,IAAI;AAEpE,MAAa,2BAA2B,aAAa,gBAAgB,IAAI;AAgBzE,MAAM,oCAAoC;AAC1C,MAAM,qCAAqC;AAE3C,SAAgB,sBACd,MACsC;AACtC,KAAI,CAAC,KAAK,mBAAmB,CAAE,OAAM,IAAI,kBAAkB,kBAAkB;CAE7E,MAAM,eAAe,KAAK,eAAe;AACzC,KAAI,iBAAiB,OACnB,OAAM,IAAI,kBAAkB,mDAAmD;AAEjF,KAAI,mBAAmB,KAAK,cAAc,gBAAgB,EAAE;EAC1D,MAAM,cAAc;AAEpB,SAAO;GACL,MAAM;GACN,WAAW,YAAY;GACvB,MAAM,YAAY;GACnB;YACQ,mBAAmB,KAAK,cAAc,2BAA2B,EAAE;EAC5E,MAAM,OAAO;EAEb,MAAM,QAAQ,OAAO,YACnB,KACG,iBAAiB,CACjB,KAAK,UAAU,CAAC,OAAO,KAAK,SAAS;GAAE;GAAO,oBAAoB;GAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CACzF;AAED,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,gCAAgC,EAAE;EACjF,MAAM,OAAO;EAEb,MAAM,QAAqC,EAAE;AAC7C,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,OACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,UAAU,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC,CAAC,SAAS;;;AAI3F,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;YACQ,mBAAmB,KAAK,cAAc,6BAA6B,EAAE;EAC9E,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAG7E,OAAK,MAAM,SAAS,KAAK,iBAAiB,CACxC,KAAI,MAAM,SAAS,kCAAkC,EAAE;GACrD,MAAM,UAAU,MAAM,MAAM,GAAG,GAA0C;GACzE,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,QAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,QAAQ,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;aAChE,MAAM,SAAS,mCAAmC,EAAE;GAC7D,MAAM,UAAU,MAAM,MAAM,GAAG,GAA2C;GAC1E,IAAI,OAAO,MAAM;AACjB,OAAI,SAAS,QAAW;AACtB,WAAO,EAAE;AACT,UAAM,WAAW;;AAEnB,QAAK,SAAS,KAAK,SAAS;IAAE;IAAO,oBAAoB;IAAM,CAAC,CAAC,SAAS;QACrE,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;AAI9E,OAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,EAAE;AAC/C,OAAI,KAAK,UAAU,OAAW,OAAM,IAAI,kBAAkB,qBAAqB,MAAM;AACrF,OAAI,KAAK,WAAW,OAAW,OAAM,IAAI,kBAAkB,sBAAsB,MAAM;;AAGzF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GAClB;GACR;YACQ,mBAAmB,KAAK,cAAc,kCAAkC,EAAE;EACnF,MAAM,OAAO;EAEb,MAAM,QAAqE,EAAE;AAC7E,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAC7C,MAAM,YAAY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC;GAC9E,MAAM,OAAO,UAAU,iBAAiB;AACxC,OAAI,SAAS,OACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,SAAS,KAClB,KAAI,MAAM,SAAS,kCAAkC,EAAE;IACrD,MAAM,MAAM,MAAM,MAAM,GAAG,GAA0C;IAErE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,QAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,QAAQ,UACpB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;cACH,MAAM,SAAS,mCAAmC,EAAE;IAC7D,MAAM,MAAM,MAAM,MAAM,GAAG,GAA2C;IAEtE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;IACF,IAAI,OAAO,MAAM;AACjB,QAAI,SAAS,QAAW;AACtB,YAAO,EAAE;AACT,WAAM,WAAW;;AAEnB,UAAM,SAAS,SAAS,UACrB,SAAS;KACR;KACA,oBAAoB;KACrB,CAAC,CACD,SAAS;SACP,OAAM,IAAI,kBAAkB,iCAAiC,QAAQ;;AAIhF,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACjD;GACR;YACQ,mBAAmB,KAAK,cAAc,8BAA8B,EAAE;EAC/E,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,OAAO,KAAK,iBAAiB,CAOtC,OAAM,OAAO,6BANI,KAAK,SAAS;GAC7B,OAAO;GACP,iBAAiB;GACjB,oBAAoB;GACrB,CAAC,CAEiD;AAGrD,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK;GACzB;GACD;YACQ,mBAAmB,KAAK,cAAc,mCAAmC,EAAE;EACpF,MAAM,OAAO;EAEb,MAAM,QAAmD,EAAE;AAC3D,OAAK,MAAM,YAAY,KAAK,iBAAiB,EAAE;GAE7C,MAAM,OADY,KAAK,SAAS;IAAE,OAAO;IAAU,oBAAoB;IAAM,CAAC,CACvD,iBAAiB;AACxC,OAAI,SAAS,OACX,OAAM,IAAI,kBAAkB,mCAAmC,WAAW;AAE5E,QAAK,MAAM,OAAO,MAAM;IACtB,MAAM,WAAW,KAAK,SAAS;KAAE,OAAO;KAAK,oBAAoB;KAAM,CAAC;IAExE,MAAM,UAAU,KAAK,UAAU,CAC7B,GAAI,KAAK,MAAM,SAAS,EACxB,GAAI,KAAK,MAAM,IAAI,CACpB,CAAC;AACF,UAAM,WAAW,6BAA6B,SAAS;;;AAI3D,SAAO;GACL,MAAM;GACN,oBAAoB,KAAK,0BAA0B,KAAK;GACxD;GACD;;AAGH,OAAM,IAAI,kBACR,8BAA8B,qBAAqB,KAAK,aAAa,GACtE;;AAGH,SAAgB,6BACd,UAC2B;AAC3B,KAAI,CAAC,mBAAmB,SAAS,cAAc,yBAAyB,CACtE,OAAM,IAAI,kBACR,0BAA0B,qBAAqB,SAAS,aAAa,CAAC,cACvD,qBAAqB,yBAAyB,GAC9D;CAGH,MAAM,OAAO,SACV,SAAS;EAAE,OAAO;EAAQ,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACjF,SAAS;CACZ,MAAM,WAAW,SAAS,eAAe;CACzC,MAAM,UAAU,SACb,SAAS;EAAE,OAAO;EAAW,iBAAiB;EAAW,oBAAoB;EAAM,CAAC,CACpF,eAAe;AAElB,QAAO;EACL,MAAM;EACN,GAAG;EACH,GAAG;EACJ;;AAGH,SAAgB,sBAAsB,MAAmB,MAAqC;CAC5F,MAAM,OAAO,WAAW,SAAS;AACjC,MAAK,OAAO,aAAa,KAAK,CAAE;AAChC,MAAK,OAAO,OAAO,CAAC,iBAAiB,KAAK,WAAW,GAAG,KAAK,aAAa,KAAK,GAAG,CAAC;AACnF,QAAO,KAAK,QAAQ,CAAC,SAAS,MAAM;;AAGtC,SAAgB,sBAAsB,SAAiB,YAA+B;AACpF,QAAO,aAAa;EAAE,SAAS;EAAM;EAAS,MAAM;EAAY,CAAiB;;AAGnF,SAAgB,qBAAqB,aAAuB,YAA+B;AACzF,QAAO,aAAa;EAAE;EAAa,MAAM;EAAY,CAAC"}

package/dist/pool/driver.cjs

@@ -1,217 +1,190 @@
-'use strict';
-
-var model = require('@platforma-sdk/model');
-var plModelMiddleLayer = require('@milaboratories/pl-model-middle-layer');
-var tsHelpers = require('@milaboratories/ts-helpers');
-var plTree = require('@milaboratories/pl-tree');
-var pfDriver = require('@milaboratories/pf-driver');
-var pframesRsNode = require('@milaboratories/pframes-rs-node');
-var path = require('node:path');
-var node_stream = require('node:stream');
-var data = require('./data.cjs');
-var plDrivers = require('@milaboratories/pl-drivers');
+const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
+const require_data = require('./data.cjs');
+let _platforma_sdk_model = require("@platforma-sdk/model");
+let _milaboratories_pl_model_middle_layer = require("@milaboratories/pl-model-middle-layer");
+let node_path = require("node:path");
+node_path = require_runtime.__toESM(node_path);
+let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
+let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
+let _milaboratories_pf_driver = require("@milaboratories/pf-driver");
+let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
+let node_stream = require("node:stream");
+let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
 
+//#region src/pool/driver.ts
 function makeBlobId(res) {
-    return String(res.rid);
-}
-class LocalBlobProviderImpl extends tsHelpers.RefCountPoolBase {
-    blobDriver;
-    logger;
-    constructor(blobDriver, logger) {
-        super();
-        this.blobDriver = blobDriver;
-        this.logger = logger;
-    }
-    calculateParamsKey(params) {
-        return makeBlobId(params);
-    }
-    createNewResource(params, _key) {
-        return this.blobDriver.getDownloadedBlob(params);
-    }
-    getByKey(blobId) {
-        const resource = super.tryGetByKey(blobId);
-        if (!resource)
-            throw new model.PFrameDriverError(`Local blob with id ${blobId} not found.`);
-        return resource;
-    }
-    makeDataSource(signal) {
-        return {
-            preloadBlob: async (blobIds) => {
-                try {
-                    await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));
-                }
-                catch (err) {
-                    if (!model.isAbortError(err))
-                        throw err;
-                }
-            },
-            resolveBlobContent: async (blobId) => {
-                const computable = this.getByKey(blobId);
-                const blob = await computable.awaitStableValue(signal);
-                return await this.blobDriver.getContent(blob.handle, { signal });
-            },
-        };
-    }
-}
-class RemoteBlobPool extends tsHelpers.RefCountPoolBase {
-    blobDriver;
-    logger;
-    constructor(blobDriver, logger) {
-        super();
-        this.blobDriver = blobDriver;
-        this.logger = logger;
-    }
-    calculateParamsKey(params) {
-        return makeBlobId(params);
-    }
-    createNewResource(params, _key) {
-        return this.blobDriver.getOnDemandBlob(params);
-    }
-    getByKey(blobId) {
-        const resource = super.tryGetByKey(blobId);
-        if (!resource)
-            throw new model.PFrameDriverError(`Remote blob with id ${blobId} not found.`);
-        return resource;
-    }
-    async withContent(handle, options) {
-        return await this.blobDriver.withContent(handle, {
-            range: {
-                from: options.range.start,
-                to: options.range.end + 1,
-            },
-            signal: options.signal,
-            handler: options.handler,
-        });
-    }
-}
-class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
-    remoteBlobProvider;
-    constructor(options) {
-        super(options);
-        this.remoteBlobProvider = options.remoteBlobProvider;
-    }
-    async request(filename, params) {
-        const blobId = filename.slice(0, -plModelMiddleLayer.PFrameInternal.ParquetExtension.length);
-        const respond = async (response) => {
-            try {
-                await params.callback(response);
-            }
-            catch (error) {
-                this.logger("warn", `PFrames blob store received unhandled rejection from HTTP handler: ${model.ensureError(error)}`);
-            }
-        };
-        try {
-            const computable = this.remoteBlobProvider.tryGetByKey(blobId);
-            if (!computable)
-                return await respond({ type: "NotFound" });
-            let blob;
-            try {
-                blob = await computable.getValue();
-            }
-            catch (error) {
-                this.logger("error", `PFrames blob store failed to get blob from computable: ${model.ensureError(error)}`);
-                return await respond({ type: "InternalError" });
-            }
-            params.signal.throwIfAborted();
-            const translatedRange = this.translate(blob.size, params.range);
-            if (!translatedRange) {
-                return await respond({
-                    type: "RangeNotSatisfiable",
-                    size: blob.size,
-                });
-            }
-            if (params.method === "HEAD") {
-                return await respond({
-                    type: "Ok",
-                    size: blob.size,
-                    range: translatedRange,
-                });
-            }
-            this.logger("info", `PFrames blob store requesting content for ${blobId}, ` +
-                `range [${translatedRange.start}..=${translatedRange.end}]`);
-            return await this.remoteBlobProvider.withContent(blob.handle, {
-                range: translatedRange,
-                signal: params.signal,
-                handler: async (data) => {
-                    return await respond({
-                        type: "Ok",
-                        size: blob.size,
-                        range: translatedRange,
-                        // eslint-disable-next-line n/no-unsupported-features/node-builtins
-                        data: node_stream.Readable.fromWeb(data),
-                    });
-                },
-            });
-        }
-        catch (error) {
-            if (!model.isAbortError(error)) {
-                if (plDrivers.isDownloadNetworkError400(error) && error.statusCode === 404) {
-                    this.logger("info", `PFrames blob store known race error: ${model.ensureError(error)}`);
-                }
-                else {
-                    this.logger("warn", `PFrames blob store unhandled error: ${model.ensureError(error)}`);
-                }
-            }
-            return await respond({ type: "InternalError" });
-        }
-    }
-}
-class RemoteBlobProviderImpl {
-    pool;
-    server;
-    constructor(pool, server) {
-        this.pool = pool;
-        this.server = server;
-    }
-    static async init(blobDriver, logger, serverOptions) {
-        const pool = new RemoteBlobPool(blobDriver, logger);
-        const store = new BlobStore({ remoteBlobProvider: pool, logger });
-        const handler = pframesRsNode.HttpHelpers.createRequestHandler({ store });
-        const server = await pframesRsNode.HttpHelpers.createHttpServer({ ...serverOptions, handler });
-        logger("info", `PFrames HTTP server started on ${server.info.url}`);
-        return new RemoteBlobProviderImpl(pool, server);
-    }
-    acquire(params) {
-        return this.pool.acquire(params);
-    }
-    httpServerInfo() {
-        return this.server.info;
-    }
-    async [Symbol.asyncDispose]() {
-        await this.server.stop();
-    }
+	return String(res.rid);
 }
+var LocalBlobProviderImpl = class extends _milaboratories_ts_helpers.RefCountPoolBase {
+	constructor(blobDriver, logger) {
+		super();
+		this.blobDriver = blobDriver;
+		this.logger = logger;
+	}
+	calculateParamsKey(params) {
+		return makeBlobId(params);
+	}
+	createNewResource(params, _key) {
+		return this.blobDriver.getDownloadedBlob(params);
+	}
+	getByKey(blobId) {
+		const resource = super.tryGetByKey(blobId);
+		if (!resource) throw new _platforma_sdk_model.PFrameDriverError(`Local blob with id ${blobId} not found.`);
+		return resource;
+	}
+	makeDataSource(signal) {
+		return {
+			preloadBlob: async (blobIds) => {
+				try {
+					await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));
+				} catch (err) {
+					if (!(0, _platforma_sdk_model.isAbortError)(err)) throw err;
+				}
+			},
+			resolveBlobContent: async (blobId) => {
+				const blob = await this.getByKey(blobId).awaitStableValue(signal);
+				return await this.blobDriver.getContent(blob.handle, { signal });
+			}
+		};
+	}
+};
+var RemoteBlobPool = class extends _milaboratories_ts_helpers.RefCountPoolBase {
+	constructor(blobDriver, logger) {
+		super();
+		this.blobDriver = blobDriver;
+		this.logger = logger;
+	}
+	calculateParamsKey(params) {
+		return makeBlobId(params);
+	}
+	createNewResource(params, _key) {
+		return this.blobDriver.getOnDemandBlob(params);
+	}
+	getByKey(blobId) {
+		const resource = super.tryGetByKey(blobId);
+		if (!resource) throw new _platforma_sdk_model.PFrameDriverError(`Remote blob with id ${blobId} not found.`);
+		return resource;
+	}
+	async withContent(handle, options) {
+		return await this.blobDriver.withContent(handle, {
+			range: {
+				from: options.range.start,
+				to: options.range.end + 1
+			},
+			signal: options.signal,
+			handler: options.handler
+		});
+	}
+};
+var BlobStore = class extends _milaboratories_pl_model_middle_layer.PFrameInternal.BaseObjectStore {
+	remoteBlobProvider;
+	constructor(options) {
+		super(options);
+		this.remoteBlobProvider = options.remoteBlobProvider;
+	}
+	async request(filename, params) {
+		const blobId = filename.slice(0, -_milaboratories_pl_model_middle_layer.PFrameInternal.ParquetExtension.length);
+		const respond = async (response) => {
+			try {
+				await params.callback(response);
+			} catch (error) {
+				this.logger("warn", `PFrames blob store received unhandled rejection from HTTP handler: ${(0, _platforma_sdk_model.ensureError)(error)}`);
+			}
+		};
+		try {
+			const computable = this.remoteBlobProvider.tryGetByKey(blobId);
+			if (!computable) return await respond({ type: "NotFound" });
+			let blob;
+			try {
+				blob = await computable.getValue();
+			} catch (error) {
+				this.logger("error", `PFrames blob store failed to get blob from computable: ${(0, _platforma_sdk_model.ensureError)(error)}`);
+				return await respond({ type: "InternalError" });
+			}
+			params.signal.throwIfAborted();
+			const translatedRange = this.translate(blob.size, params.range);
+			if (!translatedRange) return await respond({
+				type: "RangeNotSatisfiable",
+				size: blob.size
+			});
+			if (params.method === "HEAD") return await respond({
+				type: "Ok",
+				size: blob.size,
+				range: translatedRange
+			});
+			this.logger("info", `PFrames blob store requesting content for ${blobId}, range [${translatedRange.start}..=${translatedRange.end}]`);
+			return await this.remoteBlobProvider.withContent(blob.handle, {
+				range: translatedRange,
+				signal: params.signal,
+				handler: async (data) => {
+					return await respond({
+						type: "Ok",
+						size: blob.size,
+						range: translatedRange,
+						data: node_stream.Readable.fromWeb(data)
+					});
+				}
+			});
+		} catch (error) {
+			if (!(0, _platforma_sdk_model.isAbortError)(error)) if ((0, _milaboratories_pl_drivers.isDownloadNetworkError400)(error) && error.statusCode === 404) this.logger("info", `PFrames blob store known race error: ${(0, _platforma_sdk_model.ensureError)(error)}`);
+			else this.logger("warn", `PFrames blob store unhandled error: ${(0, _platforma_sdk_model.ensureError)(error)}`);
+			return await respond({ type: "InternalError" });
+		}
+	}
+};
+var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
+	constructor(pool, server) {
+		this.pool = pool;
+		this.server = server;
+	}
+	static async init(blobDriver, logger, serverOptions) {
+		const pool = new RemoteBlobPool(blobDriver, logger);
+		const store = new BlobStore({
+			remoteBlobProvider: pool,
+			logger
+		});
+		const handler = _milaboratories_pframes_rs_node.HttpHelpers.createRequestHandler({ store });
+		const server = await _milaboratories_pframes_rs_node.HttpHelpers.createHttpServer({
+			...serverOptions,
+			handler
+		});
+		logger("info", `PFrames HTTP server started on ${server.info.url}`);
+		return new RemoteBlobProviderImpl(pool, server);
+	}
+	acquire(params) {
+		return this.pool.acquire(params);
+	}
+	httpServerInfo() {
+		return this.server.info;
+	}
+	async [Symbol.asyncDispose]() {
+		await this.server.stop();
+	}
+};
 const PFrameDriverOpsDefaults = {
-    ...pfDriver.AbstractPFrameDriverOpsDefaults,
-    parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS
+	..._milaboratories_pf_driver.AbstractPFrameDriverOpsDefaults,
+	parquetServerPort: 0
 };
 async function createPFrameDriver(params) {
-    const resolvedSpillPath = path.resolve(params.spillPath);
-    await tsHelpers.emptyDir(resolvedSpillPath);
-    const logger = (level, message) => params.logger[level](message);
-    const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
-    const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {
-        port: params.options.parquetServerPort,
-    });
-    const resolveDataInfo = (spec, data$1) => {
-        return plTree.isPlTreeNodeAccessor(data$1)
-            ? data.parseDataInfoResource(data$1)
-            : model.isDataInfo(data$1)
-                ? data$1.type === "ParquetPartitioned"
-                    ? model.mapDataInfo(data$1, (a) => data.traverseParquetChunkResource(a))
-                    : model.mapDataInfo(data$1, (a) => a.persist())
-                : pfDriver.makeJsonDataInfo(spec, data$1);
-    };
-    return new pfDriver.AbstractPFrameDriver({
-        logger,
-        localBlobProvider,
-        remoteBlobProvider,
-        spillPath: resolvedSpillPath,
-        options: params.options,
-        resolveDataInfo,
-    });
+	const resolvedSpillPath = node_path.default.resolve(params.spillPath);
+	await (0, _milaboratories_ts_helpers.emptyDir)(resolvedSpillPath);
+	const logger = (level, message) => params.logger[level](message);
+	const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
+	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
+	const resolveDataInfo = (spec, data) => {
+		return (0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data) ? require_data.parseDataInfoResource(data) : (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => a.persist()) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);
+	};
+	return new _milaboratories_pf_driver.AbstractPFrameDriver({
+		logger,
+		localBlobProvider,
+		remoteBlobProvider,
+		spillPath: resolvedSpillPath,
+		options: params.options,
+		resolveDataInfo
+	});
 }
 
+//#endregion
 exports.PFrameDriverOpsDefaults = PFrameDriverOpsDefaults;
 exports.createPFrameDriver = createPFrameDriver;
-//# sourceMappingURL=driver.cjs.map
+//# sourceMappingURL=driver.cjs.map

package/dist/pool/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  PlTreeEntry,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"names":["RefCountPoolBase","PFrameDriverError","isAbortError","PFrameInternal","ensureError","Readable","isDownloadNetworkError400","HttpHelpers","AbstractPFrameDriverOpsDefaults","emptyDir","data","isPlTreeNodeAccessor","parseDataInfoResource","isDataInfo","mapDataInfo","traverseParquetChunkResource","makeJsonDataInfo","AbstractPFrameDriver"],"mappings":";;;;;;;;;;;;;AA0CA,SAAS,UAAU,CAAC,GAAgB,EAAA;AAClC,IAAA,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAgC;AACvD;AAGA,MAAM,qBACJ,SAAQA,0BAAqE,CAAA;AAI1D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;IAClD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,sBAAsB,MAAM,CAAA,WAAA,CAAa,CAAC;AACrF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,cAAc,CACnB,MAAmB,EAAA;QAEnB,OAAO;AACL,YAAA,WAAW,EAAE,OAAO,OAAsC,KAAI;AAC5D,gBAAA,IAAI;oBACF,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAC5E;gBACH;gBAAE,OAAO,GAAY,EAAE;AACrB,oBAAA,IAAI,CAACC,kBAAY,CAAC,GAAG,CAAC;AAAE,wBAAA,MAAM,GAAG;gBACnC;YACF,CAAC;AACD,YAAA,kBAAkB,EAAE,OAAO,MAAmC,KAAI;gBAChE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC;AACtD,gBAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAClE,CAAC;SACF;IACH;AACD;AAGD,MAAM,cAAe,SAAQF,0BAI5B,CAAA;AAEoB,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;IAChD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,uBAAuB,MAAM,CAAA,WAAA,CAAa,CAAC;AACtF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,MAAM,WAAW,CACtB,MAAwB,EACxB,OAIC,EAAA;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE;AAC/C,YAAA,KAAK,EAAE;AACL,gBAAA,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;AACzB,gBAAA,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC1B,aAAA;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,SAAA,CAAC;IACJ;AACD;AAMD,MAAM,SAAU,SAAQE,iCAAc,CAAC,eAAe,CAAA;AACnC,IAAA,kBAAkB;AAEnC,IAAA,WAAA,CAAY,OAAyB,EAAA;QACnC,KAAK,CAAC,OAAO,CAAC;AACd,QAAA,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;IACtD;AAEgB,IAAA,MAAM,OAAO,CAC3B,QAAwC,EACxC,MAKC,EAAA;AAED,QAAA,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAC3B,CAAC,EACD,CAACA,iCAAc,CAAC,gBAAgB,CAAC,MAAM,CACT;AAChC,QAAA,MAAM,OAAO,GAAG,OAAO,QAA4C,KAAmB;AACpF,YAAA,IAAI;AACF,gBAAA,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CACT,MAAM,EACN,CAAA,mEAAA,EAAsEC,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC3F;YACH;AACF,QAAA,CAAC;AAED,QAAA,IAAI;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;AAC9D,YAAA,IAAI,CAAC,UAAU;gBAAE,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAE3D,YAAA,IAAI,IAA6B;AACjC,YAAA,IAAI;AACF,gBAAA,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE;YACpC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CACT,OAAO,EACP,CAAA,uDAAA,EAA0DA,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC/E;gBACD,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACjD;AACA,YAAA,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;AAE9B,YAAA,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC5B,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,oBAAA,KAAK,EAAE,eAAe;AACvB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,CAAC,MAAM,CACT,MAAM,EACN,CAAA,0CAAA,EAA6C,MAAM,CAAA,EAAA,CAAI;gBACrD,CAAA,OAAA,EAAU,eAAe,CAAC,KAAK,CAAA,GAAA,EAAM,eAAe,CAAC,GAAG,CAAA,CAAA,CAAG,CAC9D;YACD,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;AAC5D,gBAAA,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;AACrB,gBAAA,OAAO,EAAE,OAAO,IAAI,KAAI;oBACtB,OAAO,MAAM,OAAO,CAAC;AACnB,wBAAA,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,wBAAA,KAAK,EAAE,eAAe;;AAEtB,wBAAA,IAAI,EAAEC,oBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;AAC7B,qBAAA,CAAC;gBACJ,CAAC;AACF,aAAA,CAAC;QACJ;QAAE,OAAO,KAAc,EAAE;AACvB,YAAA,IAAI,CAACH,kBAAY,CAAC,KAAK,CAAC,EAAE;gBACxB,IAAII,mCAAyB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,EAAE;AAChE,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA,qCAAA,EAAwCF,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;gBACnF;qBAAO;AACL,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA,oCAAA,EAAuCA,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;gBAClF;YACF;YACA,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACjD;IACF;AACD;AAED,MAAM,sBAAsB,CAAA;AAEP,IAAA,IAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,IAAoB,EACpB,MAAiC,EAAA;QADjC,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,MAAM,GAAN,MAAM;IACtB;IAEI,aAAa,IAAI,CACtB,UAA0B,EAC1B,MAA6B,EAC7B,aAAgE,EAAA;QAEhE,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC;AACnD,QAAA,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAGG,yBAAW,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3D,QAAA,MAAM,MAAM,GAAG,MAAMA,yBAAW,CAAC,gBAAgB,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,EAAE,CAAA,+BAAA,EAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA,CAAE,CAAC;AAEnE,QAAA,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD;AAEO,IAAA,OAAO,CAAC,MAAmB,EAAA;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAClC;IAEO,cAAc,GAAA;AACnB,QAAA,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI;IACzB;AAEA,IAAA,OAAO,MAAM,CAAC,YAAY,CAAC,GAAA;AACzB,QAAA,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B;AACD;AAWM,MAAM,uBAAuB,GAAoB;AACtD,IAAA,GAAGC,wCAA+B;IAClC,iBAAiB,EAAE,CAAC;;AAGf,eAAe,kBAAkB,CAAC,MAKxC,EAAA;IACC,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;AACxD,IAAA,MAAMC,kBAAQ,CAAC,iBAAiB,CAAC;AAEjC,IAAA,MAAM,MAAM,GAA0B,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;IACvF,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC;AAC9E,IAAA,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE;AACtF,QAAA,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,iBAAiB;AACvC,KAAA,CAAC;AAEF,IAAA,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAEC,MAA8C,KAAI;QAC5F,OAAOC,2BAAoB,CAACD,MAAI;AAC9B,cAAEE,0BAAqB,CAACF,MAAI;AAC5B,cAAEG,gBAAU,CAACH,MAAI;AACf,kBAAEA,MAAI,CAAC,IAAI,KAAK;AACd,sBAAEI,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAKK,iCAA4B,CAAC,CAAC,CAAC;AAC1D,sBAAED,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;AACxC,kBAAEM,yBAAgB,CAAC,IAAI,EAAEN,MAAI,CAAC;AACpC,IAAA,CAAC;IAED,OAAO,IAAIO,6BAAoB,CAAC;QAC9B,MAAM;QACN,iBAAiB;QACjB,kBAAkB;AAClB,QAAA,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,eAAe;AAChB,KAAA,CAAC;AACJ;;;;;"}
+{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  PlTreeEntry,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === \"ParquetPartitioned\"\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;AA0CA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACUA,4CAEV;CACE,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,AAAO,SAAS,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,AAAO,eACL,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,wCAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,4CAI3B;CACA,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,AAAO,SAAS,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,qDAAe,gBAAgB;CACrD,AAAiB;CAEjB,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,qDAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,4GAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,gGAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MAEP,MAAMC,qBAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,wCAAc,MAAM,CACtB,+DAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,8EAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,6EAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,4CAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,4CAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,AAAO,QAAQ,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,AAAO,iBAAgD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;;;AAa5B,MAAa,0BAA2C;CACtD,GAAGC;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,kBAAK,QAAQ,OAAO,UAAU;AACxD,gDAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,2DAA4B,KAAK,GAC7BC,mCAAsB,KAAK,wCAChB,KAAK,GACd,KAAK,SAAS,6DACA,OAAO,MAAMC,0CAA6B,EAAE,CAAC,yCAC7C,OAAO,MAAM,EAAE,SAAS,CAAC,mDACtB,MAAM,KAAK;;AAGpC,QAAO,IAAIC,+CAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/dist/pool/driver.d.ts

@@ -1,19 +1,14 @@
-import { type PColumnDataUniversal } from "@platforma-sdk/model";
-import { type MiLogger } from "@milaboratories/ts-helpers";
-import type { DownloadDriver } from "@milaboratories/pl-drivers";
-import { type PlTreeNodeAccessor } from "@milaboratories/pl-tree";
-import { type AbstractInternalPFrameDriver, type AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
-export interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {
-}
-export type PFrameDriverOps = AbstractPFrameDriverOps & {
-    /** Port to run parquet HTTP server on. */
-    parquetServerPort: number;
+import { PColumnDataUniversal } from "@platforma-sdk/model";
+import { MiLogger } from "@milaboratories/ts-helpers";
+import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
+import { DownloadDriver } from "@milaboratories/pl-drivers";
+
+//#region src/pool/driver.d.ts
+interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {}
+type PFrameDriverOps = AbstractPFrameDriverOps & {
+  /** Port to run parquet HTTP server on. */parquetServerPort: number;
 };
-export declare const PFrameDriverOpsDefaults: PFrameDriverOps;
-export declare function createPFrameDriver(params: {
-    blobDriver: DownloadDriver;
-    logger: MiLogger;
-    spillPath: string;
-    options: PFrameDriverOps;
-}): Promise<InternalPFrameDriver>;
+//#endregion
+export { InternalPFrameDriver, PFrameDriverOps };
 //# sourceMappingURL=driver.d.ts.map