@milaboratories/pl-middle-layer 1.42.1 → 1.42.3

package/dist/pool/data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.js","sources":["../../src/pool/data.ts"],"sourcesContent":["import {\n  PFrameDriverError,\n  type BinaryChunk,\n  type JsonDataInfo,\n  type ParquetChunk,\n  type ParquetChunkMapping,\n  type ParquetChunkMetadata,\n  type PColumnSpec,\n  type PColumnValue,\n  type PColumnValues,\n  type PlRef,\n  type PObjectId,\n  type PObjectSpec,\n} from '@platforma-sdk/model';\nimport type { PlTreeNodeAccessor, ResourceInfo } from '@milaboratories/pl-tree';\nimport { assertNever } from '@milaboratories/ts-helpers';\nimport canonicalize from 'canonicalize';\nimport {\n  isNullResourceId,\n  resourceType,\n  resourceTypeToString,\n  resourceTypesEqual,\n} from '@milaboratories/pl-client';\nimport type { Writable } from 'utility-types';\nimport { createHash } from 'node:crypto';\nimport type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\n\nexport function* allBlobs<B>(data: PFrameInternal.DataInfo<B>): Generator<B> {\n  switch (data.type) {\n    case 'Json':\n      return;\n    case 'JsonPartitioned':\n      for (const blob of Object.values(data.parts)) yield blob;\n      return;\n    case 'BinaryPartitioned':\n      for (const { index, values } of Object.values(data.parts)) {\n        yield index;\n        yield values;\n      }\n      return;\n    case 'ParquetPartitioned':\n      for (const { data: blob } of Object.values(data.parts)) yield blob;\n      return;\n    default:\n      assertNever(data);\n  }\n}\n\nfunction mapValues<T extends object, TResult>(\n  obj: T,\n  callback: (v: T[keyof T], key: keyof T) => TResult,\n): { [K in keyof T]: TResult } {\n  return Object.fromEntries(\n    (Object.entries(obj) as { [K in keyof T]: [K, T[K]]; }[keyof T][])\n      .map(([key, value]) => [key, callback(value, key)]),\n  ) as { [K in keyof T]: TResult };\n}\n\nexport function mapBlobs<B1, B2>(\n  data: PFrameInternal.DataInfo<B1>,\n  mapping: (blob: B1) => B2,\n): PFrameInternal.DataInfo<B2> {\n  switch (data.type) {\n    case 'Json':\n      return { ...data };\n    case 'JsonPartitioned':\n      return {\n        ...data,\n        parts: mapValues(data.parts, mapping),\n      };\n    case 'BinaryPartitioned':\n      return {\n        ...data,\n        parts: mapValues(data.parts, (v) => ({\n          index: mapping(v.index),\n          values: mapping(v.values),\n        })),\n      };\n    case 'ParquetPartitioned':\n      return {\n        ...data,\n        parts: mapValues(data.parts, (v) => ({\n          ...v,\n          data: mapping(v.data),\n        })),\n      };\n    default:\n      assertNever(data);\n  }\n}\n\nexport const PColumnDataJsonPartitioned = resourceType('PColumnData/JsonPartitioned', '1');\nexport const PColumnDataJsonSuperPartitioned = resourceType(\n  'PColumnData/Partitioned/JsonPartitioned',\n  '1',\n);\nexport const PColumnDataBinaryPartitioned = resourceType('PColumnData/BinaryPartitioned', '1');\nexport const PColumnDataBinarySuperPartitioned = resourceType(\n  'PColumnData/Partitioned/BinaryPartitioned',\n  '1',\n);\nexport const PColumnDataParquetPartitioned = resourceType('PColumnData/ParquetPartitioned', '1');\nexport const PColumnDataParquetSuperPartitioned = resourceType(\n  'PColumnData/Partitioned/ParquetPartitioned',\n  '1',\n);\nexport const PColumnDataJson = resourceType('PColumnData/Json', '1');\n\nexport type PColumnDataJsonResourceValue = {\n  keyLength: number;\n  data: Record<string, PColumnValue>;\n};\n\nexport type PColumnDataPartitionedResourceValue = {\n  partitionKeyLength: number;\n};\n\nexport type PColumnDataSuperPartitionedResourceValue = {\n  superPartitionKeyLength: number;\n  partitionKeyLength: number;\n};\n\nexport function parseDataInfoResource(\n  data: PlTreeNodeAccessor,\n): PFrameInternal.DataInfo<ResourceInfo> {\n  if (!data.getIsReadyOrError()) throw new PFrameDriverError('Data not ready.');\n\n  const resourceData = data.getDataAsJson();\n  if (resourceData === undefined)\n    throw new PFrameDriverError('unexpected data info structure, no resource data');\n\n  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {\n    const dataContent = resourceData as PColumnDataJsonResourceValue;\n\n    return {\n      type: 'Json',\n      keyLength: dataContent.keyLength,\n      data: dataContent.data,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts = Object.fromEntries(\n      data\n        .listInputFields()\n        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo]),\n    );\n\n    return {\n      type: 'JsonPartitioned',\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ResourceInfo> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const partKey = JSON.stringify([\n          ...JSON.parse(superKey) as PColumnValue[],\n          ...JSON.parse(key) as PColumnValue[]]);\n        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).resourceInfo;\n      }\n    }\n\n    return {\n      type: 'JsonPartitioned',\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<\n      string,\n      Partial<Writable<BinaryChunk<ResourceInfo>>>\n    > = {};\n\n    // parsing the structure\n    for (const field of data.listInputFields()) {\n      if (field.endsWith('.index')) {\n        const partKey = field.slice(0, field.length - 6);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.index = data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo;\n      } else if (field.endsWith('.values')) {\n        const partKey = field.slice(0, field.length - 7);\n        let part = parts[partKey];\n        if (part === undefined) {\n          part = {};\n          parts[partKey] = part;\n        }\n        part.values = data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo;\n      } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n    }\n\n    // structure validation\n    for (const [key, part] of Object.entries(parts)) {\n      if (part.index === undefined) throw new PFrameDriverError(`no index for part ${key}`);\n      if (part.values === undefined) throw new PFrameDriverError(`no values for part ${key}`);\n    }\n\n    return {\n      type: 'BinaryPartitioned',\n      partitionKeyLength: meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<ResourceInfo>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<\n      string,\n      Partial<Writable<BinaryChunk<ResourceInfo>>>\n    > = {};\n    for (const superKey of data.listInputFields()) {\n      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superData.listInputFields();\n      if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const field of keys) {\n        if (field.endsWith('.index')) {\n          const key = field.slice(0, field.length - 6);\n\n          const partKey = JSON.stringify([\n            ...JSON.parse(superKey) as PColumnValue[],\n            ...JSON.parse(key) as PColumnValue[]]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].index = superData.traverse({\n            field,\n            errorIfFieldNotSet: true,\n          }).resourceInfo;\n        } else if (field.endsWith('.values')) {\n          const key = field.slice(0, field.length - 7);\n\n          const partKey = JSON.stringify([\n            ...JSON.parse(superKey) as PColumnValue[],\n            ...JSON.parse(key) as PColumnValue[]]);\n          let part = parts[partKey];\n          if (part === undefined) {\n            part = {};\n            parts[partKey] = part;\n          }\n          parts[partKey].values = superData.traverse({\n            field,\n            errorIfFieldNotSet: true,\n          }).resourceInfo;\n        } else throw new PFrameDriverError(`unrecognized part field name: ${field}`);\n      }\n    }\n\n    return {\n      type: 'BinaryPartitioned',\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts: parts as Record<string, BinaryChunk<ResourceInfo>>,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetPartitioned)) {\n    const meta = resourceData as PColumnDataPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<ResourceInfo>> = {};\n    for (const key of data.listInputFields()) {\n      const resource = data.traverse({ field: key, assertFieldType: 'Input', errorIfFieldNotSet: true });\n\n      parts[key] = traverseParquetPartitionedResource(resource);\n    }\n\n    return {\n      type: 'ParquetPartitioned',\n      partitionKeyLength: meta.partitionKeyLength,\n      parts,\n    };\n  } else if (resourceTypesEqual(data.resourceType, PColumnDataParquetSuperPartitioned)) {\n    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;\n\n    const parts: Record<string, ParquetChunk<ResourceInfo>> = {};\n    for (const superKey of data.listInputFields()) {\n      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });\n      const keys = superPart.listInputFields();\n      if (keys === undefined) throw new PFrameDriverError(`no partition keys for super key ${superKey}`);\n\n      for (const key of keys) {\n        const resource = data.traverse({ field: key, errorIfFieldNotSet: true });\n\n        const partKey = JSON.stringify([\n          ...JSON.parse(superKey) as PColumnValue[],\n          ...JSON.parse(key) as PColumnValue[],\n        ]);\n        parts[partKey] = traverseParquetPartitionedResource(resource);\n      }\n    }\n\n    return {\n      type: 'ParquetPartitioned',\n      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,\n      parts,\n    };\n  }\n\n  throw new PFrameDriverError(`unsupported resource type: ${resourceTypeToString(data.resourceType)}`);\n}\n\nexport function traverseParquetPartitionedResource(resource: PlTreeNodeAccessor): ParquetChunk<ResourceInfo> {\n  const blob = resource.traverse(\n    { field: 'blob', assertFieldType: 'Service', errorIfFieldNotSet: true },\n  ).resourceInfo;\n  const partInfo = resource.getDataAsJson() as ParquetChunkMetadata;\n  const mapping = resource.traverse(\n    { field: 'mapping', assertFieldType: 'Service', errorIfFieldNotSet: true },\n  ).getDataAsJson() as ParquetChunkMapping;\n\n  return {\n    data: blob,\n    ...partInfo,\n    ...mapping,\n  };\n}\n\nexport function makeDataInfoFromPColumnValues(\n  spec: PColumnSpec,\n  data: PColumnValues,\n): JsonDataInfo {\n  const keyLength = spec.axesSpec.length;\n  const jsonData: Record<string, PColumnValue> = {};\n  for (const { key, val } of data) {\n    if (key.length !== keyLength)\n      throw new PFrameDriverError(`inline column key length ${key.length} differs from axes count ${keyLength}`);\n    jsonData[JSON.stringify(key)] = val;\n  }\n\n  return {\n    type: 'Json',\n    keyLength,\n    data: jsonData,\n  };\n}\n\nexport function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {\n  const hash = createHash('sha256');\n  hash.update(canonicalize(spec)!);\n  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));\n  return hash.digest().toString('hex') as PObjectId;\n}\n\nexport function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {\n  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;\n}\n\nexport function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {\n  return canonicalize({ resolvePath, name: outputName })! as PObjectId;\n}\n"],"names":[],"mappings":";;;;;;AA2BM,UAAW,QAAQ,CAAI,IAAgC,EAAA;AAC3D,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;YACT;AACF,QAAA,KAAK,iBAAiB;YACpB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;AAAE,gBAAA,MAAM,IAAI;YACxD;AACF,QAAA,KAAK,mBAAmB;AACtB,YAAA,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;AACzD,gBAAA,MAAM,KAAK;AACX,gBAAA,MAAM,MAAM;YACd;YACA;AACF,QAAA,KAAK,oBAAoB;AACvB,YAAA,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;AAAE,gBAAA,MAAM,IAAI;YAClE;AACF,QAAA;YACE,WAAW,CAAC,IAAI,CAAC;;AAEvB;AAEA,SAAS,SAAS,CAChB,GAAM,EACN,QAAkD,EAAA;IAElD,OAAO,MAAM,CAAC,WAAW,CACtB,MAAM,CAAC,OAAO,CAAC,GAAG;SAChB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CACvB;AAClC;AAEM,SAAU,QAAQ,CACtB,IAAiC,EACjC,OAAyB,EAAA;AAEzB,IAAA,QAAQ,IAAI,CAAC,IAAI;AACf,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,EAAE,GAAG,IAAI,EAAE;AACpB,QAAA,KAAK,iBAAiB;YACpB,OAAO;AACL,gBAAA,GAAG,IAAI;gBACP,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC;aACtC;AACH,QAAA,KAAK,mBAAmB;YACtB,OAAO;AACL,gBAAA,GAAG,IAAI;AACP,gBAAA,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,oBAAA,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACvB,oBAAA,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAC1B,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA,KAAK,oBAAoB;YACvB,OAAO;AACL,gBAAA,GAAG,IAAI;AACP,gBAAA,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,oBAAA,GAAG,CAAC;AACJ,oBAAA,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACtB,iBAAA,CAAC,CAAC;aACJ;AACH,QAAA;YACE,WAAW,CAAC,IAAI,CAAC;;AAEvB;AAEO,MAAM,0BAA0B,GAAG,YAAY,CAAC,6BAA6B,EAAE,GAAG;AAClF,MAAM,+BAA+B,GAAG,YAAY,CACzD,yCAAyC,EACzC,GAAG;AAEE,MAAM,4BAA4B,GAAG,YAAY,CAAC,+BAA+B,EAAE,GAAG;AACtF,MAAM,iCAAiC,GAAG,YAAY,CAC3D,2CAA2C,EAC3C,GAAG;AAEE,MAAM,6BAA6B,GAAG,YAAY,CAAC,gCAAgC,EAAE,GAAG;AACxF,MAAM,kCAAkC,GAAG,YAAY,CAC5D,4CAA4C,EAC5C,GAAG;AAEE,MAAM,eAAe,GAAG,YAAY,CAAC,kBAAkB,EAAE,GAAG;AAgB7D,SAAU,qBAAqB,CACnC,IAAwB,EAAA;AAExB,IAAA,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;AAAE,QAAA,MAAM,IAAI,iBAAiB,CAAC,iBAAiB,CAAC;AAE7E,IAAA,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE;IACzC,IAAI,YAAY,KAAK,SAAS;AAC5B,QAAA,MAAM,IAAI,iBAAiB,CAAC,kDAAkD,CAAC;IAEjF,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,EAAE;QAC1D,MAAM,WAAW,GAAG,YAA4C;QAEhE,OAAO;AACL,YAAA,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,IAAI,EAAE,WAAW,CAAC,IAAI;SACvB;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,0BAA0B,CAAC,EAAE;QAC5E,MAAM,IAAI,GAAG,YAAmD;AAEhE,QAAA,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAC9B;AACG,aAAA,eAAe;aACf,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAC5F;QAED,OAAO;AACL,YAAA,IAAI,EAAE,iBAAiB;YACvB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,+BAA+B,CAAC,EAAE;QACjF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAAiC,EAAE;QAC9C,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAElG,YAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;AACtB,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,oBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAmB;AACzC,oBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG;AAAoB,iBAAA,CAAC;gBACxC,KAAK,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,YAAY;YAC5F;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,iBAAiB;AACvB,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;YAC1E,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,4BAA4B,CAAC,EAAE;QAC9E,MAAM,IAAI,GAAG,YAAmD;QAEhE,MAAM,KAAK,GAGP,EAAE;;QAGN,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC1C,YAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;AAC5B,gBAAA,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AAChD,gBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,gBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;oBACtB,IAAI,GAAG,EAAE;AACT,oBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;gBACvB;AACA,gBAAA,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,YAAY;YAC9E;AAAO,iBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;AACpC,gBAAA,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AAChD,gBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,gBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;oBACtB,IAAI,GAAG,EAAE;AACT,oBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;gBACvB;AACA,gBAAA,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,YAAY;YAC/E;;AAAO,gBAAA,MAAM,IAAI,iBAAiB,CAAC,iCAAiC,KAAK,CAAA,CAAE,CAAC;QAC9E;;AAGA,QAAA,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AAC/C,YAAA,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,qBAAqB,GAAG,CAAA,CAAE,CAAC;AACrF,YAAA,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,sBAAsB,GAAG,CAAA,CAAE,CAAC;QACzF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,mBAAmB;YACzB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;AAC3C,YAAA,KAAK,EAAE,KAAkD;SAC1D;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,iCAAiC,CAAC,EAAE;QACnF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAGP,EAAE;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAElG,YAAA,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE;AACxB,gBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;AAC5B,oBAAA,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AAE5C,oBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,wBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAmB;AACzC,wBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG;AAAoB,qBAAA,CAAC;AACxC,oBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,oBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;wBACtB,IAAI,GAAG,EAAE;AACT,wBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;oBACvB;oBACA,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC;wBACxC,KAAK;AACL,wBAAA,kBAAkB,EAAE,IAAI;qBACzB,CAAC,CAAC,YAAY;gBACjB;AAAO,qBAAA,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;AACpC,oBAAA,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AAE5C,oBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,wBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAmB;AACzC,wBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG;AAAoB,qBAAA,CAAC;AACxC,oBAAA,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;AACzB,oBAAA,IAAI,IAAI,KAAK,SAAS,EAAE;wBACtB,IAAI,GAAG,EAAE;AACT,wBAAA,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI;oBACvB;oBACA,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;wBACzC,KAAK;AACL,wBAAA,kBAAkB,EAAE,IAAI;qBACzB,CAAC,CAAC,YAAY;gBACjB;;AAAO,oBAAA,MAAM,IAAI,iBAAiB,CAAC,iCAAiC,KAAK,CAAA,CAAE,CAAC;YAC9E;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,mBAAmB;AACzB,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;AAC1E,YAAA,KAAK,EAAE,KAAkD;SAC1D;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,6BAA6B,CAAC,EAAE;QAC/E,MAAM,IAAI,GAAG,YAAmD;QAEhE,MAAM,KAAK,GAA+C,EAAE;QAC5D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;YACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YAElG,KAAK,CAAC,GAAG,CAAC,GAAG,kCAAkC,CAAC,QAAQ,CAAC;QAC3D;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,oBAAoB;YAC1B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,KAAK;SACN;IACH;SAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,kCAAkC,CAAC,EAAE;QACpF,MAAM,IAAI,GAAG,YAAwD;QAErE,MAAM,KAAK,GAA+C,EAAE;QAC5D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;AAC7C,YAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAC9E,YAAA,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,EAAE;YACxC,IAAI,IAAI,KAAK,SAAS;AAAE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,mCAAmC,QAAQ,CAAA,CAAE,CAAC;AAElG,YAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;AACtB,gBAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;AAExE,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;AAC7B,oBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAmB;AACzC,oBAAA,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAmB;AACrC,iBAAA,CAAC;gBACF,KAAK,CAAC,OAAO,CAAC,GAAG,kCAAkC,CAAC,QAAQ,CAAC;YAC/D;QACF;QAEA,OAAO;AACL,YAAA,IAAI,EAAE,oBAAoB;AAC1B,YAAA,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB;YAC1E,KAAK;SACN;IACH;AAEA,IAAA,MAAM,IAAI,iBAAiB,CAAC,CAAA,2BAAA,EAA8B,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA,CAAE,CAAC;AACtG;AAEM,SAAU,kCAAkC,CAAC,QAA4B,EAAA;IAC7E,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAC5B,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,IAAI,EAAE,CACxE,CAAC,YAAY;AACd,IAAA,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,EAA0B;IACjE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAC/B,EAAE,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAC3E,CAAC,aAAa,EAAyB;IAExC,OAAO;AACL,QAAA,IAAI,EAAE,IAAI;AACV,QAAA,GAAG,QAAQ;AACX,QAAA,GAAG,OAAO;KACX;AACH;AAEM,SAAU,6BAA6B,CAC3C,IAAiB,EACjB,IAAmB,EAAA;AAEnB,IAAA,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;IACtC,MAAM,QAAQ,GAAiC,EAAE;IACjD,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE;AAC/B,QAAA,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAC1B,MAAM,IAAI,iBAAiB,CAAC,CAAA,yBAAA,EAA4B,GAAG,CAAC,MAAM,CAAA,yBAAA,EAA4B,SAAS,CAAA,CAAE,CAAC;QAC5G,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG;IACrC;IAEA,OAAO;AACL,QAAA,IAAI,EAAE,MAAM;QACZ,SAAS;AACT,QAAA,IAAI,EAAE,QAAQ;KACf;AACH;AAEM,SAAU,qBAAqB,CAAC,IAAiB,EAAE,IAAwB,EAAA;AAC/E,IAAA,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAE,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAc;AACnD;AAEM,SAAU,qBAAqB,CAAC,OAAe,EAAE,UAAkB,EAAA;AACvE,IAAA,OAAO,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAkB,CAAe;AACjG;AAEM,SAAU,oBAAoB,CAAC,WAAqB,EAAE,UAAkB,EAAA;IAC5E,OAAO,YAAY,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,CAAe;AACtE;;;;"}

package/dist/pool/driver.cjs

@@ -0,0 +1,620 @@
+'use strict';
+
+var plTree = require('@milaboratories/pl-tree');
+var model = require('@platforma-sdk/model');
+var lruCache = require('lru-cache');
+var ref_count_pool = require('./ref_count_pool.cjs');
+var data = require('./data.cjs');
+var node_crypto = require('node:crypto');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var canonicalize = require('canonicalize');
+var pframesRsNode = require('@milaboratories/pframes-rs-node');
+var fs = require('node:fs/promises');
+var path = require('node:path');
+var index = require('../debug/index.cjs');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
+var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
+
+function blobKey(res) {
+    return String(res.id);
+}
+const valueTypes = ['Int', 'Long', 'Float', 'Double', 'String', 'Bytes'];
+function migrateFilters(filters) {
+    const filtersV1 = [];
+    const filtersV2 = [];
+    for (const filter of filters) {
+        if (filter.type === 'bySingleColumn') {
+            filtersV1.push(filter);
+            filtersV2.push({
+                ...filter,
+                type: 'bySingleColumnV2',
+            });
+        }
+        else {
+            filtersV2.push(filter);
+        }
+    }
+    if (filtersV1.length > 0) {
+        const filtersV1Json = JSON.stringify(filtersV1);
+        console.warn(`type overriten from 'bySingleColumn' to 'bySingleColumnV2' for filters: ${filtersV1Json}`);
+    }
+    return filtersV2;
+}
+function migratePTableFilters(def) {
+    if (!('partitionFilters' in def)) {
+        // For old blocks assume all axes filters to be partition filters
+        return {
+            ...def,
+            partitionFilters: migrateFilters(def.filters.filter((f) => f.column.type === 'axis')),
+            filters: migrateFilters(def.filters.filter((f) => f.column.type === 'column')),
+        };
+    }
+    return {
+        ...def,
+        partitionFilters: migrateFilters(def.partitionFilters),
+        filters: migrateFilters(def.filters),
+    };
+}
+const bigintReplacer = (_, v) => (typeof v === 'bigint' ? v.toString() : v);
+class PTableCache {
+    logger;
+    ops;
+    perFrame = new Map();
+    global;
+    disposeListeners = new Map();
+    constructor(logger, ops) {
+        this.logger = logger;
+        this.ops = ops;
+        this.global = new lruCache.LRUCache({
+            maxSize: this.ops.pFramesCacheMaxSize,
+            dispose: (resource, key, reason) => {
+                if (reason === 'evict') {
+                    this.perFrame.get(resource.resource.pFrame)?.delete(key);
+                }
+                if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
+                    this.perFrame.delete(resource.resource.pFrame);
+                }
+                const disposeListener = this.disposeListeners.get(key);
+                this.disposeListeners.delete(key);
+                resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
+                resource.unref();
+                if (index.getDebugFlags().logPFrameRequests) {
+                    this.logger.info(`calculateTableData cache - removed PTable ${key}`);
+                }
+            },
+        });
+    }
+    cache(resource, size) {
+        const key = resource.key;
+        if (index.getDebugFlags().logPFrameRequests) {
+            this.logger.info(`calculateTableData cache - added PTable ${key} with size ${size}`);
+        }
+        this.global.set(key, resource, { size });
+        let perFrame = this.perFrame.get(resource.resource.pFrame);
+        if (!perFrame) {
+            perFrame = new lruCache.LRUCache({
+                max: this.ops.pFrameCacheMaxCount,
+                dispose: (_resource, key, reason) => {
+                    if (reason === 'evict') {
+                        this.global.delete(key);
+                    }
+                },
+            });
+            this.perFrame.set(resource.resource.pFrame, perFrame);
+        }
+        perFrame.set(key, resource);
+        const disposeListener = () => {
+            this.perFrame.get(resource.resource.pFrame)?.delete(key);
+            this.global.delete(key);
+        };
+        this.disposeListeners.set(key, disposeListener);
+        resource.resource.disposeSignal.addEventListener('abort', disposeListener);
+    }
+}
+class PFrameHolder {
+    blobDriver;
+    logger;
+    spillPath;
+    pFramePromise;
+    abortController = new AbortController();
+    blobIdToResource = new Map();
+    blobHandleComputables = new Map();
+    constructor(blobDriver, logger, spillPath, columns) {
+        this.blobDriver = blobDriver;
+        this.logger = logger;
+        this.spillPath = spillPath;
+        const logFunc = (level, message) => this.logger[level](message);
+        for (const column of columns) {
+            for (const blob of data.allBlobs(column.data)) {
+                this.blobIdToResource.set(blobKey(blob), blob);
+            }
+        }
+        const distinctСolumns = [
+            ...new Map(columns.map((column) => ({
+                ...column,
+                data: data.mapBlobs(column.data, blobKey),
+            })).map((item) => [canonicalize(item), item])).values(),
+        ];
+        try {
+            const pFrame = new pframesRsNode.PFrame(this.spillPath, logFunc);
+            pFrame.setDataSource(this);
+            const promises = [];
+            for (const column of distinctСolumns) {
+                pFrame.addColumnSpec(column.id, column.spec);
+                promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
+            }
+            this.pFramePromise = Promise.all(promises)
+                .then(() => pFrame)
+                .catch((err) => {
+                this.dispose();
+                pFrame.dispose();
+                throw new model.PFrameDriverError(`PFrame creation failed asynchronously, columns: ${JSON.stringify(distinctСolumns)}, error: ${model.ensureError(err)}`);
+            });
+        }
+        catch (err) {
+            throw new model.PFrameDriverError(`PFrame creation failed synchronously, columns: ${JSON.stringify(distinctСolumns)}, error: ${model.ensureError(err)}`);
+        }
+    }
+    getOrCreateComputableForBlob(blobId) {
+        let computable = this.blobHandleComputables.get(blobId);
+        if (computable !== undefined)
+            return computable;
+        const blobResource = this.blobIdToResource.get(blobId);
+        if (blobResource === undefined)
+            throw new model.PFrameDriverError(`Blob with id ${blobId} not found.`);
+        // precalculation of value tree will trigger the download proecess right away
+        computable = this.blobDriver.getDownloadedBlob(blobResource).withPreCalculatedValueTree();
+        this.blobHandleComputables.set(blobId, computable);
+        return computable;
+    }
+    preloadBlob = async (blobIds) => {
+        const computables = blobIds.map((blobId) => this.getOrCreateComputableForBlob(blobId));
+        for (const computable of computables) {
+            try {
+                await computable.awaitStableFullValue(this.disposeSignal);
+            }
+            catch (err) {
+                if (model.isAbortError(err)) {
+                    break; // silence abort errors
+                }
+                throw err;
+            }
+        }
+    };
+    resolveBlobContent = async (blobId) => {
+        const computable = this.getOrCreateComputableForBlob(blobId);
+        const path = this.blobDriver.getLocalPath((await computable.awaitStableValue(this.disposeSignal)).handle);
+        return await fs__namespace.readFile(path);
+    };
+    get disposeSignal() {
+        return this.abortController.signal;
+    }
+    dispose() {
+        this.abortController.abort();
+        for (const computable of this.blobHandleComputables.values())
+            computable.resetState();
+    }
+    async [Symbol.asyncDispose]() {
+        this.dispose();
+        await this.pFramePromise
+            .then((pFrame) => pFrame.dispose())
+            .catch(() => { });
+    }
+}
+class PTableHolder {
+    pFrame;
+    pTablePromise;
+    predecessor;
+    abortController = new AbortController();
+    combinedDisposeSignal;
+    constructor(pFrame, pFrameDisposeSignal, pTablePromise, predecessor) {
+        this.pFrame = pFrame;
+        this.pTablePromise = pTablePromise;
+        this.predecessor = predecessor;
+        this.combinedDisposeSignal = AbortSignal.any([pFrameDisposeSignal, this.abortController.signal]);
+    }
+    get disposeSignal() {
+        return this.combinedDisposeSignal;
+    }
+    async [Symbol.asyncDispose]() {
+        this.abortController.abort();
+        await this.pTablePromise
+            .then((pTable) => pTable.dispose())
+            .catch(() => { });
+        this.predecessor?.unref();
+    }
+}
+class PFrameDriver {
+    blobDriver;
+    logger;
+    spillPath;
+    pFrames;
+    pTables;
+    pTableCache;
+    frameConcurrencyLimiter;
+    tableConcurrencyLimiter;
+    async pprofDump() {
+        return await pframesRsNode.PFrame.pprofDump();
+    }
+    static async init(blobDriver, logger, spillPath, ops) {
+        const resolvedSpillPath = path__namespace.resolve(spillPath);
+        await tsHelpers.emptyDir(resolvedSpillPath);
+        return new PFrameDriver(blobDriver, logger, resolvedSpillPath, ops);
+    }
+    constructor(blobDriver, logger, spillPath, ops) {
+        this.blobDriver = blobDriver;
+        this.logger = logger;
+        this.spillPath = spillPath;
+        const concurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pFrameConcurrency);
+        this.frameConcurrencyLimiter = concurrencyLimiter;
+        this.tableConcurrencyLimiter = new tsHelpers.ConcurrencyLimitingExecutor(ops.pTableConcurrency);
+        this.pTableCache = new PTableCache(this.logger, ops);
+        this.pFrames = new (class extends ref_count_pool.RefCountResourcePool {
+            blobDriver;
+            logger;
+            spillPath;
+            constructor(blobDriver, logger, spillPath) {
+                super();
+                this.blobDriver = blobDriver;
+                this.logger = logger;
+                this.spillPath = spillPath;
+            }
+            acquire(params) {
+                return super.acquire(params);
+            }
+            getByKey(key) {
+                const resource = super.tryGetByKey(key);
+                if (!resource)
+                    throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
+                return resource;
+            }
+            createNewResource(params) {
+                if (index.getDebugFlags().logPFrameRequests)
+                    logger.info(`PFrame creation (pFrameHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
+                return new PFrameHolder(this.blobDriver, this.logger, this.spillPath, params);
+            }
+            calculateParamsKey(params) {
+                try {
+                    return stableKeyFromPFrameData(params);
+                }
+                catch (err) {
+                    if (model.isPFrameDriverError(err))
+                        throw err;
+                    throw new model.PFrameDriverError(`PFrame handle calculation failed, request: ${JSON.stringify(params, bigintReplacer)}, error: ${model.ensureError(err)}`);
+                }
+            }
+        })(this.blobDriver, this.logger, this.spillPath);
+        this.pTables = new (class extends ref_count_pool.RefCountResourcePool {
+            pFrames;
+            constructor(pFrames) {
+                super();
+                this.pFrames = pFrames;
+            }
+            getByKey(key) {
+                const resource = super.tryGetByKey(key);
+                if (!resource)
+                    throw new model.PFrameDriverError(`PTable not found, handle = ${key}`);
+                return resource;
+            }
+            createNewResource(params) {
+                if (index.getDebugFlags().logPFrameRequests) {
+                    logger.info(`PTable creation (pTableHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`);
+                }
+                const handle = params.pFrameHandle;
+                const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
+                // 3. Sort
+                if (params.def.sorting.length > 0) {
+                    const predecessor = this.acquire({
+                        ...params,
+                        def: {
+                            ...params.def,
+                            sorting: [],
+                        },
+                    });
+                    const { resource: { pTablePromise } } = predecessor;
+                    const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
+                    return new PTableHolder(handle, disposeSignal, sortedTable, predecessor);
+                }
+                // 2. Filter
+                if (params.def.filters.length > 0) {
+                    const predecessor = this.acquire({
+                        ...params,
+                        def: {
+                            ...params.def,
+                            filters: [],
+                        },
+                    });
+                    const { resource: { pTablePromise } } = predecessor;
+                    const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
+                    return new PTableHolder(handle, disposeSignal, filteredTable, predecessor);
+                }
+                // 1. Join
+                const table = pFramePromise.then((pFrame) => pFrame.createTable({
+                    src: joinEntryToInternal(params.def.src),
+                    filters: params.def.partitionFilters,
+                }));
+                return new PTableHolder(handle, disposeSignal, table);
+            }
+            calculateParamsKey(params) {
+                try {
+                    return stableKeyFromFullPTableDef(params);
+                }
+                catch (err) {
+                    throw new model.PFrameDriverError(`PTable handle calculation failed, request: ${JSON.stringify(params)}, error: ${model.ensureError(err)}`);
+                }
+            }
+        })(this.pFrames);
+    }
+    //
+    // Internal / Config API Methods
+    //
+    createPFrame(def, ctx) {
+        const internalData = def
+            .filter((c) => valueTypes.find((t) => t === c.spec.valueType))
+            .map((c) => model.mapPObjectData(c, (d) => plTree.isPlTreeNodeAccessor(d)
+            ? data.parseDataInfoResource(d)
+            : model.isDataInfo(d)
+                ? d.type === 'ParquetPartitioned'
+                    ? model.mapDataInfo(d, (a) => data.traverseParquetPartitionedResource(a))
+                    : model.mapDataInfo(d, (a) => a.resourceInfo)
+                : data.makeDataInfoFromPColumnValues(c.spec, d)));
+        const res = this.pFrames.acquire(internalData);
+        ctx.addOnDestroy(res.unref);
+        return res.key;
+    }
+    createPTable(rawDef, ctx) {
+        const def = migratePTableFilters(rawDef);
+        const pFrameHandle = this.createPFrame(model.extractAllColumns(def.src), ctx);
+        const defIds = model.mapPTableDef(def, (c) => c.id);
+        const res = this.pTables.acquire({ def: defIds, pFrameHandle });
+        if (index.getDebugFlags().logPFrameRequests)
+            this.logger.info(`Create PTable call (pFrameHandle = ${pFrameHandle}; pTableHandle = ${JSON.stringify(res)}): ${JSON.stringify(model.mapPTableDef(def, (c) => c.spec), bigintReplacer)}`);
+        ctx.addOnDestroy(res.unref); // in addition to pframe unref added in createPFrame above
+        return res.key;
+    }
+    //
+    // PFrame istance methods
+    //
+    async findColumns(handle, request) {
+        const iRequest = {
+            ...request,
+            compatibleWith: request.compatibleWith.length !== 0
+                ? [{
+                        axesSpec: [
+                            ...new Map(request.compatibleWith.map((item) => [canonicalize(item), item])).values(),
+                        ],
+                        qualifications: [],
+                    }]
+                : [],
+        };
+        const { pFramePromise } = this.pFrames.getByKey(handle);
+        const pFrame = await pFramePromise;
+        const responce = await pFrame.findColumns(iRequest);
+        return {
+            hits: responce.hits
+                .filter((h) => // only exactly matching columns
+             h.mappingVariants.length === 0
+                || h.mappingVariants.some((v) => v.qualifications.forHit.length === 0
+                    && v.qualifications.forQueries.every((q) => q.length === 0)))
+                .map((h) => h.hit),
+        };
+    }
+    async getColumnSpec(handle, columnId) {
+        const { pFramePromise } = this.pFrames.getByKey(handle);
+        const pFrame = await pFramePromise;
+        return await pFrame.getColumnSpec(columnId);
+    }
+    async listColumns(handle) {
+        const { pFramePromise } = this.pFrames.getByKey(handle);
+        const pFrame = await pFramePromise;
+        return await pFrame.listColumns();
+    }
+    async calculateTableData(handle, request, range, signal) {
+        if (index.getDebugFlags().logPFrameRequests) {
+            this.logger.info(`Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
+        }
+        const table = this.pTables.acquire({
+            pFrameHandle: handle,
+            def: migratePTableFilters(request),
+        });
+        const { resource: { pTablePromise, disposeSignal } } = table;
+        const pTable = await pTablePromise;
+        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
+        return await this.frameConcurrencyLimiter.run(async () => {
+            try {
+                const spec = pTable.getSpec();
+                const data = await pTable.getData([...spec.keys()], {
+                    range,
+                    signal: combinedSignal,
+                });
+                const size = await pTable.getFootprint({
+                    withPredecessors: true,
+                    signal: combinedSignal,
+                });
+                this.pTableCache.cache(table, size);
+                return spec.map((spec, i) => ({
+                    spec: spec,
+                    data: data[i],
+                }));
+            }
+            catch (err) {
+                table.unref();
+                throw err;
+            }
+        });
+    }
+    async getUniqueValues(handle, request, signal) {
+        if (index.getDebugFlags().logPFrameRequests) {
+            this.logger.info(`Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`);
+        }
+        const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
+        const pFrame = await pFramePromise;
+        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
+        return await this.frameConcurrencyLimiter.run(async () => {
+            return await pFrame.getUniqueValues({
+                ...request,
+                filters: migrateFilters(request.filters),
+            }, {
+                signal: combinedSignal,
+            });
+        });
+    }
+    //
+    // PTable istance methods
+    //
+    async getSpec(handle) {
+        const { pTablePromise } = this.pTables.getByKey(handle);
+        const pTable = await pTablePromise;
+        return pTable.getSpec();
+    }
+    async getShape(handle, signal) {
+        const { pTablePromise, disposeSignal } = this.pTables.getByKey(handle);
+        const pTable = await pTablePromise;
+        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
+        return await this.tableConcurrencyLimiter.run(async () => {
+            return await pTable.getShape({
+                signal: combinedSignal,
+            });
+        });
+    }
+    async getData(handle, columnIndices, range, signal) {
+        const { pTablePromise, disposeSignal } = this.pTables.getByKey(handle);
+        const pTable = await pTablePromise;
+        const combinedSignal = AbortSignal.any([signal, disposeSignal].filter((s) => !!s));
+        return await this.tableConcurrencyLimiter.run(async () => {
+            return await pTable.getData(columnIndices, {
+                range,
+                signal: combinedSignal,
+            });
+        });
+    }
+}
+function joinEntryToInternal(entry) {
+    switch (entry.type) {
+        case 'column':
+            return {
+                type: 'column',
+                columnId: entry.column,
+            };
+        case 'slicedColumn':
+            return {
+                type: 'slicedColumn',
+                columnId: entry.column,
+                newId: entry.newId,
+                axisFilters: entry.axisFilters,
+            };
+        case 'inlineColumn':
+            return {
+                type: 'inlineColumn',
+                newId: entry.column.id,
+                spec: entry.column.spec,
+                dataInfo: {
+                    type: 'Json',
+                    keyLength: entry.column.spec.axesSpec.length,
+                    data: entry.column.data.reduce((acc, row) => {
+                        acc[JSON.stringify(row.key)] = row.val;
+                        return acc;
+                    }, {}),
+                },
+            };
+        case 'inner':
+        case 'full':
+            return {
+                type: entry.type,
+                entries: entry.entries.map((col) => joinEntryToInternal(col)),
+            };
+        case 'outer':
+            return {
+                type: 'outer',
+                primary: joinEntryToInternal(entry.primary),
+                secondary: entry.secondary.map((col) => joinEntryToInternal(col)),
+            };
+        default:
+            tsHelpers.assertNever(entry);
+    }
+}
+function stableKeyFromFullPTableDef(data) {
+    const hash = node_crypto.createHash('sha256');
+    hash.update(canonicalize(data));
+    return hash.digest().toString('hex');
+}
+function stableKeyFromPFrameData(data) {
+    const orderedData = [...data].map((column) => model.mapPObjectData(column, (r) => {
+        let result;
+        const type = r.type;
+        switch (type) {
+            case 'Json':
+                result = {
+                    type: r.type,
+                    keyLength: r.keyLength,
+                    payload: Object.entries(r.data).map(([part, value]) => ({
+                        key: part,
+                        value,
+                    })),
+                };
+                break;
+            case 'JsonPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: blobKey(info),
+                    })),
+                };
+                break;
+            case 'BinaryPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: [blobKey(info.index), blobKey(info.values)],
+                    })),
+                };
+                break;
+            case 'ParquetPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: info.dataDigest || [
+                            blobKey(info.data),
+                            JSON.stringify({ axes: info.axes, column: info.column }),
+                        ],
+                    })),
+                };
+                break;
+            default:
+                throw new model.PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`);
+        }
+        result.payload.sort((lhs, rhs) => lhs.key.localeCompare(rhs.key));
+        return result;
+    }));
+    orderedData.sort((lhs, rhs) => lhs.id.localeCompare(rhs.id));
+    const hash = node_crypto.createHash('sha256');
+    hash.update(canonicalize(orderedData));
+    return hash.digest().toString('hex');
+}
+
+exports.PFrameDriver = PFrameDriver;
+//# sourceMappingURL=driver.cjs.map