@milaboratories/pl-middle-layer 1.64.34 → 1.64.36

package/dist/pool/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    });\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAMC,gCAAAA,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAUA,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAWA,UAAAA,QAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = (await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    })) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAgB,MAAMC,gCAAAA,YAAY,yBAAyB;GAC/D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAUA,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAWA,UAAAA,QAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/dist/pool/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    });\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAM,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAW,KAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = (await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    })) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAgB,MAAM,YAAY,yBAAyB;GAC/D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAW,KAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-middle-layer",
-  "version": "1.64.34",
+  "version": "1.64.36",
   "description": "Pl Middle Layer",
   "keywords": [],
   "license": "UNLICENSED",
@@ -19,8 +19,8 @@
     }
   },
   "dependencies": {
-    "@milaboratories/pframes-rs-node": "1.1.50",
-    "@milaboratories/pframes-rs-wasm": "1.1.50",
+    "@milaboratories/pframes-rs-node": "1.1.52",
+    "@milaboratories/pframes-rs-wasm": "1.1.52",
     "canonicalize": "~2.1.0",
     "denque": "^2.1.0",
     "es-toolkit": "^1.39.10",
@@ -32,23 +32,23 @@
     "yaml": "^2.8.0",
     "zod": "~3.25.76",
     "@milaboratories/computable": "2.9.5",
-    "@milaboratories/helpers": "1.14.2",
+    "@milaboratories/pf-spec-driver": "1.4.15",
     "@milaboratories/pl-client": "3.11.5",
-    "@milaboratories/pl-errors": "1.4.23",
+    "@milaboratories/helpers": "1.14.2",
     "@milaboratories/pl-drivers": "1.16.1",
+    "@milaboratories/pf-driver": "1.7.13",
     "@milaboratories/pl-deployments": "3.0.8",
+    "@milaboratories/pl-errors": "1.4.23",
     "@milaboratories/pl-model-backend": "1.4.8",
-    "@milaboratories/pf-driver": "1.7.12",
-    "@milaboratories/pf-spec-driver": "1.4.14",
     "@milaboratories/pl-http": "1.2.4",
     "@milaboratories/pl-model-common": "1.46.2",
-    "@milaboratories/pl-model-middle-layer": "1.30.7",
+    "@milaboratories/pl-tree": "1.12.13",
+    "@milaboratories/pl-model-middle-layer": "1.30.8",
     "@milaboratories/resolve-helper": "1.1.3",
+    "@platforma-sdk/model": "1.79.17",
+    "@platforma-sdk/block-tools": "2.11.2",
     "@milaboratories/ts-helpers": "1.8.3",
-    "@milaboratories/pl-tree": "1.12.13",
-    "@platforma-sdk/block-tools": "2.11.1",
-    "@platforma-sdk/workflow-tengo": "6.6.3",
-    "@platforma-sdk/model": "1.79.15"
+    "@platforma-sdk/workflow-tengo": "6.6.5"
   },
   "devDependencies": {
     "@types/node": "~24.5.2",
@@ -57,8 +57,8 @@
     "typescript": "~5.9.3",
     "vitest": "^4.1.3",
     "@milaboratories/build-configs": "2.0.0",
-    "@milaboratories/ts-configs": "1.2.3",
-    "@milaboratories/ts-builder": "1.5.2"
+    "@milaboratories/ts-builder": "1.5.2",
+    "@milaboratories/ts-configs": "1.2.3"
   },
   "engines": {
     "node": ">=22.19.0"

package/src/pool/driver.ts

@@ -262,11 +262,11 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
   ): Promise<RemoteBlobProviderImpl> {
     const pool = new RemoteBlobPool(blobDriver, logger);
     const store = new BlobStore({ remoteBlobProvider: pool, logger });
-    const cachingStore = await HttpHelpers.createCachingObjectStore({
+    const cachingStore = (await HttpHelpers.createCachingObjectStore({
       upstream: store,
       config: cacheConfig,
       logger,
-    });
+    })) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release
 
     const handler = HttpHelpers.createRequestHandler({ store: cachingStore });
     const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });