@milaboratories/pl-middle-layer 1.43.70 → 1.43.72

package/dist/pool/driver.cjs

@@ -9,6 +9,7 @@ var pframesRsNode = require('@milaboratories/pframes-rs-node');
 var path = require('node:path');
 var node_stream = require('node:stream');
 var data = require('./data.cjs');
+var plDrivers = require('@milaboratories/pl-drivers');
 
 function makeBlobId(res) {
     return String(res.rid);
@@ -144,7 +145,12 @@ class BlobStore extends plModelMiddleLayer.PFrameInternal.BaseObjectStore {
         }
         catch (error) {
             if (!model.isAbortError(error)) {
-                this.logger('warn', `PFrames blob store unhandled error: ${model.ensureError(error)}`);
+                if (plDrivers.isDownloadNetworkError400(error) && error.statusCode === 404) {
+                    this.logger('info', `PFrames blob store known race error: ${model.ensureError(error)}`);
+                }
+                else {
+                    this.logger('warn', `PFrames blob store unhandled error: ${model.ensureError(error)}`);
+                }
             }
             return await respond({ type: 'InternalError' });
         }

package/dist/pool/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from '@milaboratories/ts-helpers';\nimport type { DownloadDriver } from '@milaboratories/pl-drivers';\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from '@milaboratories/pl-tree';\nimport type {\n  Computable,\n  ComputableStableDefined,\n} from '@milaboratories/computable';\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from '@milaboratories/pf-driver';\nimport { HttpHelpers } from '@milaboratories/pframes-rs-node';\nimport path from 'node:path';\nimport { Readable } from 'node:stream';\nimport {\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from './data';\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, RemoteBlob> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n};\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger('warn',\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: 'NotFound' });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger('error',\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: 'InternalError' });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: 'RangeNotSatisfiable',\n          size: blob.size,\n        });\n      }\n\n      if (params.method === 'HEAD') {\n        return await respond({\n          type: 'Ok',\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger('info',\n        `PFrames blob store requesting content for ${blobId}, `\n        + `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: 'Ok',\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        this.logger('warn',\n          `PFrames blob store unhandled error: ${ensureError(error)}`,\n        );\n      }\n      return await respond({ type: 'InternalError' });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, 'handler'>,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger('info', `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver\n  extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {};\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === 'ParquetPartitioned'\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"names":["RefCountPoolBase","PFrameDriverError","isAbortError","PFrameInternal","ensureError","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","emptyDir","data","isPlTreeNodeAccessor","parseDataInfoResource","isDataInfo","mapDataInfo","traverseParquetChunkResource","makeJsonDataInfo","AbstractPFrameDriver"],"mappings":";;;;;;;;;;;;AA+CA,SAAS,UAAU,CAAC,GAAgB,EAAA;AAClC,IAAA,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAgC;AACvD;AAGA,MAAM,qBACJ,SAAQA,0BAAqE,CAAA;AAG1D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;IAClD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,sBAAsB,MAAM,CAAA,WAAA,CAAa,CAAC;AACrF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,cAAc,CAAC,MAAmB,EAAA;QACvC,OAAO;AACL,YAAA,WAAW,EAAE,OAAO,OAAsC,KAAI;AAC5D,gBAAA,IAAI;oBACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChG;gBAAE,OAAO,GAAY,EAAE;AACrB,oBAAA,IAAI,CAACC,kBAAY,CAAC,GAAG,CAAC;AAAE,wBAAA,MAAM,GAAG;gBACnC;YACF,CAAC;AACD,YAAA,kBAAkB,EAAE,OAAO,MAAmC,KAAI;gBAChE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC;AACtD,gBAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAClE,CAAC;SACF;IACH;AACD;AAGD,MAAM,cACJ,SAAQF,0BAAsE,CAAA;AAE3D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;IAChD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,uBAAuB,MAAM,CAAA,WAAA,CAAa,CAAC;AACtF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,MAAM,WAAW,CACtB,MAAwB,EACxB,OAIC,EAAA;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE;AAC/C,YAAA,KAAK,EAAE;AACL,gBAAA,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;AACzB,gBAAA,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC1B,aAAA;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,SAAA,CAAC;IACJ;AACD;AAMD,MAAM,SAAU,SAAQE,iCAAc,CAAC,eAAe,CAAA;AACnC,IAAA,kBAAkB;AAEnC,IAAA,WAAA,CAAY,OAAyB,EAAA;QACnC,KAAK,CAAC,OAAO,CAAC;AACd,QAAA,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;IACtD;AAEgB,IAAA,MAAM,OAAO,CAC3B,QAAwC,EACxC,MAKC,EAAA;AAED,QAAA,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAACA,iCAAc,CAAC,gBAAgB,CAAC,MAAM,CAAgC;AACxG,QAAA,MAAM,OAAO,GAAG,OAAO,QAA4C,KAAmB;AACpF,YAAA,IAAI;AACF,gBAAA,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,mEAAA,EAAsEC,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC3F;YACH;AACF,QAAA,CAAC;AAED,QAAA,IAAI;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;AAC9D,YAAA,IAAI,CAAC,UAAU;gBAAE,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAE3D,YAAA,IAAI,IAA6B;AACjC,YAAA,IAAI;AACF,gBAAA,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE;YACpC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EACjB,CAAA,uDAAA,EAA0DA,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC/E;gBACD,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACjD;AACA,YAAA,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;AAE9B,YAAA,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC5B,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,oBAAA,KAAK,EAAE,eAAe;AACvB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,0CAAA,EAA6C,MAAM,CAAA,EAAA;kBACjD,CAAA,OAAA,EAAU,eAAe,CAAC,KAAK,CAAA,GAAA,EAAM,eAAe,CAAC,GAAG,CAAA,CAAA,CAAG,CAC9D;YACD,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;AAC5D,gBAAA,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;AACrB,gBAAA,OAAO,EAAE,OAAO,IAAI,KAAI;oBACtB,OAAO,MAAM,OAAO,CAAC;AACnB,wBAAA,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,wBAAA,KAAK,EAAE,eAAe;;AAEtB,wBAAA,IAAI,EAAEC,oBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;AAC7B,qBAAA,CAAC;gBACJ,CAAC;AACF,aAAA,CAAC;QACJ;QAAE,OAAO,KAAc,EAAE;AACvB,YAAA,IAAI,CAACH,kBAAY,CAAC,KAAK,CAAC,EAAE;AACxB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,oCAAA,EAAuCE,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC5D;YACH;YACA,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACjD;IACF;AACD;AAED,MAAM,sBAAsB,CAAA;AAEP,IAAA,IAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,IAAoB,EACpB,MAAiC,EAAA;QADjC,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,MAAM,GAAN,MAAM;IACtB;IAEI,aAAa,IAAI,CACtB,UAA0B,EAC1B,MAA6B,EAC7B,aAAgE,EAAA;QAEhE,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC;AACnD,QAAA,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAGE,yBAAW,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3D,QAAA,MAAM,MAAM,GAAG,MAAMA,yBAAW,CAAC,gBAAgB,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,EAAE,CAAA,+BAAA,EAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA,CAAE,CAAC;AAEnE,QAAA,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD;AAEO,IAAA,OAAO,CAAC,MAAmB,EAAA;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAClC;IAEO,cAAc,GAAA;AACnB,QAAA,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI;IACzB;AAEA,IAAA,OAAO,MAAM,CAAC,YAAY,CAAC,GAAA;AACzB,QAAA,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B;AACD;AAUM,MAAM,uBAAuB,GAAoB;AACtD,IAAA,GAAGC,wCAA+B;IAClC,iBAAiB,EAAE,CAAC;;AAGf,eAAe,kBAAkB,CAAC,MAKxC,EAAA;IACC,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;AACxD,IAAA,MAAMC,kBAAQ,CAAC,iBAAiB,CAAC;AAEjC,IAAA,MAAM,MAAM,GAA0B,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;IACvF,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC;IAC9E,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAC1D,MAAM,CAAC,UAAU,EACjB,MAAM,EACN,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAC3C;AAED,IAAA,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAEC,MAA8C,KAAI;QAC5F,OAAOC,2BAAoB,CAACD,MAAI;AAC9B,cAAEE,0BAAqB,CAACF,MAAI;AAC5B,cAAEG,gBAAU,CAACH,MAAI;AACf,kBAAEA,MAAI,CAAC,IAAI,KAAK;AACd,sBAAEI,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAKK,iCAA4B,CAAC,CAAC,CAAC;AAC1D,sBAAED,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;AACxC,kBAAEM,yBAAgB,CAAC,IAAI,EAAEN,MAAI,CAAC;AACpC,IAAA,CAAC;IAED,OAAO,IAAIO,6BAAoB,CAAC;QAC9B,MAAM;QACN,iBAAiB;QACjB,kBAAkB;AAClB,QAAA,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,eAAe;AAChB,KAAA,CAAC;AACJ;;;;;"}
+{"version":3,"file":"driver.cjs","sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from '@milaboratories/ts-helpers';\nimport type { DownloadDriver } from '@milaboratories/pl-drivers';\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from '@milaboratories/pl-tree';\nimport type {\n  Computable,\n  ComputableStableDefined,\n} from '@milaboratories/computable';\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from '@milaboratories/pf-driver';\nimport { HttpHelpers } from '@milaboratories/pframes-rs-node';\nimport path from 'node:path';\nimport { Readable } from 'node:stream';\nimport {\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from './data';\nimport { isDownloadNetworkError400 } from '@milaboratories/pl-drivers';\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, RemoteBlob> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n};\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger('warn',\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: 'NotFound' });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger('error',\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: 'InternalError' });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: 'RangeNotSatisfiable',\n          size: blob.size,\n        });\n      }\n\n      if (params.method === 'HEAD') {\n        return await respond({\n          type: 'Ok',\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger('info',\n        `PFrames blob store requesting content for ${blobId}, `\n        + `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: 'Ok',\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger('info', `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger('warn',\n            `PFrames blob store unhandled error: ${ensureError(error)}`,\n          );\n        }\n      }\n      return await respond({ type: 'InternalError' });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, 'handler'>,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger('info', `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver\n  extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {};\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === 'ParquetPartitioned'\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"names":["RefCountPoolBase","PFrameDriverError","isAbortError","PFrameInternal","ensureError","Readable","isDownloadNetworkError400","HttpHelpers","AbstractPFrameDriverOpsDefaults","emptyDir","data","isPlTreeNodeAccessor","parseDataInfoResource","isDataInfo","mapDataInfo","traverseParquetChunkResource","makeJsonDataInfo","AbstractPFrameDriver"],"mappings":";;;;;;;;;;;;;AAgDA,SAAS,UAAU,CAAC,GAAgB,EAAA;AAClC,IAAA,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAgC;AACvD;AAGA,MAAM,qBACJ,SAAQA,0BAAqE,CAAA;AAG1D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;IAClD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,sBAAsB,MAAM,CAAA,WAAA,CAAa,CAAC;AACrF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,cAAc,CAAC,MAAmB,EAAA;QACvC,OAAO;AACL,YAAA,WAAW,EAAE,OAAO,OAAsC,KAAI;AAC5D,gBAAA,IAAI;oBACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChG;gBAAE,OAAO,GAAY,EAAE;AACrB,oBAAA,IAAI,CAACC,kBAAY,CAAC,GAAG,CAAC;AAAE,wBAAA,MAAM,GAAG;gBACnC;YACF,CAAC;AACD,YAAA,kBAAkB,EAAE,OAAO,MAAmC,KAAI;gBAChE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC;AACtD,gBAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAClE,CAAC;SACF;IACH;AACD;AAGD,MAAM,cACJ,SAAQF,0BAAsE,CAAA;AAE3D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;IAChD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIC,uBAAiB,CAAC,uBAAuB,MAAM,CAAA,WAAA,CAAa,CAAC;AACtF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,MAAM,WAAW,CACtB,MAAwB,EACxB,OAIC,EAAA;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE;AAC/C,YAAA,KAAK,EAAE;AACL,gBAAA,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;AACzB,gBAAA,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC1B,aAAA;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,SAAA,CAAC;IACJ;AACD;AAMD,MAAM,SAAU,SAAQE,iCAAc,CAAC,eAAe,CAAA;AACnC,IAAA,kBAAkB;AAEnC,IAAA,WAAA,CAAY,OAAyB,EAAA;QACnC,KAAK,CAAC,OAAO,CAAC;AACd,QAAA,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;IACtD;AAEgB,IAAA,MAAM,OAAO,CAC3B,QAAwC,EACxC,MAKC,EAAA;AAED,QAAA,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAACA,iCAAc,CAAC,gBAAgB,CAAC,MAAM,CAAgC;AACxG,QAAA,MAAM,OAAO,GAAG,OAAO,QAA4C,KAAmB;AACpF,YAAA,IAAI;AACF,gBAAA,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,mEAAA,EAAsEC,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC3F;YACH;AACF,QAAA,CAAC;AAED,QAAA,IAAI;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;AAC9D,YAAA,IAAI,CAAC,UAAU;gBAAE,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAE3D,YAAA,IAAI,IAA6B;AACjC,YAAA,IAAI;AACF,gBAAA,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE;YACpC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EACjB,CAAA,uDAAA,EAA0DA,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC/E;gBACD,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACjD;AACA,YAAA,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;AAE9B,YAAA,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC5B,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,oBAAA,KAAK,EAAE,eAAe;AACvB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,0CAAA,EAA6C,MAAM,CAAA,EAAA;kBACjD,CAAA,OAAA,EAAU,eAAe,CAAC,KAAK,CAAA,GAAA,EAAM,eAAe,CAAC,GAAG,CAAA,CAAA,CAAG,CAC9D;YACD,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;AAC5D,gBAAA,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;AACrB,gBAAA,OAAO,EAAE,OAAO,IAAI,KAAI;oBACtB,OAAO,MAAM,OAAO,CAAC;AACnB,wBAAA,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,wBAAA,KAAK,EAAE,eAAe;;AAEtB,wBAAA,IAAI,EAAEC,oBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;AAC7B,qBAAA,CAAC;gBACJ,CAAC;AACF,aAAA,CAAC;QACJ;QAAE,OAAO,KAAc,EAAE;AACvB,YAAA,IAAI,CAACH,kBAAY,CAAC,KAAK,CAAC,EAAE;gBACxB,IAAII,mCAAyB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,EAAE;AAChE,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA,qCAAA,EAAwCF,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;gBACnF;qBAAO;AACL,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,oCAAA,EAAuCA,iBAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC5D;gBACH;YACF;YACA,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACjD;IACF;AACD;AAED,MAAM,sBAAsB,CAAA;AAEP,IAAA,IAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,IAAoB,EACpB,MAAiC,EAAA;QADjC,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,MAAM,GAAN,MAAM;IACtB;IAEI,aAAa,IAAI,CACtB,UAA0B,EAC1B,MAA6B,EAC7B,aAAgE,EAAA;QAEhE,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC;AACnD,QAAA,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAGG,yBAAW,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3D,QAAA,MAAM,MAAM,GAAG,MAAMA,yBAAW,CAAC,gBAAgB,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,EAAE,CAAA,+BAAA,EAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA,CAAE,CAAC;AAEnE,QAAA,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD;AAEO,IAAA,OAAO,CAAC,MAAmB,EAAA;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAClC;IAEO,cAAc,GAAA;AACnB,QAAA,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI;IACzB;AAEA,IAAA,OAAO,MAAM,CAAC,YAAY,CAAC,GAAA;AACzB,QAAA,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B;AACD;AAUM,MAAM,uBAAuB,GAAoB;AACtD,IAAA,GAAGC,wCAA+B;IAClC,iBAAiB,EAAE,CAAC;;AAGf,eAAe,kBAAkB,CAAC,MAKxC,EAAA;IACC,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;AACxD,IAAA,MAAMC,kBAAQ,CAAC,iBAAiB,CAAC;AAEjC,IAAA,MAAM,MAAM,GAA0B,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;IACvF,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC;IAC9E,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAC1D,MAAM,CAAC,UAAU,EACjB,MAAM,EACN,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAC3C;AAED,IAAA,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAEC,MAA8C,KAAI;QAC5F,OAAOC,2BAAoB,CAACD,MAAI;AAC9B,cAAEE,0BAAqB,CAACF,MAAI;AAC5B,cAAEG,gBAAU,CAACH,MAAI;AACf,kBAAEA,MAAI,CAAC,IAAI,KAAK;AACd,sBAAEI,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAKK,iCAA4B,CAAC,CAAC,CAAC;AAC1D,sBAAED,iBAAW,CAACJ,MAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;AACxC,kBAAEM,yBAAgB,CAAC,IAAI,EAAEN,MAAI,CAAC;AACpC,IAAA,CAAC;IAED,OAAO,IAAIO,6BAAoB,CAAC;QAC9B,MAAM;QACN,iBAAiB;QACjB,kBAAkB;AAClB,QAAA,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,eAAe;AAChB,KAAA,CAAC;AACJ;;;;;"}

package/dist/pool/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,EAWL,KAAK,oBAAoB,EAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,QAAQ,EACd,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAGL,KAAK,kBAAkB,EACxB,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAIL,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAG7B,MAAM,2BAA2B,CAAC;AAgOnC,MAAM,WAAW,oBACf,SAAQ,4BAA4B,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;CAAG;AAEnF,MAAM,MAAM,eAAe,GAAG,uBAAuB,GAAG;IACtD,0CAA0C;IAC1C,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,eAGrC,CAAC;AAEF,wBAAsB,kBAAkB,CAAC,MAAM,EAAE;IAC/C,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,QAAQ,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,eAAe,CAAC;CAC1B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA8BhC"}
+{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,EAWL,KAAK,oBAAoB,EAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,QAAQ,EACd,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAGL,KAAK,kBAAkB,EACxB,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAIL,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAG7B,MAAM,2BAA2B,CAAC;AAqOnC,MAAM,WAAW,oBACf,SAAQ,4BAA4B,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;CAAG;AAEnF,MAAM,MAAM,eAAe,GAAG,uBAAuB,GAAG;IACtD,0CAA0C;IAC1C,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,eAGrC,CAAC;AAEF,wBAAsB,kBAAkB,CAAC,MAAM,EAAE;IAC/C,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,QAAQ,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,eAAe,CAAC;CAC1B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA8BhC"}

package/dist/pool/driver.js

@@ -7,6 +7,7 @@ import { HttpHelpers } from '@milaboratories/pframes-rs-node';
 import path from 'node:path';
 import { Readable } from 'node:stream';
 import { parseDataInfoResource, traverseParquetChunkResource } from './data.js';
+import { isDownloadNetworkError400 } from '@milaboratories/pl-drivers';
 
 function makeBlobId(res) {
     return String(res.rid);
@@ -142,7 +143,12 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
         }
         catch (error) {
             if (!isAbortError(error)) {
-                this.logger('warn', `PFrames blob store unhandled error: ${ensureError(error)}`);
+                if (isDownloadNetworkError400(error) && error.statusCode === 404) {
+                    this.logger('info', `PFrames blob store known race error: ${ensureError(error)}`);
+                }
+                else {
+                    this.logger('warn', `PFrames blob store unhandled error: ${ensureError(error)}`);
+                }
             }
             return await respond({ type: 'InternalError' });
         }

package/dist/pool/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from '@milaboratories/ts-helpers';\nimport type { DownloadDriver } from '@milaboratories/pl-drivers';\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from '@milaboratories/pl-tree';\nimport type {\n  Computable,\n  ComputableStableDefined,\n} from '@milaboratories/computable';\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from '@milaboratories/pf-driver';\nimport { HttpHelpers } from '@milaboratories/pframes-rs-node';\nimport path from 'node:path';\nimport { Readable } from 'node:stream';\nimport {\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from './data';\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, RemoteBlob> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n};\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger('warn',\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: 'NotFound' });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger('error',\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: 'InternalError' });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: 'RangeNotSatisfiable',\n          size: blob.size,\n        });\n      }\n\n      if (params.method === 'HEAD') {\n        return await respond({\n          type: 'Ok',\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger('info',\n        `PFrames blob store requesting content for ${blobId}, `\n        + `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: 'Ok',\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        this.logger('warn',\n          `PFrames blob store unhandled error: ${ensureError(error)}`,\n        );\n      }\n      return await respond({ type: 'InternalError' });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, 'handler'>,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger('info', `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver\n  extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {};\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === 'ParquetPartitioned'\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"names":[],"mappings":";;;;;;;;;;AA+CA,SAAS,UAAU,CAAC,GAAgB,EAAA;AAClC,IAAA,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAgC;AACvD;AAGA,MAAM,qBACJ,SAAQ,gBAAqE,CAAA;AAG1D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;IAClD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,sBAAsB,MAAM,CAAA,WAAA,CAAa,CAAC;AACrF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,cAAc,CAAC,MAAmB,EAAA;QACvC,OAAO;AACL,YAAA,WAAW,EAAE,OAAO,OAAsC,KAAI;AAC5D,gBAAA,IAAI;oBACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChG;gBAAE,OAAO,GAAY,EAAE;AACrB,oBAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC;AAAE,wBAAA,MAAM,GAAG;gBACnC;YACF,CAAC;AACD,YAAA,kBAAkB,EAAE,OAAO,MAAmC,KAAI;gBAChE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC;AACtD,gBAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAClE,CAAC;SACF;IACH;AACD;AAGD,MAAM,cACJ,SAAQ,gBAAsE,CAAA;AAE3D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;IAChD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,uBAAuB,MAAM,CAAA,WAAA,CAAa,CAAC;AACtF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,MAAM,WAAW,CACtB,MAAwB,EACxB,OAIC,EAAA;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE;AAC/C,YAAA,KAAK,EAAE;AACL,gBAAA,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;AACzB,gBAAA,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC1B,aAAA;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,SAAA,CAAC;IACJ;AACD;AAMD,MAAM,SAAU,SAAQ,cAAc,CAAC,eAAe,CAAA;AACnC,IAAA,kBAAkB;AAEnC,IAAA,WAAA,CAAY,OAAyB,EAAA;QACnC,KAAK,CAAC,OAAO,CAAC;AACd,QAAA,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;IACtD;AAEgB,IAAA,MAAM,OAAO,CAC3B,QAAwC,EACxC,MAKC,EAAA;AAED,QAAA,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAAC,MAAM,CAAgC;AACxG,QAAA,MAAM,OAAO,GAAG,OAAO,QAA4C,KAAmB;AACpF,YAAA,IAAI;AACF,gBAAA,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,mEAAA,EAAsE,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC3F;YACH;AACF,QAAA,CAAC;AAED,QAAA,IAAI;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;AAC9D,YAAA,IAAI,CAAC,UAAU;gBAAE,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAE3D,YAAA,IAAI,IAA6B;AACjC,YAAA,IAAI;AACF,gBAAA,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE;YACpC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EACjB,CAAA,uDAAA,EAA0D,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC/E;gBACD,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACjD;AACA,YAAA,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;AAE9B,YAAA,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC5B,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,oBAAA,KAAK,EAAE,eAAe;AACvB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,0CAAA,EAA6C,MAAM,CAAA,EAAA;kBACjD,CAAA,OAAA,EAAU,eAAe,CAAC,KAAK,CAAA,GAAA,EAAM,eAAe,CAAC,GAAG,CAAA,CAAA,CAAG,CAC9D;YACD,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;AAC5D,gBAAA,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;AACrB,gBAAA,OAAO,EAAE,OAAO,IAAI,KAAI;oBACtB,OAAO,MAAM,OAAO,CAAC;AACnB,wBAAA,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,wBAAA,KAAK,EAAE,eAAe;;AAEtB,wBAAA,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;AAC7B,qBAAA,CAAC;gBACJ,CAAC;AACF,aAAA,CAAC;QACJ;QAAE,OAAO,KAAc,EAAE;AACvB,YAAA,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;AACxB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,oCAAA,EAAuC,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC5D;YACH;YACA,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACjD;IACF;AACD;AAED,MAAM,sBAAsB,CAAA;AAEP,IAAA,IAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,IAAoB,EACpB,MAAiC,EAAA;QADjC,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,MAAM,GAAN,MAAM;IACtB;IAEI,aAAa,IAAI,CACtB,UAA0B,EAC1B,MAA6B,EAC7B,aAAgE,EAAA;QAEhE,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC;AACnD,QAAA,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAG,WAAW,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3D,QAAA,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,EAAE,CAAA,+BAAA,EAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA,CAAE,CAAC;AAEnE,QAAA,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD;AAEO,IAAA,OAAO,CAAC,MAAmB,EAAA;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAClC;IAEO,cAAc,GAAA;AACnB,QAAA,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI;IACzB;AAEA,IAAA,OAAO,MAAM,CAAC,YAAY,CAAC,GAAA;AACzB,QAAA,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B;AACD;AAUM,MAAM,uBAAuB,GAAoB;AACtD,IAAA,GAAG,+BAA+B;IAClC,iBAAiB,EAAE,CAAC;;AAGf,eAAe,kBAAkB,CAAC,MAKxC,EAAA;IACC,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;AACxD,IAAA,MAAM,QAAQ,CAAC,iBAAiB,CAAC;AAEjC,IAAA,MAAM,MAAM,GAA0B,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;IACvF,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC;IAC9E,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAC1D,MAAM,CAAC,UAAU,EACjB,MAAM,EACN,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAC3C;AAED,IAAA,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAE,IAA8C,KAAI;QAC5F,OAAO,oBAAoB,CAAC,IAAI;AAC9B,cAAE,qBAAqB,CAAC,IAAI;AAC5B,cAAE,UAAU,CAAC,IAAI;AACf,kBAAE,IAAI,CAAC,IAAI,KAAK;AACd,sBAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,4BAA4B,CAAC,CAAC,CAAC;AAC1D,sBAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;AACxC,kBAAE,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC;AACpC,IAAA,CAAC;IAED,OAAO,IAAI,oBAAoB,CAAC;QAC9B,MAAM;QACN,iBAAiB;QACjB,kBAAkB;AAClB,QAAA,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,eAAe;AAChB,KAAA,CAAC;AACJ;;;;"}
+{"version":3,"file":"driver.js","sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  emptyDir,\n  RefCountPoolBase,\n  type PoolEntry,\n  type MiLogger,\n} from '@milaboratories/ts-helpers';\nimport type { DownloadDriver } from '@milaboratories/pl-drivers';\nimport {\n  isPlTreeNodeAccessor,\n  type PlTreeEntry,\n  type PlTreeNodeAccessor,\n} from '@milaboratories/pl-tree';\nimport type {\n  Computable,\n  ComputableStableDefined,\n} from '@milaboratories/computable';\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from '@milaboratories/pf-driver';\nimport { HttpHelpers } from '@milaboratories/pframes-rs-node';\nimport path from 'node:path';\nimport { Readable } from 'node:stream';\nimport {\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from './data';\nimport { isDownloadNetworkError400 } from '@milaboratories/pl-drivers';\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n  return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)));\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool\n  extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, RemoteBlob> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n    return this.blobDriver.getOnDemandBlob(params);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n};\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger('warn',\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: 'NotFound' });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger('error',\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: 'InternalError' });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: 'RangeNotSatisfiable',\n          size: blob.size,\n        });\n      }\n\n      if (params.method === 'HEAD') {\n        return await respond({\n          type: 'Ok',\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger('info',\n        `PFrames blob store requesting content for ${blobId}, `\n        + `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: 'Ok',\n            size: blob.size,\n            range: translatedRange,\n            // eslint-disable-next-line n/no-unsupported-features/node-builtins\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger('info', `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger('warn',\n            `PFrames blob store unhandled error: ${ensureError(error)}`,\n          );\n        }\n      }\n      return await respond({ type: 'InternalError' });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, 'handler'>,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger('info', `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n  }\n}\n\nexport interface InternalPFrameDriver\n  extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {};\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    return isPlTreeNodeAccessor(data)\n      ? parseDataInfoResource(data)\n      : isDataInfo(data)\n        ? data.type === 'ParquetPartitioned'\n          ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n          : mapDataInfo(data, (a) => a.persist())\n        : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"names":[],"mappings":";;;;;;;;;;;AAgDA,SAAS,UAAU,CAAC,GAAgB,EAAA;AAClC,IAAA,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAgC;AACvD;AAGA,MAAM,qBACJ,SAAQ,gBAAqE,CAAA;AAG1D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;IAClD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,sBAAsB,MAAM,CAAA,WAAA,CAAa,CAAC;AACrF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,cAAc,CAAC,MAAmB,EAAA;QACvC,OAAO;AACL,YAAA,WAAW,EAAE,OAAO,OAAsC,KAAI;AAC5D,gBAAA,IAAI;oBACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChG;gBAAE,OAAO,GAAY,EAAE;AACrB,oBAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC;AAAE,wBAAA,MAAM,GAAG;gBACnC;YACF,CAAC;AACD,YAAA,kBAAkB,EAAE,OAAO,MAAmC,KAAI;gBAChE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC;AACtD,gBAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAClE,CAAC;SACF;IACH;AACD;AAGD,MAAM,cACJ,SAAQ,gBAAsE,CAAA;AAE3D,IAAA,UAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,UAA0B,EAC1B,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAHU,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAmB,EAAA;AAC9C,QAAA,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B;IAEU,iBAAiB,CAAC,MAAmB,EAAE,IAAiC,EAAA;QAChF,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;IAChD;AAEO,IAAA,QAAQ,CAAC,MAAmC,EAAA;QACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC;AAC1C,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,uBAAuB,MAAM,CAAA,WAAA,CAAa,CAAC;AACtF,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,MAAM,WAAW,CACtB,MAAwB,EACxB,OAIC,EAAA;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE;AAC/C,YAAA,KAAK,EAAE;AACL,gBAAA,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;AACzB,gBAAA,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC1B,aAAA;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,SAAA,CAAC;IACJ;AACD;AAMD,MAAM,SAAU,SAAQ,cAAc,CAAC,eAAe,CAAA;AACnC,IAAA,kBAAkB;AAEnC,IAAA,WAAA,CAAY,OAAyB,EAAA;QACnC,KAAK,CAAC,OAAO,CAAC;AACd,QAAA,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB;IACtD;AAEgB,IAAA,MAAM,OAAO,CAC3B,QAAwC,EACxC,MAKC,EAAA;AAED,QAAA,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAAC,MAAM,CAAgC;AACxG,QAAA,MAAM,OAAO,GAAG,OAAO,QAA4C,KAAmB;AACpF,YAAA,IAAI;AACF,gBAAA,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,mEAAA,EAAsE,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC3F;YACH;AACF,QAAA,CAAC;AAED,QAAA,IAAI;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;AAC9D,YAAA,IAAI,CAAC,UAAU;gBAAE,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAE3D,YAAA,IAAI,IAA6B;AACjC,YAAA,IAAI;AACF,gBAAA,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE;YACpC;YAAE,OAAO,KAAc,EAAE;AACvB,gBAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EACjB,CAAA,uDAAA,EAA0D,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC/E;gBACD,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACjD;AACA,YAAA,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;AAE9B,YAAA,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC5B,OAAO,MAAM,OAAO,CAAC;AACnB,oBAAA,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,oBAAA,KAAK,EAAE,eAAe;AACvB,iBAAA,CAAC;YACJ;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,0CAAA,EAA6C,MAAM,CAAA,EAAA;kBACjD,CAAA,OAAA,EAAU,eAAe,CAAC,KAAK,CAAA,GAAA,EAAM,eAAe,CAAC,GAAG,CAAA,CAAA,CAAG,CAC9D;YACD,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;AAC5D,gBAAA,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;AACrB,gBAAA,OAAO,EAAE,OAAO,IAAI,KAAI;oBACtB,OAAO,MAAM,OAAO,CAAC;AACnB,wBAAA,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,wBAAA,KAAK,EAAE,eAAe;;AAEtB,wBAAA,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;AAC7B,qBAAA,CAAC;gBACJ,CAAC;AACF,aAAA,CAAC;QACJ;QAAE,OAAO,KAAc,EAAE;AACvB,YAAA,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;gBACxB,IAAI,yBAAyB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,EAAE;AAChE,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA,qCAAA,EAAwC,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;gBACnF;qBAAO;AACL,oBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,oCAAA,EAAuC,WAAW,CAAC,KAAK,CAAC,CAAA,CAAE,CAC5D;gBACH;YACF;YACA,OAAO,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACjD;IACF;AACD;AAED,MAAM,sBAAsB,CAAA;AAEP,IAAA,IAAA;AACA,IAAA,MAAA;IAFnB,WAAA,CACmB,IAAoB,EACpB,MAAiC,EAAA;QADjC,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,MAAM,GAAN,MAAM;IACtB;IAEI,aAAa,IAAI,CACtB,UAA0B,EAC1B,MAA6B,EAC7B,aAAgE,EAAA;QAEhE,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC;AACnD,QAAA,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAG,WAAW,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3D,QAAA,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,CAAC;QAChF,MAAM,CAAC,MAAM,EAAE,CAAA,+BAAA,EAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA,CAAE,CAAC;AAEnE,QAAA,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD;AAEO,IAAA,OAAO,CAAC,MAAmB,EAAA;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAClC;IAEO,cAAc,GAAA;AACnB,QAAA,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI;IACzB;AAEA,IAAA,OAAO,MAAM,CAAC,YAAY,CAAC,GAAA;AACzB,QAAA,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;IAC1B;AACD;AAUM,MAAM,uBAAuB,GAAoB;AACtD,IAAA,GAAG,+BAA+B;IAClC,iBAAiB,EAAE,CAAC;;AAGf,eAAe,kBAAkB,CAAC,MAKxC,EAAA;IACC,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;AACxD,IAAA,MAAM,QAAQ,CAAC,iBAAiB,CAAC;AAEjC,IAAA,MAAM,MAAM,GAA0B,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;IACvF,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC;IAC9E,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAC1D,MAAM,CAAC,UAAU,EACjB,MAAM,EACN,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAC3C;AAED,IAAA,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAE,IAA8C,KAAI;QAC5F,OAAO,oBAAoB,CAAC,IAAI;AAC9B,cAAE,qBAAqB,CAAC,IAAI;AAC5B,cAAE,UAAU,CAAC,IAAI;AACf,kBAAE,IAAI,CAAC,IAAI,KAAK;AACd,sBAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,4BAA4B,CAAC,CAAC,CAAC;AAC1D,sBAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;AACxC,kBAAE,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC;AACpC,IAAA,CAAC;IAED,OAAO,IAAI,oBAAoB,CAAC;QAC9B,MAAM;QACN,iBAAiB;QACjB,kBAAkB;AAClB,QAAA,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,eAAe;AAChB,KAAA,CAAC;AACJ;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-middle-layer",
-  "version": "1.43.70",
+  "version": "1.43.72",
   "description": "Pl Middle Layer",
   "engines": {
     "node": ">=22.19.0"
@@ -26,23 +26,23 @@
     "yaml": "^2.8.0",
     "zod": "~3.23.8",
     "remeda": "^2.28.0",
-    "@milaboratories/computable": "2.7.4",
     "@milaboratories/pl-http": "1.2.0",
+    "@milaboratories/computable": "2.7.4",
+    "@milaboratories/pl-model-common": "1.21.8",
     "@milaboratories/resolve-helper": "1.1.1",
-    "@milaboratories/pl-client": "2.16.8",
-    "@platforma-sdk/block-tools": "2.6.22",
-    "@milaboratories/pl-model-common": "1.21.7",
-    "@milaboratories/pl-drivers": "1.11.19",
-    "@milaboratories/pl-model-middle-layer": "1.8.41",
-    "@milaboratories/pl-model-backend": "1.1.25",
-    "@milaboratories/pl-tree": "1.8.15",
-    "@platforma-sdk/model": "1.45.35",
+    "@milaboratories/pl-drivers": "1.11.21",
+    "@platforma-sdk/block-tools": "2.6.23",
+    "@platforma-sdk/model": "1.45.42",
+    "@milaboratories/pl-client": "2.16.9",
+    "@milaboratories/pl-model-middle-layer": "1.8.42",
+    "@milaboratories/pl-tree": "1.8.16",
     "@milaboratories/ts-helpers": "1.5.4",
+    "@milaboratories/pl-model-backend": "1.1.26",
     "@platforma-sdk/workflow-tengo": "5.6.2",
     "@milaboratories/pl-config": "1.7.9",
-    "@milaboratories/pl-errors": "1.1.40",
-    "@milaboratories/pl-deployments": "2.12.7",
-    "@milaboratories/pf-driver": "1.0.5"
+    "@milaboratories/pl-errors": "1.1.41",
+    "@milaboratories/pf-driver": "1.0.6",
+    "@milaboratories/pl-deployments": "2.12.8"
   },
   "devDependencies": {
     "semver": "^7.7.2",
@@ -52,9 +52,9 @@
     "@types/node": "~24.5.2",
     "tslib": "~2.7.0",
     "@milaboratories/build-configs": "1.0.8",
-    "@milaboratories/ts-builder": "1.0.5",
     "@milaboratories/eslint-config": "1.0.5",
-    "@milaboratories/ts-configs": "1.0.6"
+    "@milaboratories/ts-configs": "1.0.6",
+    "@milaboratories/ts-builder": "1.0.5"
   },
   "scripts": {
     "type-check": "ts-builder types --target node",

package/src/pool/driver.ts

@@ -44,6 +44,7 @@ import {
   parseDataInfoResource,
   traverseParquetChunkResource,
 } from './data';
+import { isDownloadNetworkError400 } from '@milaboratories/pl-drivers';
 
 function makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {
   return String(res.rid) as PFrameInternal.PFrameBlobId;
@@ -217,9 +218,13 @@ class BlobStore extends PFrameInternal.BaseObjectStore {
       });
     } catch (error: unknown) {
       if (!isAbortError(error)) {
-        this.logger('warn',
-          `PFrames blob store unhandled error: ${ensureError(error)}`,
-        );
+        if (isDownloadNetworkError400(error) && error.statusCode === 404) {
+          this.logger('info', `PFrames blob store known race error: ${ensureError(error)}`);
+        } else {
+          this.logger('warn',
+            `PFrames blob store unhandled error: ${ensureError(error)}`,
+          );
+        }
       }
       return await respond({ type: 'InternalError' });
     }