@milaboratories/pl-middle-layer 1.64.20 → 1.64.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pool/driver.cjs +4 -0
- package/dist/pool/driver.cjs.map +1 -1
- package/dist/pool/driver.d.ts.map +1 -1
- package/dist/pool/driver.js +4 -0
- package/dist/pool/driver.js.map +1 -1
- package/package.json +14 -14
- package/src/pool/driver.ts +11 -0
package/dist/pool/driver.cjs
CHANGED
|
@@ -160,6 +160,10 @@ var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
|
|
|
160
160
|
httpServerInfo() {
|
|
161
161
|
return this.server.info;
|
|
162
162
|
}
|
|
163
|
+
async getCacheMetrics() {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
async resetCache() {}
|
|
163
167
|
async [Symbol.asyncDispose]() {
|
|
164
168
|
await this.server.stop();
|
|
165
169
|
await this.pool[Symbol.asyncDispose]();
|
package/dist/pool/driver.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n // TODO: private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n // TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return null; // TODO: this.cache.getMetrics()\n }\n\n public async resetCache(): Promise<void> {\n // TODO: this.cache.reset()\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n // TODO: await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QAEA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAInB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAGjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO;;CAGT,MAAa,aAA4B;CAIzC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AAExB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;UAuSiB,oBAAA,SAA6B,4BAAA,CAC5C,oBAAA,CAAqB,kBAAA;AAAA,KAGX,eAAA,GAAkB,uBAAA;EAJQ,0CAMpC,iBAAA;AAAA"}
|
package/dist/pool/driver.js
CHANGED
|
@@ -158,6 +158,10 @@ var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
|
|
|
158
158
|
httpServerInfo() {
|
|
159
159
|
return this.server.info;
|
|
160
160
|
}
|
|
161
|
+
async getCacheMetrics() {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
async resetCache() {}
|
|
161
165
|
async [Symbol.asyncDispose]() {
|
|
162
166
|
await this.server.stop();
|
|
163
167
|
await this.pool[Symbol.asyncDispose]();
|
package/dist/pool/driver.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n // TODO: private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n // TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return null; // TODO: this.cache.getMetrics()\n }\n\n public async resetCache(): Promise<void> {\n // TODO: this.cache.reset()\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n // TODO: await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QAEA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAInB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAGjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO;;CAGT,MAAa,aAA4B;CAIzC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AAExB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.64.
|
|
3
|
+
"version": "1.64.22",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -31,24 +31,24 @@
|
|
|
31
31
|
"utility-types": "^3.11.0",
|
|
32
32
|
"yaml": "^2.8.0",
|
|
33
33
|
"zod": "~3.25.76",
|
|
34
|
-
"@milaboratories/computable": "2.9.5",
|
|
35
34
|
"@milaboratories/helpers": "1.14.2",
|
|
36
|
-
"@milaboratories/
|
|
37
|
-
"@milaboratories/pf-driver": "1.7.
|
|
38
|
-
"@milaboratories/pl-deployments": "3.0.7",
|
|
35
|
+
"@milaboratories/computable": "2.9.5",
|
|
36
|
+
"@milaboratories/pf-driver": "1.7.9",
|
|
39
37
|
"@milaboratories/pl-client": "3.11.4",
|
|
40
|
-
"@milaboratories/
|
|
41
|
-
"@milaboratories/pl-
|
|
42
|
-
"@milaboratories/pl-model-backend": "1.4.7",
|
|
38
|
+
"@milaboratories/pf-spec-driver": "1.4.11",
|
|
39
|
+
"@milaboratories/pl-deployments": "3.0.7",
|
|
43
40
|
"@milaboratories/pl-http": "1.2.4",
|
|
44
41
|
"@milaboratories/pl-drivers": "1.15.6",
|
|
42
|
+
"@milaboratories/pl-model-backend": "1.4.7",
|
|
43
|
+
"@milaboratories/pl-model-middle-layer": "1.30.5",
|
|
45
44
|
"@milaboratories/pl-tree": "1.12.12",
|
|
46
|
-
"@milaboratories/pl-model-
|
|
45
|
+
"@milaboratories/pl-model-common": "1.46.1",
|
|
46
|
+
"@milaboratories/pl-errors": "1.4.22",
|
|
47
47
|
"@milaboratories/resolve-helper": "1.1.3",
|
|
48
|
-
"@platforma-sdk/
|
|
48
|
+
"@platforma-sdk/block-tools": "2.10.15",
|
|
49
|
+
"@platforma-sdk/model": "1.79.3",
|
|
49
50
|
"@milaboratories/ts-helpers": "1.8.3",
|
|
50
|
-
"@platforma-sdk/
|
|
51
|
-
"@platforma-sdk/model": "1.79.2"
|
|
51
|
+
"@platforma-sdk/workflow-tengo": "6.5.1"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@types/node": "~24.5.2",
|
|
@@ -56,9 +56,9 @@
|
|
|
56
56
|
"@vitest/coverage-istanbul": "^4.1.3",
|
|
57
57
|
"typescript": "~5.9.3",
|
|
58
58
|
"vitest": "^4.1.3",
|
|
59
|
+
"@milaboratories/ts-builder": "1.5.0",
|
|
59
60
|
"@milaboratories/build-configs": "2.0.0",
|
|
60
|
-
"@milaboratories/ts-configs": "1.2.3"
|
|
61
|
-
"@milaboratories/ts-builder": "1.5.0"
|
|
61
|
+
"@milaboratories/ts-configs": "1.2.3"
|
|
62
62
|
},
|
|
63
63
|
"engines": {
|
|
64
64
|
"node": ">=22.19.0"
|
package/src/pool/driver.ts
CHANGED
|
@@ -251,6 +251,7 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
|
|
|
251
251
|
constructor(
|
|
252
252
|
private readonly pool: RemoteBlobPool,
|
|
253
253
|
private readonly server: PFrameInternal.HttpServer,
|
|
254
|
+
// TODO: private readonly cache: PFrameInternal.CachingObjectStore,
|
|
254
255
|
) {}
|
|
255
256
|
|
|
256
257
|
public static async init(
|
|
@@ -260,6 +261,7 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
|
|
|
260
261
|
): Promise<RemoteBlobProviderImpl> {
|
|
261
262
|
const pool = new RemoteBlobPool(blobDriver, logger);
|
|
262
263
|
const store = new BlobStore({ remoteBlobProvider: pool, logger });
|
|
264
|
+
// TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });
|
|
263
265
|
|
|
264
266
|
const handler = HttpHelpers.createRequestHandler({ store });
|
|
265
267
|
const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });
|
|
@@ -276,8 +278,17 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
|
|
|
276
278
|
return this.server.info;
|
|
277
279
|
}
|
|
278
280
|
|
|
281
|
+
public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {
|
|
282
|
+
return null; // TODO: this.cache.getMetrics()
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
public async resetCache(): Promise<void> {
|
|
286
|
+
// TODO: this.cache.reset()
|
|
287
|
+
}
|
|
288
|
+
|
|
279
289
|
async [Symbol.asyncDispose](): Promise<void> {
|
|
280
290
|
await this.server.stop();
|
|
291
|
+
// TODO: await this.cache[Symbol.asyncDispose]();
|
|
281
292
|
await this.pool[Symbol.asyncDispose]();
|
|
282
293
|
}
|
|
283
294
|
}
|