@milaboratories/pl-middle-layer 1.54.1 → 1.54.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/middle_layer/driver_kit.cjs +2 -1
- package/dist/middle_layer/driver_kit.cjs.map +1 -1
- package/dist/middle_layer/driver_kit.js +2 -1
- package/dist/middle_layer/driver_kit.js.map +1 -1
- package/dist/pool/driver.cjs +3 -2
- package/dist/pool/driver.cjs.map +1 -1
- package/dist/pool/driver.d.ts +1 -1
- package/dist/pool/driver.js +2 -1
- package/dist/pool/driver.js.map +1 -1
- package/package.json +19 -18
- package/src/middle_layer/driver_kit.ts +2 -1
- package/src/pool/driver.ts +2 -6
|
@@ -3,6 +3,7 @@ const require_driver = require('../pool/driver.cjs');
|
|
|
3
3
|
require('../pool/index.cjs');
|
|
4
4
|
const require_ops = require('./ops.cjs');
|
|
5
5
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
6
|
+
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
6
7
|
let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
|
|
7
8
|
|
|
8
9
|
//#region src/middle_layer/driver_kit.ts
|
|
@@ -37,7 +38,7 @@ async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
|
|
|
37
38
|
frontendDriver: new _milaboratories_pl_drivers.DownloadUrlDriver(ops.logger, pl.httpDispatcher, frontendDownloadPath, signer, ops.frontendDownloadDriverOps)
|
|
38
39
|
};
|
|
39
40
|
const dispose = async () => {
|
|
40
|
-
const disposePromises = Object.values(driverKit).flatMap((driver) => (0,
|
|
41
|
+
const disposePromises = Object.values(driverKit).flatMap((driver) => (0, _milaboratories_helpers.isAsyncDisposable)(driver) ? [driver[Symbol.asyncDispose]()] : []);
|
|
41
42
|
await Promise.all(disposePromises);
|
|
42
43
|
};
|
|
43
44
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport {
|
|
1
|
+
{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const pFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n options: ops.pFrameDriverOps,\n });\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAGA;EACH,GAAGC,qCAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAIC,4CAAiB,IAAI,YAAY;CAEpD,MAAM,sEAAsC,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,8DAA8B,IAAI,IAAI,OAAO;CACnD,MAAM,0EAA0C,IAAI,IAAI,OAAO;CAC/D,MAAM,kFAAkD,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAMC,0CAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAIC,mDAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAIC,wCACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAIC,4CAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA0B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA5BgB,IAAIC,sCAAW,IAAI,QAAQ,kBAAkB,WAAW;EA6BxE,UA5Be,MAAMC,oCAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAsBC;EACA;EACA,cAtBmB,MAAMC,kCAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,SAAS,IAAI;GACd,CAAC;EAkBA,gBAhB6B,IAAIC,6CACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,0DACtC,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { createPFrameDriver } from "../pool/driver.js";
|
|
2
2
|
import "../pool/index.js";
|
|
3
3
|
import { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from "./ops.js";
|
|
4
|
-
import { HmacSha256Signer
|
|
4
|
+
import { HmacSha256Signer } from "@milaboratories/ts-helpers";
|
|
5
|
+
import { isAsyncDisposable } from "@milaboratories/helpers";
|
|
5
6
|
import { DownloadBlobToURLDriver, DownloadDriver, DownloadUrlDriver, LogsDriver, LogsStreamDriver, LsDriver, UploadDriver, createDownloadClient, createLogsClient, createUploadBlobClient, createUploadProgressClient } from "@milaboratories/pl-drivers";
|
|
6
7
|
|
|
7
8
|
//#region src/middle_layer/driver_kit.ts
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport {
|
|
1
|
+
{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const pFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n options: ops.pFrameDriverOps,\n });\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAG;EACH,GAAG,yBAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAI,iBAAiB,IAAI,YAAY;CAEpD,MAAM,iBAAiB,qBAAqB,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,aAAa,iBAAiB,IAAI,IAAI,OAAO;CACnD,MAAM,mBAAmB,uBAAuB,IAAI,IAAI,OAAO;CAC/D,MAAM,uBAAuB,2BAA2B,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAM,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAI,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAI,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAI,iBAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA0B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA5BgB,IAAI,WAAW,IAAI,QAAQ,kBAAkB,WAAW;EA6BxE,UA5Be,MAAM,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAsBC;EACA;EACA,cAtBmB,MAAM,mBAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,SAAS,IAAI;GACd,CAAC;EAkBA,gBAhB6B,IAAI,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,WACxD,kBAAkB,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}
|
package/dist/pool/driver.cjs
CHANGED
|
@@ -6,6 +6,7 @@ let node_path = require("node:path");
|
|
|
6
6
|
node_path = require_runtime.__toESM(node_path);
|
|
7
7
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
8
8
|
let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
|
|
9
|
+
let _milaboratories_helpers = require("@milaboratories/helpers");
|
|
9
10
|
let _milaboratories_pf_driver = require("@milaboratories/pf-driver");
|
|
10
11
|
let _milaboratories_pframes_rs_node = require("@milaboratories/pframes-rs-node");
|
|
11
12
|
let node_stream = require("node:stream");
|
|
@@ -15,7 +16,7 @@ let _milaboratories_pl_drivers = require("@milaboratories/pl-drivers");
|
|
|
15
16
|
function makeBlobId(res) {
|
|
16
17
|
return String(res.rid);
|
|
17
18
|
}
|
|
18
|
-
var LocalBlobProviderImpl = class extends
|
|
19
|
+
var LocalBlobProviderImpl = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
19
20
|
constructor(blobDriver, logger) {
|
|
20
21
|
super();
|
|
21
22
|
this.blobDriver = blobDriver;
|
|
@@ -48,7 +49,7 @@ var LocalBlobProviderImpl = class extends _milaboratories_ts_helpers.RefCountPoo
|
|
|
48
49
|
};
|
|
49
50
|
}
|
|
50
51
|
};
|
|
51
|
-
var RemoteBlobPool = class extends
|
|
52
|
+
var RemoteBlobPool = class extends _milaboratories_helpers.RefCountPoolBase {
|
|
52
53
|
constructor(blobDriver, logger) {
|
|
53
54
|
super();
|
|
54
55
|
this.blobDriver = blobDriver;
|
package/dist/pool/driver.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n emptyDir,\n RefCountPoolBase,\n type PoolEntry,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n isPlTreeNodeAccessor,\n type PlTreeEntry,\n type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<PlTreeEntry>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n PlTreeEntry,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n return this.blobDriver.getOnDemandBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => a.persist())\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;AA0CA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACUA,4CAEV;CACE,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,AAAO,SAAS,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,AAAO,eACL,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,wCAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,4CAI3B;CACA,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,AAAO,SAAS,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,qDAAe,gBAAgB;CACrD,AAAiB;CAEjB,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,qDAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,4GAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,gGAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,qBAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,wCAAc,MAAM,CACtB,+DAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,8EAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,6EAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,4CAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,4CAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,AAAO,QAAQ,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,AAAO,iBAAgD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,kBAAK,QAAQ,OAAO,UAAU;AACxD,gDAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,2DAA4B,KAAK,GAC7BC,mCAAsB,KAAK,wCAChB,KAAK,GACd,KAAK,SAAS,6DACA,OAAO,MAAMC,0CAA6B,EAAE,CAAC,yCAC7C,OAAO,MAAM,EAAE,SAAS,CAAC,mDACtB,MAAM,KAAK;;AAGpC,QAAO,IAAIC,+CAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n isPlTreeNodeAccessor,\n type PlTreeEntry,\n type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<PlTreeEntry>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n PlTreeEntry,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n return this.blobDriver.getOnDemandBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => a.persist())\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsCA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACUA,yCAEV;CACE,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,AAAO,SAAS,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,AAAO,eACL,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,wCAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,yCAI3B;CACA,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,AAAO,SAAS,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,uCAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,qDAAe,gBAAgB;CACrD,AAAiB;CAEjB,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,qDAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,4GAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,gGAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,qBAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,wCAAc,MAAM,CACtB,+DAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,8EAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,6EAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAUC,4CAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,4CAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,AAAO,QAAQ,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,AAAO,iBAAgD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,kBAAK,QAAQ,OAAO,UAAU;AACxD,gDAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,2DAA4B,KAAK,GAC7BC,mCAAsB,KAAK,wCAChB,KAAK,GACd,KAAK,SAAS,6DACA,OAAO,MAAMC,0CAA6B,EAAE,CAAC,yCAC7C,OAAO,MAAM,EAAE,SAAS,CAAC,mDACtB,MAAM,KAAK;;AAGpC,QAAO,IAAIC,+CAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
package/dist/pool/driver.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { PColumnDataUniversal } from "@platforma-sdk/model";
|
|
2
|
-
import { MiLogger } from "@milaboratories/ts-helpers";
|
|
3
2
|
import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
|
|
3
|
+
import "@milaboratories/helpers";
|
|
4
4
|
import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
|
|
5
5
|
import { DownloadDriver } from "@milaboratories/pl-drivers";
|
|
6
6
|
|
package/dist/pool/driver.js
CHANGED
|
@@ -2,8 +2,9 @@ import { parseDataInfoResource, traverseParquetChunkResource } from "./data.js";
|
|
|
2
2
|
import { PFrameDriverError, ensureError, isAbortError, isDataInfo, mapDataInfo } from "@platforma-sdk/model";
|
|
3
3
|
import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import { emptyDir } from "@milaboratories/ts-helpers";
|
|
6
6
|
import { isPlTreeNodeAccessor } from "@milaboratories/pl-tree";
|
|
7
|
+
import { RefCountPoolBase } from "@milaboratories/helpers";
|
|
7
8
|
import { AbstractPFrameDriver, AbstractPFrameDriverOpsDefaults, makeJsonDataInfo } from "@milaboratories/pf-driver";
|
|
8
9
|
import { HttpHelpers } from "@milaboratories/pframes-rs-node";
|
|
9
10
|
import { Readable } from "node:stream";
|
package/dist/pool/driver.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport {\n emptyDir,\n RefCountPoolBase,\n type PoolEntry,\n type MiLogger,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n isPlTreeNodeAccessor,\n type PlTreeEntry,\n type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<PlTreeEntry>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n PlTreeEntry,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n return this.blobDriver.getOnDemandBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => a.persist())\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;AA0CA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,AAAO,SAAS,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,AAAO,eACL,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,AAAO,SAAS,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD,AAAiB;CAEjB,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,AAAO,QAAQ,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,AAAO,iBAAgD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,SAAO,qBAAqB,KAAK,GAC7B,sBAAsB,KAAK,GAC3B,WAAW,KAAK,GACd,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,EAAE,SAAS,CAAC,GACvC,iBAAiB,MAAM,KAAK;;AAGpC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport {\n isPlTreeNodeAccessor,\n type PlTreeEntry,\n type PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport { parseDataInfoResource, traverseParquetChunkResource } from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\n\nfunction makeBlobId(res: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return String(res.rid) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<PlTreeEntry, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<PlTreeEntry>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContent(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n PlTreeEntry,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: PlTreeEntry): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(params: PlTreeEntry, _key: PFrameInternal.PFrameBlobId): RemoteBlob {\n return this.blobDriver.getOnDemandBlob(params);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<PlTreeEntry> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n\n const handler = HttpHelpers.createRequestHandler({ store });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server);\n }\n\n public acquire(params: PlTreeEntry): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n port: params.options.parquetServerPort,\n });\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n return isPlTreeNodeAccessor(data)\n ? parseDataInfoResource(data)\n : isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => a.persist())\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAsCA,SAAS,WAAW,KAA+C;AACjE,QAAO,OAAO,IAAI,IAAI;;AAIxB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA8C;AAC7F,SAAO,KAAK,WAAW,kBAAkB,OAAO;;CAGlD,AAAO,SAAS,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,AAAO,eACL,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,AAAiB,YACjB,AAAiB,QACjB;AACA,SAAO;EAHU;EACA;;CAKnB,AAAU,mBAAmB,QAAkD;AAC7E,SAAO,WAAW,OAAO;;CAG3B,AAAU,kBAAkB,QAAqB,MAA+C;AAC9F,SAAO,KAAK,WAAW,gBAAgB,OAAO;;CAGhD,AAAO,SAAS,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD,AAAiB;CAEjB,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAkE;CACtE,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAEjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,AAAO,QAAQ,QAA6D;AAC1E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,AAAO,iBAAgD;AACrD,SAAO,KAAK,OAAO;;CAGrB,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,SAAO,qBAAqB,KAAK,GAC7B,sBAAsB,KAAK,GAC3B,WAAW,KAAK,GACd,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,EAAE,SAAS,CAAC,GACvC,iBAAiB,MAAM,KAAK;;AAGpC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.54.
|
|
3
|
+
"version": "1.54.3",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -30,23 +30,24 @@
|
|
|
30
30
|
"utility-types": "^3.11.0",
|
|
31
31
|
"yaml": "^2.8.0",
|
|
32
32
|
"zod": "~3.23.8",
|
|
33
|
-
"@milaboratories/
|
|
34
|
-
"@milaboratories/
|
|
35
|
-
"@milaboratories/
|
|
36
|
-
"@milaboratories/pl-
|
|
37
|
-
"@milaboratories/pl-
|
|
38
|
-
"@milaboratories/
|
|
39
|
-
"@milaboratories/pf-spec-driver": "1.2.1",
|
|
40
|
-
"@milaboratories/pl-model-backend": "1.2.3",
|
|
33
|
+
"@milaboratories/helpers": "1.14.1",
|
|
34
|
+
"@milaboratories/computable": "2.9.2",
|
|
35
|
+
"@milaboratories/pf-driver": "1.3.3",
|
|
36
|
+
"@milaboratories/pl-errors": "1.2.5",
|
|
37
|
+
"@milaboratories/pl-drivers": "1.12.7",
|
|
38
|
+
"@milaboratories/pf-spec-driver": "1.2.3",
|
|
41
39
|
"@milaboratories/pl-http": "1.2.4",
|
|
42
|
-
"@milaboratories/pl-model-
|
|
43
|
-
"@milaboratories/pl-
|
|
44
|
-
"@milaboratories/pl-
|
|
45
|
-
"@
|
|
46
|
-
"@milaboratories/
|
|
40
|
+
"@milaboratories/pl-model-common": "1.31.1",
|
|
41
|
+
"@milaboratories/pl-deployments": "2.16.5",
|
|
42
|
+
"@milaboratories/pl-client": "2.18.5",
|
|
43
|
+
"@milaboratories/pl-model-backend": "1.2.5",
|
|
44
|
+
"@milaboratories/pl-model-middle-layer": "1.16.3",
|
|
47
45
|
"@milaboratories/resolve-helper": "1.1.3",
|
|
48
|
-
"@
|
|
49
|
-
"@
|
|
46
|
+
"@milaboratories/pl-tree": "1.9.6",
|
|
47
|
+
"@milaboratories/ts-helpers": "1.8.1",
|
|
48
|
+
"@platforma-sdk/block-tools": "2.7.5",
|
|
49
|
+
"@platforma-sdk/model": "1.63.1",
|
|
50
|
+
"@platforma-sdk/workflow-tengo": "5.11.0"
|
|
50
51
|
},
|
|
51
52
|
"devDependencies": {
|
|
52
53
|
"@types/node": "~24.5.2",
|
|
@@ -55,9 +56,9 @@
|
|
|
55
56
|
"semver": "^7.7.2",
|
|
56
57
|
"typescript": "~5.9.3",
|
|
57
58
|
"vitest": "^4.0.18",
|
|
58
|
-
"@milaboratories/
|
|
59
|
+
"@milaboratories/build-configs": "1.5.2",
|
|
59
60
|
"@milaboratories/ts-configs": "1.2.2",
|
|
60
|
-
"@milaboratories/
|
|
61
|
+
"@milaboratories/ts-builder": "1.3.0"
|
|
61
62
|
},
|
|
62
63
|
"engines": {
|
|
63
64
|
"node": ">=22.19.0"
|
|
@@ -15,7 +15,8 @@ import {
|
|
|
15
15
|
} from "@milaboratories/pl-drivers";
|
|
16
16
|
import type * as Sdk from "@milaboratories/pl-model-common";
|
|
17
17
|
import type { Signer } from "@milaboratories/ts-helpers";
|
|
18
|
-
import {
|
|
18
|
+
import { isAsyncDisposable } from "@milaboratories/helpers";
|
|
19
|
+
import { HmacSha256Signer } from "@milaboratories/ts-helpers";
|
|
19
20
|
import type { InternalPFrameDriver } from "../pool";
|
|
20
21
|
import { createPFrameDriver } from "../pool";
|
|
21
22
|
import type { DriverKitOps, DriverKitOpsConstructor } from "./ops";
|
package/src/pool/driver.ts
CHANGED
|
@@ -12,12 +12,8 @@ import {
|
|
|
12
12
|
type PColumnDataUniversal,
|
|
13
13
|
} from "@platforma-sdk/model";
|
|
14
14
|
import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
RefCountPoolBase,
|
|
18
|
-
type PoolEntry,
|
|
19
|
-
type MiLogger,
|
|
20
|
-
} from "@milaboratories/ts-helpers";
|
|
15
|
+
import { emptyDir } from "@milaboratories/ts-helpers";
|
|
16
|
+
import { RefCountPoolBase, type PoolEntry, type MiLogger } from "@milaboratories/helpers";
|
|
21
17
|
import type { DownloadDriver } from "@milaboratories/pl-drivers";
|
|
22
18
|
import {
|
|
23
19
|
isPlTreeNodeAccessor,
|