@milaboratories/pl-middle-layer 1.74.9 → 1.74.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/middle_layer/build_stamp.cjs +1 -1
- package/dist/middle_layer/build_stamp.js +1 -1
- package/dist/middle_layer/driver_kit.cjs +5 -0
- package/dist/middle_layer/driver_kit.cjs.map +1 -1
- package/dist/middle_layer/driver_kit.d.ts.map +1 -1
- package/dist/middle_layer/driver_kit.js +6 -1
- package/dist/middle_layer/driver_kit.js.map +1 -1
- package/package.json +13 -13
- package/src/middle_layer/driver_kit.ts +16 -0
|
@@ -13,6 +13,11 @@ async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
|
|
|
13
13
|
..._ops
|
|
14
14
|
};
|
|
15
15
|
const signer = new _milaboratories_ts_helpers.HmacSha256Signer(ops.localSecret);
|
|
16
|
+
await Promise.all([
|
|
17
|
+
(0, _milaboratories_pl_drivers.trimCacheDir)(ops.logger, ops.blobDownloadPath, ops.blobDriverOps.cacheSoftSizeBytes),
|
|
18
|
+
(0, _milaboratories_pl_drivers.trimCacheDir)(ops.logger, ops.downloadBlobToURLPath, ops.downloadBlobToURLDriverOps.cacheSoftSizeBytes),
|
|
19
|
+
(0, _milaboratories_pl_drivers.trimCacheDir)(ops.logger, frontendDownloadPath, ops.frontendDownloadDriverOps.cacheSoftSizeBytes)
|
|
20
|
+
]);
|
|
16
21
|
const downloadClient = (0, _milaboratories_pl_drivers.createDownloadClient)(ops.logger, pl, ops.localProjections);
|
|
17
22
|
const logsClient = (0, _milaboratories_pl_drivers.createLogsClient)(pl, ops.logger);
|
|
18
23
|
const uploadBlobClient = (0, _milaboratories_pl_drivers.createUploadBlobClient)(pl, ops.logger);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { Recorder } from \"@milaboratories/pl-crash-recorder\";\nimport {\n openRecordingSession,\n wrapDataDriver,\n wrapModelDriver,\n} from \"@milaboratories/pl-crash-recorder\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Diagnostics log for the model layer, present only when crash recording is\n * switched on. Render spans are written through it so that a join recorded by\n * the instrumented pFrame driver can be attributed to the block that built it.\n * */\n readonly crashRecorder?: Recorder;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const rawPFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n cachePath: ops.parquetCachePath,\n options: ops.pFrameDriverOps,\n cacheOps: ops.parquetCacheOps,\n });\n\n // Every join a block model builds and every row it reads back passes through\n // this one driver, so instrumenting it here covers the whole model layer\n // without touching the call sites. The session is undefined unless recording\n // is switched on, in which case the driver is used exactly as before.\n const recordingSession = openRecordingSession({ role: \"middle-layer\" });\n const pFrameDriver: InternalPFrameDriver = recordingSession\n ? wrapDataDriver(\n wrapModelDriver(rawPFrameDriver, recordingSession.recorder, recordingSession.registry),\n recordingSession.recorder,\n recordingSession.registry,\n )\n : rawPFrameDriver;\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n crashRecorder: recordingSession?.recorder,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n recordingSession?.close(\"driver-kit-disposed\");\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;
|
|
1
|
+
{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n trimCacheDir,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { Recorder } from \"@milaboratories/pl-crash-recorder\";\nimport {\n openRecordingSession,\n wrapDataDriver,\n wrapModelDriver,\n} from \"@milaboratories/pl-crash-recorder\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Diagnostics log for the model layer, present only when crash recording is\n * switched on. Render spans are written through it so that a join recorded by\n * the instrumented pFrame driver can be attributed to the block that built it.\n * */\n readonly crashRecorder?: Recorder;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n // These drivers only count files they download in this session; trim leftovers from earlier ones first.\n await Promise.all([\n trimCacheDir(ops.logger, ops.blobDownloadPath, ops.blobDriverOps.cacheSoftSizeBytes),\n trimCacheDir(\n ops.logger,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps.cacheSoftSizeBytes,\n ),\n trimCacheDir(\n ops.logger,\n frontendDownloadPath,\n ops.frontendDownloadDriverOps.cacheSoftSizeBytes,\n ),\n ]);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const rawPFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n cachePath: ops.parquetCachePath,\n options: ops.pFrameDriverOps,\n cacheOps: ops.parquetCacheOps,\n });\n\n // Every join a block model builds and every row it reads back passes through\n // this one driver, so instrumenting it here covers the whole model layer\n // without touching the call sites. The session is undefined unless recording\n // is switched on, in which case the driver is used exactly as before.\n const recordingSession = openRecordingSession({ role: \"middle-layer\" });\n const pFrameDriver: InternalPFrameDriver = recordingSession\n ? wrapDataDriver(\n wrapModelDriver(rawPFrameDriver, recordingSession.recorder, recordingSession.registry),\n recordingSession.recorder,\n recordingSession.registry,\n )\n : rawPFrameDriver;\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n crashRecorder: recordingSession?.recorder,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n recordingSession?.close(\"driver-kit-disposed\");\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;AA2EA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAGA,YAAAA;EACH,GAAGC,YAAAA,yBAAyB,OAAO;EACnC,GAAG;CACL;CAEA,MAAM,SAAS,IAAIC,2BAAAA,iBAAiB,IAAI,WAAW;CAGnD,MAAM,QAAQ,IAAI;+CACH,IAAI,QAAQ,IAAI,kBAAkB,IAAI,cAAc,kBAAkB;+CAEjF,IAAI,QACJ,IAAI,uBACJ,IAAI,2BAA2B,kBACjC;+CAEE,IAAI,QACJ,sBACA,IAAI,0BAA0B,kBAChC;CACF,CAAC;CAED,MAAM,kBAAA,GAAA,2BAAA,qBAAA,CAAsC,IAAI,QAAQ,IAAI,IAAI,gBAAgB;CAChF,MAAM,cAAA,GAAA,2BAAA,iBAAA,CAA8B,IAAI,IAAI,MAAM;CAClD,MAAM,oBAAA,GAAA,2BAAA,uBAAA,CAA0C,IAAI,IAAI,MAAM;CAC9D,MAAM,wBAAA,GAAA,2BAAA,2BAAA,CAAkD,IAAI,IAAI,MAAM;CAEtE,MAAM,aAAa,MAAMC,2BAAAA,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,aACN;CAEA,MAAM,kBAAkB,IAAIC,2BAAAA,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,0BACN;CAEA,MAAM,eAAe,IAAIC,2BAAAA,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,eACN;CACA,MAAM,mBAAmB,IAAIC,2BAAAA,iBAAiB,IAAI,QAAQ,YAAY,IAAI,kBAAkB;CAC5F,MAAM,YAAY,IAAIC,2BAAAA,WAAW,IAAI,QAAQ,kBAAkB,UAAU;CACzE,MAAM,WAAW,MAAMC,2BAAAA,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,4BACN;CAEA,MAAM,kBAAkB,MAAMC,eAAAA,mBAAmB;EAC/C;EACA,QAAQ,IAAI;EACZ,WAAW,IAAI;EACf,WAAW,IAAI;EACf,SAAS,IAAI;EACb,UAAU,IAAI;CAChB,CAAC;CAMD,MAAM,oBAAA,GAAA,kCAAA,qBAAA,CAAwC,EAAE,MAAM,eAAe,CAAC;CAiBtE,MAAM,YAAY;EAChB;EACiB;EACjB;EACA;EACA;EACA;EACA,cAvByC,oBAAA,GAAA,kCAAA,eAAA,EAAA,GAAA,kCAAA,gBAAA,CAErB,iBAAiB,iBAAiB,UAAU,iBAAiB,QAAQ,GACrF,iBAAiB,UACjB,iBAAiB,QACnB,IACA;EAkBF,gBAAgB,IAhBiBC,2BAAAA,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,yBAWiC;EACrC,eAAe,kBAAkB;CACnC;CAEA,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,SAAS,CAAC,CAAC,SAAS,YAAA,GAAA,wBAAA,kBAAA,CACtC,MAAM,IAAI,CAAC,OAAO,OAAO,aAAa,CAAC,CAAC,IAAI,CAAC,CACjE;EACA,MAAM,QAAQ,IAAI,eAAe;EACjC,kBAAkB,MAAM,qBAAqB;CAC/C;CAEA,OAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;CACzB;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_kit.d.ts","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"mappings":";;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"driver_kit.d.ts","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"mappings":";;;;;;;;;;;;;;;;iBAsCiB,6BAA6B,IAAI,WAAW;;EAE3D,WAAW;WAGF,YAAY;WAEZ,iBAAiB;WAEjB,WAAW;WAEX,UAAU;WAEV,cAAc;WAEd,gBAAgB;;;;;;WAOhB,gBAAgB;;;;;WAMhB,QAAQ;;;;;WAMR,cAAc;;wBAGH,cACpB,IAAI,UACJ,iBACA,8BACA,MAAM,0BACL,QAAQ"}
|
|
@@ -2,7 +2,7 @@ import { createPFrameDriver } from "../pool/driver.js";
|
|
|
2
2
|
import "../pool/index.js";
|
|
3
3
|
import { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from "./ops.js";
|
|
4
4
|
import { HmacSha256Signer } from "@milaboratories/ts-helpers";
|
|
5
|
-
import { DownloadBlobToURLDriver, DownloadDriver, DownloadUrlDriver, LogsDriver, LogsStreamDriver, LsDriver, UploadDriver, createDownloadClient, createLogsClient, createUploadBlobClient, createUploadProgressClient } from "@milaboratories/pl-drivers";
|
|
5
|
+
import { DownloadBlobToURLDriver, DownloadDriver, DownloadUrlDriver, LogsDriver, LogsStreamDriver, LsDriver, UploadDriver, createDownloadClient, createLogsClient, createUploadBlobClient, createUploadProgressClient, trimCacheDir } from "@milaboratories/pl-drivers";
|
|
6
6
|
import { openRecordingSession, wrapDataDriver, wrapModelDriver } from "@milaboratories/pl-crash-recorder";
|
|
7
7
|
import { isAsyncDisposable } from "@milaboratories/helpers";
|
|
8
8
|
//#region src/middle_layer/driver_kit.ts
|
|
@@ -13,6 +13,11 @@ async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
|
|
|
13
13
|
..._ops
|
|
14
14
|
};
|
|
15
15
|
const signer = new HmacSha256Signer(ops.localSecret);
|
|
16
|
+
await Promise.all([
|
|
17
|
+
trimCacheDir(ops.logger, ops.blobDownloadPath, ops.blobDriverOps.cacheSoftSizeBytes),
|
|
18
|
+
trimCacheDir(ops.logger, ops.downloadBlobToURLPath, ops.downloadBlobToURLDriverOps.cacheSoftSizeBytes),
|
|
19
|
+
trimCacheDir(ops.logger, frontendDownloadPath, ops.frontendDownloadDriverOps.cacheSoftSizeBytes)
|
|
20
|
+
]);
|
|
16
21
|
const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);
|
|
17
22
|
const logsClient = createLogsClient(pl, ops.logger);
|
|
18
23
|
const uploadBlobClient = createUploadBlobClient(pl, ops.logger);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { Recorder } from \"@milaboratories/pl-crash-recorder\";\nimport {\n openRecordingSession,\n wrapDataDriver,\n wrapModelDriver,\n} from \"@milaboratories/pl-crash-recorder\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Diagnostics log for the model layer, present only when crash recording is\n * switched on. Render spans are written through it so that a join recorded by\n * the instrumented pFrame driver can be attributed to the block that built it.\n * */\n readonly crashRecorder?: Recorder;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const rawPFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n cachePath: ops.parquetCachePath,\n options: ops.pFrameDriverOps,\n cacheOps: ops.parquetCacheOps,\n });\n\n // Every join a block model builds and every row it reads back passes through\n // this one driver, so instrumenting it here covers the whole model layer\n // without touching the call sites. The session is undefined unless recording\n // is switched on, in which case the driver is used exactly as before.\n const recordingSession = openRecordingSession({ role: \"middle-layer\" });\n const pFrameDriver: InternalPFrameDriver = recordingSession\n ? wrapDataDriver(\n wrapModelDriver(rawPFrameDriver, recordingSession.recorder, recordingSession.registry),\n recordingSession.recorder,\n recordingSession.registry,\n )\n : rawPFrameDriver;\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n crashRecorder: recordingSession?.recorder,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n recordingSession?.close(\"driver-kit-disposed\");\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;
|
|
1
|
+
{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n createDownloadClient,\n createLogsClient,\n createUploadBlobClient,\n createUploadProgressClient,\n DownloadDriver,\n DownloadBlobToURLDriver,\n LogsDriver,\n LogsStreamDriver,\n LsDriver,\n UploadDriver,\n DownloadUrlDriver,\n trimCacheDir,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { Recorder } from \"@milaboratories/pl-crash-recorder\";\nimport {\n openRecordingSession,\n wrapDataDriver,\n wrapModelDriver,\n} from \"@milaboratories/pl-crash-recorder\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n /** Dispose the driver kit and all its resources. */\n dispose(): Promise<void>;\n\n // override with wider interface\n readonly blobDriver: DownloadDriver;\n // override with wider interface\n readonly blobToURLDriver: DownloadBlobToURLDriver;\n // override with wider interface\n readonly logDriver: LogsDriver;\n // override with wider interface\n readonly lsDriver: InternalLsDriver;\n // override with wider interface\n readonly pFrameDriver: InternalPFrameDriver;\n // override with wider interface\n readonly frontendDriver: DownloadUrlDriver;\n\n /**\n * Diagnostics log for the model layer, present only when crash recording is\n * switched on. Render spans are written through it so that a join recorded by\n * the instrumented pFrame driver can be attributed to the block that built it.\n * */\n readonly crashRecorder?: Recorder;\n\n /**\n * Signer is initialized from local secret in drivers initialization routine,\n * so constitutes a part of the driver kit\n * */\n readonly signer: Signer;\n\n /**\n * Used to retrieve upload progress, and initiate upload porecesses driven by\n * upload requests from block outputs.\n * */\n readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n pl: PlClient,\n workdir: string,\n frontendDownloadPath: string,\n _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n const ops: DriverKitOps = {\n ...DefaultDriverKitOpsSettings,\n ...DefaultDriverKitOpsPaths(workdir),\n ..._ops,\n };\n\n const signer = new HmacSha256Signer(ops.localSecret);\n\n // These drivers only count files they download in this session; trim leftovers from earlier ones first.\n await Promise.all([\n trimCacheDir(ops.logger, ops.blobDownloadPath, ops.blobDriverOps.cacheSoftSizeBytes),\n trimCacheDir(\n ops.logger,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps.cacheSoftSizeBytes,\n ),\n trimCacheDir(\n ops.logger,\n frontendDownloadPath,\n ops.frontendDownloadDriverOps.cacheSoftSizeBytes,\n ),\n ]);\n\n const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n const logsClient = createLogsClient(pl, ops.logger);\n const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n const blobDriver = await DownloadDriver.init(\n ops.logger,\n downloadClient,\n logsClient,\n ops.blobDownloadPath,\n ops.blobDownloadRangesCachePath,\n signer,\n ops.blobDriverOps,\n );\n\n const blobToURLDriver = new DownloadBlobToURLDriver(\n ops.logger,\n signer,\n downloadClient,\n ops.downloadBlobToURLPath,\n ops.downloadBlobToURLDriverOps,\n );\n\n const uploadDriver = new UploadDriver(\n ops.logger,\n signer,\n uploadBlobClient,\n uploadProgressClient,\n ops.uploadDriverOps,\n );\n const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n const lsDriver = await LsDriver.init(\n ops.logger,\n pl,\n signer,\n ops.localProjections,\n ops.openFileDialogCallback,\n ops.virtualLocalStoragesOverride,\n );\n\n const rawPFrameDriver = await createPFrameDriver({\n blobDriver,\n logger: ops.logger,\n spillPath: ops.pframesSpillPath,\n cachePath: ops.parquetCachePath,\n options: ops.pFrameDriverOps,\n cacheOps: ops.parquetCacheOps,\n });\n\n // Every join a block model builds and every row it reads back passes through\n // this one driver, so instrumenting it here covers the whole model layer\n // without touching the call sites. The session is undefined unless recording\n // is switched on, in which case the driver is used exactly as before.\n const recordingSession = openRecordingSession({ role: \"middle-layer\" });\n const pFrameDriver: InternalPFrameDriver = recordingSession\n ? wrapDataDriver(\n wrapModelDriver(rawPFrameDriver, recordingSession.recorder, recordingSession.registry),\n recordingSession.recorder,\n recordingSession.registry,\n )\n : rawPFrameDriver;\n\n const frontendDownloadDriver = new DownloadUrlDriver(\n ops.logger,\n pl.httpDispatcher,\n frontendDownloadPath,\n signer,\n ops.frontendDownloadDriverOps,\n );\n\n const driverKit = {\n blobDriver,\n blobToURLDriver: blobToURLDriver,\n logDriver,\n lsDriver,\n signer,\n uploadDriver,\n pFrameDriver,\n frontendDriver: frontendDownloadDriver,\n crashRecorder: recordingSession?.recorder,\n };\n\n const dispose = async () => {\n const disposePromises = Object.values(driverKit).flatMap((driver) =>\n isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n );\n await Promise.all(disposePromises);\n recordingSession?.close(\"driver-kit-disposed\");\n };\n\n return {\n ...driverKit,\n dispose,\n [Symbol.asyncDispose]: dispose,\n };\n}\n"],"mappings":";;;;;;;;AA2EA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAG;EACH,GAAG,yBAAyB,OAAO;EACnC,GAAG;CACL;CAEA,MAAM,SAAS,IAAI,iBAAiB,IAAI,WAAW;CAGnD,MAAM,QAAQ,IAAI;EAChB,aAAa,IAAI,QAAQ,IAAI,kBAAkB,IAAI,cAAc,kBAAkB;EACnF,aACE,IAAI,QACJ,IAAI,uBACJ,IAAI,2BAA2B,kBACjC;EACA,aACE,IAAI,QACJ,sBACA,IAAI,0BAA0B,kBAChC;CACF,CAAC;CAED,MAAM,iBAAiB,qBAAqB,IAAI,QAAQ,IAAI,IAAI,gBAAgB;CAChF,MAAM,aAAa,iBAAiB,IAAI,IAAI,MAAM;CAClD,MAAM,mBAAmB,uBAAuB,IAAI,IAAI,MAAM;CAC9D,MAAM,uBAAuB,2BAA2B,IAAI,IAAI,MAAM;CAEtE,MAAM,aAAa,MAAM,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,aACN;CAEA,MAAM,kBAAkB,IAAI,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,0BACN;CAEA,MAAM,eAAe,IAAI,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,eACN;CACA,MAAM,mBAAmB,IAAI,iBAAiB,IAAI,QAAQ,YAAY,IAAI,kBAAkB;CAC5F,MAAM,YAAY,IAAI,WAAW,IAAI,QAAQ,kBAAkB,UAAU;CACzE,MAAM,WAAW,MAAM,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,4BACN;CAEA,MAAM,kBAAkB,MAAM,mBAAmB;EAC/C;EACA,QAAQ,IAAI;EACZ,WAAW,IAAI;EACf,WAAW,IAAI;EACf,SAAS,IAAI;EACb,UAAU,IAAI;CAChB,CAAC;CAMD,MAAM,mBAAmB,qBAAqB,EAAE,MAAM,eAAe,CAAC;CAiBtE,MAAM,YAAY;EAChB;EACiB;EACjB;EACA;EACA;EACA;EACA,cAvByC,mBACvC,eACE,gBAAgB,iBAAiB,iBAAiB,UAAU,iBAAiB,QAAQ,GACrF,iBAAiB,UACjB,iBAAiB,QACnB,IACA;EAkBF,gBAAgB,IAhBiB,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,yBAWiC;EACrC,eAAe,kBAAkB;CACnC;CAEA,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,SAAS,CAAC,CAAC,SAAS,WACxD,kBAAkB,MAAM,IAAI,CAAC,OAAO,OAAO,aAAa,CAAC,CAAC,IAAI,CAAC,CACjE;EACA,MAAM,QAAQ,IAAI,eAAe;EACjC,kBAAkB,MAAM,qBAAqB;CAC/C;CAEA,OAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;CACzB;AACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.74.
|
|
3
|
+
"version": "1.74.11",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -30,25 +30,25 @@
|
|
|
30
30
|
"utility-types": "^3.11.0",
|
|
31
31
|
"yaml": "^2.8.0",
|
|
32
32
|
"zod": "~3.25.76",
|
|
33
|
-
"@milaboratories/computable": "2.9.8",
|
|
34
|
-
"@milaboratories/helpers": "1.14.5",
|
|
35
33
|
"@milaboratories/columns-collection-driver": "0.2.10",
|
|
36
|
-
"@milaboratories/
|
|
37
|
-
"@milaboratories/
|
|
34
|
+
"@milaboratories/helpers": "1.14.5",
|
|
35
|
+
"@milaboratories/computable": "2.9.8",
|
|
38
36
|
"@milaboratories/pf-driver": "1.9.10",
|
|
37
|
+
"@milaboratories/pl-client": "3.19.1",
|
|
38
|
+
"@milaboratories/pf-spec-driver": "1.5.10",
|
|
39
39
|
"@milaboratories/pl-crash-recorder": "0.3.6",
|
|
40
|
+
"@milaboratories/pl-drivers": "1.16.36",
|
|
40
41
|
"@milaboratories/pl-deployments": "3.0.22",
|
|
41
|
-
"@milaboratories/pl-
|
|
42
|
-
"@milaboratories/pl-http": "1.2.4",
|
|
43
|
-
"@milaboratories/pl-errors": "1.4.53",
|
|
42
|
+
"@milaboratories/pl-model-common": "1.54.0",
|
|
44
43
|
"@milaboratories/pl-model-backend": "1.4.38",
|
|
44
|
+
"@milaboratories/resolve-helper": "1.1.3",
|
|
45
|
+
"@milaboratories/pl-errors": "1.4.53",
|
|
46
|
+
"@milaboratories/pl-http": "1.2.4",
|
|
45
47
|
"@milaboratories/pl-tree": "1.16.5",
|
|
46
|
-
"@milaboratories/pl-model-common": "1.54.0",
|
|
47
48
|
"@milaboratories/pl-model-middle-layer": "1.34.4",
|
|
48
49
|
"@milaboratories/ts-helpers": "1.8.6",
|
|
50
|
+
"@platforma-sdk/workflow-tengo": "6.17.1",
|
|
49
51
|
"@platforma-sdk/block-tools": "2.16.17",
|
|
50
|
-
"@milaboratories/resolve-helper": "1.1.3",
|
|
51
|
-
"@platforma-sdk/workflow-tengo": "6.17.0",
|
|
52
52
|
"@platforma-sdk/model": "1.86.0"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
@@ -57,9 +57,9 @@
|
|
|
57
57
|
"@vitest/coverage-istanbul": "^4.1.3",
|
|
58
58
|
"typescript": "7.0.2",
|
|
59
59
|
"vitest": "^4.1.3",
|
|
60
|
+
"@milaboratories/ts-configs": "1.4.0",
|
|
60
61
|
"@milaboratories/build-configs": "2.0.2",
|
|
61
|
-
"@milaboratories/ts-builder": "1.7.2"
|
|
62
|
-
"@milaboratories/ts-configs": "1.4.0"
|
|
62
|
+
"@milaboratories/ts-builder": "1.7.2"
|
|
63
63
|
},
|
|
64
64
|
"engines": {
|
|
65
65
|
"node": ">=22.19.0"
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
LsDriver,
|
|
13
13
|
UploadDriver,
|
|
14
14
|
DownloadUrlDriver,
|
|
15
|
+
trimCacheDir,
|
|
15
16
|
} from "@milaboratories/pl-drivers";
|
|
16
17
|
import type * as Sdk from "@milaboratories/pl-model-common";
|
|
17
18
|
import type { Signer } from "@milaboratories/ts-helpers";
|
|
@@ -86,6 +87,21 @@ export async function initDriverKit(
|
|
|
86
87
|
|
|
87
88
|
const signer = new HmacSha256Signer(ops.localSecret);
|
|
88
89
|
|
|
90
|
+
// These drivers only count files they download in this session; trim leftovers from earlier ones first.
|
|
91
|
+
await Promise.all([
|
|
92
|
+
trimCacheDir(ops.logger, ops.blobDownloadPath, ops.blobDriverOps.cacheSoftSizeBytes),
|
|
93
|
+
trimCacheDir(
|
|
94
|
+
ops.logger,
|
|
95
|
+
ops.downloadBlobToURLPath,
|
|
96
|
+
ops.downloadBlobToURLDriverOps.cacheSoftSizeBytes,
|
|
97
|
+
),
|
|
98
|
+
trimCacheDir(
|
|
99
|
+
ops.logger,
|
|
100
|
+
frontendDownloadPath,
|
|
101
|
+
ops.frontendDownloadDriverOps.cacheSoftSizeBytes,
|
|
102
|
+
),
|
|
103
|
+
]);
|
|
104
|
+
|
|
89
105
|
const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);
|
|
90
106
|
const logsClient = createLogsClient(pl, ops.logger);
|
|
91
107
|
const uploadBlobClient = createUploadBlobClient(pl, ops.logger);
|