@milaboratories/pl-middle-layer 1.64.28 → 1.64.29

package/dist/middle_layer/driver_kit.cjs

@@ -32,7 +32,9 @@ async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
 			blobDriver,
 			logger: ops.logger,
 			spillPath: ops.pframesSpillPath,
-			options: ops.pFrameDriverOps
+			cachePath: ops.parquetCachePath,
+			options: ops.pFrameDriverOps,
+			cacheOps: ops.parquetCacheOps
 		}),
 		frontendDriver: new _milaboratories_pl_drivers.DownloadUrlDriver(ops.logger, pl.httpDispatcher, frontendDownloadPath, signer, ops.frontendDownloadDriverOps)
 	};

package/dist/middle_layer/driver_kit.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  createDownloadClient,\n  createLogsClient,\n  createUploadBlobClient,\n  createUploadProgressClient,\n  DownloadDriver,\n  DownloadBlobToURLDriver,\n  LogsDriver,\n  LogsStreamDriver,\n  LsDriver,\n  UploadDriver,\n  DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n  /** Dispose the driver kit and all its resources. */\n  dispose(): Promise<void>;\n\n  // override with wider interface\n  readonly blobDriver: DownloadDriver;\n  // override with wider interface\n  readonly blobToURLDriver: DownloadBlobToURLDriver;\n  // override with wider interface\n  readonly logDriver: LogsDriver;\n  // override with wider interface\n  readonly lsDriver: InternalLsDriver;\n  // override with wider interface\n  readonly pFrameDriver: InternalPFrameDriver;\n  // override with wider interface\n  readonly frontendDriver: DownloadUrlDriver;\n\n  /**\n   * Signer is initialized from local secret in drivers initialization routine,\n   * so constitutes a part of the driver kit\n   * */\n  readonly signer: Signer;\n\n  /**\n   * Used to retrieve upload progress, and initiate upload porecesses driven by\n   * upload requests from block outputs.\n   * */\n  readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n  pl: PlClient,\n  workdir: string,\n  frontendDownloadPath: string,\n  _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n  const ops: DriverKitOps = {\n    ...DefaultDriverKitOpsSettings,\n    ...DefaultDriverKitOpsPaths(workdir),\n    ..._ops,\n  };\n\n  const signer = new HmacSha256Signer(ops.localSecret);\n\n  const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n  const logsClient = createLogsClient(pl, ops.logger);\n  const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n  const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n  const blobDriver = await DownloadDriver.init(\n    ops.logger,\n    downloadClient,\n    logsClient,\n    ops.blobDownloadPath,\n    ops.blobDownloadRangesCachePath,\n    signer,\n    ops.blobDriverOps,\n  );\n\n  const blobToURLDriver = new DownloadBlobToURLDriver(\n    ops.logger,\n    signer,\n    downloadClient,\n    ops.downloadBlobToURLPath,\n    ops.downloadBlobToURLDriverOps,\n  );\n\n  const uploadDriver = new UploadDriver(\n    ops.logger,\n    signer,\n    uploadBlobClient,\n    uploadProgressClient,\n    ops.uploadDriverOps,\n  );\n  const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n  const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n  const lsDriver = await LsDriver.init(\n    ops.logger,\n    pl,\n    signer,\n    ops.localProjections,\n    ops.openFileDialogCallback,\n    ops.virtualLocalStoragesOverride,\n  );\n\n  const pFrameDriver = await createPFrameDriver({\n    blobDriver,\n    logger: ops.logger,\n    spillPath: ops.pframesSpillPath,\n    options: ops.pFrameDriverOps,\n  });\n\n  const frontendDownloadDriver = new DownloadUrlDriver(\n    ops.logger,\n    pl.httpDispatcher,\n    frontendDownloadPath,\n    signer,\n    ops.frontendDownloadDriverOps,\n  );\n\n  const driverKit = {\n    blobDriver,\n    blobToURLDriver: blobToURLDriver,\n    logDriver,\n    lsDriver,\n    signer,\n    uploadDriver,\n    pFrameDriver,\n    frontendDriver: frontendDownloadDriver,\n  };\n\n  const dispose = async () => {\n    const disposePromises = Object.values(driverKit).flatMap((driver) =>\n      isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n    );\n    await Promise.all(disposePromises);\n  };\n\n  return {\n    ...driverKit,\n    dispose,\n    [Symbol.asyncDispose]: dispose,\n  };\n}\n"],"mappings":";;;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAGA,YAAAA;EACH,GAAGC,YAAAA,yBAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAIC,2BAAAA,iBAAiB,IAAI,YAAY;CAEpD,MAAM,kBAAA,GAAA,2BAAA,sBAAsC,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,cAAA,GAAA,2BAAA,kBAA8B,IAAI,IAAI,OAAO;CACnD,MAAM,oBAAA,GAAA,2BAAA,wBAA0C,IAAI,IAAI,OAAO;CAC/D,MAAM,wBAAA,GAAA,2BAAA,4BAAkD,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAMC,2BAAAA,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAIC,2BAAAA,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAIC,2BAAAA,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAIC,2BAAAA,iBAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA0B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA5BgB,IAAIC,2BAAAA,WAAW,IAAI,QAAQ,kBAAkB,WAAW;EA6BxE,UA5Be,MAAMC,2BAAAA,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAsBC;EACA;EACA,cAtBmB,MAAMC,eAAAA,mBAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,SAAS,IAAI;GACd,CAAC;EAkBA,gBAhB6B,IAAIC,2BAAAA,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,YAAA,GAAA,wBAAA,mBACtC,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}
+{"version":3,"file":"driver_kit.cjs","names":["DefaultDriverKitOpsSettings","DefaultDriverKitOpsPaths","HmacSha256Signer","DownloadDriver","DownloadBlobToURLDriver","UploadDriver","LogsStreamDriver","LogsDriver","LsDriver","createPFrameDriver","DownloadUrlDriver"],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  createDownloadClient,\n  createLogsClient,\n  createUploadBlobClient,\n  createUploadProgressClient,\n  DownloadDriver,\n  DownloadBlobToURLDriver,\n  LogsDriver,\n  LogsStreamDriver,\n  LsDriver,\n  UploadDriver,\n  DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n  /** Dispose the driver kit and all its resources. */\n  dispose(): Promise<void>;\n\n  // override with wider interface\n  readonly blobDriver: DownloadDriver;\n  // override with wider interface\n  readonly blobToURLDriver: DownloadBlobToURLDriver;\n  // override with wider interface\n  readonly logDriver: LogsDriver;\n  // override with wider interface\n  readonly lsDriver: InternalLsDriver;\n  // override with wider interface\n  readonly pFrameDriver: InternalPFrameDriver;\n  // override with wider interface\n  readonly frontendDriver: DownloadUrlDriver;\n\n  /**\n   * Signer is initialized from local secret in drivers initialization routine,\n   * so constitutes a part of the driver kit\n   * */\n  readonly signer: Signer;\n\n  /**\n   * Used to retrieve upload progress, and initiate upload porecesses driven by\n   * upload requests from block outputs.\n   * */\n  readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n  pl: PlClient,\n  workdir: string,\n  frontendDownloadPath: string,\n  _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n  const ops: DriverKitOps = {\n    ...DefaultDriverKitOpsSettings,\n    ...DefaultDriverKitOpsPaths(workdir),\n    ..._ops,\n  };\n\n  const signer = new HmacSha256Signer(ops.localSecret);\n\n  const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n  const logsClient = createLogsClient(pl, ops.logger);\n  const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n  const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n  const blobDriver = await DownloadDriver.init(\n    ops.logger,\n    downloadClient,\n    logsClient,\n    ops.blobDownloadPath,\n    ops.blobDownloadRangesCachePath,\n    signer,\n    ops.blobDriverOps,\n  );\n\n  const blobToURLDriver = new DownloadBlobToURLDriver(\n    ops.logger,\n    signer,\n    downloadClient,\n    ops.downloadBlobToURLPath,\n    ops.downloadBlobToURLDriverOps,\n  );\n\n  const uploadDriver = new UploadDriver(\n    ops.logger,\n    signer,\n    uploadBlobClient,\n    uploadProgressClient,\n    ops.uploadDriverOps,\n  );\n  const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n  const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n  const lsDriver = await LsDriver.init(\n    ops.logger,\n    pl,\n    signer,\n    ops.localProjections,\n    ops.openFileDialogCallback,\n    ops.virtualLocalStoragesOverride,\n  );\n\n  const pFrameDriver = await createPFrameDriver({\n    blobDriver,\n    logger: ops.logger,\n    spillPath: ops.pframesSpillPath,\n    cachePath: ops.parquetCachePath,\n    options: ops.pFrameDriverOps,\n    cacheOps: ops.parquetCacheOps,\n  });\n\n  const frontendDownloadDriver = new DownloadUrlDriver(\n    ops.logger,\n    pl.httpDispatcher,\n    frontendDownloadPath,\n    signer,\n    ops.frontendDownloadDriverOps,\n  );\n\n  const driverKit = {\n    blobDriver,\n    blobToURLDriver: blobToURLDriver,\n    logDriver,\n    lsDriver,\n    signer,\n    uploadDriver,\n    pFrameDriver,\n    frontendDriver: frontendDownloadDriver,\n  };\n\n  const dispose = async () => {\n    const disposePromises = Object.values(driverKit).flatMap((driver) =>\n      isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n    );\n    await Promise.all(disposePromises);\n  };\n\n  return {\n    ...driverKit,\n    dispose,\n    [Symbol.asyncDispose]: dispose,\n  };\n}\n"],"mappings":";;;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAGA,YAAAA;EACH,GAAGC,YAAAA,yBAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAIC,2BAAAA,iBAAiB,IAAI,YAAY;CAEpD,MAAM,kBAAA,GAAA,2BAAA,sBAAsC,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,cAAA,GAAA,2BAAA,kBAA8B,IAAI,IAAI,OAAO;CACnD,MAAM,oBAAA,GAAA,2BAAA,wBAA0C,IAAI,IAAI,OAAO;CAC/D,MAAM,wBAAA,GAAA,2BAAA,4BAAkD,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAMC,2BAAAA,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAIC,2BAAAA,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAIC,2BAAAA,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAIC,2BAAAA,iBAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA4B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA9BgB,IAAIC,2BAAAA,WAAW,IAAI,QAAQ,kBAAkB,WAAW;EA+BxE,UA9Be,MAAMC,2BAAAA,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAwBC;EACA;EACA,cAxBmB,MAAMC,eAAAA,mBAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,WAAW,IAAI;GACf,SAAS,IAAI;GACb,UAAU,IAAI;GACf,CAAC;EAkBA,gBAhB6B,IAAIC,2BAAAA,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,YAAA,GAAA,wBAAA,mBACtC,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}

package/dist/middle_layer/driver_kit.js

@@ -31,7 +31,9 @@ async function initDriverKit(pl, workdir, frontendDownloadPath, _ops) {
 			blobDriver,
 			logger: ops.logger,
 			spillPath: ops.pframesSpillPath,
-			options: ops.pFrameDriverOps
+			cachePath: ops.parquetCachePath,
+			options: ops.pFrameDriverOps,
+			cacheOps: ops.parquetCacheOps
 		}),
 		frontendDriver: new DownloadUrlDriver(ops.logger, pl.httpDispatcher, frontendDownloadPath, signer, ops.frontendDownloadDriverOps)
 	};

package/dist/middle_layer/driver_kit.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  createDownloadClient,\n  createLogsClient,\n  createUploadBlobClient,\n  createUploadProgressClient,\n  DownloadDriver,\n  DownloadBlobToURLDriver,\n  LogsDriver,\n  LogsStreamDriver,\n  LsDriver,\n  UploadDriver,\n  DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n  /** Dispose the driver kit and all its resources. */\n  dispose(): Promise<void>;\n\n  // override with wider interface\n  readonly blobDriver: DownloadDriver;\n  // override with wider interface\n  readonly blobToURLDriver: DownloadBlobToURLDriver;\n  // override with wider interface\n  readonly logDriver: LogsDriver;\n  // override with wider interface\n  readonly lsDriver: InternalLsDriver;\n  // override with wider interface\n  readonly pFrameDriver: InternalPFrameDriver;\n  // override with wider interface\n  readonly frontendDriver: DownloadUrlDriver;\n\n  /**\n   * Signer is initialized from local secret in drivers initialization routine,\n   * so constitutes a part of the driver kit\n   * */\n  readonly signer: Signer;\n\n  /**\n   * Used to retrieve upload progress, and initiate upload porecesses driven by\n   * upload requests from block outputs.\n   * */\n  readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n  pl: PlClient,\n  workdir: string,\n  frontendDownloadPath: string,\n  _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n  const ops: DriverKitOps = {\n    ...DefaultDriverKitOpsSettings,\n    ...DefaultDriverKitOpsPaths(workdir),\n    ..._ops,\n  };\n\n  const signer = new HmacSha256Signer(ops.localSecret);\n\n  const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n  const logsClient = createLogsClient(pl, ops.logger);\n  const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n  const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n  const blobDriver = await DownloadDriver.init(\n    ops.logger,\n    downloadClient,\n    logsClient,\n    ops.blobDownloadPath,\n    ops.blobDownloadRangesCachePath,\n    signer,\n    ops.blobDriverOps,\n  );\n\n  const blobToURLDriver = new DownloadBlobToURLDriver(\n    ops.logger,\n    signer,\n    downloadClient,\n    ops.downloadBlobToURLPath,\n    ops.downloadBlobToURLDriverOps,\n  );\n\n  const uploadDriver = new UploadDriver(\n    ops.logger,\n    signer,\n    uploadBlobClient,\n    uploadProgressClient,\n    ops.uploadDriverOps,\n  );\n  const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n  const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n  const lsDriver = await LsDriver.init(\n    ops.logger,\n    pl,\n    signer,\n    ops.localProjections,\n    ops.openFileDialogCallback,\n    ops.virtualLocalStoragesOverride,\n  );\n\n  const pFrameDriver = await createPFrameDriver({\n    blobDriver,\n    logger: ops.logger,\n    spillPath: ops.pframesSpillPath,\n    options: ops.pFrameDriverOps,\n  });\n\n  const frontendDownloadDriver = new DownloadUrlDriver(\n    ops.logger,\n    pl.httpDispatcher,\n    frontendDownloadPath,\n    signer,\n    ops.frontendDownloadDriverOps,\n  );\n\n  const driverKit = {\n    blobDriver,\n    blobToURLDriver: blobToURLDriver,\n    logDriver,\n    lsDriver,\n    signer,\n    uploadDriver,\n    pFrameDriver,\n    frontendDriver: frontendDownloadDriver,\n  };\n\n  const dispose = async () => {\n    const disposePromises = Object.values(driverKit).flatMap((driver) =>\n      isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n    );\n    await Promise.all(disposePromises);\n  };\n\n  return {\n    ...driverKit,\n    dispose,\n    [Symbol.asyncDispose]: dispose,\n  };\n}\n"],"mappings":";;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAG;EACH,GAAG,yBAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAI,iBAAiB,IAAI,YAAY;CAEpD,MAAM,iBAAiB,qBAAqB,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,aAAa,iBAAiB,IAAI,IAAI,OAAO;CACnD,MAAM,mBAAmB,uBAAuB,IAAI,IAAI,OAAO;CAC/D,MAAM,uBAAuB,2BAA2B,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAM,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAI,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAI,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAI,iBAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA0B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA5BgB,IAAI,WAAW,IAAI,QAAQ,kBAAkB,WAAW;EA6BxE,UA5Be,MAAM,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAsBC;EACA;EACA,cAtBmB,MAAM,mBAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,SAAS,IAAI;GACd,CAAC;EAkBA,gBAhB6B,IAAI,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,WACxD,kBAAkB,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}
+{"version":3,"file":"driver_kit.js","names":[],"sources":["../../src/middle_layer/driver_kit.ts"],"sourcesContent":["import type { PlClient } from \"@milaboratories/pl-client\";\nimport type { InternalLsDriver } from \"@milaboratories/pl-drivers\";\nimport {\n  createDownloadClient,\n  createLogsClient,\n  createUploadBlobClient,\n  createUploadProgressClient,\n  DownloadDriver,\n  DownloadBlobToURLDriver,\n  LogsDriver,\n  LogsStreamDriver,\n  LsDriver,\n  UploadDriver,\n  DownloadUrlDriver,\n} from \"@milaboratories/pl-drivers\";\nimport type * as Sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { isAsyncDisposable } from \"@milaboratories/helpers\";\nimport { HmacSha256Signer } from \"@milaboratories/ts-helpers\";\nimport type { InternalPFrameDriver } from \"../pool\";\nimport { createPFrameDriver } from \"../pool\";\nimport type { DriverKitOps, DriverKitOpsConstructor } from \"./ops\";\nimport { DefaultDriverKitOpsPaths, DefaultDriverKitOpsSettings } from \"./ops\";\n\n/**\n * Drivers offered by the middle-layer for internal consumers,\n * like configuration rendering routines.\n *\n * This intertface is basically a version of the DriverKit from\n * UI SDK with extended API.\n * */\nexport interface MiddleLayerDriverKit extends Sdk.DriverKit, AsyncDisposable {\n  /** Dispose the driver kit and all its resources. */\n  dispose(): Promise<void>;\n\n  // override with wider interface\n  readonly blobDriver: DownloadDriver;\n  // override with wider interface\n  readonly blobToURLDriver: DownloadBlobToURLDriver;\n  // override with wider interface\n  readonly logDriver: LogsDriver;\n  // override with wider interface\n  readonly lsDriver: InternalLsDriver;\n  // override with wider interface\n  readonly pFrameDriver: InternalPFrameDriver;\n  // override with wider interface\n  readonly frontendDriver: DownloadUrlDriver;\n\n  /**\n   * Signer is initialized from local secret in drivers initialization routine,\n   * so constitutes a part of the driver kit\n   * */\n  readonly signer: Signer;\n\n  /**\n   * Used to retrieve upload progress, and initiate upload porecesses driven by\n   * upload requests from block outputs.\n   * */\n  readonly uploadDriver: UploadDriver;\n}\n\nexport async function initDriverKit(\n  pl: PlClient,\n  workdir: string,\n  frontendDownloadPath: string,\n  _ops: DriverKitOpsConstructor,\n): Promise<MiddleLayerDriverKit> {\n  const ops: DriverKitOps = {\n    ...DefaultDriverKitOpsSettings,\n    ...DefaultDriverKitOpsPaths(workdir),\n    ..._ops,\n  };\n\n  const signer = new HmacSha256Signer(ops.localSecret);\n\n  const downloadClient = createDownloadClient(ops.logger, pl, ops.localProjections);\n  const logsClient = createLogsClient(pl, ops.logger);\n  const uploadBlobClient = createUploadBlobClient(pl, ops.logger);\n  const uploadProgressClient = createUploadProgressClient(pl, ops.logger);\n\n  const blobDriver = await DownloadDriver.init(\n    ops.logger,\n    downloadClient,\n    logsClient,\n    ops.blobDownloadPath,\n    ops.blobDownloadRangesCachePath,\n    signer,\n    ops.blobDriverOps,\n  );\n\n  const blobToURLDriver = new DownloadBlobToURLDriver(\n    ops.logger,\n    signer,\n    downloadClient,\n    ops.downloadBlobToURLPath,\n    ops.downloadBlobToURLDriverOps,\n  );\n\n  const uploadDriver = new UploadDriver(\n    ops.logger,\n    signer,\n    uploadBlobClient,\n    uploadProgressClient,\n    ops.uploadDriverOps,\n  );\n  const logsStreamDriver = new LogsStreamDriver(ops.logger, logsClient, ops.logStreamDriverOps);\n  const logDriver = new LogsDriver(ops.logger, logsStreamDriver, blobDriver);\n  const lsDriver = await LsDriver.init(\n    ops.logger,\n    pl,\n    signer,\n    ops.localProjections,\n    ops.openFileDialogCallback,\n    ops.virtualLocalStoragesOverride,\n  );\n\n  const pFrameDriver = await createPFrameDriver({\n    blobDriver,\n    logger: ops.logger,\n    spillPath: ops.pframesSpillPath,\n    cachePath: ops.parquetCachePath,\n    options: ops.pFrameDriverOps,\n    cacheOps: ops.parquetCacheOps,\n  });\n\n  const frontendDownloadDriver = new DownloadUrlDriver(\n    ops.logger,\n    pl.httpDispatcher,\n    frontendDownloadPath,\n    signer,\n    ops.frontendDownloadDriverOps,\n  );\n\n  const driverKit = {\n    blobDriver,\n    blobToURLDriver: blobToURLDriver,\n    logDriver,\n    lsDriver,\n    signer,\n    uploadDriver,\n    pFrameDriver,\n    frontendDriver: frontendDownloadDriver,\n  };\n\n  const dispose = async () => {\n    const disposePromises = Object.values(driverKit).flatMap((driver) =>\n      isAsyncDisposable(driver) ? [driver[Symbol.asyncDispose]()] : [],\n    );\n    await Promise.all(disposePromises);\n  };\n\n  return {\n    ...driverKit,\n    dispose,\n    [Symbol.asyncDispose]: dispose,\n  };\n}\n"],"mappings":";;;;;;;AA6DA,eAAsB,cACpB,IACA,SACA,sBACA,MAC+B;CAC/B,MAAM,MAAoB;EACxB,GAAG;EACH,GAAG,yBAAyB,QAAQ;EACpC,GAAG;EACJ;CAED,MAAM,SAAS,IAAI,iBAAiB,IAAI,YAAY;CAEpD,MAAM,iBAAiB,qBAAqB,IAAI,QAAQ,IAAI,IAAI,iBAAiB;CACjF,MAAM,aAAa,iBAAiB,IAAI,IAAI,OAAO;CACnD,MAAM,mBAAmB,uBAAuB,IAAI,IAAI,OAAO;CAC/D,MAAM,uBAAuB,2BAA2B,IAAI,IAAI,OAAO;CAEvE,MAAM,aAAa,MAAM,eAAe,KACtC,IAAI,QACJ,gBACA,YACA,IAAI,kBACJ,IAAI,6BACJ,QACA,IAAI,cACL;CAED,MAAM,kBAAkB,IAAI,wBAC1B,IAAI,QACJ,QACA,gBACA,IAAI,uBACJ,IAAI,2BACL;CAED,MAAM,eAAe,IAAI,aACvB,IAAI,QACJ,QACA,kBACA,sBACA,IAAI,gBACL;CACD,MAAM,mBAAmB,IAAI,iBAAiB,IAAI,QAAQ,YAAY,IAAI,mBAAmB;CA4B7F,MAAM,YAAY;EAChB;EACiB;EACjB,WA9BgB,IAAI,WAAW,IAAI,QAAQ,kBAAkB,WAAW;EA+BxE,UA9Be,MAAM,SAAS,KAC9B,IAAI,QACJ,IACA,QACA,IAAI,kBACJ,IAAI,wBACJ,IAAI,6BACL;EAwBC;EACA;EACA,cAxBmB,MAAM,mBAAmB;GAC5C;GACA,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf,WAAW,IAAI;GACf,SAAS,IAAI;GACb,UAAU,IAAI;GACf,CAAC;EAkBA,gBAhB6B,IAAI,kBACjC,IAAI,QACJ,GAAG,gBACH,sBACA,QACA,IAAI,0BACL;EAWA;CAED,MAAM,UAAU,YAAY;EAC1B,MAAM,kBAAkB,OAAO,OAAO,UAAU,CAAC,SAAS,WACxD,kBAAkB,OAAO,GAAG,CAAC,OAAO,OAAO,eAAe,CAAC,GAAG,EAAE,CACjE;AACD,QAAM,QAAQ,IAAI,gBAAgB;;AAGpC,QAAO;EACL,GAAG;EACH;GACC,OAAO,eAAe;EACxB"}

package/dist/middle_layer/ops.cjs

@@ -32,14 +32,20 @@ const DefaultDriverKitOpsSettings = {
 		pollingInterval: 1e3,
 		stopPollingDelay: 1e3
 	},
-	pFrameDriverOps: require_driver.PFrameDriverOpsDefaults
+	pFrameDriverOps: require_driver.PFrameDriverOpsDefaults,
+	parquetCacheOps: {
+		maxSizeBytes: 8 * 1024 * 1024 * 1024,
+		admissionFraction: .2,
+		maxFilesTracked: 5e4
+	}
 };
 function DefaultDriverKitOpsPaths(workDir) {
 	return {
 		blobDownloadPath: node_path.default.join(workDir, "download"),
 		blobDownloadRangesCachePath: node_path.default.join(workDir, "downloadRangesCache"),
 		downloadBlobToURLPath: node_path.default.join(workDir, "downloadToURL"),
-		pframesSpillPath: node_path.default.join(workDir, "pframes")
+		pframesSpillPath: node_path.default.join(workDir, "pframes"),
+		parquetCachePath: node_path.default.join(workDir, "parquetCache")
 	};
 }
 /** Some defaults fot MiddleLayerOps. */

package/dist/middle_layer/ops.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"ops.cjs","names":["ConsoleLoggerAdapter","PFrameDriverOpsDefaults","path"],"sources":["../../src/middle_layer/ops.ts"],"sourcesContent":["import type { TemporalSynchronizedTreeOps } from \"./types\";\nimport type {\n  DownloadBlobToURLDriverOps,\n  DownloadDriverOps,\n  DownloadUrlDriverOps,\n  OpenFileDialogCallback,\n  VirtualLocalStorageSpec,\n} from \"@milaboratories/pl-drivers\";\nimport type { UploadDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { LogsStreamDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConsoleLoggerAdapter } from \"@milaboratories/ts-helpers\";\nimport type { LocalStorageProjection } from \"@milaboratories/pl-drivers\";\nimport path from \"node:path\";\nimport { PFrameDriverOpsDefaults, type PFrameDriverOps } from \"../pool\";\n\n/** Paths part of {@link DriverKitOps}. */\nexport type DriverKitOpsPaths = {\n  /** Common root where to put downloaded blobs / downloaded blob cache */\n  readonly blobDownloadPath: string;\n\n  /** Common root for a cache for range queries. */\n  readonly blobDownloadRangesCachePath: string;\n\n  /** Common root where to put downloaded blobs with */\n  readonly downloadBlobToURLPath: string;\n\n  /**\n   * List of pl storages that have projections in local file system.\n   *\n   * This option affect two drivers:\n   *\n   *   (1) LS driver generates \"index\" handles instead of \"upload\" for paths inside those locations\n   *\n   *   (2) Download driver directly serves content retrieval requests for blobs from listed storages,\n   *       and don't apply any caching for such blobs (i.e. preventing duplication of files for Downloaded\n   *       type handles, making OnDemand and Downloaded handles equivalent)\n   *\n   * */\n  readonly localProjections: LocalStorageProjection[];\n\n  /**\n   * List of virtual storages that will allow homogeneous access to local FSs through LS API.\n   * If undefined, default list will be created.\n   * */\n  readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];\n\n  /** Path to the directory where pframes will spill temporary files and store materialized views */\n  readonly pframesSpillPath: string;\n};\n\n/** Options required to initialize full set of middle layer driver kit */\nexport type DriverKitOpsSettings = {\n  //\n  // Common\n  //\n\n  readonly logger: MiLogger;\n\n  //\n  // Signer\n  //\n\n  /**\n   * Local secret, that is used to sign and verify different pieces of information\n   * that can be used to access local data, like local paths for ongoing uploads.\n   *\n   * Use {@link MiddleLayer.generateLocalSecret} to generate sufficiently random string.\n   * */\n  readonly localSecret: string;\n\n  //\n  // Blob Driver\n  //\n\n  /**\n   * Settings related to the download driver making operations with blobs. This driver is also used\n   * to download logs when source process terminates and log terns into a blob\n   */\n  readonly blobDriverOps: DownloadDriverOps;\n\n  //\n  // Frontend Driver\n  //\n\n  /** Settings related to the frontend driver that downloads frontends. */\n  readonly frontendDownloadDriverOps: DownloadUrlDriverOps;\n\n  //\n  // Blob To URL Driver\n  //\n\n  readonly downloadBlobToURLDriverOps: DownloadBlobToURLDriverOps;\n\n  //\n  // Upload Driver\n  //\n\n  /**\n   * Settings related to the upload driver that actually performs upload and helps render upload\n   * and indexing progresses from related pl resources.\n   * */\n  readonly uploadDriverOps: UploadDriverOps;\n\n  //\n  // Log streaming ops\n  // (static logs are served via the blob driver)\n  //\n\n  /** Settings related to the streaming log driver */\n  readonly logStreamDriverOps: LogsStreamDriverOps;\n\n  //\n  // LS Driver\n  //\n\n  /**\n   * Callback to access system file open dialog, must be provided by the environment,\n   * to allow for {@link showOpenSingleFileDialog} / {@link showOpenMultipleFilesDialog}\n   * calls from the UI.\n   */\n  readonly openFileDialogCallback: OpenFileDialogCallback;\n\n  //\n  // PFrame Driver\n  //\n\n  /** Settings related to the PFrame driver */\n  readonly pFrameDriverOps: PFrameDriverOps;\n};\n\nexport type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultDriverKitOpsSettings: Pick<\n  DriverKitOpsSettings,\n  | \"logger\"\n  | \"blobDriverOps\"\n  | \"frontendDownloadDriverOps\"\n  | \"downloadBlobToURLDriverOps\"\n  | \"uploadDriverOps\"\n  | \"logStreamDriverOps\"\n  | \"pFrameDriverOps\"\n> = {\n  logger: new ConsoleLoggerAdapter(),\n  blobDriverOps: {\n    cacheSoftSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    rangesCacheMaxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    nConcurrentDownloads: 10,\n  },\n  frontendDownloadDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    withGunzip: true,\n    nConcurrentDownloads: 50,\n  },\n  downloadBlobToURLDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    nConcurrentDownloads: 10,\n  },\n  uploadDriverOps: {\n    nConcurrentPartUploads: 10,\n    nConcurrentGetProgresses: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  logStreamDriverOps: {\n    nConcurrentGetLogs: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  pFrameDriverOps: PFrameDriverOpsDefaults,\n};\n\nexport function DefaultDriverKitOpsPaths(\n  workDir: string,\n): Pick<\n  DriverKitOpsPaths,\n  \"blobDownloadPath\" | \"blobDownloadRangesCachePath\" | \"downloadBlobToURLPath\" | \"pframesSpillPath\"\n> {\n  return {\n    blobDownloadPath: path.join(workDir, \"download\"),\n    blobDownloadRangesCachePath: path.join(workDir, \"downloadRangesCache\"),\n    downloadBlobToURLPath: path.join(workDir, \"downloadToURL\"),\n    pframesSpillPath: path.join(workDir, \"pframes\"),\n  };\n}\n\n/** Fields with default values are marked as optional here. */\nexport type DriverKitOpsConstructor = Omit<\n  DriverKitOpsSettings,\n  keyof typeof DefaultDriverKitOpsSettings\n> &\n  Partial<typeof DefaultDriverKitOpsSettings> &\n  Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> &\n  Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;\n\nexport type MiddleLayerOpsPaths = DriverKitOpsPaths & {\n  /** Common root where to put frontend code. */\n  readonly frontendDownloadPath: string;\n};\n\n/** Debug options for middle layer. */\nexport type MiddleLayerDebugOptions = {\n  /** If true, will dump initial tree state to the file with root resource id as name. */\n  dumpInitialTreeState: boolean;\n};\n\n/** Configuration controlling different aspects of middle layer behaviour. */\nexport type MiddleLayerOpsSettings = DriverKitOpsSettings & {\n  /** Debug options. */\n  readonly debugOps: MiddleLayerDebugOptions;\n\n  /** Contain temporal options controlling how often should pl trees be\n   * synchronized with the pl server. */\n  readonly defaultTreeOptions: TemporalSynchronizedTreeOps;\n\n  /** Defines interval in milliseconds for running periodic project maintenance job.\n   * Project maintenance includes background staging rendering and cached outputs cleanup. */\n  readonly projectRefreshInterval: number;\n\n  /** How often to check for dev block updates */\n  readonly devBlockUpdateRecheckInterval: number;\n\n  /** Prioritize this channel if update is available in this block */\n  readonly preferredUpdateChannel?: string;\n};\n\nexport type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultMiddleLayerOpsSettings: Pick<\n  MiddleLayerOps,\n  | keyof typeof DefaultDriverKitOpsSettings\n  | \"defaultTreeOptions\"\n  | \"projectRefreshInterval\"\n  | \"devBlockUpdateRecheckInterval\"\n  | \"debugOps\"\n> = {\n  ...DefaultDriverKitOpsSettings,\n  defaultTreeOptions: {\n    pollingInterval: 200,\n    stopPollingDelay: 2500,\n    initialTreeLoadingTimeout: 100 * 60 * 60 * 1000, // disable timeout for loading project tree (100 hours)\n  },\n  debugOps: {\n    dumpInitialTreeState: false,\n  },\n  devBlockUpdateRecheckInterval: 1000,\n  projectRefreshInterval: 2000,\n};\n\nexport function DefaultMiddleLayerOpsPaths(\n  workDir: string,\n): Pick<\n  MiddleLayerOpsPaths,\n  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | \"frontendDownloadPath\"\n> {\n  return {\n    ...DefaultDriverKitOpsPaths(workDir),\n    frontendDownloadPath: path.join(workDir, \"frontend\"),\n  };\n}\n\nexport type MiddleLayerOpsConstructor = Omit<\n  MiddleLayerOpsSettings,\n  keyof typeof DefaultMiddleLayerOpsSettings\n> &\n  Partial<typeof DefaultMiddleLayerOpsSettings> &\n  Omit<MiddleLayerOpsPaths, keyof Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>> &\n  Partial<Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>>;\n"],"mappings":";;;;;;;AAsIA,MAAa,8BAST;CACF,QAAQ,4CAAIA,sBAAsB;CAClC,eAAe;EACb,oBAAoB,IAAI,OAAO,OAAO;EACtC,yBAAyB,IAAI,OAAO,OAAO;EAC3C,sBAAsB;EACvB;CACD,2BAA2B;EACzB,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB;CACD,4BAA4B;EAC1B,oBAAoB,IAAI,OAAO,OAAO;EACtC,sBAAsB;EACvB;CACD,iBAAiB;EACf,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB;CACD,oBAAoB;EAClB,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB;CACD,iBAAiBC,eAAAA;CAClB;AAED,SAAgB,yBACd,SAIA;AACA,QAAO;EACL,kBAAkBC,UAAAA,QAAK,KAAK,SAAS,WAAW;EAChD,6BAA6BA,UAAAA,QAAK,KAAK,SAAS,sBAAsB;EACtE,uBAAuBA,UAAAA,QAAK,KAAK,SAAS,gBAAgB;EAC1D,kBAAkBA,UAAAA,QAAK,KAAK,SAAS,UAAU;EAChD;;;AA8CH,MAAa,gCAOT;CACF,GAAG;CACH,oBAAoB;EAClB,iBAAiB;EACjB,kBAAkB;EAClB,2BAA2B,MAAW,KAAK;EAC5C;CACD,UAAU,EACR,sBAAsB,OACvB;CACD,+BAA+B;CAC/B,wBAAwB;CACzB;AAED,SAAgB,2BACd,SAIA;AACA,QAAO;EACL,GAAG,yBAAyB,QAAQ;EACpC,sBAAsBA,UAAAA,QAAK,KAAK,SAAS,WAAW;EACrD"}
+{"version":3,"file":"ops.cjs","names":["ConsoleLoggerAdapter","PFrameDriverOpsDefaults","path"],"sources":["../../src/middle_layer/ops.ts"],"sourcesContent":["import type { TemporalSynchronizedTreeOps } from \"./types\";\nimport type {\n  DownloadBlobToURLDriverOps,\n  DownloadDriverOps,\n  DownloadUrlDriverOps,\n  OpenFileDialogCallback,\n  VirtualLocalStorageSpec,\n} from \"@milaboratories/pl-drivers\";\nimport type { UploadDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { LogsStreamDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConsoleLoggerAdapter } from \"@milaboratories/ts-helpers\";\nimport type { LocalStorageProjection } from \"@milaboratories/pl-drivers\";\nimport path from \"node:path\";\nimport { PFrameDriverOpsDefaults, type PFrameDriverOps, type ParquetCacheOps } from \"../pool\";\n\n/** Paths part of {@link DriverKitOps}. */\nexport type DriverKitOpsPaths = {\n  /** Common root where to put downloaded blobs / downloaded blob cache */\n  readonly blobDownloadPath: string;\n\n  /** Common root for a cache for range queries. */\n  readonly blobDownloadRangesCachePath: string;\n\n  /** Common root where to put downloaded blobs with */\n  readonly downloadBlobToURLPath: string;\n\n  /**\n   * List of pl storages that have projections in local file system.\n   *\n   * This option affect two drivers:\n   *\n   *   (1) LS driver generates \"index\" handles instead of \"upload\" for paths inside those locations\n   *\n   *   (2) Download driver directly serves content retrieval requests for blobs from listed storages,\n   *       and don't apply any caching for such blobs (i.e. preventing duplication of files for Downloaded\n   *       type handles, making OnDemand and Downloaded handles equivalent)\n   *\n   * */\n  readonly localProjections: LocalStorageProjection[];\n\n  /**\n   * List of virtual storages that will allow homogeneous access to local FSs through LS API.\n   * If undefined, default list will be created.\n   * */\n  readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];\n\n  /** Path to the directory where pframes will spill temporary files and store materialized views */\n  readonly pframesSpillPath: string;\n\n  /**\n   * Path to the directory backing the persistent parquet blob cache. Unlike\n   * {@link pframesSpillPath} this directory is NOT emptied on startup - the cache\n   * (and its lifetime counters) is meant to survive restarts.\n   */\n  readonly parquetCachePath: string;\n};\n\n/** Options required to initialize full set of middle layer driver kit */\nexport type DriverKitOpsSettings = {\n  //\n  // Common\n  //\n\n  readonly logger: MiLogger;\n\n  //\n  // Signer\n  //\n\n  /**\n   * Local secret, that is used to sign and verify different pieces of information\n   * that can be used to access local data, like local paths for ongoing uploads.\n   *\n   * Use {@link MiddleLayer.generateLocalSecret} to generate sufficiently random string.\n   * */\n  readonly localSecret: string;\n\n  //\n  // Blob Driver\n  //\n\n  /**\n   * Settings related to the download driver making operations with blobs. This driver is also used\n   * to download logs when source process terminates and log terns into a blob\n   */\n  readonly blobDriverOps: DownloadDriverOps;\n\n  //\n  // Frontend Driver\n  //\n\n  /** Settings related to the frontend driver that downloads frontends. */\n  readonly frontendDownloadDriverOps: DownloadUrlDriverOps;\n\n  //\n  // Blob To URL Driver\n  //\n\n  readonly downloadBlobToURLDriverOps: DownloadBlobToURLDriverOps;\n\n  //\n  // Upload Driver\n  //\n\n  /**\n   * Settings related to the upload driver that actually performs upload and helps render upload\n   * and indexing progresses from related pl resources.\n   * */\n  readonly uploadDriverOps: UploadDriverOps;\n\n  //\n  // Log streaming ops\n  // (static logs are served via the blob driver)\n  //\n\n  /** Settings related to the streaming log driver */\n  readonly logStreamDriverOps: LogsStreamDriverOps;\n\n  //\n  // LS Driver\n  //\n\n  /**\n   * Callback to access system file open dialog, must be provided by the environment,\n   * to allow for {@link showOpenSingleFileDialog} / {@link showOpenMultipleFilesDialog}\n   * calls from the UI.\n   */\n  readonly openFileDialogCallback: OpenFileDialogCallback;\n\n  //\n  // PFrame Driver\n  //\n\n  /** Settings related to the PFrame driver */\n  readonly pFrameDriverOps: PFrameDriverOps;\n\n  /**\n   * Tuning for the persistent parquet blob cache. Its directory is\n   * {@link DriverKitOpsPaths.parquetCachePath}; this carries only the\n   * size/eviction knobs.\n   */\n  readonly parquetCacheOps: ParquetCacheOps;\n};\n\nexport type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultDriverKitOpsSettings: Pick<\n  DriverKitOpsSettings,\n  | \"logger\"\n  | \"blobDriverOps\"\n  | \"frontendDownloadDriverOps\"\n  | \"downloadBlobToURLDriverOps\"\n  | \"uploadDriverOps\"\n  | \"logStreamDriverOps\"\n  | \"pFrameDriverOps\"\n  | \"parquetCacheOps\"\n> = {\n  logger: new ConsoleLoggerAdapter(),\n  blobDriverOps: {\n    cacheSoftSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    rangesCacheMaxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    nConcurrentDownloads: 10,\n  },\n  frontendDownloadDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    withGunzip: true,\n    nConcurrentDownloads: 50,\n  },\n  downloadBlobToURLDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    nConcurrentDownloads: 10,\n  },\n  uploadDriverOps: {\n    nConcurrentPartUploads: 10,\n    nConcurrentGetProgresses: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  logStreamDriverOps: {\n    nConcurrentGetLogs: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  pFrameDriverOps: PFrameDriverOpsDefaults,\n  parquetCacheOps: {\n    maxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    admissionFraction: 0.2, // a single file may occupy at most 20% of the budget\n    maxFilesTracked: 50_000, // ghost (previously-evicted) files remembered to guide readmission\n  },\n};\n\nexport function DefaultDriverKitOpsPaths(\n  workDir: string,\n): Pick<\n  DriverKitOpsPaths,\n  | \"blobDownloadPath\"\n  | \"blobDownloadRangesCachePath\"\n  | \"downloadBlobToURLPath\"\n  | \"pframesSpillPath\"\n  | \"parquetCachePath\"\n> {\n  return {\n    blobDownloadPath: path.join(workDir, \"download\"),\n    blobDownloadRangesCachePath: path.join(workDir, \"downloadRangesCache\"),\n    downloadBlobToURLPath: path.join(workDir, \"downloadToURL\"),\n    pframesSpillPath: path.join(workDir, \"pframes\"),\n    parquetCachePath: path.join(workDir, \"parquetCache\"),\n  };\n}\n\n/** Fields with default values are marked as optional here. */\nexport type DriverKitOpsConstructor = Omit<\n  DriverKitOpsSettings,\n  keyof typeof DefaultDriverKitOpsSettings\n> &\n  Partial<typeof DefaultDriverKitOpsSettings> &\n  Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> &\n  Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;\n\nexport type MiddleLayerOpsPaths = DriverKitOpsPaths & {\n  /** Common root where to put frontend code. */\n  readonly frontendDownloadPath: string;\n};\n\n/** Debug options for middle layer. */\nexport type MiddleLayerDebugOptions = {\n  /** If true, will dump initial tree state to the file with root resource id as name. */\n  dumpInitialTreeState: boolean;\n};\n\n/** Configuration controlling different aspects of middle layer behaviour. */\nexport type MiddleLayerOpsSettings = DriverKitOpsSettings & {\n  /** Debug options. */\n  readonly debugOps: MiddleLayerDebugOptions;\n\n  /** Contain temporal options controlling how often should pl trees be\n   * synchronized with the pl server. */\n  readonly defaultTreeOptions: TemporalSynchronizedTreeOps;\n\n  /** Defines interval in milliseconds for running periodic project maintenance job.\n   * Project maintenance includes background staging rendering and cached outputs cleanup. */\n  readonly projectRefreshInterval: number;\n\n  /** How often to check for dev block updates */\n  readonly devBlockUpdateRecheckInterval: number;\n\n  /** Prioritize this channel if update is available in this block */\n  readonly preferredUpdateChannel?: string;\n};\n\nexport type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultMiddleLayerOpsSettings: Pick<\n  MiddleLayerOps,\n  | keyof typeof DefaultDriverKitOpsSettings\n  | \"defaultTreeOptions\"\n  | \"projectRefreshInterval\"\n  | \"devBlockUpdateRecheckInterval\"\n  | \"debugOps\"\n> = {\n  ...DefaultDriverKitOpsSettings,\n  defaultTreeOptions: {\n    pollingInterval: 200,\n    stopPollingDelay: 2500,\n    initialTreeLoadingTimeout: 100 * 60 * 60 * 1000, // disable timeout for loading project tree (100 hours)\n  },\n  debugOps: {\n    dumpInitialTreeState: false,\n  },\n  devBlockUpdateRecheckInterval: 1000,\n  projectRefreshInterval: 2000,\n};\n\nexport function DefaultMiddleLayerOpsPaths(\n  workDir: string,\n): Pick<\n  MiddleLayerOpsPaths,\n  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | \"frontendDownloadPath\"\n> {\n  return {\n    ...DefaultDriverKitOpsPaths(workDir),\n    frontendDownloadPath: path.join(workDir, \"frontend\"),\n  };\n}\n\nexport type MiddleLayerOpsConstructor = Omit<\n  MiddleLayerOpsSettings,\n  keyof typeof DefaultMiddleLayerOpsSettings\n> &\n  Partial<typeof DefaultMiddleLayerOpsSettings> &\n  Omit<MiddleLayerOpsPaths, keyof Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>> &\n  Partial<Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>>;\n"],"mappings":";;;;;;;AAoJA,MAAa,8BAUT;CACF,QAAQ,4CAAIA,sBAAsB;CAClC,eAAe;EACb,oBAAoB,IAAI,OAAO,OAAO;EACtC,yBAAyB,IAAI,OAAO,OAAO;EAC3C,sBAAsB;EACvB;CACD,2BAA2B;EACzB,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB;CACD,4BAA4B;EAC1B,oBAAoB,IAAI,OAAO,OAAO;EACtC,sBAAsB;EACvB;CACD,iBAAiB;EACf,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB;CACD,oBAAoB;EAClB,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB;CACD,iBAAiBC,eAAAA;CACjB,iBAAiB;EACf,cAAc,IAAI,OAAO,OAAO;EAChC,mBAAmB;EACnB,iBAAiB;EAClB;CACF;AAED,SAAgB,yBACd,SAQA;AACA,QAAO;EACL,kBAAkBC,UAAAA,QAAK,KAAK,SAAS,WAAW;EAChD,6BAA6BA,UAAAA,QAAK,KAAK,SAAS,sBAAsB;EACtE,uBAAuBA,UAAAA,QAAK,KAAK,SAAS,gBAAgB;EAC1D,kBAAkBA,UAAAA,QAAK,KAAK,SAAS,UAAU;EAC/C,kBAAkBA,UAAAA,QAAK,KAAK,SAAS,eAAe;EACrD;;;AA8CH,MAAa,gCAOT;CACF,GAAG;CACH,oBAAoB;EAClB,iBAAiB;EACjB,kBAAkB;EAClB,2BAA2B,MAAW,KAAK;EAC5C;CACD,UAAU,EACR,sBAAsB,OACvB;CACD,+BAA+B;CAC/B,wBAAwB;CACzB;AAED,SAAgB,2BACd,SAIA;AACA,QAAO;EACL,GAAG,yBAAyB,QAAQ;EACpC,sBAAsBA,UAAAA,QAAK,KAAK,SAAS,WAAW;EACrD"}

package/dist/middle_layer/ops.d.ts

@@ -1,5 +1,5 @@
 import { TemporalSynchronizedTreeOps } from "./types.js";
-import { PFrameDriverOps } from "../pool/driver.js";
+import { PFrameDriverOps, ParquetCacheOps } from "../pool/driver.js";
 import { MiLogger } from "@milaboratories/ts-helpers";
 import { DownloadBlobToURLDriverOps, DownloadDriverOps, DownloadUrlDriverOps, LocalStorageProjection, LogsStreamDriverOps, OpenFileDialogCallback, UploadDriverOps, VirtualLocalStorageSpec } from "@milaboratories/pl-drivers";
 
@@ -28,6 +28,12 @@ type DriverKitOpsPaths = {
    * */
   readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[]; /** Path to the directory where pframes will spill temporary files and store materialized views */
   readonly pframesSpillPath: string;
+  /**
+   * Path to the directory backing the persistent parquet blob cache. Unlike
+   * {@link pframesSpillPath} this directory is NOT emptied on startup - the cache
+   * (and its lifetime counters) is meant to survive restarts.
+   */
+  readonly parquetCachePath: string;
 };
 /** Options required to initialize full set of middle layer driver kit */
 type DriverKitOpsSettings = {
@@ -59,11 +65,17 @@ type DriverKitOpsSettings = {
    */
   readonly openFileDialogCallback: OpenFileDialogCallback; /** Settings related to the PFrame driver */
   readonly pFrameDriverOps: PFrameDriverOps;
+  /**
+   * Tuning for the persistent parquet blob cache. Its directory is
+   * {@link DriverKitOpsPaths.parquetCachePath}; this carries only the
+   * size/eviction knobs.
+   */
+  readonly parquetCacheOps: ParquetCacheOps;
 };
 type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;
 /** Some defaults fot MiddleLayerOps. */
-declare const DefaultDriverKitOpsSettings: Pick<DriverKitOpsSettings, "logger" | "blobDriverOps" | "frontendDownloadDriverOps" | "downloadBlobToURLDriverOps" | "uploadDriverOps" | "logStreamDriverOps" | "pFrameDriverOps">;
-declare function DefaultDriverKitOpsPaths(workDir: string): Pick<DriverKitOpsPaths, "blobDownloadPath" | "blobDownloadRangesCachePath" | "downloadBlobToURLPath" | "pframesSpillPath">;
+declare const DefaultDriverKitOpsSettings: Pick<DriverKitOpsSettings, "logger" | "blobDriverOps" | "frontendDownloadDriverOps" | "downloadBlobToURLDriverOps" | "uploadDriverOps" | "logStreamDriverOps" | "pFrameDriverOps" | "parquetCacheOps">;
+declare function DefaultDriverKitOpsPaths(workDir: string): Pick<DriverKitOpsPaths, "blobDownloadPath" | "blobDownloadRangesCachePath" | "downloadBlobToURLPath" | "pframesSpillPath" | "parquetCachePath">;
 /** Fields with default values are marked as optional here. */
 type DriverKitOpsConstructor = Omit<DriverKitOpsSettings, keyof typeof DefaultDriverKitOpsSettings> & Partial<typeof DefaultDriverKitOpsSettings> & Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> & Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;
 type MiddleLayerOpsPaths = DriverKitOpsPaths & {

package/dist/middle_layer/ops.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ops.d.ts","names":[],"sources":["../../src/middle_layer/ops.ts"],"mappings":";;;;;;AAiBA;AAAA,KAAY,iBAAA;mFAED,gBAAA;EAAA,SAGA,2BAAA,UAGA;EAAA,SAAA,qBAAA;EAckB;;;;;;AAa7B;;;;;;EAb6B,SAAlB,gBAAA,EAAkB,sBAAA;EA+DD;;;;EAAA,SAzDjB,4BAAA,GAA+B,uBAAA,IAmFC;EAAA,SAhFhC,gBAAA;AAAA;;KAIC,oBAAA;EAAA,SAKD,MAAA,EAAQ,QAAA;EA6BR;;;;;;EAAA,SAjBA,WAAA;EAyCoB;;;;EAAA,SA/BpB,aAAA,EAAe,iBAAA,EAiDiB;EAAA,SA1ChC,yBAAA,EAA2B,oBAAA;EAAA,SAM3B,0BAAA,EAA4B,0BAAA;EAuCf;;;;EAAA,SA7Bb,eAAA,EAAiB,eAAA,EAqE3B;EAAA,SA7DU,kBAAA,EAAoB,mBAAA;EAwBW;;AAuC1C;;;EAvC0C,SAb/B,sBAAA,EAAwB,sBAAA,EAqDjC;EAAA,SA9CS,eAAA,EAAiB,eAAA;AAAA;AAAA,KAGhB,YAAA,GAAe,iBAAA,GAAoB,oBAAA;;cAGlC,2BAAA,EAA6B,IAAA,CACxC,oBAAA;AAAA,iBAsCc,wBAAA,CACd,OAAA,WACC,IAAA,CACD,iBAAA;;KAYU,uBAAA,GAA0B,IAAA,CACpC,oBAAA,eACa,2BAAA,IAEb,OAAA,QAAe,2BAAA,IACf,IAAA,CAAK,iBAAA,QAAyB,UAAA,QAAkB,wBAAA,KAChD,OAAA,CAAQ,UAAA,QAAkB,wBAAA;AAAA,KAEhB,mBAAA,GAAsB,iBAAA;EARI,uDAU3B,oBAAA;AAAA;;KAIC,uBAAA;EAToB,uFAW9B,oBAAA;AAAA;;KAIU,sBAAA,GAAyB,oBAAA;EAd5B,8BAgBE,QAAA,EAAU,uBAAA;EAtBiB;;EAAA,SA0B3B,kBAAA,EAAoB,2BAAA;EAtB7B;;EAAA,SA0BS,sBAAA,UAzBJ;EAAA,SA4BI,6BAAA,UA5BuC;EAAA,SA+BvC,sBAAA;AAAA;AAAA,KAGC,cAAA,GAAiB,sBAAA,GAAyB,mBAAA;;cAGzC,6BAAA,EAA+B,IAAA,CAC1C,cAAA,eACe,2BAAA;AAAA,iBAmBD,0BAAA,CACd,OAAA,WACC,IAAA,CACD,mBAAA,QACM,UAAA,QAAkB,wBAAA;AAAA,KAQd,yBAAA,GAA4B,IAAA,CACtC,sBAAA,eACa,6BAAA,IAEb,OAAA,QAAe,6BAAA,IACf,IAAA,CAAK,mBAAA,QAA2B,OAAA,CAAQ,UAAA,QAAkB,0BAAA,MAC1D,OAAA,CAAQ,OAAA,CAAQ,UAAA,QAAkB,0BAAA"}
+{"version":3,"file":"ops.d.ts","names":[],"sources":["../../src/middle_layer/ops.ts"],"mappings":";;;;;;AAiBA;AAAA,KAAY,iBAAA;mFAED,gBAAA;EAAA,SAGA,2BAAA,UAGA;EAAA,SAAA,qBAAA;EAckB;;;;;;;AAoB7B;;;;;EApB6B,SAAlB,gBAAA,EAAkB,sBAAA;EA4DU;;;;EAAA,SAtD5B,4BAAA,GAA+B,uBAAA,IAiGd;EAAA,SA9FjB,gBAAA;EA8FgC;;;;;EAAA,SAvFhC,gBAAA;AAAA;;KAIC,oBAAA;EAAA,SAKD,MAAA,EAAQ,QAAA;EA6CR;;;;;;EAAA,SAjCA,WAAA;EA2DiB;;;;EAAA,SAjDjB,aAAA,EAAe,iBAAA,EA2Dd;EAAA,SApDD,yBAAA,EAA2B,oBAAA;EAAA,SAM3B,0BAAA,EAA4B,0BAAA;EA8CZ;;AAG3B;;EAH2B,SApChB,eAAA,EAAiB,eAAA,EAuCc;EAAA,SA/B/B,kBAAA,EAAoB,mBAAA;EA4Ef;;;;;EAAA,SAjEL,sBAAA,EAAwB,sBAAA,EAoEjC;EAAA,SA7DS,eAAA,EAAiB,eAAA;EA6DT;AAiBnB;;;;EAjBmB,SAtDR,eAAA,EAAiB,eAAA;AAAA;AAAA,KAGhB,YAAA,GAAe,iBAAA,GAAoB,oBAAA;;cAGlC,2BAAA,EAA6B,IAAA,CACxC,oBAAA;AAAA,iBA4Cc,wBAAA,CACd,OAAA,WACC,IAAA,CACD,iBAAA;;KAiBU,uBAAA,GAA0B,IAAA,CACpC,oBAAA,eACa,2BAAA,IAEb,OAAA,QAAe,2BAAA,IACf,IAAA,CAAK,iBAAA,QAAyB,UAAA,QAAkB,wBAAA,KAChD,OAAA,CAAQ,UAAA,QAAkB,wBAAA;AAAA,KAEhB,mBAAA,GAAsB,iBAAA;EAFxB,uDAIC,oBAAA;AAAA;;KAIC,uBAAA;EAbV,uFAeA,oBAAA;AAAA;;KAIU,sBAAA,GAAyB,oBAAA;EAf9B,8BAiBI,QAAA,EAAU,uBAAA;EAjB6B;;EAAA,SAqBvC,kBAAA,EAAoB,2BAAA;EApBH;;EAAA,SAwBjB,sBAAA,UAtBC;EAAA,SAyBD,6BAAA;WAGA,sBAAA;AAAA;AAAA,KAGC,cAAA,GAAiB,sBAAA,GAAyB,mBAAA;;cAGzC,6BAAA,EAA+B,IAAA,CAC1C,cAAA,eACe,2BAAA;AAAA,iBAmBD,0BAAA,CACd,OAAA,WACC,IAAA,CACD,mBAAA,QACM,UAAA,QAAkB,wBAAA;AAAA,KAQd,yBAAA,GAA4B,IAAA,CACtC,sBAAA,eACa,6BAAA,IAEb,OAAA,QAAe,6BAAA,IACf,IAAA,CAAK,mBAAA,QAA2B,OAAA,CAAQ,UAAA,QAAkB,0BAAA,MAC1D,OAAA,CAAQ,OAAA,CAAQ,UAAA,QAAkB,0BAAA"}

package/dist/middle_layer/ops.js

@@ -31,14 +31,20 @@ const DefaultDriverKitOpsSettings = {
 		pollingInterval: 1e3,
 		stopPollingDelay: 1e3
 	},
-	pFrameDriverOps: PFrameDriverOpsDefaults
+	pFrameDriverOps: PFrameDriverOpsDefaults,
+	parquetCacheOps: {
+		maxSizeBytes: 8 * 1024 * 1024 * 1024,
+		admissionFraction: .2,
+		maxFilesTracked: 5e4
+	}
 };
 function DefaultDriverKitOpsPaths(workDir) {
 	return {
 		blobDownloadPath: path.join(workDir, "download"),
 		blobDownloadRangesCachePath: path.join(workDir, "downloadRangesCache"),
 		downloadBlobToURLPath: path.join(workDir, "downloadToURL"),
-		pframesSpillPath: path.join(workDir, "pframes")
+		pframesSpillPath: path.join(workDir, "pframes"),
+		parquetCachePath: path.join(workDir, "parquetCache")
 	};
 }
 /** Some defaults fot MiddleLayerOps. */

package/dist/middle_layer/ops.js.map

@@ -1 +1 @@
-{"version":3,"file":"ops.js","names":[],"sources":["../../src/middle_layer/ops.ts"],"sourcesContent":["import type { TemporalSynchronizedTreeOps } from \"./types\";\nimport type {\n  DownloadBlobToURLDriverOps,\n  DownloadDriverOps,\n  DownloadUrlDriverOps,\n  OpenFileDialogCallback,\n  VirtualLocalStorageSpec,\n} from \"@milaboratories/pl-drivers\";\nimport type { UploadDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { LogsStreamDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConsoleLoggerAdapter } from \"@milaboratories/ts-helpers\";\nimport type { LocalStorageProjection } from \"@milaboratories/pl-drivers\";\nimport path from \"node:path\";\nimport { PFrameDriverOpsDefaults, type PFrameDriverOps } from \"../pool\";\n\n/** Paths part of {@link DriverKitOps}. */\nexport type DriverKitOpsPaths = {\n  /** Common root where to put downloaded blobs / downloaded blob cache */\n  readonly blobDownloadPath: string;\n\n  /** Common root for a cache for range queries. */\n  readonly blobDownloadRangesCachePath: string;\n\n  /** Common root where to put downloaded blobs with */\n  readonly downloadBlobToURLPath: string;\n\n  /**\n   * List of pl storages that have projections in local file system.\n   *\n   * This option affect two drivers:\n   *\n   *   (1) LS driver generates \"index\" handles instead of \"upload\" for paths inside those locations\n   *\n   *   (2) Download driver directly serves content retrieval requests for blobs from listed storages,\n   *       and don't apply any caching for such blobs (i.e. preventing duplication of files for Downloaded\n   *       type handles, making OnDemand and Downloaded handles equivalent)\n   *\n   * */\n  readonly localProjections: LocalStorageProjection[];\n\n  /**\n   * List of virtual storages that will allow homogeneous access to local FSs through LS API.\n   * If undefined, default list will be created.\n   * */\n  readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];\n\n  /** Path to the directory where pframes will spill temporary files and store materialized views */\n  readonly pframesSpillPath: string;\n};\n\n/** Options required to initialize full set of middle layer driver kit */\nexport type DriverKitOpsSettings = {\n  //\n  // Common\n  //\n\n  readonly logger: MiLogger;\n\n  //\n  // Signer\n  //\n\n  /**\n   * Local secret, that is used to sign and verify different pieces of information\n   * that can be used to access local data, like local paths for ongoing uploads.\n   *\n   * Use {@link MiddleLayer.generateLocalSecret} to generate sufficiently random string.\n   * */\n  readonly localSecret: string;\n\n  //\n  // Blob Driver\n  //\n\n  /**\n   * Settings related to the download driver making operations with blobs. This driver is also used\n   * to download logs when source process terminates and log terns into a blob\n   */\n  readonly blobDriverOps: DownloadDriverOps;\n\n  //\n  // Frontend Driver\n  //\n\n  /** Settings related to the frontend driver that downloads frontends. */\n  readonly frontendDownloadDriverOps: DownloadUrlDriverOps;\n\n  //\n  // Blob To URL Driver\n  //\n\n  readonly downloadBlobToURLDriverOps: DownloadBlobToURLDriverOps;\n\n  //\n  // Upload Driver\n  //\n\n  /**\n   * Settings related to the upload driver that actually performs upload and helps render upload\n   * and indexing progresses from related pl resources.\n   * */\n  readonly uploadDriverOps: UploadDriverOps;\n\n  //\n  // Log streaming ops\n  // (static logs are served via the blob driver)\n  //\n\n  /** Settings related to the streaming log driver */\n  readonly logStreamDriverOps: LogsStreamDriverOps;\n\n  //\n  // LS Driver\n  //\n\n  /**\n   * Callback to access system file open dialog, must be provided by the environment,\n   * to allow for {@link showOpenSingleFileDialog} / {@link showOpenMultipleFilesDialog}\n   * calls from the UI.\n   */\n  readonly openFileDialogCallback: OpenFileDialogCallback;\n\n  //\n  // PFrame Driver\n  //\n\n  /** Settings related to the PFrame driver */\n  readonly pFrameDriverOps: PFrameDriverOps;\n};\n\nexport type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultDriverKitOpsSettings: Pick<\n  DriverKitOpsSettings,\n  | \"logger\"\n  | \"blobDriverOps\"\n  | \"frontendDownloadDriverOps\"\n  | \"downloadBlobToURLDriverOps\"\n  | \"uploadDriverOps\"\n  | \"logStreamDriverOps\"\n  | \"pFrameDriverOps\"\n> = {\n  logger: new ConsoleLoggerAdapter(),\n  blobDriverOps: {\n    cacheSoftSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    rangesCacheMaxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    nConcurrentDownloads: 10,\n  },\n  frontendDownloadDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    withGunzip: true,\n    nConcurrentDownloads: 50,\n  },\n  downloadBlobToURLDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    nConcurrentDownloads: 10,\n  },\n  uploadDriverOps: {\n    nConcurrentPartUploads: 10,\n    nConcurrentGetProgresses: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  logStreamDriverOps: {\n    nConcurrentGetLogs: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  pFrameDriverOps: PFrameDriverOpsDefaults,\n};\n\nexport function DefaultDriverKitOpsPaths(\n  workDir: string,\n): Pick<\n  DriverKitOpsPaths,\n  \"blobDownloadPath\" | \"blobDownloadRangesCachePath\" | \"downloadBlobToURLPath\" | \"pframesSpillPath\"\n> {\n  return {\n    blobDownloadPath: path.join(workDir, \"download\"),\n    blobDownloadRangesCachePath: path.join(workDir, \"downloadRangesCache\"),\n    downloadBlobToURLPath: path.join(workDir, \"downloadToURL\"),\n    pframesSpillPath: path.join(workDir, \"pframes\"),\n  };\n}\n\n/** Fields with default values are marked as optional here. */\nexport type DriverKitOpsConstructor = Omit<\n  DriverKitOpsSettings,\n  keyof typeof DefaultDriverKitOpsSettings\n> &\n  Partial<typeof DefaultDriverKitOpsSettings> &\n  Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> &\n  Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;\n\nexport type MiddleLayerOpsPaths = DriverKitOpsPaths & {\n  /** Common root where to put frontend code. */\n  readonly frontendDownloadPath: string;\n};\n\n/** Debug options for middle layer. */\nexport type MiddleLayerDebugOptions = {\n  /** If true, will dump initial tree state to the file with root resource id as name. */\n  dumpInitialTreeState: boolean;\n};\n\n/** Configuration controlling different aspects of middle layer behaviour. */\nexport type MiddleLayerOpsSettings = DriverKitOpsSettings & {\n  /** Debug options. */\n  readonly debugOps: MiddleLayerDebugOptions;\n\n  /** Contain temporal options controlling how often should pl trees be\n   * synchronized with the pl server. */\n  readonly defaultTreeOptions: TemporalSynchronizedTreeOps;\n\n  /** Defines interval in milliseconds for running periodic project maintenance job.\n   * Project maintenance includes background staging rendering and cached outputs cleanup. */\n  readonly projectRefreshInterval: number;\n\n  /** How often to check for dev block updates */\n  readonly devBlockUpdateRecheckInterval: number;\n\n  /** Prioritize this channel if update is available in this block */\n  readonly preferredUpdateChannel?: string;\n};\n\nexport type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultMiddleLayerOpsSettings: Pick<\n  MiddleLayerOps,\n  | keyof typeof DefaultDriverKitOpsSettings\n  | \"defaultTreeOptions\"\n  | \"projectRefreshInterval\"\n  | \"devBlockUpdateRecheckInterval\"\n  | \"debugOps\"\n> = {\n  ...DefaultDriverKitOpsSettings,\n  defaultTreeOptions: {\n    pollingInterval: 200,\n    stopPollingDelay: 2500,\n    initialTreeLoadingTimeout: 100 * 60 * 60 * 1000, // disable timeout for loading project tree (100 hours)\n  },\n  debugOps: {\n    dumpInitialTreeState: false,\n  },\n  devBlockUpdateRecheckInterval: 1000,\n  projectRefreshInterval: 2000,\n};\n\nexport function DefaultMiddleLayerOpsPaths(\n  workDir: string,\n): Pick<\n  MiddleLayerOpsPaths,\n  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | \"frontendDownloadPath\"\n> {\n  return {\n    ...DefaultDriverKitOpsPaths(workDir),\n    frontendDownloadPath: path.join(workDir, \"frontend\"),\n  };\n}\n\nexport type MiddleLayerOpsConstructor = Omit<\n  MiddleLayerOpsSettings,\n  keyof typeof DefaultMiddleLayerOpsSettings\n> &\n  Partial<typeof DefaultMiddleLayerOpsSettings> &\n  Omit<MiddleLayerOpsPaths, keyof Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>> &\n  Partial<Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>>;\n"],"mappings":";;;;;;AAsIA,MAAa,8BAST;CACF,QAAQ,IAAI,sBAAsB;CAClC,eAAe;EACb,oBAAoB,IAAI,OAAO,OAAO;EACtC,yBAAyB,IAAI,OAAO,OAAO;EAC3C,sBAAsB;EACvB;CACD,2BAA2B;EACzB,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB;CACD,4BAA4B;EAC1B,oBAAoB,IAAI,OAAO,OAAO;EACtC,sBAAsB;EACvB;CACD,iBAAiB;EACf,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB;CACD,oBAAoB;EAClB,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB;CACD,iBAAiB;CAClB;AAED,SAAgB,yBACd,SAIA;AACA,QAAO;EACL,kBAAkB,KAAK,KAAK,SAAS,WAAW;EAChD,6BAA6B,KAAK,KAAK,SAAS,sBAAsB;EACtE,uBAAuB,KAAK,KAAK,SAAS,gBAAgB;EAC1D,kBAAkB,KAAK,KAAK,SAAS,UAAU;EAChD;;;AA8CH,MAAa,gCAOT;CACF,GAAG;CACH,oBAAoB;EAClB,iBAAiB;EACjB,kBAAkB;EAClB,2BAA2B,MAAW,KAAK;EAC5C;CACD,UAAU,EACR,sBAAsB,OACvB;CACD,+BAA+B;CAC/B,wBAAwB;CACzB;AAED,SAAgB,2BACd,SAIA;AACA,QAAO;EACL,GAAG,yBAAyB,QAAQ;EACpC,sBAAsB,KAAK,KAAK,SAAS,WAAW;EACrD"}
+{"version":3,"file":"ops.js","names":[],"sources":["../../src/middle_layer/ops.ts"],"sourcesContent":["import type { TemporalSynchronizedTreeOps } from \"./types\";\nimport type {\n  DownloadBlobToURLDriverOps,\n  DownloadDriverOps,\n  DownloadUrlDriverOps,\n  OpenFileDialogCallback,\n  VirtualLocalStorageSpec,\n} from \"@milaboratories/pl-drivers\";\nimport type { UploadDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { LogsStreamDriverOps } from \"@milaboratories/pl-drivers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConsoleLoggerAdapter } from \"@milaboratories/ts-helpers\";\nimport type { LocalStorageProjection } from \"@milaboratories/pl-drivers\";\nimport path from \"node:path\";\nimport { PFrameDriverOpsDefaults, type PFrameDriverOps, type ParquetCacheOps } from \"../pool\";\n\n/** Paths part of {@link DriverKitOps}. */\nexport type DriverKitOpsPaths = {\n  /** Common root where to put downloaded blobs / downloaded blob cache */\n  readonly blobDownloadPath: string;\n\n  /** Common root for a cache for range queries. */\n  readonly blobDownloadRangesCachePath: string;\n\n  /** Common root where to put downloaded blobs with */\n  readonly downloadBlobToURLPath: string;\n\n  /**\n   * List of pl storages that have projections in local file system.\n   *\n   * This option affect two drivers:\n   *\n   *   (1) LS driver generates \"index\" handles instead of \"upload\" for paths inside those locations\n   *\n   *   (2) Download driver directly serves content retrieval requests for blobs from listed storages,\n   *       and don't apply any caching for such blobs (i.e. preventing duplication of files for Downloaded\n   *       type handles, making OnDemand and Downloaded handles equivalent)\n   *\n   * */\n  readonly localProjections: LocalStorageProjection[];\n\n  /**\n   * List of virtual storages that will allow homogeneous access to local FSs through LS API.\n   * If undefined, default list will be created.\n   * */\n  readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];\n\n  /** Path to the directory where pframes will spill temporary files and store materialized views */\n  readonly pframesSpillPath: string;\n\n  /**\n   * Path to the directory backing the persistent parquet blob cache. Unlike\n   * {@link pframesSpillPath} this directory is NOT emptied on startup - the cache\n   * (and its lifetime counters) is meant to survive restarts.\n   */\n  readonly parquetCachePath: string;\n};\n\n/** Options required to initialize full set of middle layer driver kit */\nexport type DriverKitOpsSettings = {\n  //\n  // Common\n  //\n\n  readonly logger: MiLogger;\n\n  //\n  // Signer\n  //\n\n  /**\n   * Local secret, that is used to sign and verify different pieces of information\n   * that can be used to access local data, like local paths for ongoing uploads.\n   *\n   * Use {@link MiddleLayer.generateLocalSecret} to generate sufficiently random string.\n   * */\n  readonly localSecret: string;\n\n  //\n  // Blob Driver\n  //\n\n  /**\n   * Settings related to the download driver making operations with blobs. This driver is also used\n   * to download logs when source process terminates and log terns into a blob\n   */\n  readonly blobDriverOps: DownloadDriverOps;\n\n  //\n  // Frontend Driver\n  //\n\n  /** Settings related to the frontend driver that downloads frontends. */\n  readonly frontendDownloadDriverOps: DownloadUrlDriverOps;\n\n  //\n  // Blob To URL Driver\n  //\n\n  readonly downloadBlobToURLDriverOps: DownloadBlobToURLDriverOps;\n\n  //\n  // Upload Driver\n  //\n\n  /**\n   * Settings related to the upload driver that actually performs upload and helps render upload\n   * and indexing progresses from related pl resources.\n   * */\n  readonly uploadDriverOps: UploadDriverOps;\n\n  //\n  // Log streaming ops\n  // (static logs are served via the blob driver)\n  //\n\n  /** Settings related to the streaming log driver */\n  readonly logStreamDriverOps: LogsStreamDriverOps;\n\n  //\n  // LS Driver\n  //\n\n  /**\n   * Callback to access system file open dialog, must be provided by the environment,\n   * to allow for {@link showOpenSingleFileDialog} / {@link showOpenMultipleFilesDialog}\n   * calls from the UI.\n   */\n  readonly openFileDialogCallback: OpenFileDialogCallback;\n\n  //\n  // PFrame Driver\n  //\n\n  /** Settings related to the PFrame driver */\n  readonly pFrameDriverOps: PFrameDriverOps;\n\n  /**\n   * Tuning for the persistent parquet blob cache. Its directory is\n   * {@link DriverKitOpsPaths.parquetCachePath}; this carries only the\n   * size/eviction knobs.\n   */\n  readonly parquetCacheOps: ParquetCacheOps;\n};\n\nexport type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultDriverKitOpsSettings: Pick<\n  DriverKitOpsSettings,\n  | \"logger\"\n  | \"blobDriverOps\"\n  | \"frontendDownloadDriverOps\"\n  | \"downloadBlobToURLDriverOps\"\n  | \"uploadDriverOps\"\n  | \"logStreamDriverOps\"\n  | \"pFrameDriverOps\"\n  | \"parquetCacheOps\"\n> = {\n  logger: new ConsoleLoggerAdapter(),\n  blobDriverOps: {\n    cacheSoftSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    rangesCacheMaxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    nConcurrentDownloads: 10,\n  },\n  frontendDownloadDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    withGunzip: true,\n    nConcurrentDownloads: 50,\n  },\n  downloadBlobToURLDriverOps: {\n    cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n    nConcurrentDownloads: 10,\n  },\n  uploadDriverOps: {\n    nConcurrentPartUploads: 10,\n    nConcurrentGetProgresses: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  logStreamDriverOps: {\n    nConcurrentGetLogs: 10,\n    pollingInterval: 1000,\n    stopPollingDelay: 1000,\n  },\n  pFrameDriverOps: PFrameDriverOpsDefaults,\n  parquetCacheOps: {\n    maxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB\n    admissionFraction: 0.2, // a single file may occupy at most 20% of the budget\n    maxFilesTracked: 50_000, // ghost (previously-evicted) files remembered to guide readmission\n  },\n};\n\nexport function DefaultDriverKitOpsPaths(\n  workDir: string,\n): Pick<\n  DriverKitOpsPaths,\n  | \"blobDownloadPath\"\n  | \"blobDownloadRangesCachePath\"\n  | \"downloadBlobToURLPath\"\n  | \"pframesSpillPath\"\n  | \"parquetCachePath\"\n> {\n  return {\n    blobDownloadPath: path.join(workDir, \"download\"),\n    blobDownloadRangesCachePath: path.join(workDir, \"downloadRangesCache\"),\n    downloadBlobToURLPath: path.join(workDir, \"downloadToURL\"),\n    pframesSpillPath: path.join(workDir, \"pframes\"),\n    parquetCachePath: path.join(workDir, \"parquetCache\"),\n  };\n}\n\n/** Fields with default values are marked as optional here. */\nexport type DriverKitOpsConstructor = Omit<\n  DriverKitOpsSettings,\n  keyof typeof DefaultDriverKitOpsSettings\n> &\n  Partial<typeof DefaultDriverKitOpsSettings> &\n  Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> &\n  Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;\n\nexport type MiddleLayerOpsPaths = DriverKitOpsPaths & {\n  /** Common root where to put frontend code. */\n  readonly frontendDownloadPath: string;\n};\n\n/** Debug options for middle layer. */\nexport type MiddleLayerDebugOptions = {\n  /** If true, will dump initial tree state to the file with root resource id as name. */\n  dumpInitialTreeState: boolean;\n};\n\n/** Configuration controlling different aspects of middle layer behaviour. */\nexport type MiddleLayerOpsSettings = DriverKitOpsSettings & {\n  /** Debug options. */\n  readonly debugOps: MiddleLayerDebugOptions;\n\n  /** Contain temporal options controlling how often should pl trees be\n   * synchronized with the pl server. */\n  readonly defaultTreeOptions: TemporalSynchronizedTreeOps;\n\n  /** Defines interval in milliseconds for running periodic project maintenance job.\n   * Project maintenance includes background staging rendering and cached outputs cleanup. */\n  readonly projectRefreshInterval: number;\n\n  /** How often to check for dev block updates */\n  readonly devBlockUpdateRecheckInterval: number;\n\n  /** Prioritize this channel if update is available in this block */\n  readonly preferredUpdateChannel?: string;\n};\n\nexport type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;\n\n/** Some defaults fot MiddleLayerOps. */\nexport const DefaultMiddleLayerOpsSettings: Pick<\n  MiddleLayerOps,\n  | keyof typeof DefaultDriverKitOpsSettings\n  | \"defaultTreeOptions\"\n  | \"projectRefreshInterval\"\n  | \"devBlockUpdateRecheckInterval\"\n  | \"debugOps\"\n> = {\n  ...DefaultDriverKitOpsSettings,\n  defaultTreeOptions: {\n    pollingInterval: 200,\n    stopPollingDelay: 2500,\n    initialTreeLoadingTimeout: 100 * 60 * 60 * 1000, // disable timeout for loading project tree (100 hours)\n  },\n  debugOps: {\n    dumpInitialTreeState: false,\n  },\n  devBlockUpdateRecheckInterval: 1000,\n  projectRefreshInterval: 2000,\n};\n\nexport function DefaultMiddleLayerOpsPaths(\n  workDir: string,\n): Pick<\n  MiddleLayerOpsPaths,\n  keyof ReturnType<typeof DefaultDriverKitOpsPaths> | \"frontendDownloadPath\"\n> {\n  return {\n    ...DefaultDriverKitOpsPaths(workDir),\n    frontendDownloadPath: path.join(workDir, \"frontend\"),\n  };\n}\n\nexport type MiddleLayerOpsConstructor = Omit<\n  MiddleLayerOpsSettings,\n  keyof typeof DefaultMiddleLayerOpsSettings\n> &\n  Partial<typeof DefaultMiddleLayerOpsSettings> &\n  Omit<MiddleLayerOpsPaths, keyof Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>> &\n  Partial<Awaited<ReturnType<typeof DefaultMiddleLayerOpsPaths>>>;\n"],"mappings":";;;;;;AAoJA,MAAa,8BAUT;CACF,QAAQ,IAAI,sBAAsB;CAClC,eAAe;EACb,oBAAoB,IAAI,OAAO,OAAO;EACtC,yBAAyB,IAAI,OAAO,OAAO;EAC3C,sBAAsB;EACvB;CACD,2BAA2B;EACzB,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB;CACD,4BAA4B;EAC1B,oBAAoB,IAAI,OAAO,OAAO;EACtC,sBAAsB;EACvB;CACD,iBAAiB;EACf,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB;CACD,oBAAoB;EAClB,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB;CACD,iBAAiB;CACjB,iBAAiB;EACf,cAAc,IAAI,OAAO,OAAO;EAChC,mBAAmB;EACnB,iBAAiB;EAClB;CACF;AAED,SAAgB,yBACd,SAQA;AACA,QAAO;EACL,kBAAkB,KAAK,KAAK,SAAS,WAAW;EAChD,6BAA6B,KAAK,KAAK,SAAS,sBAAsB;EACtE,uBAAuB,KAAK,KAAK,SAAS,gBAAgB;EAC1D,kBAAkB,KAAK,KAAK,SAAS,UAAU;EAC/C,kBAAkB,KAAK,KAAK,SAAS,eAAe;EACrD;;;AA8CH,MAAa,gCAOT;CACF,GAAG;CACH,oBAAoB;EAClB,iBAAiB;EACjB,kBAAkB;EAClB,2BAA2B,MAAW,KAAK;EAC5C;CACD,UAAU,EACR,sBAAsB,OACvB;CACD,+BAA+B;CAC/B,wBAAwB;CACzB;AAED,SAAgB,2BACd,SAIA;AACA,QAAO;EACL,GAAG,yBAAyB,QAAQ;EACpC,sBAAsB,KAAK,KAAK,SAAS,WAAW;EACrD"}

package/dist/pool/driver.cjs

@@ -45,7 +45,7 @@ var LocalBlobProviderImpl = class extends _milaboratories_helpers.RefCountPoolBa
 			},
 			resolveBlobContent: async (blobId) => {
 				const blob = await this.getByKey(blobId).awaitStableValue(signal);
-				return await this.blobDriver.getContent(blob.handle, { signal });
+				return await this.blobDriver.getContentDirect(blob.handle, { signal });
 			}
 		};
 	}
@@ -136,23 +136,29 @@ var BlobStore = class extends _milaboratories_pl_model_middle_layer.PFrameIntern
 	}
 };
 var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
-	constructor(pool, server) {
+	constructor(pool, server, cache) {
 		this.pool = pool;
 		this.server = server;
+		this.cache = cache;
 	}
-	static async init(blobDriver, logger, serverOptions) {
+	static async init(blobDriver, logger, serverOptions, cacheConfig) {
 		const pool = new RemoteBlobPool(blobDriver, logger);
 		const store = new BlobStore({
 			remoteBlobProvider: pool,
 			logger
 		});
-		const handler = _milaboratories_pframes_rs_node.HttpHelpers.createRequestHandler({ store });
+		const cachingStore = await _milaboratories_pframes_rs_node.HttpHelpers.createCachingObjectStore({
+			upstream: store,
+			config: cacheConfig,
+			logger
+		});
+		const handler = _milaboratories_pframes_rs_node.HttpHelpers.createRequestHandler({ store: cachingStore });
 		const server = await _milaboratories_pframes_rs_node.HttpHelpers.createHttpServer({
 			...serverOptions,
 			handler
 		});
 		logger("info", `PFrames HTTP server started on ${server.info.url}`);
-		return new RemoteBlobProviderImpl(pool, server);
+		return new RemoteBlobProviderImpl(pool, server, cachingStore);
 	}
 	acquire(params) {
 		return this.pool.acquire(params);
@@ -161,11 +167,14 @@ var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
 		return this.server.info;
 	}
 	async getCacheMetrics() {
-		return null;
+		return this.cache.getMetrics();
+	}
+	async resetCache() {
+		await this.cache.reset();
 	}
-	async resetCache() {}
 	async [Symbol.asyncDispose]() {
 		await this.server.stop();
+		await this.cache[Symbol.asyncDispose]();
 		await this.pool[Symbol.asyncDispose]();
 	}
 };
@@ -178,7 +187,10 @@ async function createPFrameDriver(params) {
 	await (0, _milaboratories_ts_helpers.emptyDir)(resolvedSpillPath);
 	const logger = (level, message) => params.logger[level](message);
 	const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
-	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
+	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort }, {
+		cachePath: node_path.default.resolve(params.cachePath),
+		...params.cacheOps
+	});
 	const resolveDataInfo = (spec, data) => {
 		if ((0, _milaboratories_pl_tree.isPlTreeNodeAccessor)(data)) return require_data.parseDataInfoResource(data) ?? (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, []);
 		return (0, _platforma_sdk_model.isDataInfo)(data) ? data.type === "ParquetPartitioned" ? (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.traverseParquetChunkResource(a)) : (0, _platforma_sdk_model.mapDataInfo)(data, (a) => require_data.makeLocalBlobRef(a)) : (0, _milaboratories_pf_driver.makeJsonDataInfo)(spec, data);

package/dist/pool/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    // TODO: private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    // TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return null; // TODO: this.cache.getMetrics()\n  }\n\n  public async resetCache(): Promise<void> {\n    // TODO: this.cache.reset()\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    // TODO: await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QAEA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAInB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAGjE,MAAM,UAAUC,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO;;CAGT,MAAa,aAA4B;CAIzC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AAExB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    });\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAMC,gCAAAA,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAUA,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAWA,UAAAA,QAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/dist/pool/driver.d.ts

@@ -1,4 +1,5 @@
 import { PColumnDataUniversal } from "@platforma-sdk/model";
+import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
 import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 import { DownloadDriver } from "@milaboratories/pl-drivers";
 import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
@@ -8,6 +9,12 @@ interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataU
 type PFrameDriverOps = AbstractPFrameDriverOps & {
   /** Port to run parquet HTTP server on. */parquetServerPort: number;
 };
+/**
+ * Tuning for the persistent parquet blob cache, minus its filesystem path (which
+ * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries
+ * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.
+ */
+type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, "cachePath">;
 //#endregion
-export { InternalPFrameDriver, PFrameDriverOps };
+export { InternalPFrameDriver, PFrameDriverOps, ParquetCacheOps };
 //# sourceMappingURL=driver.d.ts.map

package/dist/pool/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;UAuSiB,oBAAA,SAA6B,4BAAA,CAC5C,oBAAA,CAAqB,kBAAA;AAAA,KAGX,eAAA,GAAkB,uBAAA;EAJQ,0CAMpC,iBAAA;AAAA"}
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;;UA4SiB,oBAAA,SAA6B,4BAAA,CAC5C,oBAAA,CAAqB,kBAAA;AAAA,KAGX,eAAA,GAAkB,uBAAA;EAJQ,0CAMpC,iBAAA;AAAA;;;;;;KAaU,eAAA,GAAkB,IAAA,CAAK,cAAA,CAAe,WAAA"}

package/dist/pool/driver.js

@@ -43,7 +43,7 @@ var LocalBlobProviderImpl = class extends RefCountPoolBase {
 			},
 			resolveBlobContent: async (blobId) => {
 				const blob = await this.getByKey(blobId).awaitStableValue(signal);
-				return await this.blobDriver.getContent(blob.handle, { signal });
+				return await this.blobDriver.getContentDirect(blob.handle, { signal });
 			}
 		};
 	}
@@ -134,23 +134,29 @@ var BlobStore = class extends PFrameInternal.BaseObjectStore {
 	}
 };
 var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
-	constructor(pool, server) {
+	constructor(pool, server, cache) {
 		this.pool = pool;
 		this.server = server;
+		this.cache = cache;
 	}
-	static async init(blobDriver, logger, serverOptions) {
+	static async init(blobDriver, logger, serverOptions, cacheConfig) {
 		const pool = new RemoteBlobPool(blobDriver, logger);
 		const store = new BlobStore({
 			remoteBlobProvider: pool,
 			logger
 		});
-		const handler = HttpHelpers.createRequestHandler({ store });
+		const cachingStore = await HttpHelpers.createCachingObjectStore({
+			upstream: store,
+			config: cacheConfig,
+			logger
+		});
+		const handler = HttpHelpers.createRequestHandler({ store: cachingStore });
 		const server = await HttpHelpers.createHttpServer({
 			...serverOptions,
 			handler
 		});
 		logger("info", `PFrames HTTP server started on ${server.info.url}`);
-		return new RemoteBlobProviderImpl(pool, server);
+		return new RemoteBlobProviderImpl(pool, server, cachingStore);
 	}
 	acquire(params) {
 		return this.pool.acquire(params);
@@ -159,11 +165,14 @@ var RemoteBlobProviderImpl = class RemoteBlobProviderImpl {
 		return this.server.info;
 	}
 	async getCacheMetrics() {
-		return null;
+		return this.cache.getMetrics();
+	}
+	async resetCache() {
+		await this.cache.reset();
 	}
-	async resetCache() {}
 	async [Symbol.asyncDispose]() {
 		await this.server.stop();
+		await this.cache[Symbol.asyncDispose]();
 		await this.pool[Symbol.asyncDispose]();
 	}
 };
@@ -176,7 +185,10 @@ async function createPFrameDriver(params) {
 	await emptyDir(resolvedSpillPath);
 	const logger = (level, message) => params.logger[level](message);
 	const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
-	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort });
+	const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, { port: params.options.parquetServerPort }, {
+		cachePath: path.resolve(params.cachePath),
+		...params.cacheOps
+	});
 	const resolveDataInfo = (spec, data) => {
 		if (isPlTreeNodeAccessor(data)) return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);
 		return isDataInfo(data) ? data.type === "ParquetPartitioned" ? mapDataInfo(data, (a) => traverseParquetChunkResource(a)) : mapDataInfo(data, (a) => makeLocalBlobRef(a)) : makeJsonDataInfo(spec, data);

package/dist/pool/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContent(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    // TODO: private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    // TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });\n\n    const handler = HttpHelpers.createRequestHandler({ store });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return null; // TODO: this.cache.getMetrics()\n  }\n\n  public async resetCache(): Promise<void> {\n    // TODO: this.cache.reset()\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    // TODO: await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  options: PFrameDriverOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {\n    port: params.options.parquetServerPort,\n  });\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,WAAW,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEnE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QAEA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAInB,aAAoB,KAClB,YACA,QACA,eACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EAGjE,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,CAAC;EAC3D,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,OAAO;;CAGjD,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO;;CAGT,MAAa,aAA4B;CAIzC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AAExB,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AAED,eAAsB,mBAAmB,QAKP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KAAK,OAAO,YAAY,QAAQ,EACtF,MAAM,OAAO,QAAQ,mBACtB,CAAC;CAEF,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
+{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n  mapDataInfo,\n  isDataInfo,\n  ensureError,\n  PFrameDriverError,\n  isAbortError,\n  type LocalBlobHandleAndSize,\n  type RemoteBlobHandleAndSize,\n  type RemoteBlobHandle,\n  type ContentHandler,\n  type PColumnSpec,\n  type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n  makeJsonDataInfo,\n  AbstractPFrameDriver,\n  AbstractPFrameDriverOpsDefaults,\n  type AbstractInternalPFrameDriver,\n  type AbstractPFrameDriverOps,\n  type LocalBlobProvider,\n  type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n  BlobResourceRef,\n  makeLocalBlobRef,\n  parseDataInfoResource,\n  traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n  const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n  return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n  extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n  implements LocalBlobProvider<BlobResourceRef>\n{\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): LocalBlob {\n    return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public makeDataSource(\n    signal: AbortSignal,\n  ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n    return {\n      preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n        try {\n          await Promise.all(\n            blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n          );\n        } catch (err: unknown) {\n          if (!isAbortError(err)) throw err;\n        }\n      },\n      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n        const computable = this.getByKey(blobId);\n        const blob = await computable.awaitStableValue(signal);\n        return await this.blobDriver.getContentDirect(blob.handle, { signal });\n      },\n    };\n  }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n  BlobResourceRef,\n  PFrameInternal.PFrameBlobId,\n  RemoteBlob\n> {\n  constructor(\n    private readonly blobDriver: DownloadDriver,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n    return makeBlobId(params);\n  }\n\n  protected createNewResource(\n    params: BlobResourceRef,\n    _key: PFrameInternal.PFrameBlobId,\n  ): RemoteBlob {\n    if (params.onDemandSnapshot === undefined) {\n      throw new PFrameDriverError(\n        `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n          `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n      );\n    }\n    return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n  }\n\n  public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n    const resource = super.tryGetByKey(blobId);\n    if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n    return resource;\n  }\n\n  public async withContent<T>(\n    handle: RemoteBlobHandle,\n    options: {\n      range: PFrameInternal.FileRange;\n      signal: AbortSignal;\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    return await this.blobDriver.withContent(handle, {\n      range: {\n        from: options.range.start,\n        to: options.range.end + 1,\n      },\n      signal: options.signal,\n      handler: options.handler,\n    });\n  }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n  remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n  private readonly remoteBlobProvider: RemoteBlobPool;\n\n  constructor(options: BlobStoreOptions) {\n    super(options);\n    this.remoteBlobProvider = options.remoteBlobProvider;\n  }\n\n  public override async request(\n    filename: PFrameInternal.ParquetFileName,\n    params: {\n      method: PFrameInternal.HttpMethod;\n      range?: PFrameInternal.HttpRange;\n      signal: AbortSignal;\n      callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n    },\n  ): Promise<void> {\n    const blobId = filename.slice(\n      0,\n      -PFrameInternal.ParquetExtension.length,\n    ) as PFrameInternal.PFrameBlobId;\n    const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n      try {\n        await params.callback(response);\n      } catch (error: unknown) {\n        this.logger(\n          \"warn\",\n          `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n        );\n      }\n    };\n\n    try {\n      const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n      if (!computable) return await respond({ type: \"NotFound\" });\n\n      let blob: RemoteBlobHandleAndSize;\n      try {\n        blob = await computable.getValue();\n      } catch (error: unknown) {\n        this.logger(\n          \"error\",\n          `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n        );\n        return await respond({ type: \"InternalError\" });\n      }\n      params.signal.throwIfAborted();\n\n      const translatedRange = this.translate(blob.size, params.range);\n      if (!translatedRange) {\n        return await respond({\n          type: \"RangeNotSatisfiable\",\n          size: blob.size,\n        });\n      }\n\n      if (params.method === \"HEAD\") {\n        return await respond({\n          type: \"Ok\",\n          size: blob.size,\n          range: translatedRange,\n        });\n      }\n\n      this.logger(\n        \"info\",\n        `PFrames blob store requesting content for ${blobId}, ` +\n          `range [${translatedRange.start}..=${translatedRange.end}]`,\n      );\n      return await this.remoteBlobProvider.withContent(blob.handle, {\n        range: translatedRange,\n        signal: params.signal,\n        handler: async (data) => {\n          return await respond({\n            type: \"Ok\",\n            size: blob.size,\n            range: translatedRange,\n            data: Readable.fromWeb(data),\n          });\n        },\n      });\n    } catch (error: unknown) {\n      if (!isAbortError(error)) {\n        if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n          this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n        } else {\n          this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n        }\n      }\n      return await respond({ type: \"InternalError\" });\n    }\n  }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n  constructor(\n    private readonly pool: RemoteBlobPool,\n    private readonly server: PFrameInternal.HttpServer,\n    private readonly cache: PFrameInternal.CachingObjectStore,\n  ) {}\n\n  public static async init(\n    blobDriver: DownloadDriver,\n    logger: PFrameInternal.Logger,\n    serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n    cacheConfig: PFrameInternal.CacheConfig,\n  ): Promise<RemoteBlobProviderImpl> {\n    const pool = new RemoteBlobPool(blobDriver, logger);\n    const store = new BlobStore({ remoteBlobProvider: pool, logger });\n    const cachingStore = await HttpHelpers.createCachingObjectStore({\n      upstream: store,\n      config: cacheConfig,\n      logger,\n    });\n\n    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n    logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n    return new RemoteBlobProviderImpl(pool, server, cachingStore);\n  }\n\n  public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n    return this.pool.acquire(params);\n  }\n\n  public httpServerInfo(): PFrameInternal.HttpServerInfo {\n    return this.server.info;\n  }\n\n  public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n    return this.cache.getMetrics();\n  }\n\n  public async resetCache(): Promise<void> {\n    await this.cache.reset();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.server.stop();\n    await this.cache[Symbol.asyncDispose]();\n    await this.pool[Symbol.asyncDispose]();\n  }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n  PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n  /** Port to run parquet HTTP server on. */\n  parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n  ...AbstractPFrameDriverOpsDefaults,\n  parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n  blobDriver: DownloadDriver;\n  logger: MiLogger;\n  spillPath: string;\n  cachePath: string;\n  options: PFrameDriverOps;\n  cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n  const resolvedSpillPath = path.resolve(params.spillPath);\n  await emptyDir(resolvedSpillPath);\n\n  const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n  const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n  const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n    params.blobDriver,\n    logger,\n    { port: params.options.parquetServerPort },\n    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n  );\n\n  const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n    if (isPlTreeNodeAccessor(data)) {\n      return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n    }\n\n    return isDataInfo(data)\n      ? data.type === \"ParquetPartitioned\"\n        ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n        : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n      : makeJsonDataInfo(spec, data);\n  };\n\n  return new AbstractPFrameDriver({\n    logger,\n    localBlobProvider,\n    remoteBlobProvider,\n    spillPath: resolvedSpillPath,\n    options: params.options,\n    resolveDataInfo,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAM,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAW,KAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}

package/dist/pool/index.d.ts

@@ -1 +1 @@
-import { InternalPFrameDriver, PFrameDriverOps } from "./driver.js";
+import { InternalPFrameDriver, PFrameDriverOps, ParquetCacheOps } from "./driver.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-middle-layer",
-  "version": "1.64.28",
+  "version": "1.64.29",
   "description": "Pl Middle Layer",
   "keywords": [],
   "license": "UNLICENSED",
@@ -19,8 +19,8 @@
     }
   },
   "dependencies": {
-    "@milaboratories/pframes-rs-node": "1.1.46",
-    "@milaboratories/pframes-rs-wasm": "1.1.46",
+    "@milaboratories/pframes-rs-node": "1.1.50",
+    "@milaboratories/pframes-rs-wasm": "1.1.50",
     "canonicalize": "~2.1.0",
     "denque": "^2.1.0",
     "es-toolkit": "^1.39.10",
@@ -32,22 +32,22 @@
     "yaml": "^2.8.0",
     "zod": "~3.25.76",
     "@milaboratories/computable": "2.9.5",
+    "@milaboratories/pf-spec-driver": "1.4.13",
     "@milaboratories/helpers": "1.14.2",
+    "@milaboratories/pf-driver": "1.7.11",
     "@milaboratories/pl-client": "3.11.4",
-    "@milaboratories/pf-spec-driver": "1.4.12",
-    "@milaboratories/pf-driver": "1.7.10",
-    "@milaboratories/pl-drivers": "1.15.6",
-    "@milaboratories/pl-http": "1.2.4",
-    "@milaboratories/pl-errors": "1.4.22",
     "@milaboratories/pl-deployments": "3.0.7",
-    "@milaboratories/pl-model-backend": "1.4.7",
-    "@milaboratories/ts-helpers": "1.8.3",
+    "@milaboratories/pl-drivers": "1.16.0",
+    "@milaboratories/pl-errors": "1.4.22",
     "@milaboratories/pl-model-common": "1.46.1",
+    "@milaboratories/pl-http": "1.2.4",
     "@milaboratories/pl-model-middle-layer": "1.30.6",
+    "@milaboratories/pl-model-backend": "1.4.7",
     "@milaboratories/pl-tree": "1.12.12",
+    "@milaboratories/ts-helpers": "1.8.3",
     "@milaboratories/resolve-helper": "1.1.3",
     "@platforma-sdk/block-tools": "2.10.19",
-    "@platforma-sdk/workflow-tengo": "6.6.1",
+    "@platforma-sdk/workflow-tengo": "6.6.2",
     "@platforma-sdk/model": "1.79.6"
   },
   "devDependencies": {
@@ -57,8 +57,8 @@
     "typescript": "~5.9.3",
     "vitest": "^4.1.3",
     "@milaboratories/build-configs": "2.0.0",
-    "@milaboratories/ts-builder": "1.5.2",
-    "@milaboratories/ts-configs": "1.2.3"
+    "@milaboratories/ts-configs": "1.2.3",
+    "@milaboratories/ts-builder": "1.5.2"
   },
   "engines": {
     "node": ">=22.19.0"

package/src/middle_layer/driver_kit.ts

@@ -118,7 +118,9 @@ export async function initDriverKit(
     blobDriver,
     logger: ops.logger,
     spillPath: ops.pframesSpillPath,
+    cachePath: ops.parquetCachePath,
     options: ops.pFrameDriverOps,
+    cacheOps: ops.parquetCacheOps,
   });
 
   const frontendDownloadDriver = new DownloadUrlDriver(

package/src/middle_layer/ops.ts

@@ -12,7 +12,7 @@ import type { MiLogger } from "@milaboratories/ts-helpers";
 import { ConsoleLoggerAdapter } from "@milaboratories/ts-helpers";
 import type { LocalStorageProjection } from "@milaboratories/pl-drivers";
 import path from "node:path";
-import { PFrameDriverOpsDefaults, type PFrameDriverOps } from "../pool";
+import { PFrameDriverOpsDefaults, type PFrameDriverOps, type ParquetCacheOps } from "../pool";
 
 /** Paths part of {@link DriverKitOps}. */
 export type DriverKitOpsPaths = {
@@ -47,6 +47,13 @@ export type DriverKitOpsPaths = {
 
   /** Path to the directory where pframes will spill temporary files and store materialized views */
   readonly pframesSpillPath: string;
+
+  /**
+   * Path to the directory backing the persistent parquet blob cache. Unlike
+   * {@link pframesSpillPath} this directory is NOT emptied on startup - the cache
+   * (and its lifetime counters) is meant to survive restarts.
+   */
+  readonly parquetCachePath: string;
 };
 
 /** Options required to initialize full set of middle layer driver kit */
@@ -127,6 +134,13 @@ export type DriverKitOpsSettings = {
 
   /** Settings related to the PFrame driver */
   readonly pFrameDriverOps: PFrameDriverOps;
+
+  /**
+   * Tuning for the persistent parquet blob cache. Its directory is
+   * {@link DriverKitOpsPaths.parquetCachePath}; this carries only the
+   * size/eviction knobs.
+   */
+  readonly parquetCacheOps: ParquetCacheOps;
 };
 
 export type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;
@@ -141,6 +155,7 @@ export const DefaultDriverKitOpsSettings: Pick<
   | "uploadDriverOps"
   | "logStreamDriverOps"
   | "pFrameDriverOps"
+  | "parquetCacheOps"
 > = {
   logger: new ConsoleLoggerAdapter(),
   blobDriverOps: {
@@ -169,19 +184,29 @@ export const DefaultDriverKitOpsSettings: Pick<
     stopPollingDelay: 1000,
   },
   pFrameDriverOps: PFrameDriverOpsDefaults,
+  parquetCacheOps: {
+    maxSizeBytes: 8 * 1024 * 1024 * 1024, // 8 GB
+    admissionFraction: 0.2, // a single file may occupy at most 20% of the budget
+    maxFilesTracked: 50_000, // ghost (previously-evicted) files remembered to guide readmission
+  },
 };
 
 export function DefaultDriverKitOpsPaths(
   workDir: string,
 ): Pick<
   DriverKitOpsPaths,
-  "blobDownloadPath" | "blobDownloadRangesCachePath" | "downloadBlobToURLPath" | "pframesSpillPath"
+  | "blobDownloadPath"
+  | "blobDownloadRangesCachePath"
+  | "downloadBlobToURLPath"
+  | "pframesSpillPath"
+  | "parquetCachePath"
 > {
   return {
     blobDownloadPath: path.join(workDir, "download"),
     blobDownloadRangesCachePath: path.join(workDir, "downloadRangesCache"),
     downloadBlobToURLPath: path.join(workDir, "downloadToURL"),
     pframesSpillPath: path.join(workDir, "pframes"),
+    parquetCachePath: path.join(workDir, "parquetCache"),
   };
 }
 

package/src/pool/driver.ts

@@ -88,7 +88,7 @@ class LocalBlobProviderImpl
       resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {
         const computable = this.getByKey(blobId);
         const blob = await computable.awaitStableValue(signal);
-        return await this.blobDriver.getContent(blob.handle, { signal });
+        return await this.blobDriver.getContentDirect(blob.handle, { signal });
       },
     };
   }
@@ -251,23 +251,28 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
   constructor(
     private readonly pool: RemoteBlobPool,
     private readonly server: PFrameInternal.HttpServer,
-    // TODO: private readonly cache: PFrameInternal.CachingObjectStore,
+    private readonly cache: PFrameInternal.CachingObjectStore,
   ) {}
 
   public static async init(
     blobDriver: DownloadDriver,
     logger: PFrameInternal.Logger,
     serverOptions: Omit<PFrameInternal.HttpServerOptions, "handler">,
+    cacheConfig: PFrameInternal.CacheConfig,
   ): Promise<RemoteBlobProviderImpl> {
     const pool = new RemoteBlobPool(blobDriver, logger);
     const store = new BlobStore({ remoteBlobProvider: pool, logger });
-    // TODO: const cachingStore = HttpHelpers.createCachingObjectStore({ ... });
+    const cachingStore = await HttpHelpers.createCachingObjectStore({
+      upstream: store,
+      config: cacheConfig,
+      logger,
+    });
 
-    const handler = HttpHelpers.createRequestHandler({ store });
+    const handler = HttpHelpers.createRequestHandler({ store: cachingStore });
     const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });
     logger("info", `PFrames HTTP server started on ${server.info.url}`);
 
-    return new RemoteBlobProviderImpl(pool, server);
+    return new RemoteBlobProviderImpl(pool, server, cachingStore);
   }
 
   public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {
@@ -279,16 +284,16 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
   }
 
   public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {
-    return null; // TODO: this.cache.getMetrics()
+    return this.cache.getMetrics();
   }
 
   public async resetCache(): Promise<void> {
-    // TODO: this.cache.reset()
+    await this.cache.reset();
   }
 
   async [Symbol.asyncDispose](): Promise<void> {
     await this.server.stop();
-    // TODO: await this.cache[Symbol.asyncDispose]();
+    await this.cache[Symbol.asyncDispose]();
     await this.pool[Symbol.asyncDispose]();
   }
 }
@@ -307,20 +312,32 @@ export const PFrameDriverOpsDefaults: PFrameDriverOps = {
   parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS
 };
 
+/**
+ * Tuning for the persistent parquet blob cache, minus its filesystem path (which
+ * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries
+ * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.
+ */
+export type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, "cachePath">;
+
 export async function createPFrameDriver(params: {
   blobDriver: DownloadDriver;
   logger: MiLogger;
   spillPath: string;
+  cachePath: string;
   options: PFrameDriverOps;
+  cacheOps: ParquetCacheOps;
 }): Promise<InternalPFrameDriver> {
   const resolvedSpillPath = path.resolve(params.spillPath);
   await emptyDir(resolvedSpillPath);
 
   const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);
   const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);
-  const remoteBlobProvider = await RemoteBlobProviderImpl.init(params.blobDriver, logger, {
-    port: params.options.parquetServerPort,
-  });
+  const remoteBlobProvider = await RemoteBlobProviderImpl.init(
+    params.blobDriver,
+    logger,
+    { port: params.options.parquetServerPort },
+    { cachePath: path.resolve(params.cachePath), ...params.cacheOps },
+  );
 
   const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {
     if (isPlTreeNodeAccessor(data)) {