npm - @milaboratories/pl-drivers - Versions diffs - 1.12.8 → 1.12.9 - Mend

@milaboratories/pl-drivers 1.12.8 → 1.12.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (243) hide show

package/dist/drivers/ls.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ls.cjs","names":["fsp","parseIndexHandle","path","parseUploadHandle","createIndexImportHandle","createUploadImportHandle","createLocalStorageHandle","createRemoteStorageHandle","parseStorageHandle","createLsFilesClient","DefaultVirtualLocalStorages"],"sources":["../../src/drivers/ls.ts"],"sourcesContent":["import type { PlClient, ResourceData, ResourceId } from \"@milaboratories/pl-client\";\nimport { isNotNullResourceId } from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type {\n LocalImportFileHandle,\n LsEntry,\n OpenDialogOps,\n OpenMultipleFilesResponse,\n OpenSingleFileResponse,\n TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isImportFileHandleIndex } from \"@milaboratories/pl-model-common\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { createLsFilesClient } from \"../clients/constructors\";\nimport type { ClientLs } from \"../clients/ls_api\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport {\n createIndexImportHandle,\n createUploadImportHandle,\n parseIndexHandle,\n parseUploadHandle,\n} from \"./helpers/ls_remote_import_handle\";\nimport {\n createLocalStorageHandle,\n createRemoteStorageHandle,\n parseStorageHandle,\n} from \"./helpers/ls_storage_entry\";\nimport type { LocalStorageProjection, VirtualLocalStorageSpec } from \"./types\";\nimport { DefaultVirtualLocalStorages } from \"./virtual_storages\";\n\n/*\n Extends public and safe SDK's driver API with methods used internally in the middle\n * layer and in tests.\n /\nexport interface InternalLsDriver extends sdk.LsDriver {\n /\n Given local path, generates well-structured and signed upload handle.\n * To be used in tests and in implementation of the native file selection UI API.\n * /\n getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;\n\n listRemoteFilesWithAdditionalInfo(\n storage: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo>;\n}\n\nexport type ListRemoteFilesResultWithAdditionalInfo = {\n parent?: string;\n entries: LsEntryWithAdditionalInfo[];\n};\n\nexport type LsEntryWithAdditionalInfo = LsEntry & {\n size: number;\n};\n\nexport type OpenFileDialogCallback = (\n multipleFiles: boolean,\n ops?: OpenDialogOps,\n) => Promise<undefined \| string[]>;\n\nexport class LsDriver implements InternalLsDriver {\n private constructor(\n private readonly logger: MiLogger,\n private readonly lsClient: ClientLs,\n /* Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. /\n private readonly storageIdToResourceId: Record<string, ResourceId>,\n private readonly signer: Signer,\n /* Virtual storages by name /\n private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,\n /* Local projections by storageId /\n private readonly localProjectionsMap: Map<string, LocalStorageProjection>,\n private readonly openFileDialogCallback: OpenFileDialogCallback,\n ) {}\n\n public async getLocalFileContent(\n file: LocalImportFileHandle,\n range?: TableRange,\n ): Promise<Uint8Array> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n\n if (range) {\n const fileHandle = await fsp.open(localPath, \"r\");\n try {\n const buffer = Buffer.alloc(range.length);\n const { bytesRead } = await fileHandle.read(buffer, 0, range.length, range.offset);\n return new Uint8Array(buffer.subarray(0, bytesRead));\n } finally {\n await fileHandle.close();\n }\n }\n\n return await fsp.readFile(localPath);\n }\n\n public async getLocalFileSize(file: LocalImportFileHandle): Promise<number> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n const stat = await fsp.stat(localPath);\n return stat.size;\n }\n\n public async showOpenMultipleFilesDialog(\n ops?: OpenDialogOps,\n ): Promise<OpenMultipleFilesResponse> {\n const result = await this.openFileDialogCallback(true, ops);\n if (result === undefined) return {};\n return {\n files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),\n };\n }\n\n public async showOpenSingleFileDialog(ops?: OpenDialogOps): Promise<OpenSingleFileResponse> {\n const result = await this.openFileDialogCallback(false, ops);\n if (result === undefined) return {};\n return {\n file: await this.getLocalFileHandle(result[0]),\n };\n }\n\n /\n Resolves local handle to local file path.\n \n @param handle handle to be resolved\n * @private\n /\n private async tryResolveLocalFileHandle(handle: LocalImportFileHandle): Promise<string> {\n if (isImportFileHandleIndex(handle)) {\n const handleData = parseIndexHandle(handle);\n const localProjection = this.localProjectionsMap.get(handleData.storageId);\n if (!localProjection)\n throw new Error(`Storage ${handleData.storageId} is not mounted locally.`);\n return path.join(localProjection.localPath, handleData.path);\n } else {\n const handleData = parseUploadHandle(handle);\n // checking it is a valid local handle from out machine\n this.signer.verify(\n handleData.localPath,\n handleData.pathSignature,\n \"Failed to validate local file handle signature.\",\n );\n\n const localPath = handleData.localPath;\n\n const stat = await fsp.stat(localPath, { bigint: true });\n if (String(stat.mtimeMs / 1000n) !== handleData.modificationTime)\n throw new Error(\"File has changed since the handle was created.\");\n\n return localPath;\n }\n }\n\n public async getLocalFileHandle(\n localPath: string,\n ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {\n validateAbsolute(localPath);\n\n // Checking if local path is directly reachable by pl, because it is in one of the\n // locally mounted storages\n for (const lp of this.localProjectionsMap.values()) {\n // note: empty lp.localPath will match any address\n if (localPath.startsWith(lp.localPath)) {\n // Just in case:\n // > path.relative(\"/a/b\", \"/a/b/c\");\n // 'c'\n const pathWithinStorage =\n lp.localPath === \"\" ? localPath : path.relative(lp.localPath, localPath);\n return createIndexImportHandle(\n lp.storageId,\n pathWithinStorage,\n ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;\n }\n }\n\n // we get here if none of the local projections matched the path\n\n const stat = await fsp.stat(localPath, { bigint: true });\n return createUploadImportHandle(\n localPath,\n this.signer,\n stat.size,\n stat.mtimeMs / 1000n, // integer division\n ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;\n }\n\n public async getStorageList(): Promise<sdk.StorageEntry[]> {\n const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({\n name: s.name,\n handle: createLocalStorageHandle(s.name, s.root),\n initialFullPath: s.initialPath,\n }));\n\n const otherStorages = Object.entries(this.storageIdToResourceId!).map(\n ([storageId, resourceId]) => ({\n name: storageId,\n handle: createRemoteStorageHandle(storageId, resourceId),\n initialFullPath: \"\", // we don't have any additional information from where to start browsing remote storages\n isInitialPathHome: false,\n }),\n );\n\n // root must be a storage so we can index any file,\n // but for UI it's enough\n // to have local virtual storage on nix,\n // and local_disk_${drive} on Windows.\n const noRoot = otherStorages.filter((it) => it.name !== \"root\");\n\n return [...virtualStorages, ...noRoot];\n }\n\n public async listFiles(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<sdk.ListFilesResult> {\n const storageData = parseStorageHandle(storageHandle);\n\n if (storageData.isRemote) {\n const response = await this.lsClient.list(storageData, fullPath);\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n })),\n };\n }\n\n if (path.sep === \"/\" && fullPath === \"\") fullPath = \"/\";\n\n if (storageData.rootPath === \"\") {\n validateAbsolute(fullPath);\n }\n const lsRoot = path.isAbsolute(fullPath) ? fullPath : path.join(storageData.rootPath, fullPath);\n\n const entries: LsEntry[] = [];\n for await (const dirent of await fsp.opendir(lsRoot)) {\n if (!dirent.isFile() && !dirent.isDirectory()) continue;\n\n // We cannot use no dirent.fullPath no dirent.parentPath,\n // since the former is deprecated\n // and the later works differently on different versions.\n const absolutePath = path.join(lsRoot, dirent.name);\n\n entries.push({\n type: dirent.isFile() ? \"file\" : \"dir\",\n name: dirent.name,\n fullPath: absolutePath,\n handle: await this.getLocalFileHandle(absolutePath),\n });\n }\n\n return { entries };\n }\n\n public async listRemoteFilesWithAdditionalInfo(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo> {\n const storageData = parseStorageHandle(storageHandle);\n if (!storageData.isRemote) {\n throw new Error(`Storage ${storageData.name} is not remote`);\n }\n\n const response = await this.lsClient.list(storageData, fullPath);\n\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n size: Number(e.size),\n })),\n };\n }\n\n public async fileToImportHandle(_file: sdk.FileLike): Promise<sdk.ImportFileHandle> {\n throw new Error(\n \"Not implemented. This method must be implemented and intercepted in desktop preload script.\",\n );\n }\n\n public static async init(\n logger: MiLogger,\n client: PlClient,\n signer: Signer,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n openFileDialogCallback: OpenFileDialogCallback,\n virtualStorages?: VirtualLocalStorageSpec[],\n ): Promise<LsDriver> {\n const lsClient = createLsFilesClient(client, logger);\n\n if (!virtualStorages) virtualStorages = await DefaultVirtualLocalStorages();\n\n // validating inputs\n for (const vp of virtualStorages) validateAbsolute(vp.root);\n for (const lp of localProjections) if (lp.localPath !== \"\") validateAbsolute(lp.localPath);\n\n // creating indexed maps for quick access\n const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.name, s]));\n const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));\n\n // validating there is no intersection\n if (\n new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==\n virtualStoragesMap.size + localProjectionsMap.size\n )\n throw new Error(\n \"Intersection between local projection storage ids and virtual storages names detected.\",\n );\n\n return new LsDriver(\n logger,\n lsClient,\n await doGetAvailableStorageIds(client),\n signer,\n virtualStoragesMap,\n localProjectionsMap,\n openFileDialogCallback,\n );\n }\n}\n\nasync function doGetAvailableStorageIds(client: PlClient): Promise<Record<string, ResourceId>> {\n return client.withReadTx(\"GetAvailableStorageIds\", async (tx) => {\n const lsProviderId = await tx.getResourceByName(\"LSProvider\");\n const provider = await tx.getResourceData(lsProviderId, true);\n\n return providerToStorageIds(provider);\n });\n}\n\nfunction providerToStorageIds(provider: ResourceData) {\n return Object.fromEntries(\n provider.fields\n .filter((f) => f.type == \"Dynamic\" && isNotNullResourceId(f.value))\n .map((f) => [f.name.substring(\"storage/\".length), f.value as ResourceId]),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;AA+DA,IAAa,WAAb,MAAa,SAAqC;CAChD,AAAQ,YACN,AAAiB,QACjB,AAAiB,UAEjB,AAAiB,uBACjB,AAAiB,QAEjB,AAAiB,oBAEjB,AAAiB,qBACjB,AAAiB,wBACjB;EAViB;EACA;EAEA;EACA;EAEA;EAEA;EACA;;CAGnB,MAAa,oBACX,MACA,OACqB;EACrB,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,MAAI,OAAO;GACT,MAAM,aAAa,MAAMA,iBAAI,KAAK,WAAW,IAAI;AACjD,OAAI;IACF,MAAM,SAAS,OAAO,MAAM,MAAM,OAAO;IACzC,MAAM,EAAE,cAAc,MAAM,WAAW,KAAK,QAAQ,GAAG,MAAM,QAAQ,MAAM,OAAO;AAClF,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,UAAU,CAAC;aAC5C;AACR,UAAM,WAAW,OAAO;;;AAI5B,SAAO,MAAMA,iBAAI,SAAS,UAAU;;CAGtC,MAAa,iBAAiB,MAA8C;EAC1E,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,UADa,MAAMA,iBAAI,KAAK,UAAU,EAC1B;;CAGd,MAAa,4BACX,KACoC;EACpC,MAAM,SAAS,MAAM,KAAK,uBAAuB,MAAM,IAAI;AAC3D,MAAI,WAAW,OAAW,QAAO,EAAE;AACnC,SAAO,EACL,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,cAAc,KAAK,mBAAmB,UAAU,CAAC,CAAC,EACxF;;CAGH,MAAa,yBAAyB,KAAsD;EAC1F,MAAM,SAAS,MAAM,KAAK,uBAAuB,OAAO,IAAI;AAC5D,MAAI,WAAW,OAAW,QAAO,EAAE;AACnC,SAAO,EACL,MAAM,MAAM,KAAK,mBAAmB,OAAO,GAAG,EAC/C;;;;;;;;CASH,MAAc,0BAA0B,QAAgD;AACtF,mEAA4B,OAAO,EAAE;GACnC,MAAM,aAAaC,iDAAiB,OAAO;GAC3C,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,WAAW,UAAU;AAC1E,OAAI,CAAC,gBACH,OAAM,IAAI,MAAM,WAAW,WAAW,UAAU,0BAA0B;AAC5E,UAAOC,UAAK,KAAK,gBAAgB,WAAW,WAAW,KAAK;SACvD;GACL,MAAM,aAAaC,kDAAkB,OAAO;AAE5C,QAAK,OAAO,OACV,WAAW,WACX,WAAW,eACX,kDACD;GAED,MAAM,YAAY,WAAW;GAE7B,MAAM,OAAO,MAAMH,iBAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,OAAI,OAAO,KAAK,UAAU,MAAM,KAAK,WAAW,iBAC9C,OAAM,IAAI,MAAM,iDAAiD;AAEnE,UAAO;;;CAIX,MAAa,mBACX,WACuD;AACvD,oCAAiB,UAAU;AAI3B,OAAK,MAAM,MAAM,KAAK,oBAAoB,QAAQ,CAEhD,KAAI,UAAU,WAAW,GAAG,UAAU,EAAE;GAItC,MAAM,oBACJ,GAAG,cAAc,KAAK,YAAYE,UAAK,SAAS,GAAG,WAAW,UAAU;AAC1E,UAAOE,wDACL,GAAG,WACH,kBACD;;EAML,MAAM,OAAO,MAAMJ,iBAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,SAAOK,yDACL,WACA,KAAK,QACL,KAAK,MACL,KAAK,UAAU,MAChB;;CAGH,MAAa,iBAA8C;EACzD,MAAM,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,QAAQ,CAAC,CAAC,KAAK,OAAO;GACxE,MAAM,EAAE;GACR,QAAQC,kDAAyB,EAAE,MAAM,EAAE,KAAK;GAChD,iBAAiB,EAAE;GACpB,EAAE;EAeH,MAAM,SAbgB,OAAO,QAAQ,KAAK,sBAAuB,CAAC,KAC/D,CAAC,WAAW,iBAAiB;GAC5B,MAAM;GACN,QAAQC,mDAA0B,WAAW,WAAW;GACxD,iBAAiB;GACjB,mBAAmB;GACpB,EACF,CAM4B,QAAQ,OAAO,GAAG,SAAS,OAAO;AAE/D,SAAO,CAAC,GAAG,iBAAiB,GAAG,OAAO;;CAGxC,MAAa,UACX,eACA,UAC8B;EAC9B,MAAM,cAAcC,4CAAmB,cAAc;AAErD,MAAI,YAAY,SAEd,QAAO,EACL,UAFe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAE5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQJ,wDAAwB,YAAY,MAAM,EAAE,SAAS;GAC9D,EAAE,EACJ;AAGH,MAAIF,UAAK,QAAQ,OAAO,aAAa,GAAI,YAAW;AAEpD,MAAI,YAAY,aAAa,GAC3B,mCAAiB,SAAS;EAE5B,MAAM,SAASA,UAAK,WAAW,SAAS,GAAG,WAAWA,UAAK,KAAK,YAAY,UAAU,SAAS;EAE/F,MAAM,UAAqB,EAAE;AAC7B,aAAW,MAAM,UAAU,MAAMF,iBAAI,QAAQ,OAAO,EAAE;AACpD,OAAI,CAAC,OAAO,QAAQ,IAAI,CAAC,OAAO,aAAa,CAAE;GAK/C,MAAM,eAAeE,UAAK,KAAK,QAAQ,OAAO,KAAK;AAEnD,WAAQ,KAAK;IACX,MAAM,OAAO,QAAQ,GAAG,SAAS;IACjC,MAAM,OAAO;IACb,UAAU;IACV,QAAQ,MAAM,KAAK,mBAAmB,aAAa;IACpD,CAAC;;AAGJ,SAAO,EAAE,SAAS;;CAGpB,MAAa,kCACX,eACA,UACkD;EAClD,MAAM,cAAcM,4CAAmB,cAAc;AACrD,MAAI,CAAC,YAAY,SACf,OAAM,IAAI,MAAM,WAAW,YAAY,KAAK,gBAAgB;AAK9D,SAAO,EACL,UAHe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAG5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQJ,wDAAwB,YAAY,MAAM,EAAE,SAAS;GAC7D,MAAM,OAAO,EAAE,KAAK;GACrB,EAAE,EACJ;;CAGH,MAAa,mBAAmB,OAAoD;AAClF,QAAM,IAAI,MACR,8FACD;;CAGH,aAAoB,KAClB,QACA,QACA,QAEA,kBACA,wBACA,iBACmB;EACnB,MAAM,WAAWK,yCAAoB,QAAQ,OAAO;AAEpD,MAAI,CAAC,gBAAiB,mBAAkB,MAAMC,sDAA6B;AAG3E,OAAK,MAAM,MAAM,gBAAiB,mCAAiB,GAAG,KAAK;AAC3D,OAAK,MAAM,MAAM,iBAAkB,KAAI,GAAG,cAAc,GAAI,mCAAiB,GAAG,UAAU;EAG1F,MAAM,qBAAqB,IAAI,IAAI,gBAAgB,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;EAC3E,MAAM,sBAAsB,IAAI,IAAI,iBAAiB,KAAK,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;AAGlF,MACE,IAAI,IAAI,CAAC,GAAG,mBAAmB,MAAM,EAAE,GAAG,oBAAoB,MAAM,CAAC,CAAC,CAAC,SACvE,mBAAmB,OAAO,oBAAoB,KAE9C,OAAM,IAAI,MACR,yFACD;AAEH,SAAO,IAAI,SACT,QACA,UACA,MAAM,yBAAyB,OAAO,EACtC,QACA,oBACA,qBACA,uBACD;;;AAIL,eAAe,yBAAyB,QAAuD;AAC7F,QAAO,OAAO,WAAW,0BAA0B,OAAO,OAAO;EAC/D,MAAM,eAAe,MAAM,GAAG,kBAAkB,aAAa;AAG7D,SAAO,qBAFU,MAAM,GAAG,gBAAgB,cAAc,KAAK,CAExB;GACrC;;AAGJ,SAAS,qBAAqB,UAAwB;AACpD,QAAO,OAAO,YACZ,SAAS,OACN,QAAQ,MAAM,EAAE,QAAQ,gEAAiC,EAAE,MAAM,CAAC,CAClE,KAAK,MAAM,CAAC,EAAE,KAAK,UAAU,EAAkB,EAAE,EAAE,MAAoB,CAAC,CAC5E"}
1	+ {"version":3,"file":"ls.cjs","names":["fsp","parseIndexHandle","path","parseUploadHandle","createIndexImportHandle","createUploadImportHandle","createLocalStorageHandle","createRemoteStorageHandle","parseStorageHandle","createLsFilesClient","DefaultVirtualLocalStorages"],"sources":["../../src/drivers/ls.ts"],"sourcesContent":["import type { PlClient, ResourceData, ResourceId } from \"@milaboratories/pl-client\";\nimport { isNotNullResourceId } from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type {\n LocalImportFileHandle,\n LsEntry,\n OpenDialogOps,\n OpenMultipleFilesResponse,\n OpenSingleFileResponse,\n TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isImportFileHandleIndex } from \"@milaboratories/pl-model-common\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { createLsFilesClient } from \"../clients/constructors\";\nimport type { ClientLs } from \"../clients/ls_api\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport {\n createIndexImportHandle,\n createUploadImportHandle,\n parseIndexHandle,\n parseUploadHandle,\n} from \"./helpers/ls_remote_import_handle\";\nimport {\n createLocalStorageHandle,\n createRemoteStorageHandle,\n parseStorageHandle,\n} from \"./helpers/ls_storage_entry\";\nimport type { LocalStorageProjection, VirtualLocalStorageSpec } from \"./types\";\nimport { DefaultVirtualLocalStorages } from \"./virtual_storages\";\n\n/*\n Extends public and safe SDK's driver API with methods used internally in the middle\n * layer and in tests.\n /\nexport interface InternalLsDriver extends sdk.LsDriver {\n /\n Given local path, generates well-structured and signed upload handle.\n * To be used in tests and in implementation of the native file selection UI API.\n * /\n getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;\n\n listRemoteFilesWithAdditionalInfo(\n storage: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo>;\n}\n\nexport type ListRemoteFilesResultWithAdditionalInfo = {\n parent?: string;\n entries: LsEntryWithAdditionalInfo[];\n};\n\nexport type LsEntryWithAdditionalInfo = LsEntry & {\n size: number;\n};\n\nexport type OpenFileDialogCallback = (\n multipleFiles: boolean,\n ops?: OpenDialogOps,\n) => Promise<undefined \| string[]>;\n\nexport class LsDriver implements InternalLsDriver {\n private constructor(\n private readonly logger: MiLogger,\n private readonly lsClient: ClientLs,\n /* Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. /\n private readonly storageIdToResourceId: Record<string, ResourceId>,\n private readonly signer: Signer,\n /* Virtual storages by name /\n private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,\n /* Local projections by storageId /\n private readonly localProjectionsMap: Map<string, LocalStorageProjection>,\n private readonly openFileDialogCallback: OpenFileDialogCallback,\n ) {}\n\n public async getLocalFileContent(\n file: LocalImportFileHandle,\n range?: TableRange,\n ): Promise<Uint8Array> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n\n if (range) {\n const fileHandle = await fsp.open(localPath, \"r\");\n try {\n const buffer = Buffer.alloc(range.length);\n const { bytesRead } = await fileHandle.read(buffer, 0, range.length, range.offset);\n return new Uint8Array(buffer.subarray(0, bytesRead));\n } finally {\n await fileHandle.close();\n }\n }\n\n return await fsp.readFile(localPath);\n }\n\n public async getLocalFileSize(file: LocalImportFileHandle): Promise<number> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n const stat = await fsp.stat(localPath);\n return stat.size;\n }\n\n public async showOpenMultipleFilesDialog(\n ops?: OpenDialogOps,\n ): Promise<OpenMultipleFilesResponse> {\n const result = await this.openFileDialogCallback(true, ops);\n if (result === undefined) return {};\n return {\n files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),\n };\n }\n\n public async showOpenSingleFileDialog(ops?: OpenDialogOps): Promise<OpenSingleFileResponse> {\n const result = await this.openFileDialogCallback(false, ops);\n if (result === undefined) return {};\n return {\n file: await this.getLocalFileHandle(result[0]),\n };\n }\n\n /\n Resolves local handle to local file path.\n \n @param handle handle to be resolved\n * @private\n /\n private async tryResolveLocalFileHandle(handle: LocalImportFileHandle): Promise<string> {\n if (isImportFileHandleIndex(handle)) {\n const handleData = parseIndexHandle(handle);\n const localProjection = this.localProjectionsMap.get(handleData.storageId);\n if (!localProjection)\n throw new Error(`Storage ${handleData.storageId} is not mounted locally.`);\n return path.join(localProjection.localPath, handleData.path);\n } else {\n const handleData = parseUploadHandle(handle);\n // checking it is a valid local handle from out machine\n this.signer.verify(\n handleData.localPath,\n handleData.pathSignature,\n \"Failed to validate local file handle signature.\",\n );\n\n const localPath = handleData.localPath;\n\n const stat = await fsp.stat(localPath, { bigint: true });\n if (String(stat.mtimeMs / 1000n) !== handleData.modificationTime)\n throw new Error(\"File has changed since the handle was created.\");\n\n return localPath;\n }\n }\n\n public async getLocalFileHandle(\n localPath: string,\n ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {\n validateAbsolute(localPath);\n\n // Checking if local path is directly reachable by pl, because it is in one of the\n // locally mounted storages\n for (const lp of this.localProjectionsMap.values()) {\n // note: empty lp.localPath will match any address\n if (localPath.startsWith(lp.localPath)) {\n // Just in case:\n // > path.relative(\"/a/b\", \"/a/b/c\");\n // 'c'\n const pathWithinStorage =\n lp.localPath === \"\" ? localPath : path.relative(lp.localPath, localPath);\n return createIndexImportHandle(\n lp.storageId,\n pathWithinStorage,\n ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;\n }\n }\n\n // we get here if none of the local projections matched the path\n\n const stat = await fsp.stat(localPath, { bigint: true });\n return createUploadImportHandle(\n localPath,\n this.signer,\n stat.size,\n stat.mtimeMs / 1000n, // integer division\n ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;\n }\n\n public async getStorageList(): Promise<sdk.StorageEntry[]> {\n const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({\n name: s.name,\n handle: createLocalStorageHandle(s.name, s.root),\n initialFullPath: s.initialPath,\n }));\n\n const otherStorages = Object.entries(this.storageIdToResourceId!).map(\n ([storageId, resourceId]) => ({\n name: storageId,\n handle: createRemoteStorageHandle(storageId, resourceId),\n initialFullPath: \"\", // we don't have any additional information from where to start browsing remote storages\n isInitialPathHome: false,\n }),\n );\n\n // root must be a storage so we can index any file,\n // but for UI it's enough\n // to have local virtual storage on nix,\n // and local_disk_${drive} on Windows.\n const noRoot = otherStorages.filter((it) => it.name !== \"root\");\n\n return [...virtualStorages, ...noRoot];\n }\n\n public async listFiles(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<sdk.ListFilesResult> {\n const storageData = parseStorageHandle(storageHandle);\n\n if (storageData.isRemote) {\n const response = await this.lsClient.list(storageData, fullPath);\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n })),\n };\n }\n\n if (path.sep === \"/\" && fullPath === \"\") fullPath = \"/\";\n\n if (storageData.rootPath === \"\") {\n validateAbsolute(fullPath);\n }\n const lsRoot = path.isAbsolute(fullPath) ? fullPath : path.join(storageData.rootPath, fullPath);\n\n const entries: LsEntry[] = [];\n for await (const dirent of await fsp.opendir(lsRoot)) {\n if (!dirent.isFile() && !dirent.isDirectory()) continue;\n\n // We cannot use no dirent.fullPath no dirent.parentPath,\n // since the former is deprecated\n // and the later works differently on different versions.\n const absolutePath = path.join(lsRoot, dirent.name);\n\n entries.push({\n type: dirent.isFile() ? \"file\" : \"dir\",\n name: dirent.name,\n fullPath: absolutePath,\n handle: await this.getLocalFileHandle(absolutePath),\n });\n }\n\n return { entries };\n }\n\n public async listRemoteFilesWithAdditionalInfo(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo> {\n const storageData = parseStorageHandle(storageHandle);\n if (!storageData.isRemote) {\n throw new Error(`Storage ${storageData.name} is not remote`);\n }\n\n const response = await this.lsClient.list(storageData, fullPath);\n\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n size: Number(e.size),\n })),\n };\n }\n\n public async fileToImportHandle(_file: sdk.FileLike): Promise<sdk.ImportFileHandle> {\n throw new Error(\n \"Not implemented. This method must be implemented and intercepted in desktop preload script.\",\n );\n }\n\n public static async init(\n logger: MiLogger,\n client: PlClient,\n signer: Signer,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n openFileDialogCallback: OpenFileDialogCallback,\n virtualStorages?: VirtualLocalStorageSpec[],\n ): Promise<LsDriver> {\n const lsClient = createLsFilesClient(client, logger);\n\n if (!virtualStorages) virtualStorages = await DefaultVirtualLocalStorages();\n\n // validating inputs\n for (const vp of virtualStorages) validateAbsolute(vp.root);\n for (const lp of localProjections) if (lp.localPath !== \"\") validateAbsolute(lp.localPath);\n\n // creating indexed maps for quick access\n const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.name, s]));\n const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));\n\n // validating there is no intersection\n if (\n new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==\n virtualStoragesMap.size + localProjectionsMap.size\n )\n throw new Error(\n \"Intersection between local projection storage ids and virtual storages names detected.\",\n );\n\n return new LsDriver(\n logger,\n lsClient,\n await doGetAvailableStorageIds(client),\n signer,\n virtualStoragesMap,\n localProjectionsMap,\n openFileDialogCallback,\n );\n }\n}\n\nasync function doGetAvailableStorageIds(client: PlClient): Promise<Record<string, ResourceId>> {\n return client.withReadTx(\"GetAvailableStorageIds\", async (tx) => {\n const lsProviderId = await tx.getResourceByName(\"LSProvider\");\n const provider = await tx.getResourceData(lsProviderId, true);\n\n return providerToStorageIds(provider);\n });\n}\n\nfunction providerToStorageIds(provider: ResourceData) {\n return Object.fromEntries(\n provider.fields\n .filter((f) => f.type == \"Dynamic\" && isNotNullResourceId(f.value))\n .map((f) => [f.name.substring(\"storage/\".length), f.value as ResourceId]),\n );\n}\n"],"mappings":";;;;;;;;;;;;;AA+DA,IAAa,WAAb,MAAa,SAAqC;CAChD,YACE,QACA,UAEA,uBACA,QAEA,oBAEA,qBACA,wBACA;AAViB,OAAA,SAAA;AACA,OAAA,WAAA;AAEA,OAAA,wBAAA;AACA,OAAA,SAAA;AAEA,OAAA,qBAAA;AAEA,OAAA,sBAAA;AACA,OAAA,yBAAA;;CAGnB,MAAa,oBACX,MACA,OACqB;EACrB,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,MAAI,OAAO;GACT,MAAM,aAAa,MAAMA,iBAAI,KAAK,WAAW,IAAI;AACjD,OAAI;IACF,MAAM,SAAS,OAAO,MAAM,MAAM,OAAO;IACzC,MAAM,EAAE,cAAc,MAAM,WAAW,KAAK,QAAQ,GAAG,MAAM,QAAQ,MAAM,OAAO;AAClF,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,UAAU,CAAC;aAC5C;AACR,UAAM,WAAW,OAAO;;;AAI5B,SAAO,MAAMA,iBAAI,SAAS,UAAU;;CAGtC,MAAa,iBAAiB,MAA8C;EAC1E,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,UADa,MAAMA,iBAAI,KAAK,UAAU,EAC1B;;CAGd,MAAa,4BACX,KACoC;EACpC,MAAM,SAAS,MAAM,KAAK,uBAAuB,MAAM,IAAI;AAC3D,MAAI,WAAW,KAAA,EAAW,QAAO,EAAE;AACnC,SAAO,EACL,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,cAAc,KAAK,mBAAmB,UAAU,CAAC,CAAC,EACxF;;CAGH,MAAa,yBAAyB,KAAsD;EAC1F,MAAM,SAAS,MAAM,KAAK,uBAAuB,OAAO,IAAI;AAC5D,MAAI,WAAW,KAAA,EAAW,QAAO,EAAE;AACnC,SAAO,EACL,MAAM,MAAM,KAAK,mBAAmB,OAAO,GAAG,EAC/C;;;;;;;;CASH,MAAc,0BAA0B,QAAgD;AACtF,OAAA,GAAA,gCAAA,yBAA4B,OAAO,EAAE;GACnC,MAAM,aAAaC,gCAAAA,iBAAiB,OAAO;GAC3C,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,WAAW,UAAU;AAC1E,OAAI,CAAC,gBACH,OAAM,IAAI,MAAM,WAAW,WAAW,UAAU,0BAA0B;AAC5E,UAAOC,UAAK,KAAK,gBAAgB,WAAW,WAAW,KAAK;SACvD;GACL,MAAM,aAAaC,gCAAAA,kBAAkB,OAAO;AAE5C,QAAK,OAAO,OACV,WAAW,WACX,WAAW,eACX,kDACD;GAED,MAAM,YAAY,WAAW;GAE7B,MAAM,OAAO,MAAMH,iBAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,OAAI,OAAO,KAAK,UAAU,MAAM,KAAK,WAAW,iBAC9C,OAAM,IAAI,MAAM,iDAAiD;AAEnE,UAAO;;;CAIX,MAAa,mBACX,WACuD;AACvD,mBAAA,iBAAiB,UAAU;AAI3B,OAAK,MAAM,MAAM,KAAK,oBAAoB,QAAQ,CAEhD,KAAI,UAAU,WAAW,GAAG,UAAU,EAAE;GAItC,MAAM,oBACJ,GAAG,cAAc,KAAK,YAAYE,UAAK,SAAS,GAAG,WAAW,UAAU;AAC1E,UAAOE,gCAAAA,wBACL,GAAG,WACH,kBACD;;EAML,MAAM,OAAO,MAAMJ,iBAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,SAAOK,gCAAAA,yBACL,WACA,KAAK,QACL,KAAK,MACL,KAAK,UAAU,MAChB;;CAGH,MAAa,iBAA8C;EACzD,MAAM,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,QAAQ,CAAC,CAAC,KAAK,OAAO;GACxE,MAAM,EAAE;GACR,QAAQC,yBAAAA,yBAAyB,EAAE,MAAM,EAAE,KAAK;GAChD,iBAAiB,EAAE;GACpB,EAAE;EAeH,MAAM,SAbgB,OAAO,QAAQ,KAAK,sBAAuB,CAAC,KAC/D,CAAC,WAAW,iBAAiB;GAC5B,MAAM;GACN,QAAQC,yBAAAA,0BAA0B,WAAW,WAAW;GACxD,iBAAiB;GACjB,mBAAmB;GACpB,EACF,CAM4B,QAAQ,OAAO,GAAG,SAAS,OAAO;AAE/D,SAAO,CAAC,GAAG,iBAAiB,GAAG,OAAO;;CAGxC,MAAa,UACX,eACA,UAC8B;EAC9B,MAAM,cAAcC,yBAAAA,mBAAmB,cAAc;AAErD,MAAI,YAAY,SAEd,QAAO,EACL,UAFe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAE5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQJ,gCAAAA,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC9D,EAAE,EACJ;AAGH,MAAIF,UAAK,QAAQ,OAAO,aAAa,GAAI,YAAW;AAEpD,MAAI,YAAY,aAAa,GAC3B,kBAAA,iBAAiB,SAAS;EAE5B,MAAM,SAASA,UAAK,WAAW,SAAS,GAAG,WAAWA,UAAK,KAAK,YAAY,UAAU,SAAS;EAE/F,MAAM,UAAqB,EAAE;AAC7B,aAAW,MAAM,UAAU,MAAMF,iBAAI,QAAQ,OAAO,EAAE;AACpD,OAAI,CAAC,OAAO,QAAQ,IAAI,CAAC,OAAO,aAAa,CAAE;GAK/C,MAAM,eAAeE,UAAK,KAAK,QAAQ,OAAO,KAAK;AAEnD,WAAQ,KAAK;IACX,MAAM,OAAO,QAAQ,GAAG,SAAS;IACjC,MAAM,OAAO;IACb,UAAU;IACV,QAAQ,MAAM,KAAK,mBAAmB,aAAa;IACpD,CAAC;;AAGJ,SAAO,EAAE,SAAS;;CAGpB,MAAa,kCACX,eACA,UACkD;EAClD,MAAM,cAAcM,yBAAAA,mBAAmB,cAAc;AACrD,MAAI,CAAC,YAAY,SACf,OAAM,IAAI,MAAM,WAAW,YAAY,KAAK,gBAAgB;AAK9D,SAAO,EACL,UAHe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAG5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQJ,gCAAAA,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC7D,MAAM,OAAO,EAAE,KAAK;GACrB,EAAE,EACJ;;CAGH,MAAa,mBAAmB,OAAoD;AAClF,QAAM,IAAI,MACR,8FACD;;CAGH,aAAoB,KAClB,QACA,QACA,QAEA,kBACA,wBACA,iBACmB;EACnB,MAAM,WAAWK,qBAAAA,oBAAoB,QAAQ,OAAO;AAEpD,MAAI,CAAC,gBAAiB,mBAAkB,MAAMC,yBAAAA,6BAA6B;AAG3E,OAAK,MAAM,MAAM,gBAAiB,kBAAA,iBAAiB,GAAG,KAAK;AAC3D,OAAK,MAAM,MAAM,iBAAkB,KAAI,GAAG,cAAc,GAAI,kBAAA,iBAAiB,GAAG,UAAU;EAG1F,MAAM,qBAAqB,IAAI,IAAI,gBAAgB,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;EAC3E,MAAM,sBAAsB,IAAI,IAAI,iBAAiB,KAAK,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;AAGlF,MACE,IAAI,IAAI,CAAC,GAAG,mBAAmB,MAAM,EAAE,GAAG,oBAAoB,MAAM,CAAC,CAAC,CAAC,SACvE,mBAAmB,OAAO,oBAAoB,KAE9C,OAAM,IAAI,MACR,yFACD;AAEH,SAAO,IAAI,SACT,QACA,UACA,MAAM,yBAAyB,OAAO,EACtC,QACA,oBACA,qBACA,uBACD;;;AAIL,eAAe,yBAAyB,QAAuD;AAC7F,QAAO,OAAO,WAAW,0BAA0B,OAAO,OAAO;EAC/D,MAAM,eAAe,MAAM,GAAG,kBAAkB,aAAa;AAG7D,SAAO,qBAFU,MAAM,GAAG,gBAAgB,cAAc,KAAK,CAExB;GACrC;;AAGJ,SAAS,qBAAqB,UAAwB;AACpD,QAAO,OAAO,YACZ,SAAS,OACN,QAAQ,MAAM,EAAE,QAAQ,cAAA,GAAA,0BAAA,qBAAiC,EAAE,MAAM,CAAC,CAClE,KAAK,MAAM,CAAC,EAAE,KAAK,UAAU,EAAkB,EAAE,EAAE,MAAoB,CAAC,CAC5E"}

package/dist/drivers/ls.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ls.d.ts","names":[],"sources":["../../src/drivers/ls.ts"],"mappings":";;;;;;;;;AAoCA;;UAAiB,gBAAA,SAAyB,GAAA,CAAI,QAAA;EAKG;;;;EAA/C,kBAAA,CAAmB,SAAA,WAAoB,OAAA,CAAQ,GAAA,CAAI,qBAAA;EAEnD,iCAAA,CACE,OAAA,EAAS,GAAA,CAAI,aAAA,EACb,QAAA,WACC,OAAA,CAAQ,uCAAA;AAAA;AAAA,KAGD,uCAAA;EACV,MAAA;EACA,OAAA,EAAS,yBAAA;AAAA;AAAA,KAGC,yBAAA,GAA4B,OAAA;EACtC,IAAA;AAAA;AAAA,KAGU,sBAAA,IACV,aAAA,WACA,GAAA,GAAM,aAAA,KACH,OAAA;AAAA,cAEQ,QAAA,YAAoB,gBAAA;EAAA,iBAEZ,MAAA;EAAA,iBACA,QAAA;EAtBjB;EAAA,iBAwBiB,qBAAA;EAAA,iBACA,MAAA;EAvBR;EAAA,iBAyBQ,kBAAA;EAzB+B;EAAA,iBA2B/B,mBAAA;EAAA,iBACA,sBAAA;EAAA,QAVZ,WAAA,CAAA;EAaM,mBAAA,CACX,IAAA,EAAM,qBAAA,EACN,KAAA,GAAQ,UAAA,GACP,OAAA,CAAQ,UAAA;EAiBE,gBAAA,CAAiB,IAAA,EAAM,qBAAA,GAAwB,OAAA;EAM/C,2BAAA,CACX,GAAA,GAAM,aAAA,GACL,OAAA,CAAQ,yBAAA;EAQE,wBAAA,CAAyB,GAAA,GAAM,aAAA,GAAgB,OAAA,CAAQ,sBAAA;EA9DlC;;AAGpC;;;;EAHoC,QA4EpB,yBAAA;EA0BD,kBAAA,CACX,SAAA,WACC,OAAA,CAAQ,GAAA,CAAI,gBAAA,GAAmB,qBAAA;EA+BrB,cAAA,CAAA,GAAkB,OAAA,CAAQ,GAAA,CAAI,YAAA;EAyB9B,SAAA,CACX,aAAA,EAAe,GAAA,CAAI,aAAA,EACnB,QAAA,WACC,OAAA,CAAQ,GAAA,CAAI,eAAA;EA0CF,iCAAA,CACX,aAAA,EAAe,GAAA,CAAI,aAAA,EACnB,QAAA,WACC,OAAA,CAAQ,uCAAA;EAmBE,kBAAA,CAAmB,KAAA,EAAO,GAAA,CAAI,QAAA,GAAW,OAAA,CAAQ,GAAA,CAAI,gBAAA;EAAA,OAM9C,IAAA,CAClB,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAnOV;;EAqOE,gBAAA,EAAkB,sBAAA,IAClB,sBAAA,EAAwB,sBAAA,EACxB,eAAA,GAAkB,uBAAA,KACjB,OAAA,CAAQ,QAAA;AAAA"}

package/dist/drivers/ls.js CHANGED Viewed

@@ -7,7 +7,6 @@ import { isNotNullResourceId } from "@milaboratories/pl-client";
 import * as fsp from "node:fs/promises";
 import * as path$1 from "node:path";
 import { isImportFileHandleIndex } from "@milaboratories/pl-model-common";
 //#region src/drivers/ls.ts
 var LsDriver = class LsDriver {
 	constructor(logger, lsClient, storageIdToResourceId, signer, virtualStoragesMap, localProjectionsMap, openFileDialogCallback) {
@@ -149,7 +148,7 @@ async function doGetAvailableStorageIds(client) {
 function providerToStorageIds(provider) {
 	return Object.fromEntries(provider.fields.filter((f) => f.type == "Dynamic" && isNotNullResourceId(f.value)).map((f) => [f.name.substring(8), f.value]));
 }
 //#endregion
 export { LsDriver };
 //# sourceMappingURL=ls.js.map

package/dist/drivers/ls.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ls.js","names":["path"],"sources":["../../src/drivers/ls.ts"],"sourcesContent":["import type { PlClient, ResourceData, ResourceId } from \"@milaboratories/pl-client\";\nimport { isNotNullResourceId } from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type {\n LocalImportFileHandle,\n LsEntry,\n OpenDialogOps,\n OpenMultipleFilesResponse,\n OpenSingleFileResponse,\n TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isImportFileHandleIndex } from \"@milaboratories/pl-model-common\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { createLsFilesClient } from \"../clients/constructors\";\nimport type { ClientLs } from \"../clients/ls_api\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport {\n createIndexImportHandle,\n createUploadImportHandle,\n parseIndexHandle,\n parseUploadHandle,\n} from \"./helpers/ls_remote_import_handle\";\nimport {\n createLocalStorageHandle,\n createRemoteStorageHandle,\n parseStorageHandle,\n} from \"./helpers/ls_storage_entry\";\nimport type { LocalStorageProjection, VirtualLocalStorageSpec } from \"./types\";\nimport { DefaultVirtualLocalStorages } from \"./virtual_storages\";\n\n/*\n Extends public and safe SDK's driver API with methods used internally in the middle\n * layer and in tests.\n /\nexport interface InternalLsDriver extends sdk.LsDriver {\n /\n Given local path, generates well-structured and signed upload handle.\n * To be used in tests and in implementation of the native file selection UI API.\n * /\n getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;\n\n listRemoteFilesWithAdditionalInfo(\n storage: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo>;\n}\n\nexport type ListRemoteFilesResultWithAdditionalInfo = {\n parent?: string;\n entries: LsEntryWithAdditionalInfo[];\n};\n\nexport type LsEntryWithAdditionalInfo = LsEntry & {\n size: number;\n};\n\nexport type OpenFileDialogCallback = (\n multipleFiles: boolean,\n ops?: OpenDialogOps,\n) => Promise<undefined \| string[]>;\n\nexport class LsDriver implements InternalLsDriver {\n private constructor(\n private readonly logger: MiLogger,\n private readonly lsClient: ClientLs,\n /* Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. /\n private readonly storageIdToResourceId: Record<string, ResourceId>,\n private readonly signer: Signer,\n /* Virtual storages by name /\n private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,\n /* Local projections by storageId /\n private readonly localProjectionsMap: Map<string, LocalStorageProjection>,\n private readonly openFileDialogCallback: OpenFileDialogCallback,\n ) {}\n\n public async getLocalFileContent(\n file: LocalImportFileHandle,\n range?: TableRange,\n ): Promise<Uint8Array> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n\n if (range) {\n const fileHandle = await fsp.open(localPath, \"r\");\n try {\n const buffer = Buffer.alloc(range.length);\n const { bytesRead } = await fileHandle.read(buffer, 0, range.length, range.offset);\n return new Uint8Array(buffer.subarray(0, bytesRead));\n } finally {\n await fileHandle.close();\n }\n }\n\n return await fsp.readFile(localPath);\n }\n\n public async getLocalFileSize(file: LocalImportFileHandle): Promise<number> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n const stat = await fsp.stat(localPath);\n return stat.size;\n }\n\n public async showOpenMultipleFilesDialog(\n ops?: OpenDialogOps,\n ): Promise<OpenMultipleFilesResponse> {\n const result = await this.openFileDialogCallback(true, ops);\n if (result === undefined) return {};\n return {\n files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),\n };\n }\n\n public async showOpenSingleFileDialog(ops?: OpenDialogOps): Promise<OpenSingleFileResponse> {\n const result = await this.openFileDialogCallback(false, ops);\n if (result === undefined) return {};\n return {\n file: await this.getLocalFileHandle(result[0]),\n };\n }\n\n /\n Resolves local handle to local file path.\n \n @param handle handle to be resolved\n * @private\n /\n private async tryResolveLocalFileHandle(handle: LocalImportFileHandle): Promise<string> {\n if (isImportFileHandleIndex(handle)) {\n const handleData = parseIndexHandle(handle);\n const localProjection = this.localProjectionsMap.get(handleData.storageId);\n if (!localProjection)\n throw new Error(`Storage ${handleData.storageId} is not mounted locally.`);\n return path.join(localProjection.localPath, handleData.path);\n } else {\n const handleData = parseUploadHandle(handle);\n // checking it is a valid local handle from out machine\n this.signer.verify(\n handleData.localPath,\n handleData.pathSignature,\n \"Failed to validate local file handle signature.\",\n );\n\n const localPath = handleData.localPath;\n\n const stat = await fsp.stat(localPath, { bigint: true });\n if (String(stat.mtimeMs / 1000n) !== handleData.modificationTime)\n throw new Error(\"File has changed since the handle was created.\");\n\n return localPath;\n }\n }\n\n public async getLocalFileHandle(\n localPath: string,\n ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {\n validateAbsolute(localPath);\n\n // Checking if local path is directly reachable by pl, because it is in one of the\n // locally mounted storages\n for (const lp of this.localProjectionsMap.values()) {\n // note: empty lp.localPath will match any address\n if (localPath.startsWith(lp.localPath)) {\n // Just in case:\n // > path.relative(\"/a/b\", \"/a/b/c\");\n // 'c'\n const pathWithinStorage =\n lp.localPath === \"\" ? localPath : path.relative(lp.localPath, localPath);\n return createIndexImportHandle(\n lp.storageId,\n pathWithinStorage,\n ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;\n }\n }\n\n // we get here if none of the local projections matched the path\n\n const stat = await fsp.stat(localPath, { bigint: true });\n return createUploadImportHandle(\n localPath,\n this.signer,\n stat.size,\n stat.mtimeMs / 1000n, // integer division\n ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;\n }\n\n public async getStorageList(): Promise<sdk.StorageEntry[]> {\n const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({\n name: s.name,\n handle: createLocalStorageHandle(s.name, s.root),\n initialFullPath: s.initialPath,\n }));\n\n const otherStorages = Object.entries(this.storageIdToResourceId!).map(\n ([storageId, resourceId]) => ({\n name: storageId,\n handle: createRemoteStorageHandle(storageId, resourceId),\n initialFullPath: \"\", // we don't have any additional information from where to start browsing remote storages\n isInitialPathHome: false,\n }),\n );\n\n // root must be a storage so we can index any file,\n // but for UI it's enough\n // to have local virtual storage on nix,\n // and local_disk_${drive} on Windows.\n const noRoot = otherStorages.filter((it) => it.name !== \"root\");\n\n return [...virtualStorages, ...noRoot];\n }\n\n public async listFiles(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<sdk.ListFilesResult> {\n const storageData = parseStorageHandle(storageHandle);\n\n if (storageData.isRemote) {\n const response = await this.lsClient.list(storageData, fullPath);\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n })),\n };\n }\n\n if (path.sep === \"/\" && fullPath === \"\") fullPath = \"/\";\n\n if (storageData.rootPath === \"\") {\n validateAbsolute(fullPath);\n }\n const lsRoot = path.isAbsolute(fullPath) ? fullPath : path.join(storageData.rootPath, fullPath);\n\n const entries: LsEntry[] = [];\n for await (const dirent of await fsp.opendir(lsRoot)) {\n if (!dirent.isFile() && !dirent.isDirectory()) continue;\n\n // We cannot use no dirent.fullPath no dirent.parentPath,\n // since the former is deprecated\n // and the later works differently on different versions.\n const absolutePath = path.join(lsRoot, dirent.name);\n\n entries.push({\n type: dirent.isFile() ? \"file\" : \"dir\",\n name: dirent.name,\n fullPath: absolutePath,\n handle: await this.getLocalFileHandle(absolutePath),\n });\n }\n\n return { entries };\n }\n\n public async listRemoteFilesWithAdditionalInfo(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo> {\n const storageData = parseStorageHandle(storageHandle);\n if (!storageData.isRemote) {\n throw new Error(`Storage ${storageData.name} is not remote`);\n }\n\n const response = await this.lsClient.list(storageData, fullPath);\n\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n size: Number(e.size),\n })),\n };\n }\n\n public async fileToImportHandle(_file: sdk.FileLike): Promise<sdk.ImportFileHandle> {\n throw new Error(\n \"Not implemented. This method must be implemented and intercepted in desktop preload script.\",\n );\n }\n\n public static async init(\n logger: MiLogger,\n client: PlClient,\n signer: Signer,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n openFileDialogCallback: OpenFileDialogCallback,\n virtualStorages?: VirtualLocalStorageSpec[],\n ): Promise<LsDriver> {\n const lsClient = createLsFilesClient(client, logger);\n\n if (!virtualStorages) virtualStorages = await DefaultVirtualLocalStorages();\n\n // validating inputs\n for (const vp of virtualStorages) validateAbsolute(vp.root);\n for (const lp of localProjections) if (lp.localPath !== \"\") validateAbsolute(lp.localPath);\n\n // creating indexed maps for quick access\n const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.name, s]));\n const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));\n\n // validating there is no intersection\n if (\n new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==\n virtualStoragesMap.size + localProjectionsMap.size\n )\n throw new Error(\n \"Intersection between local projection storage ids and virtual storages names detected.\",\n );\n\n return new LsDriver(\n logger,\n lsClient,\n await doGetAvailableStorageIds(client),\n signer,\n virtualStoragesMap,\n localProjectionsMap,\n openFileDialogCallback,\n );\n }\n}\n\nasync function doGetAvailableStorageIds(client: PlClient): Promise<Record<string, ResourceId>> {\n return client.withReadTx(\"GetAvailableStorageIds\", async (tx) => {\n const lsProviderId = await tx.getResourceByName(\"LSProvider\");\n const provider = await tx.getResourceData(lsProviderId, true);\n\n return providerToStorageIds(provider);\n });\n}\n\nfunction providerToStorageIds(provider: ResourceData) {\n return Object.fromEntries(\n provider.fields\n .filter((f) => f.type == \"Dynamic\" && isNotNullResourceId(f.value))\n .map((f) => [f.name.substring(\"storage/\".length), f.value as ResourceId]),\n );\n}\n"],"mappings":";;;;;;;;;;;AA+DA,IAAa,WAAb,MAAa,SAAqC;CAChD,AAAQ,YACN,AAAiB,QACjB,AAAiB,UAEjB,AAAiB,uBACjB,AAAiB,QAEjB,AAAiB,oBAEjB,AAAiB,qBACjB,AAAiB,wBACjB;EAViB;EACA;EAEA;EACA;EAEA;EAEA;EACA;;CAGnB,MAAa,oBACX,MACA,OACqB;EACrB,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,MAAI,OAAO;GACT,MAAM,aAAa,MAAM,IAAI,KAAK,WAAW,IAAI;AACjD,OAAI;IACF,MAAM,SAAS,OAAO,MAAM,MAAM,OAAO;IACzC,MAAM,EAAE,cAAc,MAAM,WAAW,KAAK,QAAQ,GAAG,MAAM,QAAQ,MAAM,OAAO;AAClF,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,UAAU,CAAC;aAC5C;AACR,UAAM,WAAW,OAAO;;;AAI5B,SAAO,MAAM,IAAI,SAAS,UAAU;;CAGtC,MAAa,iBAAiB,MAA8C;EAC1E,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,UADa,MAAM,IAAI,KAAK,UAAU,EAC1B;;CAGd,MAAa,4BACX,KACoC;EACpC,MAAM,SAAS,MAAM,KAAK,uBAAuB,MAAM,IAAI;AAC3D,MAAI,WAAW,OAAW,QAAO,EAAE;AACnC,SAAO,EACL,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,cAAc,KAAK,mBAAmB,UAAU,CAAC,CAAC,EACxF;;CAGH,MAAa,yBAAyB,KAAsD;EAC1F,MAAM,SAAS,MAAM,KAAK,uBAAuB,OAAO,IAAI;AAC5D,MAAI,WAAW,OAAW,QAAO,EAAE;AACnC,SAAO,EACL,MAAM,MAAM,KAAK,mBAAmB,OAAO,GAAG,EAC/C;;;;;;;;CASH,MAAc,0BAA0B,QAAgD;AACtF,MAAI,wBAAwB,OAAO,EAAE;GACnC,MAAM,aAAa,iBAAiB,OAAO;GAC3C,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,WAAW,UAAU;AAC1E,OAAI,CAAC,gBACH,OAAM,IAAI,MAAM,WAAW,WAAW,UAAU,0BAA0B;AAC5E,UAAOA,OAAK,KAAK,gBAAgB,WAAW,WAAW,KAAK;SACvD;GACL,MAAM,aAAa,kBAAkB,OAAO;AAE5C,QAAK,OAAO,OACV,WAAW,WACX,WAAW,eACX,kDACD;GAED,MAAM,YAAY,WAAW;GAE7B,MAAM,OAAO,MAAM,IAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,OAAI,OAAO,KAAK,UAAU,MAAM,KAAK,WAAW,iBAC9C,OAAM,IAAI,MAAM,iDAAiD;AAEnE,UAAO;;;CAIX,MAAa,mBACX,WACuD;AACvD,mBAAiB,UAAU;AAI3B,OAAK,MAAM,MAAM,KAAK,oBAAoB,QAAQ,CAEhD,KAAI,UAAU,WAAW,GAAG,UAAU,EAAE;GAItC,MAAM,oBACJ,GAAG,cAAc,KAAK,YAAYA,OAAK,SAAS,GAAG,WAAW,UAAU;AAC1E,UAAO,wBACL,GAAG,WACH,kBACD;;EAML,MAAM,OAAO,MAAM,IAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,SAAO,yBACL,WACA,KAAK,QACL,KAAK,MACL,KAAK,UAAU,MAChB;;CAGH,MAAa,iBAA8C;EACzD,MAAM,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,QAAQ,CAAC,CAAC,KAAK,OAAO;GACxE,MAAM,EAAE;GACR,QAAQ,yBAAyB,EAAE,MAAM,EAAE,KAAK;GAChD,iBAAiB,EAAE;GACpB,EAAE;EAeH,MAAM,SAbgB,OAAO,QAAQ,KAAK,sBAAuB,CAAC,KAC/D,CAAC,WAAW,iBAAiB;GAC5B,MAAM;GACN,QAAQ,0BAA0B,WAAW,WAAW;GACxD,iBAAiB;GACjB,mBAAmB;GACpB,EACF,CAM4B,QAAQ,OAAO,GAAG,SAAS,OAAO;AAE/D,SAAO,CAAC,GAAG,iBAAiB,GAAG,OAAO;;CAGxC,MAAa,UACX,eACA,UAC8B;EAC9B,MAAM,cAAc,mBAAmB,cAAc;AAErD,MAAI,YAAY,SAEd,QAAO,EACL,UAFe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAE5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQ,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC9D,EAAE,EACJ;AAGH,MAAIA,OAAK,QAAQ,OAAO,aAAa,GAAI,YAAW;AAEpD,MAAI,YAAY,aAAa,GAC3B,kBAAiB,SAAS;EAE5B,MAAM,SAASA,OAAK,WAAW,SAAS,GAAG,WAAWA,OAAK,KAAK,YAAY,UAAU,SAAS;EAE/F,MAAM,UAAqB,EAAE;AAC7B,aAAW,MAAM,UAAU,MAAM,IAAI,QAAQ,OAAO,EAAE;AACpD,OAAI,CAAC,OAAO,QAAQ,IAAI,CAAC,OAAO,aAAa,CAAE;GAK/C,MAAM,eAAeA,OAAK,KAAK,QAAQ,OAAO,KAAK;AAEnD,WAAQ,KAAK;IACX,MAAM,OAAO,QAAQ,GAAG,SAAS;IACjC,MAAM,OAAO;IACb,UAAU;IACV,QAAQ,MAAM,KAAK,mBAAmB,aAAa;IACpD,CAAC;;AAGJ,SAAO,EAAE,SAAS;;CAGpB,MAAa,kCACX,eACA,UACkD;EAClD,MAAM,cAAc,mBAAmB,cAAc;AACrD,MAAI,CAAC,YAAY,SACf,OAAM,IAAI,MAAM,WAAW,YAAY,KAAK,gBAAgB;AAK9D,SAAO,EACL,UAHe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAG5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQ,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC7D,MAAM,OAAO,EAAE,KAAK;GACrB,EAAE,EACJ;;CAGH,MAAa,mBAAmB,OAAoD;AAClF,QAAM,IAAI,MACR,8FACD;;CAGH,aAAoB,KAClB,QACA,QACA,QAEA,kBACA,wBACA,iBACmB;EACnB,MAAM,WAAW,oBAAoB,QAAQ,OAAO;AAEpD,MAAI,CAAC,gBAAiB,mBAAkB,MAAM,6BAA6B;AAG3E,OAAK,MAAM,MAAM,gBAAiB,kBAAiB,GAAG,KAAK;AAC3D,OAAK,MAAM,MAAM,iBAAkB,KAAI,GAAG,cAAc,GAAI,kBAAiB,GAAG,UAAU;EAG1F,MAAM,qBAAqB,IAAI,IAAI,gBAAgB,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;EAC3E,MAAM,sBAAsB,IAAI,IAAI,iBAAiB,KAAK,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;AAGlF,MACE,IAAI,IAAI,CAAC,GAAG,mBAAmB,MAAM,EAAE,GAAG,oBAAoB,MAAM,CAAC,CAAC,CAAC,SACvE,mBAAmB,OAAO,oBAAoB,KAE9C,OAAM,IAAI,MACR,yFACD;AAEH,SAAO,IAAI,SACT,QACA,UACA,MAAM,yBAAyB,OAAO,EACtC,QACA,oBACA,qBACA,uBACD;;;AAIL,eAAe,yBAAyB,QAAuD;AAC7F,QAAO,OAAO,WAAW,0BAA0B,OAAO,OAAO;EAC/D,MAAM,eAAe,MAAM,GAAG,kBAAkB,aAAa;AAG7D,SAAO,qBAFU,MAAM,GAAG,gBAAgB,cAAc,KAAK,CAExB;GACrC;;AAGJ,SAAS,qBAAqB,UAAwB;AACpD,QAAO,OAAO,YACZ,SAAS,OACN,QAAQ,MAAM,EAAE,QAAQ,aAAa,oBAAoB,EAAE,MAAM,CAAC,CAClE,KAAK,MAAM,CAAC,EAAE,KAAK,UAAU,EAAkB,EAAE,EAAE,MAAoB,CAAC,CAC5E"}
1	+ {"version":3,"file":"ls.js","names":["path"],"sources":["../../src/drivers/ls.ts"],"sourcesContent":["import type { PlClient, ResourceData, ResourceId } from \"@milaboratories/pl-client\";\nimport { isNotNullResourceId } from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type {\n LocalImportFileHandle,\n LsEntry,\n OpenDialogOps,\n OpenMultipleFilesResponse,\n OpenSingleFileResponse,\n TableRange,\n} from \"@milaboratories/pl-model-common\";\nimport { isImportFileHandleIndex } from \"@milaboratories/pl-model-common\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { createLsFilesClient } from \"../clients/constructors\";\nimport type { ClientLs } from \"../clients/ls_api\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport {\n createIndexImportHandle,\n createUploadImportHandle,\n parseIndexHandle,\n parseUploadHandle,\n} from \"./helpers/ls_remote_import_handle\";\nimport {\n createLocalStorageHandle,\n createRemoteStorageHandle,\n parseStorageHandle,\n} from \"./helpers/ls_storage_entry\";\nimport type { LocalStorageProjection, VirtualLocalStorageSpec } from \"./types\";\nimport { DefaultVirtualLocalStorages } from \"./virtual_storages\";\n\n/*\n Extends public and safe SDK's driver API with methods used internally in the middle\n * layer and in tests.\n /\nexport interface InternalLsDriver extends sdk.LsDriver {\n /\n Given local path, generates well-structured and signed upload handle.\n * To be used in tests and in implementation of the native file selection UI API.\n * /\n getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;\n\n listRemoteFilesWithAdditionalInfo(\n storage: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo>;\n}\n\nexport type ListRemoteFilesResultWithAdditionalInfo = {\n parent?: string;\n entries: LsEntryWithAdditionalInfo[];\n};\n\nexport type LsEntryWithAdditionalInfo = LsEntry & {\n size: number;\n};\n\nexport type OpenFileDialogCallback = (\n multipleFiles: boolean,\n ops?: OpenDialogOps,\n) => Promise<undefined \| string[]>;\n\nexport class LsDriver implements InternalLsDriver {\n private constructor(\n private readonly logger: MiLogger,\n private readonly lsClient: ClientLs,\n /* Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. /\n private readonly storageIdToResourceId: Record<string, ResourceId>,\n private readonly signer: Signer,\n /* Virtual storages by name /\n private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,\n /* Local projections by storageId /\n private readonly localProjectionsMap: Map<string, LocalStorageProjection>,\n private readonly openFileDialogCallback: OpenFileDialogCallback,\n ) {}\n\n public async getLocalFileContent(\n file: LocalImportFileHandle,\n range?: TableRange,\n ): Promise<Uint8Array> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n\n if (range) {\n const fileHandle = await fsp.open(localPath, \"r\");\n try {\n const buffer = Buffer.alloc(range.length);\n const { bytesRead } = await fileHandle.read(buffer, 0, range.length, range.offset);\n return new Uint8Array(buffer.subarray(0, bytesRead));\n } finally {\n await fileHandle.close();\n }\n }\n\n return await fsp.readFile(localPath);\n }\n\n public async getLocalFileSize(file: LocalImportFileHandle): Promise<number> {\n const localPath = await this.tryResolveLocalFileHandle(file);\n const stat = await fsp.stat(localPath);\n return stat.size;\n }\n\n public async showOpenMultipleFilesDialog(\n ops?: OpenDialogOps,\n ): Promise<OpenMultipleFilesResponse> {\n const result = await this.openFileDialogCallback(true, ops);\n if (result === undefined) return {};\n return {\n files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),\n };\n }\n\n public async showOpenSingleFileDialog(ops?: OpenDialogOps): Promise<OpenSingleFileResponse> {\n const result = await this.openFileDialogCallback(false, ops);\n if (result === undefined) return {};\n return {\n file: await this.getLocalFileHandle(result[0]),\n };\n }\n\n /\n Resolves local handle to local file path.\n \n @param handle handle to be resolved\n * @private\n /\n private async tryResolveLocalFileHandle(handle: LocalImportFileHandle): Promise<string> {\n if (isImportFileHandleIndex(handle)) {\n const handleData = parseIndexHandle(handle);\n const localProjection = this.localProjectionsMap.get(handleData.storageId);\n if (!localProjection)\n throw new Error(`Storage ${handleData.storageId} is not mounted locally.`);\n return path.join(localProjection.localPath, handleData.path);\n } else {\n const handleData = parseUploadHandle(handle);\n // checking it is a valid local handle from out machine\n this.signer.verify(\n handleData.localPath,\n handleData.pathSignature,\n \"Failed to validate local file handle signature.\",\n );\n\n const localPath = handleData.localPath;\n\n const stat = await fsp.stat(localPath, { bigint: true });\n if (String(stat.mtimeMs / 1000n) !== handleData.modificationTime)\n throw new Error(\"File has changed since the handle was created.\");\n\n return localPath;\n }\n }\n\n public async getLocalFileHandle(\n localPath: string,\n ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {\n validateAbsolute(localPath);\n\n // Checking if local path is directly reachable by pl, because it is in one of the\n // locally mounted storages\n for (const lp of this.localProjectionsMap.values()) {\n // note: empty lp.localPath will match any address\n if (localPath.startsWith(lp.localPath)) {\n // Just in case:\n // > path.relative(\"/a/b\", \"/a/b/c\");\n // 'c'\n const pathWithinStorage =\n lp.localPath === \"\" ? localPath : path.relative(lp.localPath, localPath);\n return createIndexImportHandle(\n lp.storageId,\n pathWithinStorage,\n ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;\n }\n }\n\n // we get here if none of the local projections matched the path\n\n const stat = await fsp.stat(localPath, { bigint: true });\n return createUploadImportHandle(\n localPath,\n this.signer,\n stat.size,\n stat.mtimeMs / 1000n, // integer division\n ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;\n }\n\n public async getStorageList(): Promise<sdk.StorageEntry[]> {\n const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({\n name: s.name,\n handle: createLocalStorageHandle(s.name, s.root),\n initialFullPath: s.initialPath,\n }));\n\n const otherStorages = Object.entries(this.storageIdToResourceId!).map(\n ([storageId, resourceId]) => ({\n name: storageId,\n handle: createRemoteStorageHandle(storageId, resourceId),\n initialFullPath: \"\", // we don't have any additional information from where to start browsing remote storages\n isInitialPathHome: false,\n }),\n );\n\n // root must be a storage so we can index any file,\n // but for UI it's enough\n // to have local virtual storage on nix,\n // and local_disk_${drive} on Windows.\n const noRoot = otherStorages.filter((it) => it.name !== \"root\");\n\n return [...virtualStorages, ...noRoot];\n }\n\n public async listFiles(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<sdk.ListFilesResult> {\n const storageData = parseStorageHandle(storageHandle);\n\n if (storageData.isRemote) {\n const response = await this.lsClient.list(storageData, fullPath);\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n })),\n };\n }\n\n if (path.sep === \"/\" && fullPath === \"\") fullPath = \"/\";\n\n if (storageData.rootPath === \"\") {\n validateAbsolute(fullPath);\n }\n const lsRoot = path.isAbsolute(fullPath) ? fullPath : path.join(storageData.rootPath, fullPath);\n\n const entries: LsEntry[] = [];\n for await (const dirent of await fsp.opendir(lsRoot)) {\n if (!dirent.isFile() && !dirent.isDirectory()) continue;\n\n // We cannot use no dirent.fullPath no dirent.parentPath,\n // since the former is deprecated\n // and the later works differently on different versions.\n const absolutePath = path.join(lsRoot, dirent.name);\n\n entries.push({\n type: dirent.isFile() ? \"file\" : \"dir\",\n name: dirent.name,\n fullPath: absolutePath,\n handle: await this.getLocalFileHandle(absolutePath),\n });\n }\n\n return { entries };\n }\n\n public async listRemoteFilesWithAdditionalInfo(\n storageHandle: sdk.StorageHandle,\n fullPath: string,\n ): Promise<ListRemoteFilesResultWithAdditionalInfo> {\n const storageData = parseStorageHandle(storageHandle);\n if (!storageData.isRemote) {\n throw new Error(`Storage ${storageData.name} is not remote`);\n }\n\n const response = await this.lsClient.list(storageData, fullPath);\n\n return {\n entries: response.items.map((e) => ({\n type: e.isDir ? \"dir\" : \"file\",\n name: e.name,\n fullPath: e.fullName,\n handle: createIndexImportHandle(storageData.name, e.fullName),\n size: Number(e.size),\n })),\n };\n }\n\n public async fileToImportHandle(_file: sdk.FileLike): Promise<sdk.ImportFileHandle> {\n throw new Error(\n \"Not implemented. This method must be implemented and intercepted in desktop preload script.\",\n );\n }\n\n public static async init(\n logger: MiLogger,\n client: PlClient,\n signer: Signer,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n openFileDialogCallback: OpenFileDialogCallback,\n virtualStorages?: VirtualLocalStorageSpec[],\n ): Promise<LsDriver> {\n const lsClient = createLsFilesClient(client, logger);\n\n if (!virtualStorages) virtualStorages = await DefaultVirtualLocalStorages();\n\n // validating inputs\n for (const vp of virtualStorages) validateAbsolute(vp.root);\n for (const lp of localProjections) if (lp.localPath !== \"\") validateAbsolute(lp.localPath);\n\n // creating indexed maps for quick access\n const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.name, s]));\n const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));\n\n // validating there is no intersection\n if (\n new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==\n virtualStoragesMap.size + localProjectionsMap.size\n )\n throw new Error(\n \"Intersection between local projection storage ids and virtual storages names detected.\",\n );\n\n return new LsDriver(\n logger,\n lsClient,\n await doGetAvailableStorageIds(client),\n signer,\n virtualStoragesMap,\n localProjectionsMap,\n openFileDialogCallback,\n );\n }\n}\n\nasync function doGetAvailableStorageIds(client: PlClient): Promise<Record<string, ResourceId>> {\n return client.withReadTx(\"GetAvailableStorageIds\", async (tx) => {\n const lsProviderId = await tx.getResourceByName(\"LSProvider\");\n const provider = await tx.getResourceData(lsProviderId, true);\n\n return providerToStorageIds(provider);\n });\n}\n\nfunction providerToStorageIds(provider: ResourceData) {\n return Object.fromEntries(\n provider.fields\n .filter((f) => f.type == \"Dynamic\" && isNotNullResourceId(f.value))\n .map((f) => [f.name.substring(\"storage/\".length), f.value as ResourceId]),\n );\n}\n"],"mappings":";;;;;;;;;;AA+DA,IAAa,WAAb,MAAa,SAAqC;CAChD,YACE,QACA,UAEA,uBACA,QAEA,oBAEA,qBACA,wBACA;AAViB,OAAA,SAAA;AACA,OAAA,WAAA;AAEA,OAAA,wBAAA;AACA,OAAA,SAAA;AAEA,OAAA,qBAAA;AAEA,OAAA,sBAAA;AACA,OAAA,yBAAA;;CAGnB,MAAa,oBACX,MACA,OACqB;EACrB,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,MAAI,OAAO;GACT,MAAM,aAAa,MAAM,IAAI,KAAK,WAAW,IAAI;AACjD,OAAI;IACF,MAAM,SAAS,OAAO,MAAM,MAAM,OAAO;IACzC,MAAM,EAAE,cAAc,MAAM,WAAW,KAAK,QAAQ,GAAG,MAAM,QAAQ,MAAM,OAAO;AAClF,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,UAAU,CAAC;aAC5C;AACR,UAAM,WAAW,OAAO;;;AAI5B,SAAO,MAAM,IAAI,SAAS,UAAU;;CAGtC,MAAa,iBAAiB,MAA8C;EAC1E,MAAM,YAAY,MAAM,KAAK,0BAA0B,KAAK;AAE5D,UADa,MAAM,IAAI,KAAK,UAAU,EAC1B;;CAGd,MAAa,4BACX,KACoC;EACpC,MAAM,SAAS,MAAM,KAAK,uBAAuB,MAAM,IAAI;AAC3D,MAAI,WAAW,KAAA,EAAW,QAAO,EAAE;AACnC,SAAO,EACL,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,cAAc,KAAK,mBAAmB,UAAU,CAAC,CAAC,EACxF;;CAGH,MAAa,yBAAyB,KAAsD;EAC1F,MAAM,SAAS,MAAM,KAAK,uBAAuB,OAAO,IAAI;AAC5D,MAAI,WAAW,KAAA,EAAW,QAAO,EAAE;AACnC,SAAO,EACL,MAAM,MAAM,KAAK,mBAAmB,OAAO,GAAG,EAC/C;;;;;;;;CASH,MAAc,0BAA0B,QAAgD;AACtF,MAAI,wBAAwB,OAAO,EAAE;GACnC,MAAM,aAAa,iBAAiB,OAAO;GAC3C,MAAM,kBAAkB,KAAK,oBAAoB,IAAI,WAAW,UAAU;AAC1E,OAAI,CAAC,gBACH,OAAM,IAAI,MAAM,WAAW,WAAW,UAAU,0BAA0B;AAC5E,UAAOA,OAAK,KAAK,gBAAgB,WAAW,WAAW,KAAK;SACvD;GACL,MAAM,aAAa,kBAAkB,OAAO;AAE5C,QAAK,OAAO,OACV,WAAW,WACX,WAAW,eACX,kDACD;GAED,MAAM,YAAY,WAAW;GAE7B,MAAM,OAAO,MAAM,IAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,OAAI,OAAO,KAAK,UAAU,MAAM,KAAK,WAAW,iBAC9C,OAAM,IAAI,MAAM,iDAAiD;AAEnE,UAAO;;;CAIX,MAAa,mBACX,WACuD;AACvD,mBAAiB,UAAU;AAI3B,OAAK,MAAM,MAAM,KAAK,oBAAoB,QAAQ,CAEhD,KAAI,UAAU,WAAW,GAAG,UAAU,EAAE;GAItC,MAAM,oBACJ,GAAG,cAAc,KAAK,YAAYA,OAAK,SAAS,GAAG,WAAW,UAAU;AAC1E,UAAO,wBACL,GAAG,WACH,kBACD;;EAML,MAAM,OAAO,MAAM,IAAI,KAAK,WAAW,EAAE,QAAQ,MAAM,CAAC;AACxD,SAAO,yBACL,WACA,KAAK,QACL,KAAK,MACL,KAAK,UAAU,MAChB;;CAGH,MAAa,iBAA8C;EACzD,MAAM,kBAAkB,CAAC,GAAG,KAAK,mBAAmB,QAAQ,CAAC,CAAC,KAAK,OAAO;GACxE,MAAM,EAAE;GACR,QAAQ,yBAAyB,EAAE,MAAM,EAAE,KAAK;GAChD,iBAAiB,EAAE;GACpB,EAAE;EAeH,MAAM,SAbgB,OAAO,QAAQ,KAAK,sBAAuB,CAAC,KAC/D,CAAC,WAAW,iBAAiB;GAC5B,MAAM;GACN,QAAQ,0BAA0B,WAAW,WAAW;GACxD,iBAAiB;GACjB,mBAAmB;GACpB,EACF,CAM4B,QAAQ,OAAO,GAAG,SAAS,OAAO;AAE/D,SAAO,CAAC,GAAG,iBAAiB,GAAG,OAAO;;CAGxC,MAAa,UACX,eACA,UAC8B;EAC9B,MAAM,cAAc,mBAAmB,cAAc;AAErD,MAAI,YAAY,SAEd,QAAO,EACL,UAFe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAE5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQ,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC9D,EAAE,EACJ;AAGH,MAAIA,OAAK,QAAQ,OAAO,aAAa,GAAI,YAAW;AAEpD,MAAI,YAAY,aAAa,GAC3B,kBAAiB,SAAS;EAE5B,MAAM,SAASA,OAAK,WAAW,SAAS,GAAG,WAAWA,OAAK,KAAK,YAAY,UAAU,SAAS;EAE/F,MAAM,UAAqB,EAAE;AAC7B,aAAW,MAAM,UAAU,MAAM,IAAI,QAAQ,OAAO,EAAE;AACpD,OAAI,CAAC,OAAO,QAAQ,IAAI,CAAC,OAAO,aAAa,CAAE;GAK/C,MAAM,eAAeA,OAAK,KAAK,QAAQ,OAAO,KAAK;AAEnD,WAAQ,KAAK;IACX,MAAM,OAAO,QAAQ,GAAG,SAAS;IACjC,MAAM,OAAO;IACb,UAAU;IACV,QAAQ,MAAM,KAAK,mBAAmB,aAAa;IACpD,CAAC;;AAGJ,SAAO,EAAE,SAAS;;CAGpB,MAAa,kCACX,eACA,UACkD;EAClD,MAAM,cAAc,mBAAmB,cAAc;AACrD,MAAI,CAAC,YAAY,SACf,OAAM,IAAI,MAAM,WAAW,YAAY,KAAK,gBAAgB;AAK9D,SAAO,EACL,UAHe,MAAM,KAAK,SAAS,KAAK,aAAa,SAAS,EAG5C,MAAM,KAAK,OAAO;GAClC,MAAM,EAAE,QAAQ,QAAQ;GACxB,MAAM,EAAE;GACR,UAAU,EAAE;GACZ,QAAQ,wBAAwB,YAAY,MAAM,EAAE,SAAS;GAC7D,MAAM,OAAO,EAAE,KAAK;GACrB,EAAE,EACJ;;CAGH,MAAa,mBAAmB,OAAoD;AAClF,QAAM,IAAI,MACR,8FACD;;CAGH,aAAoB,KAClB,QACA,QACA,QAEA,kBACA,wBACA,iBACmB;EACnB,MAAM,WAAW,oBAAoB,QAAQ,OAAO;AAEpD,MAAI,CAAC,gBAAiB,mBAAkB,MAAM,6BAA6B;AAG3E,OAAK,MAAM,MAAM,gBAAiB,kBAAiB,GAAG,KAAK;AAC3D,OAAK,MAAM,MAAM,iBAAkB,KAAI,GAAG,cAAc,GAAI,kBAAiB,GAAG,UAAU;EAG1F,MAAM,qBAAqB,IAAI,IAAI,gBAAgB,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;EAC3E,MAAM,sBAAsB,IAAI,IAAI,iBAAiB,KAAK,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;AAGlF,MACE,IAAI,IAAI,CAAC,GAAG,mBAAmB,MAAM,EAAE,GAAG,oBAAoB,MAAM,CAAC,CAAC,CAAC,SACvE,mBAAmB,OAAO,oBAAoB,KAE9C,OAAM,IAAI,MACR,yFACD;AAEH,SAAO,IAAI,SACT,QACA,UACA,MAAM,yBAAyB,OAAO,EACtC,QACA,oBACA,qBACA,uBACD;;;AAIL,eAAe,yBAAyB,QAAuD;AAC7F,QAAO,OAAO,WAAW,0BAA0B,OAAO,OAAO;EAC/D,MAAM,eAAe,MAAM,GAAG,kBAAkB,aAAa;AAG7D,SAAO,qBAFU,MAAM,GAAG,gBAAgB,cAAc,KAAK,CAExB;GACrC;;AAGJ,SAAS,qBAAqB,UAAwB;AACpD,QAAO,OAAO,YACZ,SAAS,OACN,QAAQ,MAAM,EAAE,QAAQ,aAAa,oBAAoB,EAAE,MAAM,CAAC,CAClE,KAAK,MAAM,CAAC,EAAE,KAAK,UAAU,EAAkB,EAAE,EAAE,MAAoB,CAAC,CAC5E"}

package/dist/drivers/types.cjs CHANGED Viewed

@@ -1,7 +1,6 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
+require("../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 let zod = require("zod");
 //#region src/drivers/types.ts
 /** ResourceSnapshot that can be passed to OnDemandBlob */
 const OnDemandBlobResourceSnapshot = (0, _milaboratories_pl_tree.rsSchema)({ kv: { "ctl/file/blobInfo": zod.z.object({ sizeBytes: zod.z.coerce.number() }) } });
@@ -32,7 +31,6 @@ const UploadResourceSnapshot = (0, _milaboratories_pl_tree.rsSchema)({
 	fields: { blob: false }
 });
 const IndexResourceSnapshot = (0, _milaboratories_pl_tree.rsSchema)({ fields: { incarnation: false } });
 //#endregion
 exports.ImportFileHandleData = ImportFileHandleData;
 exports.ImportFileHandleIndexData = ImportFileHandleIndexData;
@@ -41,4 +39,5 @@ exports.IndexResourceSnapshot = IndexResourceSnapshot;
 exports.OnDemandBlobResourceSnapshot = OnDemandBlobResourceSnapshot;
 exports.UploadResourceSnapshot = UploadResourceSnapshot;
 exports.getSize = getSize;
 //# sourceMappingURL=types.cjs.map

package/dist/drivers/types.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.cjs","names":["z"],"sources":["../../src/drivers/types.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { InferSnapshot } from \"@milaboratories/pl-tree\";\nimport { rsSchema } from \"@milaboratories/pl-tree\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\n\n//\n// download\n//\n/** ResourceSnapshot that can be passed to OnDemandBlob /\nexport const OnDemandBlobResourceSnapshot = rsSchema({\n kv: {\n \"ctl/file/blobInfo\": z.object({\n sizeBytes: z.coerce.number(),\n }),\n },\n});\n\nexport type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;\n\nexport function getSize(bs: OnDemandBlobResourceSnapshot, range?: RangeBytes): number {\n const size = bs.kv[\"ctl/file/blobInfo\"].sizeBytes;\n if (range) {\n const newSize = range.to - range.from;\n if (newSize > size) {\n throw new Error(\n `getSize: range (${JSON.stringify(range)}, newSize: ${newSize}) is greater than size (${size})`,\n );\n }\n\n return newSize;\n }\n\n return size;\n}\n\n//\n// upload\n//\n\nexport const ImportFileHandleUploadData = z.object({\n /* Local file path, to take data for upload /\n localPath: z.string(),\n /* Path signature, to check this data was generated by us /\n pathSignature: z.string(),\n /* File size in bytes /\n sizeBytes: z.string(),\n /* Modification time unix timestamp in seconds /\n modificationTime: z.string(),\n});\nexport type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;\n\nexport const ImportFileHandleIndexData = z.object({\n /* Pl storage id /\n storageId: z.string(),\n /* Path inside storage /\n path: z.string(),\n});\nexport type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;\n\nexport const ImportFileHandleData = z.union([\n ImportFileHandleUploadData,\n ImportFileHandleIndexData,\n]);\nexport type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;\n\n/* Options from BlobUpload resource that have to be passed to getProgress. /\n\n/* ResourceSnapshot that can be passed to GetProgressID /\nexport const UploadResourceSnapshot = rsSchema({\n data: ImportFileHandleUploadData,\n fields: {\n blob: false,\n },\n});\n\nexport const IndexResourceSnapshot = rsSchema({\n fields: {\n incarnation: false,\n },\n});\n\nexport type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;\nexport type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;\n\nexport type ImportResourceSnapshot = UploadResourceSnapshot \| IndexResourceSnapshot;\n\n//\n// ls\n//\n\n/* Defines which storages from pl are available via local paths /\nexport type LocalStorageProjection = {\n /* Pl storage id /\n readonly storageId: string;\n\n /\n Local path, the storage is mounted at.\n \n Empty string means that this storage accepts absolute paths, and operates inside the same OS.\n * This matches the behaviour how pl interprets FS storage config.\n * /\n readonly localPath: string;\n};\n\n/* Allows to add parts of local FS as virtual storages, presenting homogeneous API to UI /\nexport type VirtualLocalStorageSpec = {\n /* Virtual storage ID, must not intersect with other storage ids /\n readonly name: string;\n\n /* Local path to \"chroot\" the API in /\n readonly root: string;\n\n /* Used as hint to UI controls to, set as initial path during browsing */\n readonly initialPath: string;\n};\n"],"mappings":"~~;;;;;;~~AASA,MAAa,~~qEAAwC~~,EACnD,IAAI,EACF,qBAAqBA,~~MAAE~~,OAAO,EAC5B,WAAWA,~~MAAE~~,OAAO,QAAQ,EAC7B,CAAC,EACH,EACF,CAAC;AAIF,SAAgB,QAAQ,IAAkC,OAA4B;CACpF,MAAM,OAAO,GAAG,GAAG,qBAAqB;AACxC,KAAI,OAAO;EACT,MAAM,UAAU,MAAM,KAAK,MAAM;AACjC,MAAI,UAAU,KACZ,OAAM,IAAI,MACR,mBAAmB,KAAK,UAAU,MAAM,CAAC,aAAa,QAAQ,0BAA0B,KAAK,GAC9F;AAGH,SAAO;;AAGT,QAAO;;AAOT,MAAa,6BAA6BA,~~MAAE~~,OAAO;CAEjD,WAAWA,~~MAAE~~,QAAQ;CAErB,eAAeA,~~MAAE~~,QAAQ;CAEzB,WAAWA,~~MAAE~~,QAAQ;CAErB,kBAAkBA,~~MAAE~~,QAAQ;CAC7B,CAAC;AAGF,MAAa,4BAA4BA,~~MAAE~~,OAAO;CAEhD,WAAWA,~~MAAE~~,QAAQ;CAErB,MAAMA,~~MAAE~~,QAAQ;CACjB,CAAC;AAGF,MAAa,uBAAuBA,~~MAAE~~,MAAM,CAC1C,4BACA,0BACD,CAAC;;;AAMF,MAAa~~,+DAAkC~~;CAC7C,MAAM;CACN,QAAQ,EACN,MAAM,OACP;CACF,CAAC;AAEF,MAAa,~~8DAAiC~~,EAC5C,QAAQ,EACN,aAAa,OACd,EACF,CAAC"}
1	+ {"version":3,"file":"types.cjs","names":["z"],"sources":["../../src/drivers/types.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { InferSnapshot } from \"@milaboratories/pl-tree\";\nimport { rsSchema } from \"@milaboratories/pl-tree\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\n\n//\n// download\n//\n/** ResourceSnapshot that can be passed to OnDemandBlob /\nexport const OnDemandBlobResourceSnapshot = rsSchema({\n kv: {\n \"ctl/file/blobInfo\": z.object({\n sizeBytes: z.coerce.number(),\n }),\n },\n});\n\nexport type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;\n\nexport function getSize(bs: OnDemandBlobResourceSnapshot, range?: RangeBytes): number {\n const size = bs.kv[\"ctl/file/blobInfo\"].sizeBytes;\n if (range) {\n const newSize = range.to - range.from;\n if (newSize > size) {\n throw new Error(\n `getSize: range (${JSON.stringify(range)}, newSize: ${newSize}) is greater than size (${size})`,\n );\n }\n\n return newSize;\n }\n\n return size;\n}\n\n//\n// upload\n//\n\nexport const ImportFileHandleUploadData = z.object({\n /* Local file path, to take data for upload /\n localPath: z.string(),\n /* Path signature, to check this data was generated by us /\n pathSignature: z.string(),\n /* File size in bytes /\n sizeBytes: z.string(),\n /* Modification time unix timestamp in seconds /\n modificationTime: z.string(),\n});\nexport type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;\n\nexport const ImportFileHandleIndexData = z.object({\n /* Pl storage id /\n storageId: z.string(),\n /* Path inside storage /\n path: z.string(),\n});\nexport type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;\n\nexport const ImportFileHandleData = z.union([\n ImportFileHandleUploadData,\n ImportFileHandleIndexData,\n]);\nexport type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;\n\n/* Options from BlobUpload resource that have to be passed to getProgress. /\n\n/* ResourceSnapshot that can be passed to GetProgressID /\nexport const UploadResourceSnapshot = rsSchema({\n data: ImportFileHandleUploadData,\n fields: {\n blob: false,\n },\n});\n\nexport const IndexResourceSnapshot = rsSchema({\n fields: {\n incarnation: false,\n },\n});\n\nexport type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;\nexport type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;\n\nexport type ImportResourceSnapshot = UploadResourceSnapshot \| IndexResourceSnapshot;\n\n//\n// ls\n//\n\n/* Defines which storages from pl are available via local paths /\nexport type LocalStorageProjection = {\n /* Pl storage id /\n readonly storageId: string;\n\n /\n Local path, the storage is mounted at.\n \n Empty string means that this storage accepts absolute paths, and operates inside the same OS.\n * This matches the behaviour how pl interprets FS storage config.\n * /\n readonly localPath: string;\n};\n\n/* Allows to add parts of local FS as virtual storages, presenting homogeneous API to UI /\nexport type VirtualLocalStorageSpec = {\n /* Virtual storage ID, must not intersect with other storage ids /\n readonly name: string;\n\n /* Local path to \"chroot\" the API in /\n readonly root: string;\n\n /* Used as hint to UI controls to, set as initial path during browsing */\n readonly initialPath: string;\n};\n"],"mappings":";;;;;AASA,MAAa,gCAAA,GAAA,wBAAA,UAAwC,EACnD,IAAI,EACF,qBAAqBA,IAAAA,EAAE,OAAO,EAC5B,WAAWA,IAAAA,EAAE,OAAO,QAAQ,EAC7B,CAAC,EACH,EACF,CAAC;AAIF,SAAgB,QAAQ,IAAkC,OAA4B;CACpF,MAAM,OAAO,GAAG,GAAG,qBAAqB;AACxC,KAAI,OAAO;EACT,MAAM,UAAU,MAAM,KAAK,MAAM;AACjC,MAAI,UAAU,KACZ,OAAM,IAAI,MACR,mBAAmB,KAAK,UAAU,MAAM,CAAC,aAAa,QAAQ,0BAA0B,KAAK,GAC9F;AAGH,SAAO;;AAGT,QAAO;;AAOT,MAAa,6BAA6BA,IAAAA,EAAE,OAAO;CAEjD,WAAWA,IAAAA,EAAE,QAAQ;CAErB,eAAeA,IAAAA,EAAE,QAAQ;CAEzB,WAAWA,IAAAA,EAAE,QAAQ;CAErB,kBAAkBA,IAAAA,EAAE,QAAQ;CAC7B,CAAC;AAGF,MAAa,4BAA4BA,IAAAA,EAAE,OAAO;CAEhD,WAAWA,IAAAA,EAAE,QAAQ;CAErB,MAAMA,IAAAA,EAAE,QAAQ;CACjB,CAAC;AAGF,MAAa,uBAAuBA,IAAAA,EAAE,MAAM,CAC1C,4BACA,0BACD,CAAC;;;AAMF,MAAa,0BAAA,GAAA,wBAAA,UAAkC;CAC7C,MAAM;CACN,QAAQ,EACN,MAAM,OACP;CACF,CAAC;AAEF,MAAa,yBAAA,GAAA,wBAAA,UAAiC,EAC5C,QAAQ,EACN,aAAa,OACd,EACF,CAAC"}

package/dist/drivers/types.d.ts CHANGED Viewed

@@ -1,11 +1,11 @@
 import { RangeBytes } from "@milaboratories/pl-model-common";
-import * as _milaboratories_pl_tree0 from "@milaboratories/pl-tree";
+import * as _$_milaboratories_pl_tree0 from "@milaboratories/pl-tree";
 import { InferSnapshot } from "@milaboratories/pl-tree";
 import { z } from "zod";
 //#region src/drivers/types.d.ts
 /** ResourceSnapshot that can be passed to OnDemandBlob */
-declare const OnDemandBlobResourceSnapshot: _milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, {
+declare const OnDemandBlobResourceSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, {
   readonly "ctl/file/blobInfo": z.ZodObject<{
     sizeBytes: z.ZodNumber;
   }, "strip", z.ZodTypeAny, {
@@ -72,7 +72,7 @@ declare const ImportFileHandleData: z.ZodUnion<[z.ZodObject<{
 type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
 /** Options from BlobUpload resource that have to be passed to getProgress. */
 /** ResourceSnapshot that can be passed to GetProgressID */
-declare const UploadResourceSnapshot: _milaboratories_pl_tree0.ResourceSnapshotSchema<z.ZodObject<{
+declare const UploadResourceSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<z.ZodObject<{
   /** Local file path, to take data for upload */localPath: z.ZodString; /** Path signature, to check this data was generated by us */
   pathSignature: z.ZodString; /** File size in bytes */
   sizeBytes: z.ZodString; /** Modification time unix timestamp in seconds */
@@ -90,7 +90,7 @@ declare const UploadResourceSnapshot: _milaboratories_pl_tree0.ResourceSnapshotS
 }>, {
   readonly blob: false;
 }, undefined>;
-declare const IndexResourceSnapshot: _milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, {
+declare const IndexResourceSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, {
   readonly incarnation: false;
 }, undefined>;
 type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;

package/dist/drivers/types.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"types.d.ts","names":[],"sources":["../../src/drivers/types.ts"],"mappings":";;;;;;;cASa,4BAAA,6BAA4B,sBAAA;EAAA;;;;;;;;KAQ7B,4BAAA,GAA+B,aAAA,QAAqB,4BAAA;AAAA,iBAEhD,OAAA,CAAQ,EAAA,EAAI,4BAAA,EAA8B,KAAA,GAAQ,UAAA;AAAA,cAoBrD,0BAAA,EAA0B,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;KAU3B,0BAAA,GAA6B,CAAA,CAAE,KAAA,QAAa,0BAAA;AAAA,cAE3C,yBAAA,EAAyB,CAAA,CAAA,SAAA;EAhCV,4CAAsC;;;;;;;;;KAsCtD,yBAAA,GAA4B,CAAA,CAAE,KAAA,QAAa,yBAAA;AAAA,cAE1C,oBAAA,EAAoB,CAAA,CAAA,QAAA,EAAA,CAAA,CAAA,SAAA;yEApBM;8BAAA;;;;;;;;;;;;;;;;;;;;;;;KAwB3B,oBAAA,GAAuB,CAAA,CAAE,KAAA,QAAa,oBAAA;;;cAKrC,sBAAA,EAAsB,0BAAA,CAAA,sBAAA,CAAA,CAAA,CAAA,SAAA;EAnBM,uEAAe;8BAA0B;0BAOhF;;;;;;;;;;;;;;;cAmBW,qBAAA,EAIX,0BAAA,CAJgC,sBAAA;EAAA,SAAA,WAAA;AAAA;AAAA,KAMtB,sBAAA,GAAyB,aAAA,QAAqB,sBAAA;AAAA,KAC9C,qBAAA,GAAwB,aAAA,QAAqB,qBAAA;AAAA,KAE7C,sBAAA,GAAyB,sBAAA,GAAyB,qBAAA;;KAOlD,sBAAA;+BAED,SAAA;EApC0B;;;;;;EAAA,SA4C1B,SAAA;AAAA;AA1CX;AAAA,KA8CY,uBAAA;+EAED,IAAA;WAGA,IAAA;WAGA,WAAA;AAAA"}

package/dist/drivers/types.js CHANGED Viewed

@@ -1,6 +1,5 @@
 import { rsSchema } from "@milaboratories/pl-tree";
 import { z } from "zod";
 //#region src/drivers/types.ts
 /** ResourceSnapshot that can be passed to OnDemandBlob */
 const OnDemandBlobResourceSnapshot = rsSchema({ kv: { "ctl/file/blobInfo": z.object({ sizeBytes: z.coerce.number() }) } });
@@ -31,7 +30,7 @@ const UploadResourceSnapshot = rsSchema({
 	fields: { blob: false }
 });
 const IndexResourceSnapshot = rsSchema({ fields: { incarnation: false } });
 //#endregion
 export { ImportFileHandleData, ImportFileHandleIndexData, ImportFileHandleUploadData, IndexResourceSnapshot, OnDemandBlobResourceSnapshot, UploadResourceSnapshot, getSize };
 //# sourceMappingURL=types.js.map

package/dist/drivers/types.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.js","names":[],"sources":["../../src/drivers/types.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { InferSnapshot } from \"@milaboratories/pl-tree\";\nimport { rsSchema } from \"@milaboratories/pl-tree\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\n\n//\n// download\n//\n/** ResourceSnapshot that can be passed to OnDemandBlob /\nexport const OnDemandBlobResourceSnapshot = rsSchema({\n kv: {\n \"ctl/file/blobInfo\": z.object({\n sizeBytes: z.coerce.number(),\n }),\n },\n});\n\nexport type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;\n\nexport function getSize(bs: OnDemandBlobResourceSnapshot, range?: RangeBytes): number {\n const size = bs.kv[\"ctl/file/blobInfo\"].sizeBytes;\n if (range) {\n const newSize = range.to - range.from;\n if (newSize > size) {\n throw new Error(\n `getSize: range (${JSON.stringify(range)}, newSize: ${newSize}) is greater than size (${size})`,\n );\n }\n\n return newSize;\n }\n\n return size;\n}\n\n//\n// upload\n//\n\nexport const ImportFileHandleUploadData = z.object({\n /* Local file path, to take data for upload /\n localPath: z.string(),\n /* Path signature, to check this data was generated by us /\n pathSignature: z.string(),\n /* File size in bytes /\n sizeBytes: z.string(),\n /* Modification time unix timestamp in seconds /\n modificationTime: z.string(),\n});\nexport type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;\n\nexport const ImportFileHandleIndexData = z.object({\n /* Pl storage id /\n storageId: z.string(),\n /* Path inside storage /\n path: z.string(),\n});\nexport type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;\n\nexport const ImportFileHandleData = z.union([\n ImportFileHandleUploadData,\n ImportFileHandleIndexData,\n]);\nexport type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;\n\n/* Options from BlobUpload resource that have to be passed to getProgress. /\n\n/* ResourceSnapshot that can be passed to GetProgressID /\nexport const UploadResourceSnapshot = rsSchema({\n data: ImportFileHandleUploadData,\n fields: {\n blob: false,\n },\n});\n\nexport const IndexResourceSnapshot = rsSchema({\n fields: {\n incarnation: false,\n },\n});\n\nexport type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;\nexport type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;\n\nexport type ImportResourceSnapshot = UploadResourceSnapshot \| IndexResourceSnapshot;\n\n//\n// ls\n//\n\n/* Defines which storages from pl are available via local paths /\nexport type LocalStorageProjection = {\n /* Pl storage id /\n readonly storageId: string;\n\n /\n Local path, the storage is mounted at.\n \n Empty string means that this storage accepts absolute paths, and operates inside the same OS.\n * This matches the behaviour how pl interprets FS storage config.\n * /\n readonly localPath: string;\n};\n\n/* Allows to add parts of local FS as virtual storages, presenting homogeneous API to UI /\nexport type VirtualLocalStorageSpec = {\n /* Virtual storage ID, must not intersect with other storage ids /\n readonly name: string;\n\n /* Local path to \"chroot\" the API in /\n readonly root: string;\n\n /* Used as hint to UI controls to, set as initial path during browsing */\n readonly initialPath: string;\n};\n"],"mappings":"~~;;;;;~~AASA,MAAa,+BAA+B,SAAS,EACnD,IAAI,EACF,qBAAqB,EAAE,OAAO,EAC5B,WAAW,EAAE,OAAO,QAAQ,EAC7B,CAAC,EACH,EACF,CAAC;AAIF,SAAgB,QAAQ,IAAkC,OAA4B;CACpF,MAAM,OAAO,GAAG,GAAG,qBAAqB;AACxC,KAAI,OAAO;EACT,MAAM,UAAU,MAAM,KAAK,MAAM;AACjC,MAAI,UAAU,KACZ,OAAM,IAAI,MACR,mBAAmB,KAAK,UAAU,MAAM,CAAC,aAAa,QAAQ,0BAA0B,KAAK,GAC9F;AAGH,SAAO;;AAGT,QAAO;;AAOT,MAAa,6BAA6B,EAAE,OAAO;CAEjD,WAAW,EAAE,QAAQ;CAErB,eAAe,EAAE,QAAQ;CAEzB,WAAW,EAAE,QAAQ;CAErB,kBAAkB,EAAE,QAAQ;CAC7B,CAAC;AAGF,MAAa,4BAA4B,EAAE,OAAO;CAEhD,WAAW,EAAE,QAAQ;CAErB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAGF,MAAa,uBAAuB,EAAE,MAAM,CAC1C,4BACA,0BACD,CAAC;;;AAMF,MAAa,yBAAyB,SAAS;CAC7C,MAAM;CACN,QAAQ,EACN,MAAM,OACP;CACF,CAAC;AAEF,MAAa,wBAAwB,SAAS,EAC5C,QAAQ,EACN,aAAa,OACd,EACF,CAAC"}
1	+ {"version":3,"file":"types.js","names":[],"sources":["../../src/drivers/types.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { InferSnapshot } from \"@milaboratories/pl-tree\";\nimport { rsSchema } from \"@milaboratories/pl-tree\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\n\n//\n// download\n//\n/** ResourceSnapshot that can be passed to OnDemandBlob /\nexport const OnDemandBlobResourceSnapshot = rsSchema({\n kv: {\n \"ctl/file/blobInfo\": z.object({\n sizeBytes: z.coerce.number(),\n }),\n },\n});\n\nexport type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;\n\nexport function getSize(bs: OnDemandBlobResourceSnapshot, range?: RangeBytes): number {\n const size = bs.kv[\"ctl/file/blobInfo\"].sizeBytes;\n if (range) {\n const newSize = range.to - range.from;\n if (newSize > size) {\n throw new Error(\n `getSize: range (${JSON.stringify(range)}, newSize: ${newSize}) is greater than size (${size})`,\n );\n }\n\n return newSize;\n }\n\n return size;\n}\n\n//\n// upload\n//\n\nexport const ImportFileHandleUploadData = z.object({\n /* Local file path, to take data for upload /\n localPath: z.string(),\n /* Path signature, to check this data was generated by us /\n pathSignature: z.string(),\n /* File size in bytes /\n sizeBytes: z.string(),\n /* Modification time unix timestamp in seconds /\n modificationTime: z.string(),\n});\nexport type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;\n\nexport const ImportFileHandleIndexData = z.object({\n /* Pl storage id /\n storageId: z.string(),\n /* Path inside storage /\n path: z.string(),\n});\nexport type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;\n\nexport const ImportFileHandleData = z.union([\n ImportFileHandleUploadData,\n ImportFileHandleIndexData,\n]);\nexport type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;\n\n/* Options from BlobUpload resource that have to be passed to getProgress. /\n\n/* ResourceSnapshot that can be passed to GetProgressID /\nexport const UploadResourceSnapshot = rsSchema({\n data: ImportFileHandleUploadData,\n fields: {\n blob: false,\n },\n});\n\nexport const IndexResourceSnapshot = rsSchema({\n fields: {\n incarnation: false,\n },\n});\n\nexport type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;\nexport type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;\n\nexport type ImportResourceSnapshot = UploadResourceSnapshot \| IndexResourceSnapshot;\n\n//\n// ls\n//\n\n/* Defines which storages from pl are available via local paths /\nexport type LocalStorageProjection = {\n /* Pl storage id /\n readonly storageId: string;\n\n /\n Local path, the storage is mounted at.\n \n Empty string means that this storage accepts absolute paths, and operates inside the same OS.\n * This matches the behaviour how pl interprets FS storage config.\n * /\n readonly localPath: string;\n};\n\n/* Allows to add parts of local FS as virtual storages, presenting homogeneous API to UI /\nexport type VirtualLocalStorageSpec = {\n /* Virtual storage ID, must not intersect with other storage ids /\n readonly name: string;\n\n /* Local path to \"chroot\" the API in /\n readonly root: string;\n\n /* Used as hint to UI controls to, set as initial path during browsing */\n readonly initialPath: string;\n};\n"],"mappings":";;;;AASA,MAAa,+BAA+B,SAAS,EACnD,IAAI,EACF,qBAAqB,EAAE,OAAO,EAC5B,WAAW,EAAE,OAAO,QAAQ,EAC7B,CAAC,EACH,EACF,CAAC;AAIF,SAAgB,QAAQ,IAAkC,OAA4B;CACpF,MAAM,OAAO,GAAG,GAAG,qBAAqB;AACxC,KAAI,OAAO;EACT,MAAM,UAAU,MAAM,KAAK,MAAM;AACjC,MAAI,UAAU,KACZ,OAAM,IAAI,MACR,mBAAmB,KAAK,UAAU,MAAM,CAAC,aAAa,QAAQ,0BAA0B,KAAK,GAC9F;AAGH,SAAO;;AAGT,QAAO;;AAOT,MAAa,6BAA6B,EAAE,OAAO;CAEjD,WAAW,EAAE,QAAQ;CAErB,eAAe,EAAE,QAAQ;CAEzB,WAAW,EAAE,QAAQ;CAErB,kBAAkB,EAAE,QAAQ;CAC7B,CAAC;AAGF,MAAa,4BAA4B,EAAE,OAAO;CAEhD,WAAW,EAAE,QAAQ;CAErB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAGF,MAAa,uBAAuB,EAAE,MAAM,CAC1C,4BACA,0BACD,CAAC;;;AAMF,MAAa,yBAAyB,SAAS;CAC7C,MAAM;CACN,QAAQ,EACN,MAAM,OACP;CACF,CAAC;AAEF,MAAa,wBAAwB,SAAS,EAC5C,QAAQ,EACN,aAAa,OACd,EACF,CAAC"}

package/dist/drivers/upload.cjs CHANGED Viewed

@@ -1,13 +1,12 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
-const require_types = require('./types.cjs');
-const require_helpers = require('./helpers/helpers.cjs');
-const require_upload_task = require('./upload_task.cjs');
+require("../_virtual/_rolldown/runtime.cjs");
+const require_types = require("./types.cjs");
+const require_helpers = require("./helpers/helpers.cjs");
+const require_upload_task = require("./upload_task.cjs");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 let node_crypto = require("node:crypto");
 let node_timers_promises = require("node:timers/promises");
 //#region src/drivers/upload.ts
 function makeBlobImportSnapshot(entryOrAccessor, ctx) {
 	const node = (0, _milaboratories_pl_tree.isPlTreeEntry)(entryOrAccessor) ? ctx.accessor(entryOrAccessor).node() : (0, _milaboratories_pl_tree.isPlTreeEntryAccessor)(entryOrAccessor) ? entryOrAccessor.node() : entryOrAccessor;
@@ -131,8 +130,8 @@ function isProgressDone(p) {
 function validateResourceType(methodName, rType) {
 	if (!rType.name.startsWith("BlobUpload") && !rType.name.startsWith("BlobIndex")) throw new require_helpers.WrongResourceTypeError(`${methodName}: wrong resource type: ${rType.name}, expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`);
 }
 //#endregion
 exports.UploadDriver = UploadDriver;
 exports.makeBlobImportSnapshot = makeBlobImportSnapshot;
 //# sourceMappingURL=upload.cjs.map

package/dist/drivers/upload.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upload.cjs","names":["UploadResourceSnapshot","IndexResourceSnapshot","TaskProcessor","PollingComputableHooks","Computable","UploadTask","nonRecoverableError","scheduler","WrongResourceTypeError"],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): ImportResourceSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n if (node.resourceType.name.startsWith(\"BlobUpload\"))\n return makeResourceSnapshot(node, UploadResourceSnapshot);\n return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n /** How much parts of a file can be multipart-uploaded to S3 at once. /\n nConcurrentPartUploads: number;\n /* How much upload/indexing statuses of blobs can the driver ask\n * from the platform gRPC at once. /\n nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/* Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. /\nexport class UploadDriver {\n private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n /* Holds a queue that upload blobs. /\n private readonly uploadQueue: TaskProcessor;\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly opts: UploadDriverOps = {\n nConcurrentPartUploads: 10,\n nConcurrentGetProgresses: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.uploadQueue = new TaskProcessor(this.logger, 1, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 20,\n maxDelay: 15000, // 15 seconds\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n /* Returns a progress id and schedules an upload task if it's necessary. /\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ): Computable<sdk.ImportProgress>;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n ): sdk.ImportProgress;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.ImportProgress> \| sdk.ImportProgress {\n if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n ? makeBlobImportSnapshot(handleResource, ctx)\n : handleResource;\n\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n return result;\n }\n\n private getProgressIdNoCtx(\n w: Watcher,\n res: ImportResourceSnapshot,\n callerId: string,\n ): sdk.ImportProgress {\n validateResourceType(\"getProgressId\", res.type);\n\n const task = this.idToProgress.get(res.id);\n\n if (task != undefined) {\n task.setDoneIfOutputSet(res);\n return task.getProgress(w, callerId);\n }\n\n const newTask = new UploadTask(\n this.logger,\n this.clientBlob,\n this.clientProgress,\n this.opts.nConcurrentPartUploads,\n this.signer,\n res,\n );\n\n this.idToProgress.set(res.id, newTask);\n\n if (newTask.shouldScheduleUpload()) {\n this.uploadQueue.push({\n fn: () => newTask.uploadBlobTask(),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n }\n\n newTask.setDoneIfOutputSet(res);\n return newTask.getProgress(w, callerId);\n }\n\n /* Decrement counters for the file and remove an uploading if counter == 0. /\n private async release(id: ResourceId, callerId: string) {\n const task = this.idToProgress.get(id);\n if (task === undefined) return;\n\n const deleted = task.decCounter(callerId);\n if (deleted) this.idToProgress.delete(id);\n }\n\n /* Must be called when the driver is closing. /\n public async releaseAll() {\n this.uploadQueue.stop();\n }\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* Stops polling loop and waits for it to finish /\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. */\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n await asyncPool(\n this.opts.nConcurrentGetProgresses,\n this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllNotDoneProgresses(): Array<UploadTask> {\n return Array.from(this.idToProgress.entries())\n .filter(([_, p]) => !isProgressDone(p.progress))\n .map(([_, p]) => p);\n }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,kDAAqB,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,sDACd,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,0DAA4B,MAAMA,qCAAuB;AAC3D,0DAA4B,MAAMC,oCAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,AAAiB,+BAA4C,IAAI,KAAK;;CAGtE,AAAiB;CACjB,AAAiB;CAEjB,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAiB,YACjB,AAAiB,gBACjB,AAAiB,OAAwB;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;EAViB;EACA;EACA;EACA;EACA;AAOjB,OAAK,cAAc,IAAIC,yCAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAIC,wDACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,OAAW,QAAOC,sCAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,mDAA8C,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,wCAAuB;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,AAAQ,mBACN,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,QAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAIC,+BAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAACC,wCAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,OAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,AAAQ,uBAA2C,EAAE;CAErD,AAAQ,oBAAoB,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,AAAQ,gBAAsB;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,AAAQ,eAAqB;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,AAAQ,cAAc;;CAEtB,AAAQ,cAAyC;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,oDACE,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAMC,+BAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc;;CAGrB,AAAQ,0BAA6C;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAIC,uCACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}
1	+ {"version":3,"file":"upload.cjs","names":["UploadResourceSnapshot","IndexResourceSnapshot","TaskProcessor","PollingComputableHooks","Computable","UploadTask","nonRecoverableError","scheduler","WrongResourceTypeError"],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): ImportResourceSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n if (node.resourceType.name.startsWith(\"BlobUpload\"))\n return makeResourceSnapshot(node, UploadResourceSnapshot);\n return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n /** How much parts of a file can be multipart-uploaded to S3 at once. /\n nConcurrentPartUploads: number;\n /* How much upload/indexing statuses of blobs can the driver ask\n * from the platform gRPC at once. /\n nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/* Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. /\nexport class UploadDriver {\n private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n /* Holds a queue that upload blobs. /\n private readonly uploadQueue: TaskProcessor;\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly opts: UploadDriverOps = {\n nConcurrentPartUploads: 10,\n nConcurrentGetProgresses: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.uploadQueue = new TaskProcessor(this.logger, 1, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 20,\n maxDelay: 15000, // 15 seconds\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n /* Returns a progress id and schedules an upload task if it's necessary. /\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ): Computable<sdk.ImportProgress>;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n ): sdk.ImportProgress;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.ImportProgress> \| sdk.ImportProgress {\n if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n ? makeBlobImportSnapshot(handleResource, ctx)\n : handleResource;\n\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n return result;\n }\n\n private getProgressIdNoCtx(\n w: Watcher,\n res: ImportResourceSnapshot,\n callerId: string,\n ): sdk.ImportProgress {\n validateResourceType(\"getProgressId\", res.type);\n\n const task = this.idToProgress.get(res.id);\n\n if (task != undefined) {\n task.setDoneIfOutputSet(res);\n return task.getProgress(w, callerId);\n }\n\n const newTask = new UploadTask(\n this.logger,\n this.clientBlob,\n this.clientProgress,\n this.opts.nConcurrentPartUploads,\n this.signer,\n res,\n );\n\n this.idToProgress.set(res.id, newTask);\n\n if (newTask.shouldScheduleUpload()) {\n this.uploadQueue.push({\n fn: () => newTask.uploadBlobTask(),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n }\n\n newTask.setDoneIfOutputSet(res);\n return newTask.getProgress(w, callerId);\n }\n\n /* Decrement counters for the file and remove an uploading if counter == 0. /\n private async release(id: ResourceId, callerId: string) {\n const task = this.idToProgress.get(id);\n if (task === undefined) return;\n\n const deleted = task.decCounter(callerId);\n if (deleted) this.idToProgress.delete(id);\n }\n\n /* Must be called when the driver is closing. /\n public async releaseAll() {\n this.uploadQueue.stop();\n }\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* Stops polling loop and waits for it to finish /\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. */\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n await asyncPool(\n this.opts.nConcurrentGetProgresses,\n this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllNotDoneProgresses(): Array<UploadTask> {\n return Array.from(this.idToProgress.entries())\n .filter(([_, p]) => !isProgressDone(p.progress))\n .map(([_, p]) => p);\n }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,QAAA,GAAA,wBAAA,eAAqB,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,IAAA,GAAA,wBAAA,uBACd,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,SAAA,GAAA,wBAAA,sBAA4B,MAAMA,cAAAA,uBAAuB;AAC3D,SAAA,GAAA,wBAAA,sBAA4B,MAAMC,cAAAA,sBAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,+BAA6D,IAAI,KAAK;;CAGtE;CACA;CAEA,YACE,QACA,QACA,YACA,gBACA,OAAyC;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;AAViB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,iBAAA;AACA,OAAA,OAAA;AAOjB,OAAK,cAAc,IAAIC,2BAAAA,cAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAIC,2BAAAA,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,SAAA,GAAA,wBAAA,eAA8C,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,mBACE,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAIC,oBAAAA,WAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAACC,oBAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,KAAA,EAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,uBAAmD,EAAE;CAErD,oBAA4B,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,gBAA8B;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,eAA6B;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,WAAA,GAAA,2BAAA,WACE,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAMC,qBAAAA,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc,KAAA;;CAGrB,0BAAqD;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAIC,gBAAAA,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}

package/dist/drivers/upload.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"upload.d.ts","names":[],"sources":["../../src/drivers/upload.ts"],"mappings":";;;;;;;;;;iBAsBgB,sBAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,sBAAA;AAAA,KAYS,eAAA,GAAkB,UAAA;EAfQ,wEAiBpC,sBAAA;EAhBiB;;EAmBjB,wBAAA;AAAA;;;;;cASW,YAAA;EAAA,iBAQQ,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,UAAA;EAAA,iBACA,cAAA;EAAA,iBACA,IAAA;EAAA,iBAXF,YAAA;EA3BM;EAAA,iBA8BN,WAAA;EAAA,iBACA,KAAA;cAGE,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,UAAA,EAAY,YAAA,EACZ,cAAA,EAAgB,cAAA,EAChB,IAAA,GAAM,eAAA;EA1Ba;EAkDtC,aAAA,CACE,cAAA,EAAgB,sBAAA,GAAyB,WAAA,GACxC,UAAA,CAAW,GAAA,CAAI,cAAA;EAClB,aAAA,CACE,cAAA,EAAgB,sBAAA,GAAyB,WAAA,EACzC,GAAA,EAAK,aAAA,GACJ,GAAA,CAAI,cAAA;EAAA,QAoBC,kBAAA;EAvEgB;EAAA,QA4GV,OAAA;EAnGH;EA4GE,UAAA,CAAA,GAAU,OAAA;EAAA,QAIf,oBAAA;EAAA,QAEA,mBAAA;EAzGmB;EAAA,QA8GnB,aAAA;EA5G2B;EAAA,QAkH3B,YAAA;EAxFU;EA6FL,SAAA,CAAA,GAAa,OAAA;EAAA,CAOnB,MAAA,CAAO,YAAA,KAAiB,OAAA;EAnG5B;EAAA,QAwGK,WAAA;EAtGmC;EAAA,QAwGnC,WAAA;EAAA,QAEM,QAAA;EAAA,QAwBN,uBAAA;AAAA"}

package/dist/drivers/upload.js CHANGED Viewed

@@ -6,7 +6,6 @@ import { Computable, PollingComputableHooks } from "@milaboratories/computable";
 import { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot } from "@milaboratories/pl-tree";
 import { randomUUID } from "node:crypto";
 import { scheduler } from "node:timers/promises";
 //#region src/drivers/upload.ts
 function makeBlobImportSnapshot(entryOrAccessor, ctx) {
 	const node = isPlTreeEntry(entryOrAccessor) ? ctx.accessor(entryOrAccessor).node() : isPlTreeEntryAccessor(entryOrAccessor) ? entryOrAccessor.node() : entryOrAccessor;
@@ -130,7 +129,7 @@ function isProgressDone(p) {
 function validateResourceType(methodName, rType) {
 	if (!rType.name.startsWith("BlobUpload") && !rType.name.startsWith("BlobIndex")) throw new WrongResourceTypeError(`${methodName}: wrong resource type: ${rType.name}, expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`);
 }
 //#endregion
 export { UploadDriver, makeBlobImportSnapshot };
 //# sourceMappingURL=upload.js.map

package/dist/drivers/upload.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upload.js","names":[],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): ImportResourceSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n if (node.resourceType.name.startsWith(\"BlobUpload\"))\n return makeResourceSnapshot(node, UploadResourceSnapshot);\n return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n /** How much parts of a file can be multipart-uploaded to S3 at once. /\n nConcurrentPartUploads: number;\n /* How much upload/indexing statuses of blobs can the driver ask\n * from the platform gRPC at once. /\n nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/* Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. /\nexport class UploadDriver {\n private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n /* Holds a queue that upload blobs. /\n private readonly uploadQueue: TaskProcessor;\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly opts: UploadDriverOps = {\n nConcurrentPartUploads: 10,\n nConcurrentGetProgresses: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.uploadQueue = new TaskProcessor(this.logger, 1, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 20,\n maxDelay: 15000, // 15 seconds\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n /* Returns a progress id and schedules an upload task if it's necessary. /\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ): Computable<sdk.ImportProgress>;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n ): sdk.ImportProgress;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.ImportProgress> \| sdk.ImportProgress {\n if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n ? makeBlobImportSnapshot(handleResource, ctx)\n : handleResource;\n\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n return result;\n }\n\n private getProgressIdNoCtx(\n w: Watcher,\n res: ImportResourceSnapshot,\n callerId: string,\n ): sdk.ImportProgress {\n validateResourceType(\"getProgressId\", res.type);\n\n const task = this.idToProgress.get(res.id);\n\n if (task != undefined) {\n task.setDoneIfOutputSet(res);\n return task.getProgress(w, callerId);\n }\n\n const newTask = new UploadTask(\n this.logger,\n this.clientBlob,\n this.clientProgress,\n this.opts.nConcurrentPartUploads,\n this.signer,\n res,\n );\n\n this.idToProgress.set(res.id, newTask);\n\n if (newTask.shouldScheduleUpload()) {\n this.uploadQueue.push({\n fn: () => newTask.uploadBlobTask(),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n }\n\n newTask.setDoneIfOutputSet(res);\n return newTask.getProgress(w, callerId);\n }\n\n /* Decrement counters for the file and remove an uploading if counter == 0. /\n private async release(id: ResourceId, callerId: string) {\n const task = this.idToProgress.get(id);\n if (task === undefined) return;\n\n const deleted = task.decCounter(callerId);\n if (deleted) this.idToProgress.delete(id);\n }\n\n /* Must be called when the driver is closing. /\n public async releaseAll() {\n this.uploadQueue.stop();\n }\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* Stops polling loop and waits for it to finish /\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. */\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n await asyncPool(\n this.opts.nConcurrentGetProgresses,\n this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllNotDoneProgresses(): Array<UploadTask> {\n return Array.from(this.idToProgress.entries())\n .filter(([_, p]) => !isProgressDone(p.progress))\n .map(([_, p]) => p);\n }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,OAAO,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,QAAO,qBAAqB,MAAM,uBAAuB;AAC3D,QAAO,qBAAqB,MAAM,sBAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,AAAiB,+BAA4C,IAAI,KAAK;;CAGtE,AAAiB;CACjB,AAAiB;CAEjB,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAiB,YACjB,AAAiB,gBACjB,AAAiB,OAAwB;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;EAViB;EACA;EACA;EACA;EACA;AAOjB,OAAK,cAAc,IAAI,cAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAI,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,OAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,QAAgC,cAAc,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,AAAQ,mBACN,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,QAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAI,WAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,OAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,AAAQ,uBAA2C,EAAE;CAErD,AAAQ,oBAAoB,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,AAAQ,gBAAsB;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,AAAQ,eAAqB;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,AAAQ,cAAc;;CAEtB,AAAQ,cAAyC;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,UAAM,UACJ,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAM,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc;;CAGrB,AAAQ,0BAA6C;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}
1	+ {"version":3,"file":"upload.js","names":[],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): ImportResourceSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n if (node.resourceType.name.startsWith(\"BlobUpload\"))\n return makeResourceSnapshot(node, UploadResourceSnapshot);\n return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n /** How much parts of a file can be multipart-uploaded to S3 at once. /\n nConcurrentPartUploads: number;\n /* How much upload/indexing statuses of blobs can the driver ask\n * from the platform gRPC at once. /\n nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/* Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. /\nexport class UploadDriver {\n private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n /* Holds a queue that upload blobs. /\n private readonly uploadQueue: TaskProcessor;\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientBlob: ClientUpload,\n private readonly clientProgress: ClientProgress,\n private readonly opts: UploadDriverOps = {\n nConcurrentPartUploads: 10,\n nConcurrentGetProgresses: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.uploadQueue = new TaskProcessor(this.logger, 1, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 20,\n maxDelay: 15000, // 15 seconds\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n /* Returns a progress id and schedules an upload task if it's necessary. /\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ): Computable<sdk.ImportProgress>;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n ): sdk.ImportProgress;\n getProgressId(\n handleResource: ImportResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.ImportProgress> \| sdk.ImportProgress {\n if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n ? makeBlobImportSnapshot(handleResource, ctx)\n : handleResource;\n\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n return result;\n }\n\n private getProgressIdNoCtx(\n w: Watcher,\n res: ImportResourceSnapshot,\n callerId: string,\n ): sdk.ImportProgress {\n validateResourceType(\"getProgressId\", res.type);\n\n const task = this.idToProgress.get(res.id);\n\n if (task != undefined) {\n task.setDoneIfOutputSet(res);\n return task.getProgress(w, callerId);\n }\n\n const newTask = new UploadTask(\n this.logger,\n this.clientBlob,\n this.clientProgress,\n this.opts.nConcurrentPartUploads,\n this.signer,\n res,\n );\n\n this.idToProgress.set(res.id, newTask);\n\n if (newTask.shouldScheduleUpload()) {\n this.uploadQueue.push({\n fn: () => newTask.uploadBlobTask(),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n }\n\n newTask.setDoneIfOutputSet(res);\n return newTask.getProgress(w, callerId);\n }\n\n /* Decrement counters for the file and remove an uploading if counter == 0. /\n private async release(id: ResourceId, callerId: string) {\n const task = this.idToProgress.get(id);\n if (task === undefined) return;\n\n const deleted = task.decCounter(callerId);\n if (deleted) this.idToProgress.delete(id);\n }\n\n /* Must be called when the driver is closing. /\n public async releaseAll() {\n this.uploadQueue.stop();\n }\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* Stops polling loop and waits for it to finish /\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. */\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n await asyncPool(\n this.opts.nConcurrentGetProgresses,\n this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllNotDoneProgresses(): Array<UploadTask> {\n return Array.from(this.idToProgress.entries())\n .filter(([_, p]) => !isProgressDone(p.progress))\n .map(([_, p]) => p);\n }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,OAAO,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,QAAO,qBAAqB,MAAM,uBAAuB;AAC3D,QAAO,qBAAqB,MAAM,sBAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,+BAA6D,IAAI,KAAK;;CAGtE;CACA;CAEA,YACE,QACA,QACA,YACA,gBACA,OAAyC;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;AAViB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,iBAAA;AACA,OAAA,OAAA;AAOjB,OAAK,cAAc,IAAI,cAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAI,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,QAAgC,cAAc,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,mBACE,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAI,WAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,KAAA,EAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,uBAAmD,EAAE;CAErD,oBAA4B,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,gBAA8B;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,eAA6B;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,UAAM,UACJ,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAM,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc,KAAA;;CAGrB,0BAAqD;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}

package/dist/drivers/upload_task.cjs CHANGED Viewed

@@ -1,12 +1,11 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
-const require_upload = require('../clients/upload.cjs');
-const require_types = require('./types.cjs');
+const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
+const require_upload = require("../clients/upload.cjs");
+const require_types = require("./types.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_computable = require("@milaboratories/computable");
 let node_assert = require("node:assert");
 node_assert = require_runtime.__toESM(node_assert);
 //#region src/drivers/upload_task.ts
 /** Holds all info needed to upload a file and a status of uploading
 * and indexing. Also, has a method to update a status of the progress.
@@ -255,7 +254,6 @@ function decreaseConcurrency(logger, current, min) {
 	if (newConcurrency != current) logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);
 	return newConcurrency;
 }
 //#endregion
 exports.UploadTask = UploadTask;
 exports.isMyUpload = isMyUpload;
@@ -264,4 +262,5 @@ exports.isTerminalUploadError = isTerminalUploadError;
 exports.isUpload = isUpload;
 exports.nonRecoverableError = nonRecoverableError;
 exports.uploadBlob = uploadBlob;
 //# sourceMappingURL=upload_task.cjs.map