@milaboratories/pl-drivers 1.16.1 → 1.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.cjs.map +1 -1
- package/dist/clients/constructors.js.map +1 -1
- package/dist/clients/crc32c.cjs.map +1 -1
- package/dist/clients/crc32c.js.map +1 -1
- package/dist/clients/download.cjs +5 -3
- package/dist/clients/download.cjs.map +1 -1
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/download.js +2 -0
- package/dist/clients/download.js.map +1 -1
- package/dist/clients/download_url_cache.cjs +3 -1
- package/dist/clients/download_url_cache.cjs.map +1 -1
- package/dist/clients/download_url_cache.js +2 -1
- package/dist/clients/download_url_cache.js.map +1 -1
- package/dist/clients/logs.cjs +2 -1
- package/dist/clients/logs.cjs.map +1 -1
- package/dist/clients/logs.d.ts.map +1 -1
- package/dist/clients/logs.js +2 -0
- package/dist/clients/logs.js.map +1 -1
- package/dist/clients/ls_api.cjs +1 -1
- package/dist/clients/ls_api.cjs.map +1 -1
- package/dist/clients/ls_api.d.ts.map +1 -1
- package/dist/clients/ls_api.js +1 -0
- package/dist/clients/ls_api.js.map +1 -1
- package/dist/clients/progress.cjs +2 -1
- package/dist/clients/progress.cjs.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/progress.js +2 -0
- package/dist/clients/progress.js.map +1 -1
- package/dist/clients/upload.cjs +7 -5
- package/dist/clients/upload.cjs.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/clients/upload.js +6 -4
- package/dist/clients/upload.js.map +1 -1
- package/dist/drivers/download_blob/blob_key.cjs +1 -1
- package/dist/drivers/download_blob/blob_key.cjs.map +1 -1
- package/dist/drivers/download_blob/blob_key.js.map +1 -1
- package/dist/drivers/download_blob/download_blob.cjs +17 -6
- package/dist/drivers/download_blob/download_blob.cjs.map +1 -1
- package/dist/drivers/download_blob/download_blob.d.ts.map +1 -1
- package/dist/drivers/download_blob/download_blob.js +11 -0
- package/dist/drivers/download_blob/download_blob.js.map +1 -1
- package/dist/drivers/download_blob/download_blob_task.cjs +9 -3
- package/dist/drivers/download_blob/download_blob_task.cjs.map +1 -1
- package/dist/drivers/download_blob/download_blob_task.js +6 -1
- package/dist/drivers/download_blob/download_blob_task.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/cache.cjs +9 -1
- package/dist/drivers/download_blob/sparse_cache/cache.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/cache.js +8 -0
- package/dist/drivers/download_blob/sparse_cache/cache.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.cjs +2 -1
- package/dist/drivers/download_blob/sparse_cache/file.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.js +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs +3 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.js +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.js.map +1 -1
- package/dist/drivers/download_blob_url/driver.cjs +6 -1
- package/dist/drivers/download_blob_url/driver.cjs.map +1 -1
- package/dist/drivers/download_blob_url/driver.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/driver.js +5 -0
- package/dist/drivers/download_blob_url/driver.js.map +1 -1
- package/dist/drivers/download_blob_url/driver_id.cjs +0 -1
- package/dist/drivers/download_blob_url/driver_id.cjs.map +1 -1
- package/dist/drivers/download_blob_url/driver_id.js.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.cjs +0 -1
- package/dist/drivers/download_blob_url/snapshot.cjs.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.d.ts +1 -2
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.js.map +1 -1
- package/dist/drivers/download_blob_url/task.cjs +14 -6
- package/dist/drivers/download_blob_url/task.cjs.map +1 -1
- package/dist/drivers/download_blob_url/task.d.ts +1 -1
- package/dist/drivers/download_blob_url/task.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/task.js +8 -1
- package/dist/drivers/download_blob_url/task.js.map +1 -1
- package/dist/drivers/download_url/driver.cjs +41 -19
- package/dist/drivers/download_url/driver.cjs.map +1 -1
- package/dist/drivers/download_url/driver.d.ts +10 -5
- package/dist/drivers/download_url/driver.d.ts.map +1 -1
- package/dist/drivers/download_url/driver.js +40 -18
- package/dist/drivers/download_url/driver.js.map +1 -1
- package/dist/drivers/download_url/task.cjs +39 -23
- package/dist/drivers/download_url/task.cjs.map +1 -1
- package/dist/drivers/download_url/task.d.ts +20 -6
- package/dist/drivers/download_url/task.d.ts.map +1 -1
- package/dist/drivers/download_url/task.js +36 -20
- package/dist/drivers/download_url/task.js.map +1 -1
- package/dist/drivers/helpers/download_local_handle.cjs.map +1 -1
- package/dist/drivers/helpers/download_local_handle.js.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.cjs +0 -1
- package/dist/drivers/helpers/download_remote_handle.cjs.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.js.map +1 -1
- package/dist/drivers/helpers/files_cache.cjs +1 -1
- package/dist/drivers/helpers/files_cache.cjs.map +1 -1
- package/dist/drivers/helpers/files_cache.js +1 -0
- package/dist/drivers/helpers/files_cache.js.map +1 -1
- package/dist/drivers/helpers/helpers.cjs +1 -0
- package/dist/drivers/helpers/helpers.cjs.map +1 -1
- package/dist/drivers/helpers/helpers.d.ts.map +1 -1
- package/dist/drivers/helpers/helpers.js +1 -0
- package/dist/drivers/helpers/helpers.js.map +1 -1
- package/dist/drivers/helpers/logs_handle.cjs +3 -1
- package/dist/drivers/helpers/logs_handle.cjs.map +1 -1
- package/dist/drivers/helpers/logs_handle.js +1 -1
- package/dist/drivers/helpers/logs_handle.js.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.cjs.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.js.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.cjs +2 -1
- package/dist/drivers/helpers/ls_storage_entry.cjs.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.js +1 -1
- package/dist/drivers/helpers/ls_storage_entry.js.map +1 -1
- package/dist/drivers/helpers/polling_ops.d.ts.map +1 -1
- package/dist/drivers/helpers/read_file.cjs +2 -2
- package/dist/drivers/helpers/read_file.cjs.map +1 -1
- package/dist/drivers/helpers/read_file.js.map +1 -1
- package/dist/drivers/logs.cjs +3 -1
- package/dist/drivers/logs.cjs.map +1 -1
- package/dist/drivers/logs.js +3 -0
- package/dist/drivers/logs.js.map +1 -1
- package/dist/drivers/logs_stream.cjs +8 -1
- package/dist/drivers/logs_stream.cjs.map +1 -1
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/logs_stream.js +8 -0
- package/dist/drivers/logs_stream.js.map +1 -1
- package/dist/drivers/ls.cjs +10 -3
- package/dist/drivers/ls.cjs.map +1 -1
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/ls.js +8 -1
- package/dist/drivers/ls.js.map +1 -1
- package/dist/drivers/types.cjs +10 -1
- package/dist/drivers/types.cjs.map +1 -1
- package/dist/drivers/types.d.ts +3 -4
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/types.js +10 -0
- package/dist/drivers/types.js.map +1 -1
- package/dist/drivers/upload.cjs +5 -1
- package/dist/drivers/upload.cjs.map +1 -1
- package/dist/drivers/upload.d.ts.map +1 -1
- package/dist/drivers/upload.js +5 -0
- package/dist/drivers/upload.js.map +1 -1
- package/dist/drivers/upload_task.cjs +6 -1
- package/dist/drivers/upload_task.cjs.map +1 -1
- package/dist/drivers/upload_task.d.ts.map +1 -1
- package/dist/drivers/upload_task.js +5 -0
- package/dist/drivers/upload_task.js.map +1 -1
- package/dist/drivers/urls/url.cjs +1 -1
- package/dist/drivers/urls/url.cjs.map +1 -1
- package/dist/drivers/urls/url.js.map +1 -1
- package/dist/drivers/virtual_storages.cjs +3 -3
- package/dist/drivers/virtual_storages.cjs.map +1 -1
- package/dist/drivers/virtual_storages.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.js.map +1 -1
- package/dist/helpers/download.cjs +1 -1
- package/dist/helpers/download.cjs.map +1 -1
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/helpers/download.js +1 -0
- package/dist/helpers/download.js.map +1 -1
- package/dist/helpers/download_errors.cjs.map +1 -1
- package/dist/helpers/download_errors.d.ts.map +1 -1
- package/dist/helpers/download_errors.js.map +1 -1
- package/dist/helpers/validate.cjs +1 -1
- package/dist/helpers/validate.cjs.map +1 -1
- package/dist/helpers/validate.d.ts.map +1 -1
- package/dist/helpers/validate.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs +4 -2
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs +4 -2
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js +3 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs +8 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js +5 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs +17 -7
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js +7 -7
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.cjs +0 -1
- package/dist/proto-grpc/google/protobuf/duration.cjs.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.d.ts.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.js.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.cjs +0 -1
- package/dist/proto-grpc/google/protobuf/timestamp.cjs.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.d.ts.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.js.map +1 -1
- package/dist/proto-rest/downloadapi.d.ts +1 -1
- package/dist/proto-rest/downloadapi.d.ts.map +1 -1
- package/dist/proto-rest/index.d.ts +1 -1
- package/dist/proto-rest/index.d.ts.map +1 -1
- package/dist/proto-rest/progressapi.d.ts +1 -1
- package/dist/proto-rest/progressapi.d.ts.map +1 -1
- package/dist/proto-rest/streamingapi.d.ts +1 -1
- package/dist/proto-rest/streamingapi.d.ts.map +1 -1
- package/package.json +8 -8
- package/src/drivers/download_url/driver.ts +44 -19
- package/src/drivers/download_url/get_local_tgz.test.ts +106 -0
- package/src/drivers/download_url/task.ts +53 -30
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs_handle.js","names":[],"sources":["../../../src/drivers/helpers/logs_handle.ts"],"sourcesContent":["/** Handle of logs. This handle should be passed\n * to the driver for retrieving logs. */\n\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\n\nexport function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {\n const { globalId, signature } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(signature);\n if (live) {\n return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.LiveLogHandle;\n }\n\n return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.ReadyLogHandle;\n}\n\n/** Handle of the live logs of a program.\n * The resource that represents a log can be deleted,\n * in this case the handle should be refreshed. */\n\nexport const liveHandleRegex =\n /^log\\+live:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>[^/]+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {\n return liveHandleRegex.test(handle);\n}\n\n/** Handle of the ready logs of a program. */\n\nexport const readyHandleRegex =\n /^log\\+ready:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>.+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {\n return readyHandleRegex.test(handle);\n}\n\nexport function getResourceInfoFromLogHandle(handle: sdk.AnyLogHandle): ResourceInfo {\n let parsed: RegExpMatchArray | null;\n\n if (isLiveLogHandle(handle)) {\n parsed = handle.match(liveHandleRegex);\n } else if (isReadyLogHandle(handle)) {\n parsed = handle.match(readyHandleRegex);\n } else throw new Error(`Log handle is malformed: ${handle}`);\n if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);\n\n const { resourceType, resourceVersion, resourceId, resourceSig } = parsed.groups!;\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n };\n}\n"],"mappings":";;AAYA,SAAgB,aAAa,MAAe,OAAuC;CACjF,MAAM,EAAE,UAAU,cAAc,sBAAsB,MAAM,
|
|
1
|
+
{"version":3,"file":"logs_handle.js","names":[],"sources":["../../../src/drivers/helpers/logs_handle.ts"],"sourcesContent":["/** Handle of logs. This handle should be passed\n * to the driver for retrieving logs. */\n\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\n\nexport function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {\n const { globalId, signature } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(signature);\n if (live) {\n return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.LiveLogHandle;\n }\n\n return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.ReadyLogHandle;\n}\n\n/** Handle of the live logs of a program.\n * The resource that represents a log can be deleted,\n * in this case the handle should be refreshed. */\n\nexport const liveHandleRegex =\n /^log\\+live:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>[^/]+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {\n return liveHandleRegex.test(handle);\n}\n\n/** Handle of the ready logs of a program. */\n\nexport const readyHandleRegex =\n /^log\\+ready:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>.+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {\n return readyHandleRegex.test(handle);\n}\n\nexport function getResourceInfoFromLogHandle(handle: sdk.AnyLogHandle): ResourceInfo {\n let parsed: RegExpMatchArray | null;\n\n if (isLiveLogHandle(handle)) {\n parsed = handle.match(liveHandleRegex);\n } else if (isReadyLogHandle(handle)) {\n parsed = handle.match(readyHandleRegex);\n } else throw new Error(`Log handle is malformed: ${handle}`);\n if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);\n\n const { resourceType, resourceVersion, resourceId, resourceSig } = parsed.groups!;\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n };\n}\n"],"mappings":";;AAYA,SAAgB,aAAa,MAAe,OAAuC;CACjF,MAAM,EAAE,UAAU,cAAc,sBAAsB,MAAM,EAAE;CAC9D,MAAM,aAAa,qBAAqB,SAAS;CACjD,IAAI,MACF,OAAO,kBAAkB,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG;CAGhF,OAAO,mBAAmB,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG;AACjF;;;;AAMA,MAAa,kBACX;AAEF,SAAgB,gBAAgB,QAA6C;CAC3E,OAAO,gBAAgB,KAAK,MAAM;AACpC;;AAIA,MAAa,mBACX;AAEF,SAAgB,iBAAiB,QAA8C;CAC7E,OAAO,iBAAiB,KAAK,MAAM;AACrC;AAEA,SAAgB,6BAA6B,QAAwC;CACnF,IAAI;CAEJ,IAAI,gBAAgB,MAAM,GACxB,SAAS,OAAO,MAAM,eAAe;MAChC,IAAI,iBAAiB,MAAM,GAChC,SAAS,OAAO,MAAM,gBAAgB;MACjC,MAAM,IAAI,MAAM,4BAA4B,QAAQ;CAC3D,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,6BAA6B,QAAQ;CAEzE,MAAM,EAAE,cAAc,iBAAiB,YAAY,gBAAgB,OAAO;CAE1E,MAAM,MAAM,cAAc,qBAAqB,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,IAAI,uBAAuB,OAAO,UAAU,GAAG,GAAG;EAClD,MAAM;GAAE,MAAM;GAAc,SAAS;EAAgB;CACvD;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ls_remote_import_handle.cjs","names":["ImportFileHandleUploadData","ImportFileHandleIndexData"],"sources":["../../../src/drivers/helpers/ls_remote_import_handle.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { ImportFileHandleIndexData, ImportFileHandleUploadData } from \"../types\";\n\nexport function createIndexImportHandle(\n storageId: string,\n path: string,\n additionalInfo?: Record<string, string>,\n): sdk.ImportFileHandleIndex {\n const data: ImportFileHandleIndexData = {\n storageId: storageId,\n path: path,\n };\n\n // Only embed the envelope when non-empty; preserves byte-identical URL for non-federative storages.\n if (additionalInfo && Object.keys(additionalInfo).length > 0) {\n data.additionalInfo = additionalInfo;\n }\n\n return `index://index/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function createUploadImportHandle(\n localPath: string,\n signer: Signer,\n sizeBytes: bigint,\n modificationTimeSeconds: bigint,\n): sdk.ImportFileHandleUpload {\n const data: ImportFileHandleUploadData = {\n localPath,\n pathSignature: signer.sign(localPath),\n sizeBytes: String(sizeBytes),\n modificationTime: String(modificationTimeSeconds),\n };\n\n return `upload://upload/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function parseUploadHandle(handle: sdk.ImportFileHandleUpload): ImportFileHandleUploadData {\n const url = new URL(handle);\n return ImportFileHandleUploadData.parse(\n JSON.parse(decodeURIComponent(url.pathname.substring(1))),\n );\n}\n\nexport function parseIndexHandle(handle: sdk.ImportFileHandleIndex): ImportFileHandleIndexData {\n const url = new URL(handle);\n return ImportFileHandleIndexData.parse(JSON.parse(decodeURIComponent(url.pathname.substring(1))));\n}\n"],"mappings":";;AAIA,SAAgB,wBACd,WACA,MACA,gBAC2B;CAC3B,MAAM,OAAkC;EAC3B;EACL;
|
|
1
|
+
{"version":3,"file":"ls_remote_import_handle.cjs","names":["ImportFileHandleUploadData","ImportFileHandleIndexData"],"sources":["../../../src/drivers/helpers/ls_remote_import_handle.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { ImportFileHandleIndexData, ImportFileHandleUploadData } from \"../types\";\n\nexport function createIndexImportHandle(\n storageId: string,\n path: string,\n additionalInfo?: Record<string, string>,\n): sdk.ImportFileHandleIndex {\n const data: ImportFileHandleIndexData = {\n storageId: storageId,\n path: path,\n };\n\n // Only embed the envelope when non-empty; preserves byte-identical URL for non-federative storages.\n if (additionalInfo && Object.keys(additionalInfo).length > 0) {\n data.additionalInfo = additionalInfo;\n }\n\n return `index://index/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function createUploadImportHandle(\n localPath: string,\n signer: Signer,\n sizeBytes: bigint,\n modificationTimeSeconds: bigint,\n): sdk.ImportFileHandleUpload {\n const data: ImportFileHandleUploadData = {\n localPath,\n pathSignature: signer.sign(localPath),\n sizeBytes: String(sizeBytes),\n modificationTime: String(modificationTimeSeconds),\n };\n\n return `upload://upload/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function parseUploadHandle(handle: sdk.ImportFileHandleUpload): ImportFileHandleUploadData {\n const url = new URL(handle);\n return ImportFileHandleUploadData.parse(\n JSON.parse(decodeURIComponent(url.pathname.substring(1))),\n );\n}\n\nexport function parseIndexHandle(handle: sdk.ImportFileHandleIndex): ImportFileHandleIndexData {\n const url = new URL(handle);\n return ImportFileHandleIndexData.parse(JSON.parse(decodeURIComponent(url.pathname.substring(1))));\n}\n"],"mappings":";;AAIA,SAAgB,wBACd,WACA,MACA,gBAC2B;CAC3B,MAAM,OAAkC;EAC3B;EACL;CACR;CAGA,IAAI,kBAAkB,OAAO,KAAK,cAAc,CAAC,CAAC,SAAS,GACzD,KAAK,iBAAiB;CAGxB,OAAO,iBAAiB,mBAAmB,KAAK,UAAU,IAAI,CAAC;AACjE;AAEA,SAAgB,yBACd,WACA,QACA,WACA,yBAC4B;CAC5B,MAAM,OAAmC;EACvC;EACA,eAAe,OAAO,KAAK,SAAS;EACpC,WAAW,OAAO,SAAS;EAC3B,kBAAkB,OAAO,uBAAuB;CAClD;CAEA,OAAO,mBAAmB,mBAAmB,KAAK,UAAU,IAAI,CAAC;AACnE;AAEA,SAAgB,kBAAkB,QAAgE;CAChG,MAAM,MAAM,IAAI,IAAI,MAAM;CAC1B,OAAOA,cAAAA,2BAA2B,MAChC,KAAK,MAAM,mBAAmB,IAAI,SAAS,UAAU,CAAC,CAAC,CAAC,CAC1D;AACF;AAEA,SAAgB,iBAAiB,QAA8D;CAC7F,MAAM,MAAM,IAAI,IAAI,MAAM;CAC1B,OAAOC,cAAAA,0BAA0B,MAAM,KAAK,MAAM,mBAAmB,IAAI,SAAS,UAAU,CAAC,CAAC,CAAC,CAAC;AAClG"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ls_remote_import_handle.js","names":[],"sources":["../../../src/drivers/helpers/ls_remote_import_handle.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { ImportFileHandleIndexData, ImportFileHandleUploadData } from \"../types\";\n\nexport function createIndexImportHandle(\n storageId: string,\n path: string,\n additionalInfo?: Record<string, string>,\n): sdk.ImportFileHandleIndex {\n const data: ImportFileHandleIndexData = {\n storageId: storageId,\n path: path,\n };\n\n // Only embed the envelope when non-empty; preserves byte-identical URL for non-federative storages.\n if (additionalInfo && Object.keys(additionalInfo).length > 0) {\n data.additionalInfo = additionalInfo;\n }\n\n return `index://index/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function createUploadImportHandle(\n localPath: string,\n signer: Signer,\n sizeBytes: bigint,\n modificationTimeSeconds: bigint,\n): sdk.ImportFileHandleUpload {\n const data: ImportFileHandleUploadData = {\n localPath,\n pathSignature: signer.sign(localPath),\n sizeBytes: String(sizeBytes),\n modificationTime: String(modificationTimeSeconds),\n };\n\n return `upload://upload/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function parseUploadHandle(handle: sdk.ImportFileHandleUpload): ImportFileHandleUploadData {\n const url = new URL(handle);\n return ImportFileHandleUploadData.parse(\n JSON.parse(decodeURIComponent(url.pathname.substring(1))),\n );\n}\n\nexport function parseIndexHandle(handle: sdk.ImportFileHandleIndex): ImportFileHandleIndexData {\n const url = new URL(handle);\n return ImportFileHandleIndexData.parse(JSON.parse(decodeURIComponent(url.pathname.substring(1))));\n}\n"],"mappings":";;AAIA,SAAgB,wBACd,WACA,MACA,gBAC2B;CAC3B,MAAM,OAAkC;EAC3B;EACL;
|
|
1
|
+
{"version":3,"file":"ls_remote_import_handle.js","names":[],"sources":["../../../src/drivers/helpers/ls_remote_import_handle.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport { ImportFileHandleIndexData, ImportFileHandleUploadData } from \"../types\";\n\nexport function createIndexImportHandle(\n storageId: string,\n path: string,\n additionalInfo?: Record<string, string>,\n): sdk.ImportFileHandleIndex {\n const data: ImportFileHandleIndexData = {\n storageId: storageId,\n path: path,\n };\n\n // Only embed the envelope when non-empty; preserves byte-identical URL for non-federative storages.\n if (additionalInfo && Object.keys(additionalInfo).length > 0) {\n data.additionalInfo = additionalInfo;\n }\n\n return `index://index/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function createUploadImportHandle(\n localPath: string,\n signer: Signer,\n sizeBytes: bigint,\n modificationTimeSeconds: bigint,\n): sdk.ImportFileHandleUpload {\n const data: ImportFileHandleUploadData = {\n localPath,\n pathSignature: signer.sign(localPath),\n sizeBytes: String(sizeBytes),\n modificationTime: String(modificationTimeSeconds),\n };\n\n return `upload://upload/${encodeURIComponent(JSON.stringify(data))}`;\n}\n\nexport function parseUploadHandle(handle: sdk.ImportFileHandleUpload): ImportFileHandleUploadData {\n const url = new URL(handle);\n return ImportFileHandleUploadData.parse(\n JSON.parse(decodeURIComponent(url.pathname.substring(1))),\n );\n}\n\nexport function parseIndexHandle(handle: sdk.ImportFileHandleIndex): ImportFileHandleIndexData {\n const url = new URL(handle);\n return ImportFileHandleIndexData.parse(JSON.parse(decodeURIComponent(url.pathname.substring(1))));\n}\n"],"mappings":";;AAIA,SAAgB,wBACd,WACA,MACA,gBAC2B;CAC3B,MAAM,OAAkC;EAC3B;EACL;CACR;CAGA,IAAI,kBAAkB,OAAO,KAAK,cAAc,CAAC,CAAC,SAAS,GACzD,KAAK,iBAAiB;CAGxB,OAAO,iBAAiB,mBAAmB,KAAK,UAAU,IAAI,CAAC;AACjE;AAEA,SAAgB,yBACd,WACA,QACA,WACA,yBAC4B;CAC5B,MAAM,OAAmC;EACvC;EACA,eAAe,OAAO,KAAK,SAAS;EACpC,WAAW,OAAO,SAAS;EAC3B,kBAAkB,OAAO,uBAAuB;CAClD;CAEA,OAAO,mBAAmB,mBAAmB,KAAK,UAAU,IAAI,CAAC;AACnE;AAEA,SAAgB,kBAAkB,QAAgE;CAChG,MAAM,MAAM,IAAI,IAAI,MAAM;CAC1B,OAAO,2BAA2B,MAChC,KAAK,MAAM,mBAAmB,IAAI,SAAS,UAAU,CAAC,CAAC,CAAC,CAC1D;AACF;AAEA,SAAgB,iBAAiB,QAA8D;CAC7F,MAAM,MAAM,IAAI,IAAI,MAAM;CAC1B,OAAO,0BAA0B,MAAM,KAAK,MAAM,mBAAmB,IAAI,SAAS,UAAU,CAAC,CAAC,CAAC,CAAC;AAClG"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
1
|
let _milaboratories_pl_client = require("@milaboratories/pl-client");
|
|
3
2
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
4
3
|
//#region src/drivers/helpers/ls_storage_entry.ts
|
|
@@ -48,6 +47,8 @@ function parseRemoteStorageHandle(handle) {
|
|
|
48
47
|
//#endregion
|
|
49
48
|
exports.createLocalStorageHandle = createLocalStorageHandle;
|
|
50
49
|
exports.createRemoteStorageHandle = createRemoteStorageHandle;
|
|
50
|
+
exports.isLocalStorageHandle = isLocalStorageHandle;
|
|
51
|
+
exports.isRemoteStorageHandle = isRemoteStorageHandle;
|
|
51
52
|
exports.parseStorageHandle = parseStorageHandle;
|
|
52
53
|
|
|
53
54
|
//# sourceMappingURL=ls_storage_entry.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ls_storage_entry.cjs","names":[],"sources":["../../../src/drivers/helpers/ls_storage_entry.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ResourceType, SignedResourceId, StorageInfo } from \"@milaboratories/pl-client\";\nimport {\n resourceTypeToString,\n parseResourceType,\n asSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport { assertNever } from \"@milaboratories/ts-helpers\";\n\nexport type StorageHandleData = RemoteStorageHandleData | LocalStorageHandleData;\n\n/**\n * Gets a storage handle and gives an underlying data from it.\n */\nexport function parseStorageHandle(handle: sdk.StorageHandle): StorageHandleData {\n if (isRemoteStorageHandle(handle)) {\n return parseRemoteStorageHandle(handle);\n } else if (isLocalStorageHandle(handle)) {\n return parseLocalStorageHandle(handle);\n }\n\n assertNever(handle);\n}\n\n//\n// Local storage:\n//\n\nexport type LocalStorageHandleData = {\n isRemote: false;\n name: string;\n rootPath: string;\n};\n\nconst localHandleRegex = /^local:\\/\\/(?<name>.*)\\/(?<path>.*)$/;\n\nexport function isLocalStorageHandle(handle: sdk.StorageHandle): handle is sdk.StorageHandleLocal {\n return localHandleRegex.test(handle);\n}\n\nexport function createLocalStorageHandle(name: string, path: string): sdk.StorageHandleLocal {\n return `local://${name}/${encodeURIComponent(path)}`;\n}\n\nfunction parseLocalStorageHandle(handle: string): LocalStorageHandleData {\n const parsed = handle.match(localHandleRegex);\n if (parsed == null) throw new Error(`Local list handle wasn't parsed: ${handle}`);\n\n const { name, path } = parsed.groups!;\n\n return {\n isRemote: false,\n rootPath: decodeURIComponent(path),\n name,\n };\n}\n\n//\n// Remote storage:\n//\n\nexport type RemoteStorageHandleData = {\n isRemote: true;\n storageId: string;\n resourceType: ResourceType;\n resourceId: SignedResourceId;\n};\n\nconst remoteHandleRegex = /^remote:\\/\\/(?<storageId>.*)\\/(?<resourceType>.*)\\/(?<resourceId>.*)$/;\n\nexport function isRemoteStorageHandle(\n handle: sdk.StorageHandle,\n): handle is sdk.StorageHandleRemote {\n return remoteHandleRegex.test(handle);\n}\n\nexport function createRemoteStorageHandle(info: StorageInfo): sdk.StorageHandleRemote {\n return `remote://${info.storageId}/${encodeURIComponent(resourceTypeToString(info.resourceType))}/${encodeURIComponent(info.resourceId)}`;\n}\n\nfunction parseRemoteStorageHandle(handle: string): RemoteStorageHandleData {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed == null) throw new Error(`Remote list handle wasn't parsed: ${handle}`);\n const { storageId, resourceType: encodedRType, resourceId: encodedRId } = parsed.groups!;\n\n const resourceType = parseResourceType(decodeURIComponent(encodedRType));\n const resourceId = asSignedResourceId(decodeURIComponent(encodedRId));\n return {\n isRemote: true,\n storageId,\n resourceType: resourceType,\n resourceId: resourceId,\n };\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"ls_storage_entry.cjs","names":[],"sources":["../../../src/drivers/helpers/ls_storage_entry.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ResourceType, SignedResourceId, StorageInfo } from \"@milaboratories/pl-client\";\nimport {\n resourceTypeToString,\n parseResourceType,\n asSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport { assertNever } from \"@milaboratories/ts-helpers\";\n\nexport type StorageHandleData = RemoteStorageHandleData | LocalStorageHandleData;\n\n/**\n * Gets a storage handle and gives an underlying data from it.\n */\nexport function parseStorageHandle(handle: sdk.StorageHandle): StorageHandleData {\n if (isRemoteStorageHandle(handle)) {\n return parseRemoteStorageHandle(handle);\n } else if (isLocalStorageHandle(handle)) {\n return parseLocalStorageHandle(handle);\n }\n\n assertNever(handle);\n}\n\n//\n// Local storage:\n//\n\nexport type LocalStorageHandleData = {\n isRemote: false;\n name: string;\n rootPath: string;\n};\n\nconst localHandleRegex = /^local:\\/\\/(?<name>.*)\\/(?<path>.*)$/;\n\nexport function isLocalStorageHandle(handle: sdk.StorageHandle): handle is sdk.StorageHandleLocal {\n return localHandleRegex.test(handle);\n}\n\nexport function createLocalStorageHandle(name: string, path: string): sdk.StorageHandleLocal {\n return `local://${name}/${encodeURIComponent(path)}`;\n}\n\nfunction parseLocalStorageHandle(handle: string): LocalStorageHandleData {\n const parsed = handle.match(localHandleRegex);\n if (parsed == null) throw new Error(`Local list handle wasn't parsed: ${handle}`);\n\n const { name, path } = parsed.groups!;\n\n return {\n isRemote: false,\n rootPath: decodeURIComponent(path),\n name,\n };\n}\n\n//\n// Remote storage:\n//\n\nexport type RemoteStorageHandleData = {\n isRemote: true;\n storageId: string;\n resourceType: ResourceType;\n resourceId: SignedResourceId;\n};\n\nconst remoteHandleRegex = /^remote:\\/\\/(?<storageId>.*)\\/(?<resourceType>.*)\\/(?<resourceId>.*)$/;\n\nexport function isRemoteStorageHandle(\n handle: sdk.StorageHandle,\n): handle is sdk.StorageHandleRemote {\n return remoteHandleRegex.test(handle);\n}\n\nexport function createRemoteStorageHandle(info: StorageInfo): sdk.StorageHandleRemote {\n return `remote://${info.storageId}/${encodeURIComponent(resourceTypeToString(info.resourceType))}/${encodeURIComponent(info.resourceId)}`;\n}\n\nfunction parseRemoteStorageHandle(handle: string): RemoteStorageHandleData {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed == null) throw new Error(`Remote list handle wasn't parsed: ${handle}`);\n const { storageId, resourceType: encodedRType, resourceId: encodedRId } = parsed.groups!;\n\n const resourceType = parseResourceType(decodeURIComponent(encodedRType));\n const resourceId = asSignedResourceId(decodeURIComponent(encodedRId));\n return {\n isRemote: true,\n storageId,\n resourceType: resourceType,\n resourceId: resourceId,\n };\n}\n"],"mappings":";;;;;;AAcA,SAAgB,mBAAmB,QAA8C;CAC/E,IAAI,sBAAsB,MAAM,GAC9B,OAAO,yBAAyB,MAAM;MACjC,IAAI,qBAAqB,MAAM,GACpC,OAAO,wBAAwB,MAAM;CAGvC,CAAA,GAAA,2BAAA,YAAA,CAAY,MAAM;AACpB;AAYA,MAAM,mBAAmB;AAEzB,SAAgB,qBAAqB,QAA6D;CAChG,OAAO,iBAAiB,KAAK,MAAM;AACrC;AAEA,SAAgB,yBAAyB,MAAc,MAAsC;CAC3F,OAAO,WAAW,KAAK,GAAG,mBAAmB,IAAI;AACnD;AAEA,SAAS,wBAAwB,QAAwC;CACvE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAC5C,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,oCAAoC,QAAQ;CAEhF,MAAM,EAAE,MAAM,SAAS,OAAO;CAE9B,OAAO;EACL,UAAU;EACV,UAAU,mBAAmB,IAAI;EACjC;CACF;AACF;AAaA,MAAM,oBAAoB;AAE1B,SAAgB,sBACd,QACmC;CACnC,OAAO,kBAAkB,KAAK,MAAM;AACtC;AAEA,SAAgB,0BAA0B,MAA4C;CACpF,OAAO,YAAY,KAAK,UAAU,GAAG,oBAAA,GAAA,0BAAA,qBAAA,CAAwC,KAAK,YAAY,CAAC,EAAE,GAAG,mBAAmB,KAAK,UAAU;AACxI;AAEA,SAAS,yBAAyB,QAAyC;CACzE,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,qCAAqC,QAAQ;CACjF,MAAM,EAAE,WAAW,cAAc,cAAc,YAAY,eAAe,OAAO;CAIjF,OAAO;EACL,UAAU;EACV;EACA,eAAA,GAAA,0BAAA,kBAAA,CALqC,mBAAmB,YAAY,CAK3C;EACzB,aAAA,GAAA,0BAAA,mBAAA,CALoC,mBAAmB,UAAU,CAK5C;CACvB;AACF"}
|
|
@@ -45,6 +45,6 @@ function parseRemoteStorageHandle(handle) {
|
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
47
|
//#endregion
|
|
48
|
-
export { createLocalStorageHandle, createRemoteStorageHandle, parseStorageHandle };
|
|
48
|
+
export { createLocalStorageHandle, createRemoteStorageHandle, isLocalStorageHandle, isRemoteStorageHandle, parseStorageHandle };
|
|
49
49
|
|
|
50
50
|
//# sourceMappingURL=ls_storage_entry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ls_storage_entry.js","names":[],"sources":["../../../src/drivers/helpers/ls_storage_entry.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ResourceType, SignedResourceId, StorageInfo } from \"@milaboratories/pl-client\";\nimport {\n resourceTypeToString,\n parseResourceType,\n asSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport { assertNever } from \"@milaboratories/ts-helpers\";\n\nexport type StorageHandleData = RemoteStorageHandleData | LocalStorageHandleData;\n\n/**\n * Gets a storage handle and gives an underlying data from it.\n */\nexport function parseStorageHandle(handle: sdk.StorageHandle): StorageHandleData {\n if (isRemoteStorageHandle(handle)) {\n return parseRemoteStorageHandle(handle);\n } else if (isLocalStorageHandle(handle)) {\n return parseLocalStorageHandle(handle);\n }\n\n assertNever(handle);\n}\n\n//\n// Local storage:\n//\n\nexport type LocalStorageHandleData = {\n isRemote: false;\n name: string;\n rootPath: string;\n};\n\nconst localHandleRegex = /^local:\\/\\/(?<name>.*)\\/(?<path>.*)$/;\n\nexport function isLocalStorageHandle(handle: sdk.StorageHandle): handle is sdk.StorageHandleLocal {\n return localHandleRegex.test(handle);\n}\n\nexport function createLocalStorageHandle(name: string, path: string): sdk.StorageHandleLocal {\n return `local://${name}/${encodeURIComponent(path)}`;\n}\n\nfunction parseLocalStorageHandle(handle: string): LocalStorageHandleData {\n const parsed = handle.match(localHandleRegex);\n if (parsed == null) throw new Error(`Local list handle wasn't parsed: ${handle}`);\n\n const { name, path } = parsed.groups!;\n\n return {\n isRemote: false,\n rootPath: decodeURIComponent(path),\n name,\n };\n}\n\n//\n// Remote storage:\n//\n\nexport type RemoteStorageHandleData = {\n isRemote: true;\n storageId: string;\n resourceType: ResourceType;\n resourceId: SignedResourceId;\n};\n\nconst remoteHandleRegex = /^remote:\\/\\/(?<storageId>.*)\\/(?<resourceType>.*)\\/(?<resourceId>.*)$/;\n\nexport function isRemoteStorageHandle(\n handle: sdk.StorageHandle,\n): handle is sdk.StorageHandleRemote {\n return remoteHandleRegex.test(handle);\n}\n\nexport function createRemoteStorageHandle(info: StorageInfo): sdk.StorageHandleRemote {\n return `remote://${info.storageId}/${encodeURIComponent(resourceTypeToString(info.resourceType))}/${encodeURIComponent(info.resourceId)}`;\n}\n\nfunction parseRemoteStorageHandle(handle: string): RemoteStorageHandleData {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed == null) throw new Error(`Remote list handle wasn't parsed: ${handle}`);\n const { storageId, resourceType: encodedRType, resourceId: encodedRId } = parsed.groups!;\n\n const resourceType = parseResourceType(decodeURIComponent(encodedRType));\n const resourceId = asSignedResourceId(decodeURIComponent(encodedRId));\n return {\n isRemote: true,\n storageId,\n resourceType: resourceType,\n resourceId: resourceId,\n };\n}\n"],"mappings":";;;;;;AAcA,SAAgB,mBAAmB,QAA8C;
|
|
1
|
+
{"version":3,"file":"ls_storage_entry.js","names":[],"sources":["../../../src/drivers/helpers/ls_storage_entry.ts"],"sourcesContent":["import type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ResourceType, SignedResourceId, StorageInfo } from \"@milaboratories/pl-client\";\nimport {\n resourceTypeToString,\n parseResourceType,\n asSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport { assertNever } from \"@milaboratories/ts-helpers\";\n\nexport type StorageHandleData = RemoteStorageHandleData | LocalStorageHandleData;\n\n/**\n * Gets a storage handle and gives an underlying data from it.\n */\nexport function parseStorageHandle(handle: sdk.StorageHandle): StorageHandleData {\n if (isRemoteStorageHandle(handle)) {\n return parseRemoteStorageHandle(handle);\n } else if (isLocalStorageHandle(handle)) {\n return parseLocalStorageHandle(handle);\n }\n\n assertNever(handle);\n}\n\n//\n// Local storage:\n//\n\nexport type LocalStorageHandleData = {\n isRemote: false;\n name: string;\n rootPath: string;\n};\n\nconst localHandleRegex = /^local:\\/\\/(?<name>.*)\\/(?<path>.*)$/;\n\nexport function isLocalStorageHandle(handle: sdk.StorageHandle): handle is sdk.StorageHandleLocal {\n return localHandleRegex.test(handle);\n}\n\nexport function createLocalStorageHandle(name: string, path: string): sdk.StorageHandleLocal {\n return `local://${name}/${encodeURIComponent(path)}`;\n}\n\nfunction parseLocalStorageHandle(handle: string): LocalStorageHandleData {\n const parsed = handle.match(localHandleRegex);\n if (parsed == null) throw new Error(`Local list handle wasn't parsed: ${handle}`);\n\n const { name, path } = parsed.groups!;\n\n return {\n isRemote: false,\n rootPath: decodeURIComponent(path),\n name,\n };\n}\n\n//\n// Remote storage:\n//\n\nexport type RemoteStorageHandleData = {\n isRemote: true;\n storageId: string;\n resourceType: ResourceType;\n resourceId: SignedResourceId;\n};\n\nconst remoteHandleRegex = /^remote:\\/\\/(?<storageId>.*)\\/(?<resourceType>.*)\\/(?<resourceId>.*)$/;\n\nexport function isRemoteStorageHandle(\n handle: sdk.StorageHandle,\n): handle is sdk.StorageHandleRemote {\n return remoteHandleRegex.test(handle);\n}\n\nexport function createRemoteStorageHandle(info: StorageInfo): sdk.StorageHandleRemote {\n return `remote://${info.storageId}/${encodeURIComponent(resourceTypeToString(info.resourceType))}/${encodeURIComponent(info.resourceId)}`;\n}\n\nfunction parseRemoteStorageHandle(handle: string): RemoteStorageHandleData {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed == null) throw new Error(`Remote list handle wasn't parsed: ${handle}`);\n const { storageId, resourceType: encodedRType, resourceId: encodedRId } = parsed.groups!;\n\n const resourceType = parseResourceType(decodeURIComponent(encodedRType));\n const resourceId = asSignedResourceId(decodeURIComponent(encodedRId));\n return {\n isRemote: true,\n storageId,\n resourceType: resourceType,\n resourceId: resourceId,\n };\n}\n"],"mappings":";;;;;;AAcA,SAAgB,mBAAmB,QAA8C;CAC/E,IAAI,sBAAsB,MAAM,GAC9B,OAAO,yBAAyB,MAAM;MACjC,IAAI,qBAAqB,MAAM,GACpC,OAAO,wBAAwB,MAAM;CAGvC,YAAY,MAAM;AACpB;AAYA,MAAM,mBAAmB;AAEzB,SAAgB,qBAAqB,QAA6D;CAChG,OAAO,iBAAiB,KAAK,MAAM;AACrC;AAEA,SAAgB,yBAAyB,MAAc,MAAsC;CAC3F,OAAO,WAAW,KAAK,GAAG,mBAAmB,IAAI;AACnD;AAEA,SAAS,wBAAwB,QAAwC;CACvE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAC5C,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,oCAAoC,QAAQ;CAEhF,MAAM,EAAE,MAAM,SAAS,OAAO;CAE9B,OAAO;EACL,UAAU;EACV,UAAU,mBAAmB,IAAI;EACjC;CACF;AACF;AAaA,MAAM,oBAAoB;AAE1B,SAAgB,sBACd,QACmC;CACnC,OAAO,kBAAkB,KAAK,MAAM;AACtC;AAEA,SAAgB,0BAA0B,MAA4C;CACpF,OAAO,YAAY,KAAK,UAAU,GAAG,mBAAmB,qBAAqB,KAAK,YAAY,CAAC,EAAE,GAAG,mBAAmB,KAAK,UAAU;AACxI;AAEA,SAAS,yBAAyB,QAAyC;CACzE,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,qCAAqC,QAAQ;CACjF,MAAM,EAAE,WAAW,cAAc,cAAc,YAAY,eAAe,OAAO;CAIjF,OAAO;EACL,UAAU;EACV;EACA,cALmB,kBAAkB,mBAAmB,YAAY,CAK3C;EACzB,YALiB,mBAAmB,mBAAmB,UAAU,CAK5C;CACvB;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"polling_ops.d.ts","names":[],"sources":["../../../src/drivers/helpers/polling_ops.ts"],"mappings":";;KACY,UAAA;EAAU,6EAEpB,eAAA;EAEA,
|
|
1
|
+
{"version":3,"file":"polling_ops.d.ts","names":[],"sources":["../../../src/drivers/helpers/polling_ops.ts"],"mappings":";;KACY,UAAA;EAAU,6EAEpB,eAAA;EAEA,gBAAgB;AAAA"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
2
|
let node_fs_promises = require("node:fs/promises");
|
|
3
|
-
node_fs_promises = require_runtime.__toESM(node_fs_promises);
|
|
3
|
+
node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
|
|
4
4
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
5
5
|
let node_fs = require("node:fs");
|
|
6
|
-
node_fs = require_runtime.__toESM(node_fs);
|
|
6
|
+
node_fs = require_runtime.__toESM(node_fs, 1);
|
|
7
7
|
let node_stream = require("node:stream");
|
|
8
8
|
//#region src/drivers/helpers/read_file.ts
|
|
9
9
|
const fileReadLimiter = new _milaboratories_ts_helpers.ConcurrencyLimitingExecutor(32);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read_file.cjs","names":["ConcurrencyLimitingExecutor","fsp","fs","Readable"],"sources":["../../../src/drivers/helpers/read_file.ts"],"sourcesContent":["import { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport { Readable } from \"node:stream\";\n\n// Global concurrency limiter for file reads - limit to 32 parallel reads\nconst fileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n/**\n * Reads file content with concurrency limiting and proper error handling.\n * Ensures file descriptors are properly cleaned up even in error cases.\n */\nexport async function withFileContent<T>({\n path,\n range,\n signal,\n handler,\n}: {\n path: string;\n range?: RangeBytes;\n signal?: AbortSignal;\n handler: (content: ReadableStream, size: number) => Promise<T>;\n}): Promise<T> {\n return await fileReadLimiter.run(async () => {\n const readOps = {\n start: range?.from,\n end: range?.to !== undefined ? range.to - 1 : undefined,\n signal: signal,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(path);\n stream = fs.createReadStream(path, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n}\n"],"mappings":";;;;;;;;AAOA,MAAM,kBAAkB,IAAIA,2BAAAA,4BAA4B,
|
|
1
|
+
{"version":3,"file":"read_file.cjs","names":["ConcurrencyLimitingExecutor","fsp","fs","Readable"],"sources":["../../../src/drivers/helpers/read_file.ts"],"sourcesContent":["import { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport { Readable } from \"node:stream\";\n\n// Global concurrency limiter for file reads - limit to 32 parallel reads\nconst fileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n/**\n * Reads file content with concurrency limiting and proper error handling.\n * Ensures file descriptors are properly cleaned up even in error cases.\n */\nexport async function withFileContent<T>({\n path,\n range,\n signal,\n handler,\n}: {\n path: string;\n range?: RangeBytes;\n signal?: AbortSignal;\n handler: (content: ReadableStream, size: number) => Promise<T>;\n}): Promise<T> {\n return await fileReadLimiter.run(async () => {\n const readOps = {\n start: range?.from,\n end: range?.to !== undefined ? range.to - 1 : undefined,\n signal: signal,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(path);\n stream = fs.createReadStream(path, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n}\n"],"mappings":";;;;;;;;AAOA,MAAM,kBAAkB,IAAIA,2BAAAA,4BAA4B,EAAE;;;;;AAM1D,eAAsB,gBAAmB,EACvC,MACA,OACA,QACA,WAMa;CACb,OAAO,MAAM,gBAAgB,IAAI,YAAY;EAC3C,MAAM,UAAU;GACd,OAAO,OAAO;GACd,KAAK,OAAO,OAAO,KAAA,IAAY,MAAM,KAAK,IAAI,KAAA;GACtC;EACV;EACA,IAAI;EACJ,IAAI,iBAAiB;EAErB,IAAI;GACF,MAAM,OAAO,MAAMC,iBAAI,KAAK,IAAI;GAChC,SAASC,QAAG,iBAAiB,MAAM,OAAO;GAG1C,MAAM,SAAS,MAAM,QAFHC,YAAAA,SAAS,MAAM,MAEI,GAAG,KAAK,IAAI;GACjD,iBAAiB;GACjB,OAAO;EACT,SAAS,OAAO;GAEd,IAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,WACvC,OAAO,QAAQ;GAEjB,MAAM;EACR;CACF,CAAC;AACH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read_file.js","names":["fs"],"sources":["../../../src/drivers/helpers/read_file.ts"],"sourcesContent":["import { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport { Readable } from \"node:stream\";\n\n// Global concurrency limiter for file reads - limit to 32 parallel reads\nconst fileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n/**\n * Reads file content with concurrency limiting and proper error handling.\n * Ensures file descriptors are properly cleaned up even in error cases.\n */\nexport async function withFileContent<T>({\n path,\n range,\n signal,\n handler,\n}: {\n path: string;\n range?: RangeBytes;\n signal?: AbortSignal;\n handler: (content: ReadableStream, size: number) => Promise<T>;\n}): Promise<T> {\n return await fileReadLimiter.run(async () => {\n const readOps = {\n start: range?.from,\n end: range?.to !== undefined ? range.to - 1 : undefined,\n signal: signal,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(path);\n stream = fs.createReadStream(path, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n}\n"],"mappings":";;;;;AAOA,MAAM,kBAAkB,IAAI,4BAA4B,
|
|
1
|
+
{"version":3,"file":"read_file.js","names":["fs"],"sources":["../../../src/drivers/helpers/read_file.ts"],"sourcesContent":["import { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport { Readable } from \"node:stream\";\n\n// Global concurrency limiter for file reads - limit to 32 parallel reads\nconst fileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n/**\n * Reads file content with concurrency limiting and proper error handling.\n * Ensures file descriptors are properly cleaned up even in error cases.\n */\nexport async function withFileContent<T>({\n path,\n range,\n signal,\n handler,\n}: {\n path: string;\n range?: RangeBytes;\n signal?: AbortSignal;\n handler: (content: ReadableStream, size: number) => Promise<T>;\n}): Promise<T> {\n return await fileReadLimiter.run(async () => {\n const readOps = {\n start: range?.from,\n end: range?.to !== undefined ? range.to - 1 : undefined,\n signal: signal,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(path);\n stream = fs.createReadStream(path, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n}\n"],"mappings":";;;;;AAOA,MAAM,kBAAkB,IAAI,4BAA4B,EAAE;;;;;AAM1D,eAAsB,gBAAmB,EACvC,MACA,OACA,QACA,WAMa;CACb,OAAO,MAAM,gBAAgB,IAAI,YAAY;EAC3C,MAAM,UAAU;GACd,OAAO,OAAO;GACd,KAAK,OAAO,OAAO,KAAA,IAAY,MAAM,KAAK,IAAI,KAAA;GACtC;EACV;EACA,IAAI;EACJ,IAAI,iBAAiB;EAErB,IAAI;GACF,MAAM,OAAO,MAAM,IAAI,KAAK,IAAI;GAChC,SAASA,KAAG,iBAAiB,MAAM,OAAO;GAG1C,MAAM,SAAS,MAAM,QAFH,SAAS,MAAM,MAEI,GAAG,KAAK,IAAI;GACjD,iBAAiB;GACjB,OAAO;EACT,SAAS,OAAO;GAEd,IAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,WACvC,OAAO,QAAQ;GAEjB,MAAM;EACR;CACF,CAAC;AACH"}
|
package/dist/drivers/logs.cjs
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
require("../_virtual/_rolldown/runtime.cjs");
|
|
2
1
|
const require_logs_handle = require("./helpers/logs_handle.cjs");
|
|
3
2
|
let _milaboratories_pl_client = require("@milaboratories/pl-client");
|
|
4
3
|
let _milaboratories_computable = require("@milaboratories/computable");
|
|
5
4
|
//#region src/drivers/logs.ts
|
|
6
5
|
var LogsDriver = class {
|
|
6
|
+
logger;
|
|
7
|
+
logsStreamDriver;
|
|
8
|
+
downloadDriver;
|
|
7
9
|
constructor(logger, logsStreamDriver, downloadDriver) {
|
|
8
10
|
this.logger = logger;
|
|
9
11
|
this.logsStreamDriver = logsStreamDriver;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs.cjs","names":["Computable","isLiveLogHandle"],"sources":["../../src/drivers/logs.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { LogsStreamDriver } from \"./logs_stream\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"./download_blob/download_blob\";\nimport { isLiveLogHandle } from \"./helpers/logs_handle\";\nimport { isNotFoundError } from \"@milaboratories/pl-client\";\n\nexport class LogsDriver implements sdk.LogsDriver {\n constructor(\n private readonly logger: MiLogger,\n private readonly logsStreamDriver: LogsStreamDriver,\n private readonly downloadDriver: DownloadDriver,\n ) {}\n\n /** Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. */\n getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(res: PlTreeEntry, lines: number, ctx: ComputableCtx): Computable<string | undefined>;\n getLastLogs(\n res: PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLastLogs(stream, lines, ctx);\n\n try {\n return this.logsStreamDriver.getLastLogs(stream, lines, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting last logs: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(res: PlTreeEntry, patternToSearch: string): Computable<string | undefined>;\n getProgressLog(res: PlTreeEntry, patternToSearch: string, ctx: ComputableCtx): string | undefined;\n getProgressLog(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n\n try {\n return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting a progress log: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Same as getProgressLog but also returns a liveliness of the log.\n * The previous getProgressLog couldn't be extended.\n * Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ): Computable<sdk.ProgressLogWithInfo | undefined>;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): sdk.ProgressLogWithInfo | undefined;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<sdk.ProgressLogWithInfo | undefined> | sdk.ProgressLogWithInfo | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLogWithInfo(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) {\n const log = this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: false,\n };\n }\n\n try {\n const log = this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: true,\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(\n `NOT_FOUND in logs stream driver while getting a progress log with info: ${e}`,\n );\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns an Id of a smart object, that can read logs directly from\n * the platform. */\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle | undefined>;\n getLogHandle(res: PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle | undefined;\n getLogHandle(\n res: PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLogHandle(stream, ctx);\n\n return this.logsStreamDriver.getLogHandle(stream, ctx);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.readText(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.readText(handle, lineCount, offsetBytes, searchStr);\n }\n}\n\nfunction isBlob(rInfo: ResourceInfo) {\n return !rInfo.type.name.startsWith(\"StreamWorkdir\");\n}\n\nfunction streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {\n return ctx.accessor(manager).node().traverse(\"stream\")?.resourceInfo;\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"logs.cjs","names":["Computable","isLiveLogHandle"],"sources":["../../src/drivers/logs.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { LogsStreamDriver } from \"./logs_stream\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"./download_blob/download_blob\";\nimport { isLiveLogHandle } from \"./helpers/logs_handle\";\nimport { isNotFoundError } from \"@milaboratories/pl-client\";\n\nexport class LogsDriver implements sdk.LogsDriver {\n constructor(\n private readonly logger: MiLogger,\n private readonly logsStreamDriver: LogsStreamDriver,\n private readonly downloadDriver: DownloadDriver,\n ) {}\n\n /** Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. */\n getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(res: PlTreeEntry, lines: number, ctx: ComputableCtx): Computable<string | undefined>;\n getLastLogs(\n res: PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLastLogs(stream, lines, ctx);\n\n try {\n return this.logsStreamDriver.getLastLogs(stream, lines, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting last logs: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(res: PlTreeEntry, patternToSearch: string): Computable<string | undefined>;\n getProgressLog(res: PlTreeEntry, patternToSearch: string, ctx: ComputableCtx): string | undefined;\n getProgressLog(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n\n try {\n return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting a progress log: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Same as getProgressLog but also returns a liveliness of the log.\n * The previous getProgressLog couldn't be extended.\n * Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ): Computable<sdk.ProgressLogWithInfo | undefined>;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): sdk.ProgressLogWithInfo | undefined;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<sdk.ProgressLogWithInfo | undefined> | sdk.ProgressLogWithInfo | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLogWithInfo(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) {\n const log = this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: false,\n };\n }\n\n try {\n const log = this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: true,\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(\n `NOT_FOUND in logs stream driver while getting a progress log with info: ${e}`,\n );\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns an Id of a smart object, that can read logs directly from\n * the platform. */\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle | undefined>;\n getLogHandle(res: PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle | undefined;\n getLogHandle(\n res: PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLogHandle(stream, ctx);\n\n return this.logsStreamDriver.getLogHandle(stream, ctx);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.readText(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.readText(handle, lineCount, offsetBytes, searchStr);\n }\n}\n\nfunction isBlob(rInfo: ResourceInfo) {\n return !rInfo.type.name.startsWith(\"StreamWorkdir\");\n}\n\nfunction streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {\n return ctx.accessor(manager).node().traverse(\"stream\")?.resourceInfo;\n}\n"],"mappings":";;;;AAUA,IAAa,aAAb,MAAkD;CAE7B;CACA;CACA;CAHnB,YACE,QACA,kBACA,gBACA;EAHiB,KAAA,SAAA;EACA,KAAA,mBAAA;EACA,KAAA,iBAAA;CAChB;CAMH,YACE,KACA,OACA,KACqD;EACrD,IAAI,QAAQ,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAExF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,YAAY,QAAQ,OAAO,GAAG;EAE7E,IAAI;GACF,OAAO,KAAK,iBAAiB,YAAY,QAAQ,OAAO,GAAG;EAC7D,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GAAG;IACtB,IAAI,aAAa,4DAA4D,GAAG;IAChF;GACF;GACA,MAAM;EACR;CACF;CAMA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,QAAQ,KAAA,GACV,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,eAAe,QAAQ,iBAAiB,GAAG;EAE1F,IAAI;GACF,OAAO,KAAK,iBAAiB,eAAe,QAAQ,iBAAiB,GAAG;EAC1E,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GAAG;IACtB,IAAI,aAAa,iEAAiE,GAAG;IACrF;GACF;GACA,MAAM;EACR;CACF;CAeA,uBACE,KACA,iBACA,KACuF;EACvF,IAAI,QAAQ,KAAA,GACV,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,uBAAuB,KAAK,iBAAiB,GAAG,CAAC;EAExF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAEf,OAAO;GACL,cAFU,KAAK,eAAe,eAAe,QAAQ,iBAAiB,GAEtD;GAChB,MAAM;EACR;EAGF,IAAI;GAEF,OAAO;IACL,cAFU,KAAK,iBAAiB,eAAe,QAAQ,iBAAiB,GAExD;IAChB,MAAM;GACR;EACF,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GAAG;IACtB,IAAI,aACF,2EAA2E,GAC7E;IACA;GACF;GACA,MAAM;EACR;CACF;CAMA,aACE,KACA,KACyE;EACzE,IAAI,QAAQ,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAElF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,aAAa,QAAQ,GAAG;EAEvE,OAAO,KAAK,iBAAiB,aAAa,QAAQ,GAAG;CACvD;CAEA,MAAM,UACJ,QACA,WACA,aACA,WACmC;EACnC,IAAIC,oBAAAA,gBAAgB,MAAM,GACxB,OAAO,MAAM,KAAK,iBAAiB,UAAU,QAAQ,WAAW,aAAa,SAAS;EACxF,OAAO,MAAM,KAAK,eAAe,UAAU,QAAQ,WAAW,aAAa,SAAS;CACtF;CAEA,MAAM,SACJ,QACA,WACA,aACA,WACmC;EACnC,IAAIA,oBAAAA,gBAAgB,MAAM,GACxB,OAAO,MAAM,KAAK,iBAAiB,SAAS,QAAQ,WAAW,aAAa,SAAS;EACvF,OAAO,MAAM,KAAK,eAAe,SAAS,QAAQ,WAAW,aAAa,SAAS;CACrF;AACF;AAEA,SAAS,OAAO,OAAqB;CACnC,OAAO,CAAC,MAAM,KAAK,KAAK,WAAW,eAAe;AACpD;AAEA,SAAS,uBAAuB,KAAoB,SAAsB;CACxE,OAAO,IAAI,SAAS,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,QAAQ,CAAC,EAAE;AAC1D"}
|
package/dist/drivers/logs.js
CHANGED
|
@@ -3,6 +3,9 @@ import { isNotFoundError } from "@milaboratories/pl-client";
|
|
|
3
3
|
import { Computable } from "@milaboratories/computable";
|
|
4
4
|
//#region src/drivers/logs.ts
|
|
5
5
|
var LogsDriver = class {
|
|
6
|
+
logger;
|
|
7
|
+
logsStreamDriver;
|
|
8
|
+
downloadDriver;
|
|
6
9
|
constructor(logger, logsStreamDriver, downloadDriver) {
|
|
7
10
|
this.logger = logger;
|
|
8
11
|
this.logsStreamDriver = logsStreamDriver;
|
package/dist/drivers/logs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs.js","names":[],"sources":["../../src/drivers/logs.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { LogsStreamDriver } from \"./logs_stream\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"./download_blob/download_blob\";\nimport { isLiveLogHandle } from \"./helpers/logs_handle\";\nimport { isNotFoundError } from \"@milaboratories/pl-client\";\n\nexport class LogsDriver implements sdk.LogsDriver {\n constructor(\n private readonly logger: MiLogger,\n private readonly logsStreamDriver: LogsStreamDriver,\n private readonly downloadDriver: DownloadDriver,\n ) {}\n\n /** Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. */\n getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(res: PlTreeEntry, lines: number, ctx: ComputableCtx): Computable<string | undefined>;\n getLastLogs(\n res: PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLastLogs(stream, lines, ctx);\n\n try {\n return this.logsStreamDriver.getLastLogs(stream, lines, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting last logs: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(res: PlTreeEntry, patternToSearch: string): Computable<string | undefined>;\n getProgressLog(res: PlTreeEntry, patternToSearch: string, ctx: ComputableCtx): string | undefined;\n getProgressLog(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n\n try {\n return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting a progress log: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Same as getProgressLog but also returns a liveliness of the log.\n * The previous getProgressLog couldn't be extended.\n * Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ): Computable<sdk.ProgressLogWithInfo | undefined>;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): sdk.ProgressLogWithInfo | undefined;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<sdk.ProgressLogWithInfo | undefined> | sdk.ProgressLogWithInfo | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLogWithInfo(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) {\n const log = this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: false,\n };\n }\n\n try {\n const log = this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: true,\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(\n `NOT_FOUND in logs stream driver while getting a progress log with info: ${e}`,\n );\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns an Id of a smart object, that can read logs directly from\n * the platform. */\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle | undefined>;\n getLogHandle(res: PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle | undefined;\n getLogHandle(\n res: PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLogHandle(stream, ctx);\n\n return this.logsStreamDriver.getLogHandle(stream, ctx);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.readText(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.readText(handle, lineCount, offsetBytes, searchStr);\n }\n}\n\nfunction isBlob(rInfo: ResourceInfo) {\n return !rInfo.type.name.startsWith(\"StreamWorkdir\");\n}\n\nfunction streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {\n return ctx.accessor(manager).node().traverse(\"stream\")?.resourceInfo;\n}\n"],"mappings":";;;;AAUA,IAAa,aAAb,MAAkD;
|
|
1
|
+
{"version":3,"file":"logs.js","names":[],"sources":["../../src/drivers/logs.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type { LogsStreamDriver } from \"./logs_stream\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadDriver } from \"./download_blob/download_blob\";\nimport { isLiveLogHandle } from \"./helpers/logs_handle\";\nimport { isNotFoundError } from \"@milaboratories/pl-client\";\n\nexport class LogsDriver implements sdk.LogsDriver {\n constructor(\n private readonly logger: MiLogger,\n private readonly logsStreamDriver: LogsStreamDriver,\n private readonly downloadDriver: DownloadDriver,\n ) {}\n\n /** Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. */\n getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(res: PlTreeEntry, lines: number, ctx: ComputableCtx): Computable<string | undefined>;\n getLastLogs(\n res: PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLastLogs(stream, lines, ctx);\n\n try {\n return this.logsStreamDriver.getLastLogs(stream, lines, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting last logs: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(res: PlTreeEntry, patternToSearch: string): Computable<string | undefined>;\n getProgressLog(res: PlTreeEntry, patternToSearch: string, ctx: ComputableCtx): string | undefined;\n getProgressLog(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n\n try {\n return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(`NOT_FOUND in logs stream driver while getting a progress log: ${e}`);\n return undefined;\n }\n throw e;\n }\n }\n\n /** Same as getProgressLog but also returns a liveliness of the log.\n * The previous getProgressLog couldn't be extended.\n * Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ): Computable<sdk.ProgressLogWithInfo | undefined>;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): sdk.ProgressLogWithInfo | undefined;\n getProgressLogWithInfo(\n res: PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<sdk.ProgressLogWithInfo | undefined> | sdk.ProgressLogWithInfo | undefined {\n if (ctx === undefined)\n return Computable.make((ctx) => this.getProgressLogWithInfo(res, patternToSearch, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) {\n const log = this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: false,\n };\n }\n\n try {\n const log = this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);\n return {\n progressLine: log,\n live: true,\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n ctx.markUnstable(\n `NOT_FOUND in logs stream driver while getting a progress log with info: ${e}`,\n );\n return undefined;\n }\n throw e;\n }\n }\n\n /** Returns an Id of a smart object, that can read logs directly from\n * the platform. */\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle | undefined>;\n getLogHandle(res: PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle | undefined;\n getLogHandle(\n res: PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const stream = streamManagerGetStream(ctx, res);\n if (stream === undefined) {\n ctx.markUnstable(\"no stream in stream manager\");\n return undefined;\n }\n\n if (isBlob(stream)) return this.downloadDriver.getLogHandle(stream, ctx);\n\n return this.logsStreamDriver.getLogHandle(stream, ctx);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.lastLines(handle, lineCount, offsetBytes, searchStr);\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<sdk.StreamingApiResponse> {\n if (isLiveLogHandle(handle))\n return await this.logsStreamDriver.readText(handle, lineCount, offsetBytes, searchStr);\n return await this.downloadDriver.readText(handle, lineCount, offsetBytes, searchStr);\n }\n}\n\nfunction isBlob(rInfo: ResourceInfo) {\n return !rInfo.type.name.startsWith(\"StreamWorkdir\");\n}\n\nfunction streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {\n return ctx.accessor(manager).node().traverse(\"stream\")?.resourceInfo;\n}\n"],"mappings":";;;;AAUA,IAAa,aAAb,MAAkD;CAE7B;CACA;CACA;CAHnB,YACE,QACA,kBACA,gBACA;EAHiB,KAAA,SAAA;EACA,KAAA,mBAAA;EACA,KAAA,iBAAA;CAChB;CAMH,YACE,KACA,OACA,KACqD;EACrD,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAExF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,YAAY,QAAQ,OAAO,GAAG;EAE7E,IAAI;GACF,OAAO,KAAK,iBAAiB,YAAY,QAAQ,OAAO,GAAG;EAC7D,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GAAG;IACtB,IAAI,aAAa,4DAA4D,GAAG;IAChF;GACF;GACA,MAAM;EACR;CACF;CAMA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,QAAQ,KAAA,GACV,OAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,eAAe,QAAQ,iBAAiB,GAAG;EAE1F,IAAI;GACF,OAAO,KAAK,iBAAiB,eAAe,QAAQ,iBAAiB,GAAG;EAC1E,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GAAG;IACtB,IAAI,aAAa,iEAAiE,GAAG;IACrF;GACF;GACA,MAAM;EACR;CACF;CAeA,uBACE,KACA,iBACA,KACuF;EACvF,IAAI,QAAQ,KAAA,GACV,OAAO,WAAW,MAAM,QAAQ,KAAK,uBAAuB,KAAK,iBAAiB,GAAG,CAAC;EAExF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAEf,OAAO;GACL,cAFU,KAAK,eAAe,eAAe,QAAQ,iBAAiB,GAEtD;GAChB,MAAM;EACR;EAGF,IAAI;GAEF,OAAO;IACL,cAFU,KAAK,iBAAiB,eAAe,QAAQ,iBAAiB,GAExD;IAChB,MAAM;GACR;EACF,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GAAG;IACtB,IAAI,aACF,2EAA2E,GAC7E;IACA;GACF;GACA,MAAM;EACR;CACF;CAMA,aACE,KACA,KACyE;EACzE,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAElF,MAAM,SAAS,uBAAuB,KAAK,GAAG;EAC9C,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,aAAa,6BAA6B;GAC9C;EACF;EAEA,IAAI,OAAO,MAAM,GAAG,OAAO,KAAK,eAAe,aAAa,QAAQ,GAAG;EAEvE,OAAO,KAAK,iBAAiB,aAAa,QAAQ,GAAG;CACvD;CAEA,MAAM,UACJ,QACA,WACA,aACA,WACmC;EACnC,IAAI,gBAAgB,MAAM,GACxB,OAAO,MAAM,KAAK,iBAAiB,UAAU,QAAQ,WAAW,aAAa,SAAS;EACxF,OAAO,MAAM,KAAK,eAAe,UAAU,QAAQ,WAAW,aAAa,SAAS;CACtF;CAEA,MAAM,SACJ,QACA,WACA,aACA,WACmC;EACnC,IAAI,gBAAgB,MAAM,GACxB,OAAO,MAAM,KAAK,iBAAiB,SAAS,QAAQ,WAAW,aAAa,SAAS;EACvF,OAAO,MAAM,KAAK,eAAe,SAAS,QAAQ,WAAW,aAAa,SAAS;CACrF;AACF;AAEA,SAAS,OAAO,OAAqB;CACnC,OAAO,CAAC,MAAM,KAAK,KAAK,WAAW,eAAe;AACpD;AAEA,SAAS,uBAAuB,KAAoB,SAAsB;CACxE,OAAO,IAAI,SAAS,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,QAAQ,CAAC,EAAE;AAC1D"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require("../_virtual/_rolldown/runtime.cjs");
|
|
2
1
|
const require_helpers = require("./helpers/helpers.cjs");
|
|
3
2
|
const require_logs_handle = require("./helpers/logs_handle.cjs");
|
|
4
3
|
let _milaboratories_pl_client = require("@milaboratories/pl-client");
|
|
@@ -9,6 +8,9 @@ let node_crypto = require("node:crypto");
|
|
|
9
8
|
let node_timers_promises = require("node:timers/promises");
|
|
10
9
|
//#region src/drivers/logs_stream.ts
|
|
11
10
|
var LogsStreamDriver = class {
|
|
11
|
+
logger;
|
|
12
|
+
clientLogs;
|
|
13
|
+
opts;
|
|
12
14
|
/** Holds a map of StreamManager Resource Id to all logs of this stream. */
|
|
13
15
|
idToLastLines = /* @__PURE__ */ new Map();
|
|
14
16
|
/** Holds a map of StreamManager Resource Id to the last log line of this stream. */
|
|
@@ -162,6 +164,11 @@ var LogsStreamDriver = class {
|
|
|
162
164
|
};
|
|
163
165
|
/** A job that gets last lines from a StreamWorkdir resource. */
|
|
164
166
|
var LogGetter = class {
|
|
167
|
+
logger;
|
|
168
|
+
clientLogs;
|
|
169
|
+
rInfo;
|
|
170
|
+
lines;
|
|
171
|
+
patternToSearch;
|
|
165
172
|
logs;
|
|
166
173
|
error = void 0;
|
|
167
174
|
change = new _milaboratories_computable.ChangeSource();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs_stream.cjs","names":["PollingComputableHooks","Computable","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","scheduler","ChangeSource","CallersCounter","WrongResourceTypeError"],"sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientLogs } from \"../clients/logs\";\nimport { randomUUID } from \"node:crypto\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { treeEntryToResourceInfo } from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { StreamingAPI_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from \"./helpers/logs_handle\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states */\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n private readonly idToLastLines: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n private readonly idToProgressLog: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to log id smart object. */\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n \"The logs are from stream, so we consider them unstable. Final values will be got from blobs.\",\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getLastLogs\", rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ): Computable<string | undefined>;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string | undefined;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n \"The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.\",\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getProgressLog\", rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo | PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType(\"getLogHandle\", rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /** Called from observer */\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /** Called from observer */\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /** Stops polling loop and waits for it to finish */\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /** If true, main loop will continue polling pl state. */\n private keepRunning = false;\n /** Actual state of main loop. */\n private currentLoop: Promise<void> | undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string | undefined;\n private error: any | undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string | undefined;\n error?: any | undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs)\n this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.logs = \"\";\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"StreamWorkdir\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA0BA,IAAa,mBAAb,MAAwD;;CAEtD,gCAAmE,IAAI,KAAK;;CAG5E,kCAAqE,IAAI,KAAK;;CAG9E;CAEA,YACE,QACA,YACA,OAA6C;EAC3C,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB,EACD;AAPiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,OAAA;AAMjB,OAAK,QAAQ,IAAIA,2BAAAA,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CASH,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,gBAAgB,EAAE,IAAI,SAAS,CAAC;EAE5D,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAG,OAAO,SAAS;AACrE,MAAI,aACF,+FACD;AAED,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,uBAAqB,eAAe,MAAM,KAAK;EAE/C,IAAI,YAAY,KAAK,cAAc,IAAI,MAAM,GAAG;AAEhD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,MAAM;AAC9E,QAAK,cAAc,IAAI,MAAM,IAAI,aAAa;AAE9C,eAAY;;AAGd,YAAU,OAAO,GAAG,SAAS;EAC7B,MAAM,SAAS,UAAU,QAAQ;AACjC,MAAI,OAAO,SAAS,KAAA,EAAW,OAAM,OAAO;AAE5C,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,mBAAmB,EAAE,IAAI,SAAS,CAAC;EAE/D,MAAM,SAAS,KAAK,oBAAoB,IAAI,SAAS,GAAG,iBAAiB,SAAS;AAClF,MAAI,aACF,uGACD;AAED,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,uBAAqB,kBAAkB,MAAM,KAAK;EAElD,IAAI,YAAY,KAAK,gBAAgB,IAAI,MAAM,GAAG;AAElD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,GAAG,gBAAgB;AAC3F,QAAK,gBAAgB,IAAI,MAAM,IAAI,aAAa;AAEhD,eAAY;;AAGd,YAAU,OAAO,GAAG,SAAS;EAC7B,MAAM,SAAS,UAAU,QAAQ;AACjC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAKhB,aACE,KACA,KACiD;AACjD,MAAI,OAAO,KAAA,EAAW,QAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAE3C,MAAM,SAAS,KAAK,kBAAkB,EAAE;AAIxC,MAAI,aAAa,aAAA,GAAA,0BAAA,oBAA+B,EAAE,GAAG,GAAG;AAExD,SAAO;;CAGT,kBAA0B,OAAuC;AAC/D,uBAAqB,gBAAgB,MAAM,KAAK;AAEhD,SAAOC,oBAAAA,aAAa,MAAM,MAAM;;CAGlC,MAAM,UACJ,QACA,WACA,aACA,WACA;AACA,SAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,UACdC,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD,CACF;;CAGH,MAAM,SACJ,QACA,WACA,aACA,WACA;AACA,SAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,SACdA,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD,CACF;;CAGH,MAAc,gBACZ,QACA,QACmC;AACnC,MAAI,CAACC,oBAAAA,gBAAgB,OAAO,CAC1B,OAAM,IAAI,MAAM,8DAA8D,SAAS;AAEzF,MAAI;GACF,MAAM,OAAO,MAAM,QAAQ;AAC3B,UAAO;IACL,MAAM;IACN,oBAAoB;IACpB,MAAM,KAAK;IACX,MAAM,OAAO,KAAK,KAAK;IACvB,WAAW,OAAO,KAAK,UAAU;IAClC;WACM,GAAQ;AACf,QAAA,GAAA,0BAAA,iBAAoB,EAAE,CACpB,QAAO,EAAE,oBAAoB,MAAM;AAGrC,SAAM;;;CAIV,MAAc,gBAAgB,KAAuB,UAAkB;AAErE,MADgB,KAAK,cAAc,IAAI,IAAI,EAAE,QAAQ,SAAS,CACjD,MAAK,cAAc,OAAO,IAAI;;CAG7C,MAAc,mBAAmB,KAAuB,UAAkB;AAExE,MADgB,KAAK,gBAAgB,IAAI,IAAI,EAAE,QAAQ,SAAS,CACnD,MAAK,gBAAgB,OAAO,IAAI;;CAG/C,MAAM,aAAa;CAEnB,uBAAmD,EAAE;CAErD,oBAA4B,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,gBAA8B;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,eAA6B;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;IACF,MAAM,OAAO,KAAK,YAAY;AAC9B,WAAA,GAAA,2BAAA,WACE,KAAK,KAAK,oBACV,KAAK,KAAK,WAAW,YAAY,MAAM,OAAO,QAAQ,CAAC,CACxD;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAMC,qBAAAA,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc,KAAA;;CAGrB,aAAuC;AACrC,SAAO,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAC5C,OAAO,MAAM,KAAK,KAAK,gBAAgB,SAAS,CAAC,CAAC,CAClD,KAAK,CAAC,GAAG,YAAY,OAAO;;;;AAKnC,IAAM,YAAN,MAAgB;CACd;CACA,QAAiC,KAAA;CAEjC,SAAwC,IAAIC,2BAAAA,cAAc;CAC1D,UAA2C,IAAIC,2BAAAA,gBAAgB;CAE/D,YACE,QACA,YACA,OACA,OACA,iBACA;AALiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,QAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;;CAGnB,SAGE;AACA,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAQ,UAA2B;AACjC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,MAAM,SAAS;AACb,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,KAAK,OACL,KAAK,OACL,IACA,KAAK,gBACN;GAED,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,KAAK,KAAK;AACnD,OAAI,KAAK,QAAQ,QACf,MAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,oBAA+B,KAAK,MAAM,GAAG,CAAC,UAAU;AAClF,QAAK,OAAO;AACZ,QAAK,QAAQ,KAAA;AAEb;WACO,GAAQ;AACf,QAAA,GAAA,0BAAA,iBAAoB,EAAE,EAAE;AAEtB,SAAK,OAAO;AACZ,SAAK,QAAQ;AACb,SAAK,OAAO,aAAa;AACzB;;AAGF,QAAK,OAAO,MACV,yBAAA,GAAA,0BAAA,yBAAgD,KAAK,MAAM,GAAG,CAAC,mBAAmB,KAAK,UAAU,EAAE,GACpG;AACD,SAAM;;;;AAUZ,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,gBAAgB,CACzC,OAAM,IAAIC,gBAAAA,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,iDAEnD"}
|
|
1
|
+
{"version":3,"file":"logs_stream.cjs","names":["PollingComputableHooks","Computable","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","scheduler","ChangeSource","CallersCounter","WrongResourceTypeError"],"sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientLogs } from \"../clients/logs\";\nimport { randomUUID } from \"node:crypto\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { treeEntryToResourceInfo } from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { StreamingAPI_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from \"./helpers/logs_handle\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states */\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n private readonly idToLastLines: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n private readonly idToProgressLog: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to log id smart object. */\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n \"The logs are from stream, so we consider them unstable. Final values will be got from blobs.\",\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getLastLogs\", rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ): Computable<string | undefined>;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string | undefined;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n \"The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.\",\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getProgressLog\", rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo | PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType(\"getLogHandle\", rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /** Called from observer */\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /** Called from observer */\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /** Stops polling loop and waits for it to finish */\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /** If true, main loop will continue polling pl state. */\n private keepRunning = false;\n /** Actual state of main loop. */\n private currentLoop: Promise<void> | undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string | undefined;\n private error: any | undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string | undefined;\n error?: any | undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs)\n this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.logs = \"\";\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"StreamWorkdir\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;AA0BA,IAAa,mBAAb,MAAwD;CAWnC;CACA;CACA;;CAXnB,gCAAmE,IAAI,IAAI;;CAG3E,kCAAqE,IAAI,IAAI;;CAG7E;CAEA,YACE,QACA,YACA,OAA6C;EAC3C,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;CACpB,GACA;EAPiB,KAAA,SAAA;EACA,KAAA,aAAA;EACA,KAAA,OAAA;EAMjB,KAAK,QAAQ,IAAIA,2BAAAA,6BACT,KAAK,cAAc,SACnB,KAAK,aAAa,GACxB,EAAE,cAAc,KAAK,iBAAiB,IACrC,SAAS,WAAW,KAAK,oBAAoB,SAAS,MAAM,CAC/D;CACF;CAQA,YACE,KACA,OACA,KACqD;EACrD,IAAI,OAAO,KAAA,GAAW,OAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAEvF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAC1C,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,aAAa,KAAK,KAAK;EAC3B,IAAI,mBAAmB,KAAK,gBAAgB,EAAE,IAAI,QAAQ,CAAC;EAE3D,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAG,OAAO,QAAQ;EACpE,IAAI,aACF,8FACF;EAEA,OAAO;CACT;CAEA,iBACE,GACA,OACA,OACA,UACoB;EACpB,qBAAqB,eAAe,MAAM,IAAI;EAE9C,IAAI,YAAY,KAAK,cAAc,IAAI,MAAM,EAAE;EAE/C,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,KAAK;GAC7E,KAAK,cAAc,IAAI,MAAM,IAAI,YAAY;GAE7C,YAAY;EACd;EAEA,UAAU,OAAO,GAAG,QAAQ;EAC5B,MAAM,SAAS,UAAU,OAAO;EAChC,IAAI,OAAO,SAAS,KAAA,GAAW,MAAM,OAAO;EAE5C,OAAO,OAAO;CAChB;CAaA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,OAAO,KAAA,GACT,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAC1C,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,aAAa,KAAK,KAAK;EAC3B,IAAI,mBAAmB,KAAK,mBAAmB,EAAE,IAAI,QAAQ,CAAC;EAE9D,MAAM,SAAS,KAAK,oBAAoB,IAAI,SAAS,GAAG,iBAAiB,QAAQ;EACjF,IAAI,aACF,sGACF;EAEA,OAAO;CACT;CAEA,oBACE,GACA,OACA,iBACA,UACoB;EACpB,qBAAqB,kBAAkB,MAAM,IAAI;EAEjD,IAAI,YAAY,KAAK,gBAAgB,IAAI,MAAM,EAAE;EAEjD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,GAAG,eAAe;GAC1F,KAAK,gBAAgB,IAAI,MAAM,IAAI,YAAY;GAE/C,YAAY;EACd;EAEA,UAAU,OAAO,GAAG,QAAQ;EAC5B,MAAM,SAAS,UAAU,OAAO;EAChC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAIA,aACE,KACA,KACiD;EACjD,IAAI,OAAO,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAE1C,MAAM,SAAS,KAAK,kBAAkB,CAAC;EAIvC,IAAI,aAAa,aAAA,GAAA,0BAAA,mBAAA,CAA+B,EAAE,EAAE,GAAG;EAEvD,OAAO;CACT;CAEA,kBAA0B,OAAuC;EAC/D,qBAAqB,gBAAgB,MAAM,IAAI;EAE/C,OAAOC,oBAAAA,aAAa,MAAM,KAAK;CACjC;CAEA,MAAM,UACJ,QACA,WACA,aACA,WACA;EACA,OAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,UACdC,oBAAAA,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF,CACF;CACF;CAEA,MAAM,SACJ,QACA,WACA,aACA,WACA;EACA,OAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,SACdA,oBAAAA,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF,CACF;CACF;CAEA,MAAc,gBACZ,QACA,QACmC;EACnC,IAAI,CAACC,oBAAAA,gBAAgB,MAAM,GACzB,MAAM,IAAI,MAAM,8DAA8D,QAAQ;EAExF,IAAI;GACF,MAAM,OAAO,MAAM,OAAO;GAC1B,OAAO;IACL,MAAM;IACN,oBAAoB;IACpB,MAAM,KAAK;IACX,MAAM,OAAO,KAAK,IAAI;IACtB,WAAW,OAAO,KAAK,SAAS;GAClC;EACF,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GACnB,OAAO,EAAE,oBAAoB,KAAK;GAGpC,MAAM;EACR;CACF;CAEA,MAAc,gBAAgB,KAAuB,UAAkB;EAErE,IADgB,KAAK,cAAc,IAAI,GAAG,CAAC,EAAE,QAAQ,QAAQ,GAChD,KAAK,cAAc,OAAO,GAAG;CAC5C;CAEA,MAAc,mBAAmB,KAAuB,UAAkB;EAExE,IADgB,KAAK,gBAAgB,IAAI,GAAG,CAAC,EAAE,QAAQ,QAAQ,GAClD,KAAK,gBAAgB,OAAO,GAAG;CAC9C;CAEA,MAAM,aAAa,CAAC;CAEpB,uBAAmD,CAAC;CAEpD,oBAA4B,SAAqB,QAAkC;EACjF,KAAK,qBAAqB,KAAK;GAAE;GAAS;EAAO,CAAC;CACpD;;CAGA,gBAA8B;EAC5B,KAAK,cAAc;EACnB,IAAI,KAAK,gBAAgB,KAAA,GAAW,KAAK,cAAc,KAAK,SAAS;CACvE;;CAGA,eAA6B;EAC3B,KAAK,cAAc;CACrB;;CAGA,MAAa,YAA2B;EACtC,KAAK,aAAa;EAClB,IAAI,KAAK,gBAAgB,KAAA,GACvB,MAAM,KAAK;CAEf;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,UAAU;CACvB;;CAGA,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;EACvB,OAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;GACtB,KAAK,uBAAuB,CAAC;GAE7B,IAAI;IACF,MAAM,OAAO,KAAK,WAAW;IAC7B,OAAA,GAAA,2BAAA,UAAA,CACE,KAAK,KAAK,oBACV,KAAK,KAAK,WAAW,YAAY,MAAM,OAAO,OAAO,CAAC,CACxD;IAEA,SAAS,SAAS,MAAM,EAAE,QAAQ,CAAC;GACrC,SAAS,GAAQ;IACf,QAAQ,MAAM,CAAC;IACf,SAAS,SAAS,MAAM,EAAE,OAAO,CAAC,CAAC;GACrC;GAEA,IAAI,CAAC,KAAK,aAAa;GACvB,MAAMC,qBAAAA,UAAU,KAAK,KAAK,KAAK,eAAe;EAChD;EAEA,KAAK,cAAc,KAAA;CACrB;CAEA,aAAuC;EACrC,OAAO,MAAM,KAAK,KAAK,cAAc,QAAQ,CAAC,CAAC,CAC5C,OAAO,MAAM,KAAK,KAAK,gBAAgB,QAAQ,CAAC,CAAC,CAAC,CAClD,KAAK,CAAC,GAAG,YAAY,MAAM;CAChC;AACF;;AAGA,IAAM,YAAN,MAAgB;CAQK;CACA;CACA;CACA;CACA;CAXnB;CACA,QAAiC,KAAA;CAEjC,SAAwC,IAAIC,2BAAAA,aAAa;CACzD,UAA2C,IAAIC,2BAAAA,eAAe;CAE9D,YACE,QACA,YACA,OACA,OACA,iBACA;EALiB,KAAA,SAAA;EACA,KAAA,aAAA;EACA,KAAA,QAAA;EACA,KAAA,QAAA;EACA,KAAA,kBAAA;CAChB;CAEH,SAGE;EACA,OAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,OAAO,cAAc,CAAC;EAC3B,KAAK,QAAQ,IAAI,QAAQ;CAC3B;CAEA,QAAQ,UAA2B;EACjC,OAAO,KAAK,QAAQ,IAAI,QAAQ;CAClC;CAEA,MAAM,SAAS;EACb,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,KAAK,OACL,KAAK,OACL,IACA,KAAK,eACP;GAEA,MAAM,UAAU,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK,IAAI;GAClD,IAAI,KAAK,QAAQ,SACf,KAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,mBAAA,CAA+B,KAAK,MAAM,EAAE,EAAE,SAAS;GACjF,KAAK,OAAO;GACZ,KAAK,QAAQ,KAAA;GAEb;EACF,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GAAG;IAEtB,KAAK,OAAO;IACZ,KAAK,QAAQ;IACb,KAAK,OAAO,YAAY;IACxB;GACF;GAEA,KAAK,OAAO,MACV,yBAAA,GAAA,0BAAA,wBAAA,CAAgD,KAAK,MAAM,EAAE,EAAE,mBAAmB,KAAK,UAAU,CAAC,GACpG;GACA,MAAM;EACR;CACF;AACF;AAOA,SAAS,qBAAqB,YAAoB,OAAqB;CACrE,IAAI,CAAC,MAAM,KAAK,WAAW,eAAe,GACxC,MAAM,IAAIC,gBAAAA,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,gDAEpD;AAEJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs_stream.d.ts","names":[],"sources":["../../src/drivers/logs_stream.ts"],"mappings":";;;;;;;;KAqBY,mBAAA,GAAsB,
|
|
1
|
+
{"version":3,"file":"logs_stream.d.ts","names":[],"sources":["../../src/drivers/logs_stream.ts"],"mappings":";;;;;;;;KAqBY,mBAAA,GAAsB,UAAU;EAAhC,qGAEV,kBAAkB;AAAA;AAAA,cAGP,gBAAA,YAA4B,GAAA,CAAI,UAAA;EAAA,iBAWxB,MAAA;EAAA,iBACA,UAAA;EAAA,iBACA,IAAA;;mBAXF,aAAA;EAUc;EAAA,iBAPd,eAAA;EAsBA;EAAA,iBAnBA,KAAA;cAGE,MAAA,EAAQ,QAAA,EACR,UAAA,EAAY,UAAA,EACZ,IAAA,GAAM,mBAAA;EAczB,WAAA,CAAY,GAAA,EAAK,YAAA,GAAe,WAAA,EAAa,KAAA,WAAgB,UAAA;EAC7D,WAAA,CACE,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,UACA,GAAA,EAAK,aAAA,GACJ,UAAA;EAAA,QAqBK,gBAAA;EArBL;;EA+CH,cAAA,CACE,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,WACC,UAAA;EACH,cAAA,CACE,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,UACA,GAAA,EAAK,aAAA;EAAA,QAuBC,mBAAA;EAwBR,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,GAAc,UAAA,CAAW,GAAA,CAAI,YAAA;EAC9D,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,EAAa,GAAA,EAAK,aAAA,GAAgB,GAAA,CAAI,YAAA;EAAA,QAkB/D,iBAAA;EAMF,SAAA,CACJ,MAAA,EAAQ,GAAA,CAAI,YAAA,EACZ,SAAA,UACA,WAAA,WACA,SAAA,wBAA8B,OAAA,CAAA,GAAA,CAAA,oBAAA;EAY1B,QAAA,CACJ,MAAA,EAAQ,GAAA,CAAI,YAAA,EACZ,SAAA,UACA,WAAA,WACA,SAAA,wBAA8B,OAAA,CAAA,GAAA,CAAA,oBAAA;EAAA,QAYlB,eAAA;EAAA,QAyBA,eAAA;EAAA,QAKA,kBAAA;EAKR,UAAA,IAAU,OAAA;EAAA,QAER,oBAAA;EAAA,QAEA,mBAAA;EAnEwB;EAAA,QAwExB,aAAA;EA3DE;EAAA,QAiEF,YAAA;EA9DwB;EAmEnB,SAAA,IAAa,OAAA;EAAA,CAOnB,MAAA,CAAO,YAAA,KAAiB,OAAA;EAAA;EAAA,QAKvB,WAAA;EAjQ+B;EAAA,QAmQ/B,WAAA;EAAA,QAEM,QAAA;EAAA,QAyBN,UAAA;AAAA"}
|
|
@@ -8,6 +8,9 @@ import { randomUUID } from "node:crypto";
|
|
|
8
8
|
import { scheduler } from "node:timers/promises";
|
|
9
9
|
//#region src/drivers/logs_stream.ts
|
|
10
10
|
var LogsStreamDriver = class {
|
|
11
|
+
logger;
|
|
12
|
+
clientLogs;
|
|
13
|
+
opts;
|
|
11
14
|
/** Holds a map of StreamManager Resource Id to all logs of this stream. */
|
|
12
15
|
idToLastLines = /* @__PURE__ */ new Map();
|
|
13
16
|
/** Holds a map of StreamManager Resource Id to the last log line of this stream. */
|
|
@@ -161,6 +164,11 @@ var LogsStreamDriver = class {
|
|
|
161
164
|
};
|
|
162
165
|
/** A job that gets last lines from a StreamWorkdir resource. */
|
|
163
166
|
var LogGetter = class {
|
|
167
|
+
logger;
|
|
168
|
+
clientLogs;
|
|
169
|
+
rInfo;
|
|
170
|
+
lines;
|
|
171
|
+
patternToSearch;
|
|
164
172
|
logs;
|
|
165
173
|
error = void 0;
|
|
166
174
|
change = new ChangeSource();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs_stream.js","names":[],"sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientLogs } from \"../clients/logs\";\nimport { randomUUID } from \"node:crypto\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { treeEntryToResourceInfo } from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { StreamingAPI_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from \"./helpers/logs_handle\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states */\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n private readonly idToLastLines: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n private readonly idToProgressLog: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to log id smart object. */\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n \"The logs are from stream, so we consider them unstable. Final values will be got from blobs.\",\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getLastLogs\", rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ): Computable<string | undefined>;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string | undefined;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n \"The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.\",\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getProgressLog\", rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo | PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType(\"getLogHandle\", rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /** Called from observer */\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /** Called from observer */\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /** Stops polling loop and waits for it to finish */\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /** If true, main loop will continue polling pl state. */\n private keepRunning = false;\n /** Actual state of main loop. */\n private currentLoop: Promise<void> | undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string | undefined;\n private error: any | undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string | undefined;\n error?: any | undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs)\n this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.logs = \"\";\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"StreamWorkdir\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;AA0BA,IAAa,mBAAb,MAAwD;;CAEtD,gCAAmE,IAAI,KAAK;;CAG5E,kCAAqE,IAAI,KAAK;;CAG9E;CAEA,YACE,QACA,YACA,OAA6C;EAC3C,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;EACnB,EACD;AAPiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,OAAA;AAMjB,OAAK,QAAQ,IAAI,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CASH,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,gBAAgB,EAAE,IAAI,SAAS,CAAC;EAE5D,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAG,OAAO,SAAS;AACrE,MAAI,aACF,+FACD;AAED,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,uBAAqB,eAAe,MAAM,KAAK;EAE/C,IAAI,YAAY,KAAK,cAAc,IAAI,MAAM,GAAG;AAEhD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,MAAM;AAC9E,QAAK,cAAc,IAAI,MAAM,IAAI,aAAa;AAE9C,eAAY;;AAGd,YAAU,OAAO,GAAG,SAAS;EAC7B,MAAM,SAAS,UAAU,QAAQ;AACjC,MAAI,OAAO,SAAS,KAAA,EAAW,OAAM,OAAO;AAE5C,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,mBAAmB,EAAE,IAAI,SAAS,CAAC;EAE/D,MAAM,SAAS,KAAK,oBAAoB,IAAI,SAAS,GAAG,iBAAiB,SAAS;AAClF,MAAI,aACF,uGACD;AAED,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,uBAAqB,kBAAkB,MAAM,KAAK;EAElD,IAAI,YAAY,KAAK,gBAAgB,IAAI,MAAM,GAAG;AAElD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,GAAG,gBAAgB;AAC3F,QAAK,gBAAgB,IAAI,MAAM,IAAI,aAAa;AAEhD,eAAY;;AAGd,YAAU,OAAO,GAAG,SAAS;EAC7B,MAAM,SAAS,UAAU,QAAQ;AACjC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAKhB,aACE,KACA,KACiD;AACjD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAE3C,MAAM,SAAS,KAAK,kBAAkB,EAAE;AAIxC,MAAI,aAAa,YAAY,mBAAmB,EAAE,GAAG,GAAG;AAExD,SAAO;;CAGT,kBAA0B,OAAuC;AAC/D,uBAAqB,gBAAgB,MAAM,KAAK;AAEhD,SAAO,aAAa,MAAM,MAAM;;CAGlC,MAAM,UACJ,QACA,WACA,aACA,WACA;AACA,SAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,UACd,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD,CACF;;CAGH,MAAM,SACJ,QACA,WACA,aACA,WACA;AACA,SAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,SACd,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD,CACF;;CAGH,MAAc,gBACZ,QACA,QACmC;AACnC,MAAI,CAAC,gBAAgB,OAAO,CAC1B,OAAM,IAAI,MAAM,8DAA8D,SAAS;AAEzF,MAAI;GACF,MAAM,OAAO,MAAM,QAAQ;AAC3B,UAAO;IACL,MAAM;IACN,oBAAoB;IACpB,MAAM,KAAK;IACX,MAAM,OAAO,KAAK,KAAK;IACvB,WAAW,OAAO,KAAK,UAAU;IAClC;WACM,GAAQ;AACf,OAAI,gBAAgB,EAAE,CACpB,QAAO,EAAE,oBAAoB,MAAM;AAGrC,SAAM;;;CAIV,MAAc,gBAAgB,KAAuB,UAAkB;AAErE,MADgB,KAAK,cAAc,IAAI,IAAI,EAAE,QAAQ,SAAS,CACjD,MAAK,cAAc,OAAO,IAAI;;CAG7C,MAAc,mBAAmB,KAAuB,UAAkB;AAExE,MADgB,KAAK,gBAAgB,IAAI,IAAI,EAAE,QAAQ,SAAS,CACnD,MAAK,gBAAgB,OAAO,IAAI;;CAG/C,MAAM,aAAa;CAEnB,uBAAmD,EAAE;CAErD,oBAA4B,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,gBAA8B;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,eAA6B;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;IACF,MAAM,OAAO,KAAK,YAAY;AAC9B,UAAM,UACJ,KAAK,KAAK,oBACV,KAAK,KAAK,WAAW,YAAY,MAAM,OAAO,QAAQ,CAAC,CACxD;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAM,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc,KAAA;;CAGrB,aAAuC;AACrC,SAAO,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAC5C,OAAO,MAAM,KAAK,KAAK,gBAAgB,SAAS,CAAC,CAAC,CAClD,KAAK,CAAC,GAAG,YAAY,OAAO;;;;AAKnC,IAAM,YAAN,MAAgB;CACd;CACA,QAAiC,KAAA;CAEjC,SAAwC,IAAI,cAAc;CAC1D,UAA2C,IAAI,gBAAgB;CAE/D,YACE,QACA,YACA,OACA,OACA,iBACA;AALiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,QAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;;CAGnB,SAGE;AACA,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAQ,UAA2B;AACjC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,MAAM,SAAS;AACb,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,KAAK,OACL,KAAK,OACL,IACA,KAAK,gBACN;GAED,MAAM,UAAU,IAAI,aAAa,CAAC,OAAO,KAAK,KAAK;AACnD,OAAI,KAAK,QAAQ,QACf,MAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,UAAU;AAClF,QAAK,OAAO;AACZ,QAAK,QAAQ,KAAA;AAEb;WACO,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,OAAO;AACZ,SAAK,QAAQ;AACb,SAAK,OAAO,aAAa;AACzB;;AAGF,QAAK,OAAO,MACV,wBAAwB,wBAAwB,KAAK,MAAM,GAAG,CAAC,mBAAmB,KAAK,UAAU,EAAE,GACpG;AACD,SAAM;;;;AAUZ,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,gBAAgB,CACzC,OAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,iDAEnD"}
|
|
1
|
+
{"version":3,"file":"logs_stream.js","names":[],"sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientLogs } from \"../clients/logs\";\nimport { randomUUID } from \"node:crypto\";\nimport type { PlTreeEntry, ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { treeEntryToResourceInfo } from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { StreamingAPI_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from \"./helpers/logs_handle\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states */\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n private readonly idToLastLines: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n private readonly idToProgressLog: Map<SignedResourceId, LogGetter> = new Map();\n\n /** Holds a map of StreamManager Resource Id to log id smart object. */\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string | undefined>;\n getLastLogs(\n res: ResourceInfo | PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n \"The logs are from stream, so we consider them unstable. Final values will be got from blobs.\",\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getLastLogs\", rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /** Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. */\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ): Computable<string | undefined>;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string | undefined;\n getProgressLog(\n res: ResourceInfo | PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string | undefined> | string | undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n \"The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.\",\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string | undefined {\n validateResourceType(\"getProgressLog\", rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo | PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType(\"getLogHandle\", rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string | undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (isNotFoundError(e)) {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: SignedResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /** Called from observer */\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /** Called from observer */\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /** Stops polling loop and waits for it to finish */\n public async terminate(): Promise<void> {\n this.stopUpdating();\n if (this.currentLoop !== undefined) {\n await this.currentLoop;\n }\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.terminate();\n }\n\n /** If true, main loop will continue polling pl state. */\n private keepRunning = false;\n /** Actual state of main loop. */\n private currentLoop: Promise<void> | undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string | undefined;\n private error: any | undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string | undefined;\n error?: any | undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs)\n this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.logs = \"\";\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"StreamWorkdir\")) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, ` +\n `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"mappings":";;;;;;;;;AA0BA,IAAa,mBAAb,MAAwD;CAWnC;CACA;CACA;;CAXnB,gCAAmE,IAAI,IAAI;;CAG3E,kCAAqE,IAAI,IAAI;;CAG7E;CAEA,YACE,QACA,YACA,OAA6C;EAC3C,oBAAoB;EACpB,iBAAiB;EACjB,kBAAkB;CACpB,GACA;EAPiB,KAAA,SAAA;EACA,KAAA,aAAA;EACA,KAAA,OAAA;EAMjB,KAAK,QAAQ,IAAI,6BACT,KAAK,cAAc,SACnB,KAAK,aAAa,GACxB,EAAE,cAAc,KAAK,iBAAiB,IACrC,SAAS,WAAW,KAAK,oBAAoB,SAAS,MAAM,CAC/D;CACF;CAQA,YACE,KACA,OACA,KACqD;EACrD,IAAI,OAAO,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAEvF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAC1C,MAAM,WAAW,WAAW;EAC5B,IAAI,aAAa,KAAK,KAAK;EAC3B,IAAI,mBAAmB,KAAK,gBAAgB,EAAE,IAAI,QAAQ,CAAC;EAE3D,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAG,OAAO,QAAQ;EACpE,IAAI,aACF,8FACF;EAEA,OAAO;CACT;CAEA,iBACE,GACA,OACA,OACA,UACoB;EACpB,qBAAqB,eAAe,MAAM,IAAI;EAE9C,IAAI,YAAY,KAAK,cAAc,IAAI,MAAM,EAAE;EAE/C,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,KAAK;GAC7E,KAAK,cAAc,IAAI,MAAM,IAAI,YAAY;GAE7C,YAAY;EACd;EAEA,UAAU,OAAO,GAAG,QAAQ;EAC5B,MAAM,SAAS,UAAU,OAAO;EAChC,IAAI,OAAO,SAAS,KAAA,GAAW,MAAM,OAAO;EAE5C,OAAO,OAAO;CAChB;CAaA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,OAAO,KAAA,GACT,OAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAC1C,MAAM,WAAW,WAAW;EAC5B,IAAI,aAAa,KAAK,KAAK;EAC3B,IAAI,mBAAmB,KAAK,mBAAmB,EAAE,IAAI,QAAQ,CAAC;EAE9D,MAAM,SAAS,KAAK,oBAAoB,IAAI,SAAS,GAAG,iBAAiB,QAAQ;EACjF,IAAI,aACF,sGACF;EAEA,OAAO;CACT;CAEA,oBACE,GACA,OACA,iBACA,UACoB;EACpB,qBAAqB,kBAAkB,MAAM,IAAI;EAEjD,IAAI,YAAY,KAAK,gBAAgB,IAAI,MAAM,EAAE;EAEjD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,UAAU,KAAK,QAAQ,KAAK,YAAY,OAAO,GAAG,eAAe;GAC1F,KAAK,gBAAgB,IAAI,MAAM,IAAI,YAAY;GAE/C,YAAY;EACd;EAEA,UAAU,OAAO,GAAG,QAAQ;EAC5B,MAAM,SAAS,UAAU,OAAO;EAChC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAIA,aACE,KACA,KACiD;EACjD,IAAI,OAAO,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAE1C,MAAM,SAAS,KAAK,kBAAkB,CAAC;EAIvC,IAAI,aAAa,YAAY,mBAAmB,EAAE,EAAE,GAAG;EAEvD,OAAO;CACT;CAEA,kBAA0B,OAAuC;EAC/D,qBAAqB,gBAAgB,MAAM,IAAI;EAE/C,OAAO,aAAa,MAAM,KAAK;CACjC;CAEA,MAAM,UACJ,QACA,WACA,aACA,WACA;EACA,OAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,UACd,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF,CACF;CACF;CAEA,MAAM,SACJ,QACA,WACA,aACA,WACA;EACA,OAAO,MAAM,KAAK,gBAAgB,cAChC,KAAK,WAAW,SACd,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF,CACF;CACF;CAEA,MAAc,gBACZ,QACA,QACmC;EACnC,IAAI,CAAC,gBAAgB,MAAM,GACzB,MAAM,IAAI,MAAM,8DAA8D,QAAQ;EAExF,IAAI;GACF,MAAM,OAAO,MAAM,OAAO;GAC1B,OAAO;IACL,MAAM;IACN,oBAAoB;IACpB,MAAM,KAAK;IACX,MAAM,OAAO,KAAK,IAAI;IACtB,WAAW,OAAO,KAAK,SAAS;GAClC;EACF,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GACnB,OAAO,EAAE,oBAAoB,KAAK;GAGpC,MAAM;EACR;CACF;CAEA,MAAc,gBAAgB,KAAuB,UAAkB;EAErE,IADgB,KAAK,cAAc,IAAI,GAAG,CAAC,EAAE,QAAQ,QAAQ,GAChD,KAAK,cAAc,OAAO,GAAG;CAC5C;CAEA,MAAc,mBAAmB,KAAuB,UAAkB;EAExE,IADgB,KAAK,gBAAgB,IAAI,GAAG,CAAC,EAAE,QAAQ,QAAQ,GAClD,KAAK,gBAAgB,OAAO,GAAG;CAC9C;CAEA,MAAM,aAAa,CAAC;CAEpB,uBAAmD,CAAC;CAEpD,oBAA4B,SAAqB,QAAkC;EACjF,KAAK,qBAAqB,KAAK;GAAE;GAAS;EAAO,CAAC;CACpD;;CAGA,gBAA8B;EAC5B,KAAK,cAAc;EACnB,IAAI,KAAK,gBAAgB,KAAA,GAAW,KAAK,cAAc,KAAK,SAAS;CACvE;;CAGA,eAA6B;EAC3B,KAAK,cAAc;CACrB;;CAGA,MAAa,YAA2B;EACtC,KAAK,aAAa;EAClB,IAAI,KAAK,gBAAgB,KAAA,GACvB,MAAM,KAAK;CAEf;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,UAAU;CACvB;;CAGA,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;EACvB,OAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;GACtB,KAAK,uBAAuB,CAAC;GAE7B,IAAI;IACF,MAAM,OAAO,KAAK,WAAW;IAC7B,MAAM,UACJ,KAAK,KAAK,oBACV,KAAK,KAAK,WAAW,YAAY,MAAM,OAAO,OAAO,CAAC,CACxD;IAEA,SAAS,SAAS,MAAM,EAAE,QAAQ,CAAC;GACrC,SAAS,GAAQ;IACf,QAAQ,MAAM,CAAC;IACf,SAAS,SAAS,MAAM,EAAE,OAAO,CAAC,CAAC;GACrC;GAEA,IAAI,CAAC,KAAK,aAAa;GACvB,MAAM,UAAU,KAAK,KAAK,KAAK,eAAe;EAChD;EAEA,KAAK,cAAc,KAAA;CACrB;CAEA,aAAuC;EACrC,OAAO,MAAM,KAAK,KAAK,cAAc,QAAQ,CAAC,CAAC,CAC5C,OAAO,MAAM,KAAK,KAAK,gBAAgB,QAAQ,CAAC,CAAC,CAAC,CAClD,KAAK,CAAC,GAAG,YAAY,MAAM;CAChC;AACF;;AAGA,IAAM,YAAN,MAAgB;CAQK;CACA;CACA;CACA;CACA;CAXnB;CACA,QAAiC,KAAA;CAEjC,SAAwC,IAAI,aAAa;CACzD,UAA2C,IAAI,eAAe;CAE9D,YACE,QACA,YACA,OACA,OACA,iBACA;EALiB,KAAA,SAAA;EACA,KAAA,aAAA;EACA,KAAA,QAAA;EACA,KAAA,QAAA;EACA,KAAA,kBAAA;CAChB;CAEH,SAGE;EACA,OAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,OAAO,cAAc,CAAC;EAC3B,KAAK,QAAQ,IAAI,QAAQ;CAC3B;CAEA,QAAQ,UAA2B;EACjC,OAAO,KAAK,QAAQ,IAAI,QAAQ;CAClC;CAEA,MAAM,SAAS;EACb,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,KAAK,OACL,KAAK,OACL,IACA,KAAK,eACP;GAEA,MAAM,UAAU,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK,IAAI;GAClD,IAAI,KAAK,QAAQ,SACf,KAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,EAAE,EAAE,SAAS;GACjF,KAAK,OAAO;GACZ,KAAK,QAAQ,KAAA;GAEb;EACF,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GAAG;IAEtB,KAAK,OAAO;IACZ,KAAK,QAAQ;IACb,KAAK,OAAO,YAAY;IACxB;GACF;GAEA,KAAK,OAAO,MACV,wBAAwB,wBAAwB,KAAK,MAAM,EAAE,EAAE,mBAAmB,KAAK,UAAU,CAAC,GACpG;GACA,MAAM;EACR;CACF;AACF;AAOA,SAAS,qBAAqB,YAAoB,OAAqB;CACrE,IAAI,CAAC,MAAM,KAAK,WAAW,eAAe,GACxC,MAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,gDAEpD;AAEJ"}
|
package/dist/drivers/ls.cjs
CHANGED
|
@@ -5,12 +5,19 @@ const require_ls_remote_import_handle = require("./helpers/ls_remote_import_hand
|
|
|
5
5
|
const require_ls_storage_entry = require("./helpers/ls_storage_entry.cjs");
|
|
6
6
|
const require_virtual_storages = require("./virtual_storages.cjs");
|
|
7
7
|
let node_fs_promises = require("node:fs/promises");
|
|
8
|
-
node_fs_promises = require_runtime.__toESM(node_fs_promises);
|
|
8
|
+
node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
|
|
9
9
|
let node_path = require("node:path");
|
|
10
|
-
node_path = require_runtime.__toESM(node_path);
|
|
10
|
+
node_path = require_runtime.__toESM(node_path, 1);
|
|
11
11
|
let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
|
|
12
12
|
//#region src/drivers/ls.ts
|
|
13
13
|
var LsDriver = class LsDriver {
|
|
14
|
+
logger;
|
|
15
|
+
lsClient;
|
|
16
|
+
userResources;
|
|
17
|
+
signer;
|
|
18
|
+
virtualStoragesMap;
|
|
19
|
+
localProjectionsMap;
|
|
20
|
+
openFileDialogCallback;
|
|
14
21
|
constructor(logger, lsClient, userResources, signer, virtualStoragesMap, localProjectionsMap, openFileDialogCallback) {
|
|
15
22
|
this.logger = logger;
|
|
16
23
|
this.lsClient = lsClient;
|
|
@@ -149,7 +156,7 @@ var LsDriver = class LsDriver {
|
|
|
149
156
|
for (const lp of localProjections) if (lp.localPath !== "") require_validate.validateAbsolute(lp.localPath);
|
|
150
157
|
const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.id, s]));
|
|
151
158
|
const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));
|
|
152
|
-
if (new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !== virtualStoragesMap.size + localProjectionsMap.size) throw new Error("Intersection between local projection storage ids and virtual storages names detected.");
|
|
159
|
+
if ((/* @__PURE__ */ new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()])).size !== virtualStoragesMap.size + localProjectionsMap.size) throw new Error("Intersection between local projection storage ids and virtual storages names detected.");
|
|
153
160
|
return new LsDriver(logger, lsClient, client.userResources, signer, virtualStoragesMap, localProjectionsMap, openFileDialogCallback);
|
|
154
161
|
}
|
|
155
162
|
};
|