@milaboratories/pl-drivers 1.11.25 → 1.11.27

package/dist/clients/ls_api.cjs

@@ -43,8 +43,7 @@ class ClientLs {
                 isDir: item.isDir,
                 fullName: item.fullName,
                 directory: item.directory,
-                // TODO: check we get value in a correct way here
-                lastModified: { seconds: BigInt(item.lastModified), nanos: 0 },
+                lastModified: parseTimestamp(item.lastModified),
                 version: item.version,
             }));
             return {
@@ -54,6 +53,21 @@ class ClientLs {
         }
     }
 }
+function parseTimestamp(timestamp) {
+    // Parse ISO 8601 format: 2025-08-08T11:12:44.145635532Z
+    const date = new Date(timestamp);
+    const seconds = BigInt(Math.floor(date.getTime() / 1000));
+    // Extract fractional seconds from the original string
+    const fractionalMatch = timestamp.match(/\.(\d+)Z?$/);
+    let nanos = 0;
+    if (fractionalMatch) {
+        const fractionalPart = fractionalMatch[1];
+        // Pad or truncate to 9 digits (nanoseconds)
+        const padded = fractionalPart.padEnd(9, '0').slice(0, 9);
+        nanos = parseInt(padded, 10);
+    }
+    return { seconds, nanos };
+}
 
 exports.ClientLs = ClientLs;
 //# sourceMappingURL=ls_api.cjs.map

package/dist/clients/ls_api.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"ls_api.cjs","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { WireClientProvider, WireClientProviderFactory } from '@milaboratories/pl-client';\nimport { RestAPI } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader } from '@milaboratories/pl-client';\nimport type { LsAPI_List_Response, LsAPI_ListItem } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { LsApiPaths, LsRestClientType } from '../proto-rest';\n\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n  private readonly wire: WireClientProvider<LsRestClientType | LSClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    private readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new LSClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<LsApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async list(\n    rInfo: ResourceInfo,\n    path: string,\n    options?: RpcOptions,\n  ): Promise<LsAPI_List_Response> {\n    const client = this.wire.get();\n\n    if (client instanceof LSClient) {\n      return await client.list(\n        {\n          resourceId: rInfo.id,\n          location: path,\n        },\n        addRTypeToMetadata(rInfo.type, options),\n      ).response;\n    } else {\n      const resp = (await client.POST('/v1/list', {\n        body: {\n          resourceId: rInfo.id.toString(),\n          location: path,\n        },\n        headers: { ...createRTypeRoutingHeader(rInfo.type) },\n      })).data!;\n\n      const items: LsAPI_ListItem[] = resp.items.map((item) => ({\n        name: item.name,\n        size: BigInt(item.size),\n        isDir: item.isDir,\n        fullName: item.fullName,\n        directory: item.directory,\n        // TODO: check we get value in a correct way here\n        lastModified: { seconds: BigInt(item.lastModified), nanos: 0 },\n        version: item.version,\n      }));\n\n      return {\n        items,\n        delimiter: resp.delimiter,\n      };\n    }\n  }\n}\n"],"names":["LSClient","RestAPI","addRTypeToMetadata","createRTypeRoutingHeader"],"mappings":";;;;;MAWa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,IAAI;IAErB,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEvB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAIA,wBAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YACrC;YAEA,OAAOC,gBAAO,CAAC,YAAY,CAAa;AACtC,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYD,wBAAQ,EAAE;AAC9B,YAAA,OAAO,MAAM,MAAM,CAAC,IAAI,CACtB;gBACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,gBAAA,QAAQ,EAAE,IAAI;aACf,EACDE,2BAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;QACZ;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;AAC1C,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,oBAAA,QAAQ,EAAE,IAAI;AACf,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;aACrD,CAAC,EAAE,IAAK;AAET,YAAA,MAAM,KAAK,GAAqB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;;AAEzB,gBAAA,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;gBAC9D,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC,CAAC;YAEH,OAAO;gBACL,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;QACH;IACF;AACD;;;;"}
+{"version":3,"file":"ls_api.cjs","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { WireClientProvider, WireClientProviderFactory } from '@milaboratories/pl-client';\nimport { RestAPI } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader } from '@milaboratories/pl-client';\nimport type { LsAPI_List_Response, LsAPI_ListItem } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { LsApiPaths, LsRestClientType } from '../proto-rest';\n\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n  private readonly wire: WireClientProvider<LsRestClientType | LSClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    private readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new LSClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<LsApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async list(\n    rInfo: ResourceInfo,\n    path: string,\n    options?: RpcOptions,\n  ): Promise<LsAPI_List_Response> {\n    const client = this.wire.get();\n\n    if (client instanceof LSClient) {\n      return await client.list(\n        {\n          resourceId: rInfo.id,\n          location: path,\n        },\n        addRTypeToMetadata(rInfo.type, options),\n      ).response;\n    } else {\n      const resp = (await client.POST('/v1/list', {\n        body: {\n          resourceId: rInfo.id.toString(),\n          location: path,\n        },\n        headers: { ...createRTypeRoutingHeader(rInfo.type) },\n      })).data!;\n\n      const items: LsAPI_ListItem[] = resp.items.map((item) => ({\n        name: item.name,\n        size: BigInt(item.size),\n        isDir: item.isDir,\n        fullName: item.fullName,\n        directory: item.directory,\n        lastModified: parseTimestamp(item.lastModified),\n        version: item.version,\n      }));\n\n      return {\n        items,\n        delimiter: resp.delimiter,\n      };\n    }\n  }\n}\n\nfunction parseTimestamp(timestamp: string): { seconds: bigint; nanos: number } {\n  // Parse ISO 8601 format: 2025-08-08T11:12:44.145635532Z\n  const date = new Date(timestamp);\n  const seconds = BigInt(Math.floor(date.getTime() / 1000));\n\n  // Extract fractional seconds from the original string\n  const fractionalMatch = timestamp.match(/\\.(\\d+)Z?$/);\n  let nanos = 0;\n  if (fractionalMatch) {\n    const fractionalPart = fractionalMatch[1];\n    // Pad or truncate to 9 digits (nanoseconds)\n    const padded = fractionalPart.padEnd(9, '0').slice(0, 9);\n    nanos = parseInt(padded, 10);\n  }\n\n  return { seconds, nanos };\n}\n"],"names":["LSClient","RestAPI","addRTypeToMetadata","createRTypeRoutingHeader"],"mappings":";;;;;MAWa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,IAAI;IAErB,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEvB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAIA,wBAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YACrC;YAEA,OAAOC,gBAAO,CAAC,YAAY,CAAa;AACtC,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAYD,wBAAQ,EAAE;AAC9B,YAAA,OAAO,MAAM,MAAM,CAAC,IAAI,CACtB;gBACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,gBAAA,QAAQ,EAAE,IAAI;aACf,EACDE,2BAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;QACZ;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;AAC1C,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,oBAAA,QAAQ,EAAE,IAAI;AACf,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAGC,iCAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;aACrD,CAAC,EAAE,IAAK;AAET,YAAA,MAAM,KAAK,GAAqB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,YAAY,EAAE,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC,CAAC;YAEH,OAAO;gBACL,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;QACH;IACF;AACD;AAED,SAAS,cAAc,CAAC,SAAiB,EAAA;;AAEvC,IAAA,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC;AAChC,IAAA,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;;IAGzD,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC;IACrD,IAAI,KAAK,GAAG,CAAC;IACb,IAAI,eAAe,EAAE;AACnB,QAAA,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC;;AAEzC,QAAA,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;AACxD,QAAA,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;IAC9B;AAEA,IAAA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE;AAC3B;;;;"}

package/dist/clients/ls_api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ls_api.d.ts","sourceRoot":"","sources":["../../src/clients/ls_api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAsB,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAG/F,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,iFAAiF,CAAC;AAI3I,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,qBAAa,QAAQ;IAKjB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkD;gBAGrE,yBAAyB,EAAE,yBAAyB,EACnC,MAAM,EAAE,QAAQ;IAkBnC,KAAK;IAEQ,IAAI,CACf,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,mBAAmB,CAAC;CAqChC"}
+{"version":3,"file":"ls_api.d.ts","sourceRoot":"","sources":["../../src/clients/ls_api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAsB,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAG/F,OAAO,KAAK,EAAE,mBAAmB,EAAkB,MAAM,iFAAiF,CAAC;AAI3I,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,qBAAa,QAAQ;IAKjB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkD;gBAGrE,yBAAyB,EAAE,yBAAyB,EACnC,MAAM,EAAE,QAAQ;IAkBnC,KAAK;IAEQ,IAAI,CACf,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,mBAAmB,CAAC;CAoChC"}

package/dist/clients/ls_api.js

@@ -41,8 +41,7 @@ class ClientLs {
                 isDir: item.isDir,
                 fullName: item.fullName,
                 directory: item.directory,
-                // TODO: check we get value in a correct way here
-                lastModified: { seconds: BigInt(item.lastModified), nanos: 0 },
+                lastModified: parseTimestamp(item.lastModified),
                 version: item.version,
             }));
             return {
@@ -52,6 +51,21 @@ class ClientLs {
         }
     }
 }
+function parseTimestamp(timestamp) {
+    // Parse ISO 8601 format: 2025-08-08T11:12:44.145635532Z
+    const date = new Date(timestamp);
+    const seconds = BigInt(Math.floor(date.getTime() / 1000));
+    // Extract fractional seconds from the original string
+    const fractionalMatch = timestamp.match(/\.(\d+)Z?$/);
+    let nanos = 0;
+    if (fractionalMatch) {
+        const fractionalPart = fractionalMatch[1];
+        // Pad or truncate to 9 digits (nanoseconds)
+        const padded = fractionalPart.padEnd(9, '0').slice(0, 9);
+        nanos = parseInt(padded, 10);
+    }
+    return { seconds, nanos };
+}
 
 export { ClientLs };
 //# sourceMappingURL=ls_api.js.map

package/dist/clients/ls_api.js.map

@@ -1 +1 @@
-{"version":3,"file":"ls_api.js","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { WireClientProvider, WireClientProviderFactory } from '@milaboratories/pl-client';\nimport { RestAPI } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader } from '@milaboratories/pl-client';\nimport type { LsAPI_List_Response, LsAPI_ListItem } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { LsApiPaths, LsRestClientType } from '../proto-rest';\n\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n  private readonly wire: WireClientProvider<LsRestClientType | LSClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    private readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new LSClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<LsApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async list(\n    rInfo: ResourceInfo,\n    path: string,\n    options?: RpcOptions,\n  ): Promise<LsAPI_List_Response> {\n    const client = this.wire.get();\n\n    if (client instanceof LSClient) {\n      return await client.list(\n        {\n          resourceId: rInfo.id,\n          location: path,\n        },\n        addRTypeToMetadata(rInfo.type, options),\n      ).response;\n    } else {\n      const resp = (await client.POST('/v1/list', {\n        body: {\n          resourceId: rInfo.id.toString(),\n          location: path,\n        },\n        headers: { ...createRTypeRoutingHeader(rInfo.type) },\n      })).data!;\n\n      const items: LsAPI_ListItem[] = resp.items.map((item) => ({\n        name: item.name,\n        size: BigInt(item.size),\n        isDir: item.isDir,\n        fullName: item.fullName,\n        directory: item.directory,\n        // TODO: check we get value in a correct way here\n        lastModified: { seconds: BigInt(item.lastModified), nanos: 0 },\n        version: item.version,\n      }));\n\n      return {\n        items,\n        delimiter: resp.delimiter,\n      };\n    }\n  }\n}\n"],"names":[],"mappings":";;;MAWa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,IAAI;IAErB,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEvB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YACrC;YAEA,OAAO,OAAO,CAAC,YAAY,CAAa;AACtC,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,QAAQ,EAAE;AAC9B,YAAA,OAAO,MAAM,MAAM,CAAC,IAAI,CACtB;gBACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,gBAAA,QAAQ,EAAE,IAAI;aACf,EACD,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;QACZ;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;AAC1C,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,oBAAA,QAAQ,EAAE,IAAI;AACf,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;aACrD,CAAC,EAAE,IAAK;AAET,YAAA,MAAM,KAAK,GAAqB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;;AAEzB,gBAAA,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;gBAC9D,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC,CAAC;YAEH,OAAO;gBACL,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;QACH;IACF;AACD;;;;"}
+{"version":3,"file":"ls_api.js","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { WireClientProvider, WireClientProviderFactory } from '@milaboratories/pl-client';\nimport { RestAPI } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, createRTypeRoutingHeader } from '@milaboratories/pl-client';\nimport type { LsAPI_List_Response, LsAPI_ListItem } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { LsApiPaths, LsRestClientType } from '../proto-rest';\n\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n  private readonly wire: WireClientProvider<LsRestClientType | LSClient>;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    private readonly logger: MiLogger,\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider(\n      (wire) => {\n        if (wire.type === 'grpc') {\n          return new LSClient(wire.Transport);\n        }\n\n        return RestAPI.createClient<LsApiPaths>({\n          hostAndPort: wire.Config.hostAndPort,\n          ssl: wire.Config.ssl,\n          dispatcher: wire.Dispatcher,\n          middlewares: wire.Middlewares,\n        });\n      },\n    );\n  }\n\n  close() {}\n\n  public async list(\n    rInfo: ResourceInfo,\n    path: string,\n    options?: RpcOptions,\n  ): Promise<LsAPI_List_Response> {\n    const client = this.wire.get();\n\n    if (client instanceof LSClient) {\n      return await client.list(\n        {\n          resourceId: rInfo.id,\n          location: path,\n        },\n        addRTypeToMetadata(rInfo.type, options),\n      ).response;\n    } else {\n      const resp = (await client.POST('/v1/list', {\n        body: {\n          resourceId: rInfo.id.toString(),\n          location: path,\n        },\n        headers: { ...createRTypeRoutingHeader(rInfo.type) },\n      })).data!;\n\n      const items: LsAPI_ListItem[] = resp.items.map((item) => ({\n        name: item.name,\n        size: BigInt(item.size),\n        isDir: item.isDir,\n        fullName: item.fullName,\n        directory: item.directory,\n        lastModified: parseTimestamp(item.lastModified),\n        version: item.version,\n      }));\n\n      return {\n        items,\n        delimiter: resp.delimiter,\n      };\n    }\n  }\n}\n\nfunction parseTimestamp(timestamp: string): { seconds: bigint; nanos: number } {\n  // Parse ISO 8601 format: 2025-08-08T11:12:44.145635532Z\n  const date = new Date(timestamp);\n  const seconds = BigInt(Math.floor(date.getTime() / 1000));\n\n  // Extract fractional seconds from the original string\n  const fractionalMatch = timestamp.match(/\\.(\\d+)Z?$/);\n  let nanos = 0;\n  if (fractionalMatch) {\n    const fractionalPart = fractionalMatch[1];\n    // Pad or truncate to 9 digits (nanoseconds)\n    const padded = fractionalPart.padEnd(9, '0').slice(0, 9);\n    nanos = parseInt(padded, 10);\n  }\n\n  return { seconds, nanos };\n}\n"],"names":[],"mappings":";;;MAWa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,IAAI;IAErB,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEvB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,wBAAwB,CAC5D,CAAC,IAAI,KAAI;AACP,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;AACxB,gBAAA,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YACrC;YAEA,OAAO,OAAO,CAAC,YAAY,CAAa;AACtC,gBAAA,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;AACpC,gBAAA,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;AAC9B,aAAA,CAAC;AACJ,QAAA,CAAC,CACF;IACH;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;AAE9B,QAAA,IAAI,MAAM,YAAY,QAAQ,EAAE;AAC9B,YAAA,OAAO,MAAM,MAAM,CAAC,IAAI,CACtB;gBACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,gBAAA,QAAQ,EAAE,IAAI;aACf,EACD,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;QACZ;aAAO;YACL,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE;AAC1C,gBAAA,IAAI,EAAE;AACJ,oBAAA,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE;AAC/B,oBAAA,QAAQ,EAAE,IAAI;AACf,iBAAA;gBACD,OAAO,EAAE,EAAE,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;aACrD,CAAC,EAAE,IAAK;AAET,YAAA,MAAM,KAAK,GAAqB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,YAAY,EAAE,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC,CAAC;YAEH,OAAO;gBACL,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;QACH;IACF;AACD;AAED,SAAS,cAAc,CAAC,SAAiB,EAAA;;AAEvC,IAAA,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC;AAChC,IAAA,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;;IAGzD,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC;IACrD,IAAI,KAAK,GAAG,CAAC;IACb,IAAI,eAAe,EAAE;AACnB,QAAA,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC;;AAEzC,QAAA,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;AACxD,QAAA,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;IAC9B;AAEA,IAAA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE;AAC3B;;;;"}

package/dist/drivers/download_blob/download_blob_task.cjs

@@ -75,7 +75,7 @@ class DownloadBlobTask {
         catch (e) {
             this.logger.error(`blob ${plClient.stringifyWithResourceId(this.rInfo)} download failed, `
                 + `state: ${JSON.stringify(this.state)}, `
-                + `error: ${e}`);
+                + `error: ${JSON.stringify(e)}`);
             if (nonRecoverableError(e)) {
                 this.setError(e);
                 this.change.markChanged(`blob ${plClient.resourceIdToString(this.rInfo.id)} download failed`);

package/dist/drivers/download_blob/download_blob_task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.cjs","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${e}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || isDownloadNetworkError400(e)\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","fsp","ensureDirExists","path","fileExists","createPathAtomically","Writable","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAIA,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAIC,wBAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQC,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,CAAC,CAAA,CAAE,CAChB;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAME,cAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQJ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAMC,cAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAMI,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAGC,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACVC,yCAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;;;"}
+{"version":3,"file":"download_blob_task.cjs","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || isDownloadNetworkError400(e)\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","fsp","ensureDirExists","path","fileExists","createPathAtomically","Writable","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAIA,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAIC,wBAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQC,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,CAAE,CAChC;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAME,cAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQJ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAMC,cAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAMI,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAGC,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACVC,yCAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;;;"}

package/dist/drivers/download_blob/download_blob_task.js

@@ -53,7 +53,7 @@ class DownloadBlobTask {
         catch (e) {
             this.logger.error(`blob ${stringifyWithResourceId(this.rInfo)} download failed, `
                 + `state: ${JSON.stringify(this.state)}, `
-                + `error: ${e}`);
+                + `error: ${JSON.stringify(e)}`);
             if (nonRecoverableError(e)) {
                 this.setError(e);
                 this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);

package/dist/drivers/download_blob/download_blob_task.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.js","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${e}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || isDownloadNetworkError400(e)\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["fs"],"mappings":";;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,CAAC,CAAA,CAAE,CAChB;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACA,WAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACV,yBAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;"}
+{"version":3,"file":"download_blob_task.js","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, `\n        + `state: ${JSON.stringify(this.state)}, `\n        + `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, `\n        + `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || isDownloadNetworkError400(e)\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["fs"],"mappings":";;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,kBAAA,CAAoB,CAAC;QACxF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,kBAAA;kBACzC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,EAAA;kBACpC,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,CAAE,CAChC;AACD,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,gBAAA,CAAkB,CAAC;;AAEpF,gBAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,CAAA,KAAA,EAAQ,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,yBAAA;AACzC,kBAAA,CAAA,MAAA,EAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CACjC;AACD,YAAA,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACA,WAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACV,yBAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;"}

package/dist/drivers/download_url/driver.cjs

@@ -5,6 +5,7 @@ var tsHelpers = require('@milaboratories/ts-helpers');
 var node_crypto = require('node:crypto');
 var path = require('node:path');
 var download = require('../../helpers/download.cjs');
+var download_errors = require('../../helpers/download_errors.cjs');
 var files_cache = require('../helpers/files_cache.cjs');
 var plClient = require('@milaboratories/pl-client');
 var plModelCommon = require('@milaboratories/pl-model-common');
@@ -71,15 +72,15 @@ class DownloadUrlDriver {
     }
     getUrlNoCtx(url, w, callerId) {
         const key = url.toString();
-        const task = this.urlToDownload.get(key);
-        if (task !== undefined) {
-            task.attach(w, callerId);
-            return task.getUrl();
+        const task$1 = this.urlToDownload.get(key);
+        if (task$1 !== undefined) {
+            task$1.attach(w, callerId);
+            return task$1.getUrl();
         }
         const newTask = this.setNewTask(w, url, callerId);
         this.downloadQueue.push({
             fn: async () => this.downloadUrl(newTask, callerId),
-            recoverableErrorPredicate: (e) => true,
+            recoverableErrorPredicate: (e) => !(e instanceof task.URLAborted) && !download_errors.isDownloadNetworkError400(e),
         });
         return newTask.getUrl();
     }

package/dist/drivers/download_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","randomUUID","url","isBlockUIURL","getPathForBlockUIURL","task","rmRFDir","stringifyWithResourceId","DownloadByUrlTask","createHash","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAACC,KAAW,EAAA;AAC3B,QAAA,IAAI,CAACC,0BAAY,CAACD,KAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsBA,KAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAOE,wBAAoB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMG,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAIA,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAAuB,KAAI;AAC7C,gBAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBE,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGF,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAEA,MAAI,CAAC,KAAI;AAChE,YAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAIG,sBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAGC,sBAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}
+{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","randomUUID","task","URLAborted","isDownloadNetworkError400","url","isBlockUIURL","getPathForBlockUIURL","rmRFDir","stringifyWithResourceId","DownloadByUrlTask","createHash","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMC,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAIA,MAAI,KAAK,SAAS,EAAE;AACtB,YAAAA,MAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAOA,MAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,YAAYC,eAAU,CAAC,IAAI,CAACC,yCAAyB,CAAC,CAAC,CAAC;AAC9F,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAACC,KAAW,EAAA;AAC3B,QAAA,IAAI,CAACC,0BAAY,CAACD,KAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsBA,KAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAOE,wBAAoB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMH,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAIA,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAAuB,KAAI;AAC7C,gBAAA,MAAMM,YAAO,CAACN,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBO,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGP,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAEA,MAAI,CAAC,KAAI;AAChE,YAAA,MAAMM,YAAO,CAACN,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYO,gCAAuB,CAACP,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAIQ,sBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAGC,sBAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}

package/dist/drivers/download_url/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_url/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EACV,QAAQ,EACR,MAAM,EACP,MAAM,4BAA4B,CAAC;AAMpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAIzC,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAGlF,OAAO,EAAE,iBAAiB,EAAW,MAAM,QAAQ,CAAC;AAEpD,MAAM,WAAW,qBAAqB;IACpC;;sEAEkE;IAClE,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,SAAS;IACxB;qDACiD;IACjD,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;SAGK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAE3B,mFAAmF;IACnF,UAAU,EAAE,OAAO,CAAC;IAEpB;;SAEK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;qCACqC;AACrC,qBAAa,iBAAkB,YAAW,qBAAqB,EAAE,cAAc;IAW3E,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAdvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IAEtD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,aAAa,CAAgB;IAErC;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAAgC;gBAG1B,MAAM,EAAE,QAAQ,EACjC,UAAU,EAAE,UAAU,EACL,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,oBAItB;IAOH,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,GAAG,SAAS,GAAG,SAAS;IAE3D,4CAA4C;IAC5C,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;IAwBnD,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAkBlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAQtC,wEAAwE;IAClE,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM;IAM3D;mCAC+B;IACzB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B5D,2CAA2C;IACrC,UAAU;IAgBhB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,WAAW;CAIpB"}
+{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../../src/drivers/download_url/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EACV,QAAQ,EACR,MAAM,EACP,MAAM,4BAA4B,CAAC;AAMpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAKzC,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAGlF,OAAO,EAAE,iBAAiB,EAAuB,MAAM,QAAQ,CAAC;AAEhE,MAAM,WAAW,qBAAqB;IACpC;;sEAEkE;IAClE,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,SAAS;IACxB;qDACiD;IACjD,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;SAGK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAE3B,mFAAmF;IACnF,UAAU,EAAE,OAAO,CAAC;IAEpB;;SAEK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;qCACqC;AACrC,qBAAa,iBAAkB,YAAW,qBAAqB,EAAE,cAAc;IAW3E,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAdvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IAEtD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,aAAa,CAAgB;IAErC;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAAgC;gBAG1B,MAAM,EAAE,QAAQ,EACjC,UAAU,EAAE,UAAU,EACL,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,oBAItB;IAOH,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,GAAG,SAAS,GAAG,SAAS;IAE3D,4CAA4C;IAC5C,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;IAwBnD,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAkBlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAQtC,wEAAwE;IAClE,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM;IAM3D;mCAC+B;IACzB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B5D,2CAA2C;IACrC,UAAU;IAgBhB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,WAAW;CAIpB"}

package/dist/drivers/download_url/driver.js

@@ -3,11 +3,12 @@ import { TaskProcessor } from '@milaboratories/ts-helpers';
 import { randomUUID, createHash } from 'node:crypto';
 import * as path from 'node:path';
 import { RemoteFileDownloader } from '../../helpers/download.js';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors.js';
 import { FilesCache } from '../helpers/files_cache.js';
 import { stringifyWithResourceId } from '@milaboratories/pl-client';
 import { isBlockUIURL } from '@milaboratories/pl-model-common';
 import { getPathForBlockUIURL } from '../urls/url.js';
-import { rmRFDir, DownloadByUrlTask } from './task.js';
+import { URLAborted, rmRFDir, DownloadByUrlTask } from './task.js';
 
 /** Downloads .tar or .tar.gz archives by given URLs
  * and extracts them into saveDir. */
@@ -58,7 +59,7 @@ class DownloadUrlDriver {
         const newTask = this.setNewTask(w, url, callerId);
         this.downloadQueue.push({
             fn: async () => this.downloadUrl(newTask, callerId),
-            recoverableErrorPredicate: (e) => true,
+            recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),
         });
         return newTask.getUrl();
     }

package/dist/drivers/download_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAAC,GAAW,EAAA;AAC3B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAuB,KAAI;AAC7C,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,KAAI;AAChE,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}
+{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAoDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,YAAY,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,CAAC;AAC9F,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAAC,GAAW,EAAA;AAC3B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAuB,KAAI;AAC7C,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,KAAI;AAChE,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}

package/dist/drivers/logs_stream.cjs

@@ -222,7 +222,7 @@ class LogGetter {
                 this.change.markChanged();
                 return;
             }
-            this.logger.error(`Stream log lines for ${plClient.stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`);
+            this.logger.error(`Stream log lines for ${plClient.stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`);
             throw e;
         }
     }

package/dist/drivers/logs_stream.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"logs_stream.cjs","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n  ComputableCtx,\n  Watcher,\n} from '@milaboratories/computable';\nimport {\n  ChangeSource,\n  Computable,\n  PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n  ResourceId,\n  ResourceType } from '@milaboratories/pl-client';\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n  /** Max number of concurrent requests to log streaming backend while calculating computable states */\n  nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n  /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n  private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n  private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to log id smart object. */\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly opts: LogsStreamDriverOps = {\n      nConcurrentGetLogs: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx\n  ): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n    ctx.markUnstable(\n      'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getLastLogs', rInfo.type);\n\n    let logGetter = this.idToLastLines.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n      this.idToLastLines.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error != undefined) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string\n  ): Computable<string | undefined>;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx\n  ): string | undefined;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n    const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n    ctx.markUnstable(\n      'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getProgressLog', rInfo.type);\n\n    let logGetter = this.idToProgressLog.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n      this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n  getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n  getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    const result = this.getLogHandleNoCtx(r);\n\n    // All logs from streams should be considered unstable,\n    // final value will be got from blobs.\n    ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n    return result;\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n    validateResourceType('getLogHandle', rInfo.type);\n\n    return newLogHandle(true, rInfo);\n  }\n\n  async lastLines(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.lastLines(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  async readText(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.readText(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  private async tryWithNotFound(\n    handle: sdk.AnyLogHandle,\n    method: () => Promise<StreamingAPI_Response>,\n  ): Promise<sdk.StreamingApiResponse> {\n    if (!isLiveLogHandle(handle))\n      throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n    try {\n      const resp = await method();\n      return {\n        live: true,\n        shouldUpdateHandle: false,\n        data: resp.data,\n        size: Number(resp.size),\n        newOffset: Number(resp.newOffset),\n      };\n    } catch (e: any) {\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        return { shouldUpdateHandle: true };\n      }\n\n      throw e;\n    }\n  }\n\n  private async releaseLastLogs(rId: ResourceId, callerId: string) {\n    const deleted = this.idToLastLines.get(rId)?.release(callerId);\n    if (deleted) this.idToLastLines.delete(rId);\n  }\n\n  private async releaseProgressLog(rId: ResourceId, callerId: string) {\n    const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n    if (deleted) this.idToProgressLog.delete(rId);\n  }\n\n  async releaseAll() {}\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        const logs = this.getAllLogs();\n        await asyncPool(\n          this.opts.nConcurrentGetLogs,\n          logs.map((getter) => async () => await getter.update()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllLogs(): Array<LogGetter> {\n    return Array.from(this.idToLastLines.entries())\n      .concat(Array.from(this.idToProgressLog.entries()))\n      .map(([_, getter]) => getter);\n  }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n  private logs: string | undefined;\n  private error: any | undefined = undefined;\n\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly rInfo: ResourceInfo,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {}\n\n  getLog(): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    return {\n      log: this.logs,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n\n  async update() {\n    try {\n      const resp = await this.clientLogs.lastLines(\n        this.rInfo,\n        this.lines,\n        0n,\n        this.patternToSearch,\n      );\n\n      const newLogs = new TextDecoder().decode(resp.data);\n      if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n      this.logs = newLogs;\n      this.error = undefined;\n\n      return;\n    } catch (e: any) {\n      e as RpcError;\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        // No resource\n        this.logs = '';\n        this.error = e;\n        this.change.markChanged();\n        return;\n      }\n\n      this.logger.error(\n        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n      );\n      throw e;\n    }\n  }\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith('StreamWorkdir')) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, `\n      + `expected: a resource of type 'StreamWorkdir'.`,\n    );\n  }\n}\n"],"names":["PollingComputableHooks","Computable","treeEntryToResourceInfo","randomUUID","resourceIdToString","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","asyncPool","scheduler","ChangeSource","CallersCounter","stringifyWithResourceId","WrongResourceTypeError"],"mappings":";;;;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAIA,iCAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAYE,2BAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAOC,wBAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvBC,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtBA,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAACC,2BAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAMC,mBAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAMC,kBAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAIC,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAYP,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwBQ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAIC,8BAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}
+{"version":3,"file":"logs_stream.cjs","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n  ComputableCtx,\n  Watcher,\n} from '@milaboratories/computable';\nimport {\n  ChangeSource,\n  Computable,\n  PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n  ResourceId,\n  ResourceType } from '@milaboratories/pl-client';\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n  /** Max number of concurrent requests to log streaming backend while calculating computable states */\n  nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n  /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n  private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n  private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to log id smart object. */\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly opts: LogsStreamDriverOps = {\n      nConcurrentGetLogs: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx\n  ): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n    ctx.markUnstable(\n      'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getLastLogs', rInfo.type);\n\n    let logGetter = this.idToLastLines.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n      this.idToLastLines.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error != undefined) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string\n  ): Computable<string | undefined>;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx\n  ): string | undefined;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n    const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n    ctx.markUnstable(\n      'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getProgressLog', rInfo.type);\n\n    let logGetter = this.idToProgressLog.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n      this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n  getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n  getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    const result = this.getLogHandleNoCtx(r);\n\n    // All logs from streams should be considered unstable,\n    // final value will be got from blobs.\n    ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n    return result;\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n    validateResourceType('getLogHandle', rInfo.type);\n\n    return newLogHandle(true, rInfo);\n  }\n\n  async lastLines(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.lastLines(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  async readText(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.readText(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  private async tryWithNotFound(\n    handle: sdk.AnyLogHandle,\n    method: () => Promise<StreamingAPI_Response>,\n  ): Promise<sdk.StreamingApiResponse> {\n    if (!isLiveLogHandle(handle))\n      throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n    try {\n      const resp = await method();\n      return {\n        live: true,\n        shouldUpdateHandle: false,\n        data: resp.data,\n        size: Number(resp.size),\n        newOffset: Number(resp.newOffset),\n      };\n    } catch (e: any) {\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        return { shouldUpdateHandle: true };\n      }\n\n      throw e;\n    }\n  }\n\n  private async releaseLastLogs(rId: ResourceId, callerId: string) {\n    const deleted = this.idToLastLines.get(rId)?.release(callerId);\n    if (deleted) this.idToLastLines.delete(rId);\n  }\n\n  private async releaseProgressLog(rId: ResourceId, callerId: string) {\n    const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n    if (deleted) this.idToProgressLog.delete(rId);\n  }\n\n  async releaseAll() {}\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        const logs = this.getAllLogs();\n        await asyncPool(\n          this.opts.nConcurrentGetLogs,\n          logs.map((getter) => async () => await getter.update()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllLogs(): Array<LogGetter> {\n    return Array.from(this.idToLastLines.entries())\n      .concat(Array.from(this.idToProgressLog.entries()))\n      .map(([_, getter]) => getter);\n  }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n  private logs: string | undefined;\n  private error: any | undefined = undefined;\n\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly rInfo: ResourceInfo,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {}\n\n  getLog(): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    return {\n      log: this.logs,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n\n  async update() {\n    try {\n      const resp = await this.clientLogs.lastLines(\n        this.rInfo,\n        this.lines,\n        0n,\n        this.patternToSearch,\n      );\n\n      const newLogs = new TextDecoder().decode(resp.data);\n      if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n      this.logs = newLogs;\n      this.error = undefined;\n\n      return;\n    } catch (e: any) {\n      e as RpcError;\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        // No resource\n        this.logs = '';\n        this.error = e;\n        this.change.markChanged();\n        return;\n      }\n\n      this.logger.error(\n        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n      );\n      throw e;\n    }\n  }\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith('StreamWorkdir')) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, `\n      + `expected: a resource of type 'StreamWorkdir'.`,\n    );\n  }\n}\n"],"names":["PollingComputableHooks","Computable","treeEntryToResourceInfo","randomUUID","resourceIdToString","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","asyncPool","scheduler","ChangeSource","CallersCounter","stringifyWithResourceId","WrongResourceTypeError"],"mappings":";;;;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAIA,iCAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAYE,2BAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAOC,wBAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvBC,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtBA,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAACC,2BAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAMC,mBAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAMC,kBAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAIC,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAYP,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwBQ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,iBAAA,EAAoB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,CAAE,CACtG;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAIC,8BAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}

package/dist/drivers/logs_stream.js

@@ -220,7 +220,7 @@ class LogGetter {
                 this.change.markChanged();
                 return;
             }
-            this.logger.error(`Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`);
+            this.logger.error(`Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`);
             throw e;
         }
     }

package/dist/drivers/logs_stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"logs_stream.js","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n  ComputableCtx,\n  Watcher,\n} from '@milaboratories/computable';\nimport {\n  ChangeSource,\n  Computable,\n  PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n  ResourceId,\n  ResourceType } from '@milaboratories/pl-client';\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n  /** Max number of concurrent requests to log streaming backend while calculating computable states */\n  nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n  /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n  private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n  private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to log id smart object. */\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly opts: LogsStreamDriverOps = {\n      nConcurrentGetLogs: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx\n  ): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n    ctx.markUnstable(\n      'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getLastLogs', rInfo.type);\n\n    let logGetter = this.idToLastLines.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n      this.idToLastLines.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error != undefined) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string\n  ): Computable<string | undefined>;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx\n  ): string | undefined;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n    const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n    ctx.markUnstable(\n      'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getProgressLog', rInfo.type);\n\n    let logGetter = this.idToProgressLog.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n      this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n  getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n  getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    const result = this.getLogHandleNoCtx(r);\n\n    // All logs from streams should be considered unstable,\n    // final value will be got from blobs.\n    ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n    return result;\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n    validateResourceType('getLogHandle', rInfo.type);\n\n    return newLogHandle(true, rInfo);\n  }\n\n  async lastLines(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.lastLines(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  async readText(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.readText(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  private async tryWithNotFound(\n    handle: sdk.AnyLogHandle,\n    method: () => Promise<StreamingAPI_Response>,\n  ): Promise<sdk.StreamingApiResponse> {\n    if (!isLiveLogHandle(handle))\n      throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n    try {\n      const resp = await method();\n      return {\n        live: true,\n        shouldUpdateHandle: false,\n        data: resp.data,\n        size: Number(resp.size),\n        newOffset: Number(resp.newOffset),\n      };\n    } catch (e: any) {\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        return { shouldUpdateHandle: true };\n      }\n\n      throw e;\n    }\n  }\n\n  private async releaseLastLogs(rId: ResourceId, callerId: string) {\n    const deleted = this.idToLastLines.get(rId)?.release(callerId);\n    if (deleted) this.idToLastLines.delete(rId);\n  }\n\n  private async releaseProgressLog(rId: ResourceId, callerId: string) {\n    const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n    if (deleted) this.idToProgressLog.delete(rId);\n  }\n\n  async releaseAll() {}\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        const logs = this.getAllLogs();\n        await asyncPool(\n          this.opts.nConcurrentGetLogs,\n          logs.map((getter) => async () => await getter.update()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllLogs(): Array<LogGetter> {\n    return Array.from(this.idToLastLines.entries())\n      .concat(Array.from(this.idToProgressLog.entries()))\n      .map(([_, getter]) => getter);\n  }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n  private logs: string | undefined;\n  private error: any | undefined = undefined;\n\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly rInfo: ResourceInfo,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {}\n\n  getLog(): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    return {\n      log: this.logs,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n\n  async update() {\n    try {\n      const resp = await this.clientLogs.lastLines(\n        this.rInfo,\n        this.lines,\n        0n,\n        this.patternToSearch,\n      );\n\n      const newLogs = new TextDecoder().decode(resp.data);\n      if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n      this.logs = newLogs;\n      this.error = undefined;\n\n      return;\n    } catch (e: any) {\n      e as RpcError;\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        // No resource\n        this.logs = '';\n        this.error = e;\n        this.change.markChanged();\n        return;\n      }\n\n      this.logger.error(\n        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n      );\n      throw e;\n    }\n  }\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith('StreamWorkdir')) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, `\n      + `expected: a resource of type 'StreamWorkdir'.`,\n    );\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAI,sBAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAO,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAI,sBAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}
+{"version":3,"file":"logs_stream.js","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n  ComputableCtx,\n  Watcher,\n} from '@milaboratories/computable';\nimport {\n  ChangeSource,\n  Computable,\n  PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n  ResourceId,\n  ResourceType } from '@milaboratories/pl-client';\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n  /** Max number of concurrent requests to log streaming backend while calculating computable states */\n  nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n  /** Holds a map of StreamManager Resource Id to all logs of this stream. */\n  private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to the last log line of this stream. */\n  private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n  /** Holds a map of StreamManager Resource Id to log id smart object. */\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly opts: LogsStreamDriverOps = {\n      nConcurrentGetLogs: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx\n  ): Computable<string | undefined>;\n  getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n    ctx.markUnstable(\n      'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getLastLogs', rInfo.type);\n\n    let logGetter = this.idToLastLines.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n      this.idToLastLines.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error != undefined) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string\n  ): Computable<string | undefined>;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx\n  ): string | undefined;\n  getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n    const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n    ctx.markUnstable(\n      'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n    );\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceInfo,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateResourceType('getProgressLog', rInfo.type);\n\n    let logGetter = this.idToProgressLog.get(rInfo.id);\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n      this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    logGetter.attach(w, callerId);\n    const result = logGetter.getLog();\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;\n  getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n  getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    const result = this.getLogHandleNoCtx(r);\n\n    // All logs from streams should be considered unstable,\n    // final value will be got from blobs.\n    ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n    return result;\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n    validateResourceType('getLogHandle', rInfo.type);\n\n    return newLogHandle(true, rInfo);\n  }\n\n  async lastLines(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.lastLines(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  async readText(\n    handle: sdk.AnyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string | undefined,\n  ) {\n    return await this.tryWithNotFound(handle, () =>\n      this.clientLogs.readText(\n        getResourceInfoFromLogHandle(handle),\n        lineCount,\n        BigInt(offsetBytes ?? 0),\n        searchStr,\n      ),\n    );\n  }\n\n  private async tryWithNotFound(\n    handle: sdk.AnyLogHandle,\n    method: () => Promise<StreamingAPI_Response>,\n  ): Promise<sdk.StreamingApiResponse> {\n    if (!isLiveLogHandle(handle))\n      throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n    try {\n      const resp = await method();\n      return {\n        live: true,\n        shouldUpdateHandle: false,\n        data: resp.data,\n        size: Number(resp.size),\n        newOffset: Number(resp.newOffset),\n      };\n    } catch (e: any) {\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        return { shouldUpdateHandle: true };\n      }\n\n      throw e;\n    }\n  }\n\n  private async releaseLastLogs(rId: ResourceId, callerId: string) {\n    const deleted = this.idToLastLines.get(rId)?.release(callerId);\n    if (deleted) this.idToLastLines.delete(rId);\n  }\n\n  private async releaseProgressLog(rId: ResourceId, callerId: string) {\n    const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n    if (deleted) this.idToProgressLog.delete(rId);\n  }\n\n  async releaseAll() {}\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        const logs = this.getAllLogs();\n        await asyncPool(\n          this.opts.nConcurrentGetLogs,\n          logs.map((getter) => async () => await getter.update()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllLogs(): Array<LogGetter> {\n    return Array.from(this.idToLastLines.entries())\n      .concat(Array.from(this.idToProgressLog.entries()))\n      .map(([_, getter]) => getter);\n  }\n}\n\n/** A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n  private logs: string | undefined;\n  private error: any | undefined = undefined;\n\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientLogs: ClientLogs,\n    private readonly rInfo: ResourceInfo,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {}\n\n  getLog(): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    return {\n      log: this.logs,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n\n  async update() {\n    try {\n      const resp = await this.clientLogs.lastLines(\n        this.rInfo,\n        this.lines,\n        0n,\n        this.patternToSearch,\n      );\n\n      const newLogs = new TextDecoder().decode(resp.data);\n      if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n      this.logs = newLogs;\n      this.error = undefined;\n\n      return;\n    } catch (e: any) {\n      e as RpcError;\n      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n        // No resource\n        this.logs = '';\n        this.error = e;\n        this.change.markChanged();\n        return;\n      }\n\n      this.logger.error(\n        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,\n      );\n      throw e;\n    }\n  }\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith('StreamWorkdir')) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, `\n      + `expected: a resource of type 'StreamWorkdir'.`,\n    );\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAI,sBAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAO,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,iBAAA,EAAoB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA,CAAE,CACtG;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAI,sBAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.11.25",
+  "version": "1.11.27",
   "engines": {
     "node": ">=22"
   },
@@ -33,11 +33,11 @@
     "upath": "^2.0.1",
     "zod": "~3.23.8",
     "@milaboratories/ts-helpers": "1.5.4",
-    "@milaboratories/computable": "2.7.4",
-    "@milaboratories/pl-client": "2.16.12",
-    "@milaboratories/pl-model-common": "1.21.9",
+    "@milaboratories/pl-tree": "1.8.20",
+    "@milaboratories/pl-client": "2.16.13",
+    "@milaboratories/pl-model-common": "1.21.10",
     "@milaboratories/helpers": "1.12.0",
-    "@milaboratories/pl-tree": "1.8.19"
+    "@milaboratories/computable": "2.7.4"
   },
   "devDependencies": {
     "@types/decompress": "^4.2.7",
@@ -49,9 +49,9 @@
     "typescript": "~5.6.3",
     "vitest": "^4.0.7",
     "@milaboratories/eslint-config": "1.0.5",
-    "@milaboratories/ts-builder": "1.0.5",
-    "@milaboratories/ts-configs": "1.0.6",
-    "@milaboratories/build-configs": "1.0.8",
+    "@milaboratories/ts-configs": "1.2.0",
+    "@milaboratories/ts-builder": "1.2.0",
+    "@milaboratories/build-configs": "1.2.0",
     "@milaboratories/test-helpers": "1.1.1"
   },
   "scripts": {

package/src/clients/ls_api.ts

@@ -64,8 +64,7 @@ export class ClientLs {
         isDir: item.isDir,
         fullName: item.fullName,
         directory: item.directory,
-        // TODO: check we get value in a correct way here
-        lastModified: { seconds: BigInt(item.lastModified), nanos: 0 },
+        lastModified: parseTimestamp(item.lastModified),
         version: item.version,
       }));
 
@@ -76,3 +75,21 @@ export class ClientLs {
     }
   }
 }
+
+function parseTimestamp(timestamp: string): { seconds: bigint; nanos: number } {
+  // Parse ISO 8601 format: 2025-08-08T11:12:44.145635532Z
+  const date = new Date(timestamp);
+  const seconds = BigInt(Math.floor(date.getTime() / 1000));
+
+  // Extract fractional seconds from the original string
+  const fractionalMatch = timestamp.match(/\.(\d+)Z?$/);
+  let nanos = 0;
+  if (fractionalMatch) {
+    const fractionalPart = fractionalMatch[1];
+    // Pad or truncate to 9 digits (nanoseconds)
+    const padded = fractionalPart.padEnd(9, '0').slice(0, 9);
+    nanos = parseInt(padded, 10);
+  }
+
+  return { seconds, nanos };
+}

package/src/drivers/download_blob/download_blob_task.ts

@@ -64,7 +64,7 @@ export class DownloadBlobTask {
       this.logger.error(
         `blob ${stringifyWithResourceId(this.rInfo)} download failed, `
         + `state: ${JSON.stringify(this.state)}, `
-        + `error: ${e}`,
+        + `error: ${JSON.stringify(e)}`,
       );
       if (nonRecoverableError(e)) {
         this.setError(e);

package/src/drivers/download_url/driver.test.ts

@@ -9,7 +9,7 @@ import * as path from 'node:path';
 import { DownloadUrlDriver } from './driver';
 import { test, expect } from 'vitest';
 
-test('should download a tar archive and extracts its content and then deleted', async () => {
+test.skip('should download a tar archive and extracts its content and then deleted', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
@@ -41,34 +41,31 @@ test('should download a tar archive and extracts its content and then deleted',
 
     c.resetState();
   });
-}, 15000);
+}, 45000);
 
 test('should show a error when 404 status code', async () => {
-  try {
-    await TestHelpers.withTempRoot(async (client) => {
-      const logger = new ConsoleLoggerAdapter();
-      const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
-      const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
-
-      const url = new URL(
-        'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND',
-      );
-
-      const c = driver.getUrl(url);
-
-      const url1 = await c.getValue();
-      expect(url1).toBeUndefined();
-
-      await c.awaitChange();
-
-      const url2 = await c.getValue();
-      expect(url2).not.toBeUndefined();
-      expect(url2?.error).not.toBeUndefined();
-    });
-  } catch (e) {
-    console.log('HERE: ', e);
-  }
-});
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
+    const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
+
+    const url = new URL(
+      'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND',
+    );
+
+    const c = driver.getUrl(url);
+
+    const url1 = await c.getValue();
+    expect(url1).toBeUndefined();
+
+    await c.awaitChange();
+
+    const url2 = await c.getValue();
+    expect(url2).not.toBeUndefined();
+    expect(url2?.error).not.toBeUndefined();
+    expect(url2?.url).toBeUndefined();
+  });
+}, 45000);
 
 test('should abort a downloading process when we reset a state of a computable', async () => {
   await TestHelpers.withTempRoot(async (client) => {

package/src/drivers/download_url/driver.ts

@@ -11,12 +11,13 @@ import { createHash, randomUUID } from 'node:crypto';
 import * as path from 'node:path';
 import type { Dispatcher } from 'undici';
 import { RemoteFileDownloader } from '../../helpers/download';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors';
 import { FilesCache } from '../helpers/files_cache';
 import { stringifyWithResourceId } from '@milaboratories/pl-client';
 import type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';
 import { isBlockUIURL } from '@milaboratories/pl-model-common';
 import { getPathForBlockUIURL } from '../urls/url';
-import { DownloadByUrlTask, rmRFDir } from './task';
+import { DownloadByUrlTask, rmRFDir, URLAborted } from './task';
 
 export interface DownloadUrlSyncReader {
   /** Returns a Computable that (when the time will come)
@@ -117,7 +118,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
     const newTask = this.setNewTask(w, url, callerId);
     this.downloadQueue.push({
       fn: async () => this.downloadUrl(newTask, callerId),
-      recoverableErrorPredicate: (e) => true,
+      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),
     });
 
     return newTask.getUrl();

package/src/drivers/logs_stream.ts

@@ -375,7 +375,7 @@ class LogGetter {
       }
 
       this.logger.error(
-        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,
+        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${JSON.stringify(e)}`,
       );
       throw e;
     }