@milaboratories/pl-drivers 1.9.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (334) hide show
  1. package/dist/clients/constructors.cjs +45 -0
  2. package/dist/clients/constructors.cjs.map +1 -0
  3. package/dist/clients/constructors.d.ts +0 -1
  4. package/dist/clients/constructors.js +39 -0
  5. package/dist/clients/constructors.js.map +1 -0
  6. package/dist/clients/download.cjs +149 -0
  7. package/dist/clients/download.cjs.map +1 -0
  8. package/dist/clients/download.d.ts +0 -1
  9. package/dist/clients/download.js +121 -0
  10. package/dist/clients/download.js.map +1 -0
  11. package/dist/clients/logs.cjs +44 -0
  12. package/dist/clients/logs.cjs.map +1 -0
  13. package/dist/clients/logs.d.ts +0 -1
  14. package/dist/clients/logs.js +42 -0
  15. package/dist/clients/logs.js.map +1 -0
  16. package/dist/clients/ls_api.cjs +23 -0
  17. package/dist/clients/ls_api.cjs.map +1 -0
  18. package/dist/clients/ls_api.d.ts +0 -1
  19. package/dist/clients/ls_api.js +21 -0
  20. package/dist/clients/ls_api.js.map +1 -0
  21. package/dist/clients/progress.cjs +58 -0
  22. package/dist/clients/progress.cjs.map +1 -0
  23. package/dist/clients/progress.d.ts +1 -3
  24. package/dist/clients/progress.js +56 -0
  25. package/dist/clients/progress.js.map +1 -0
  26. package/dist/clients/upload.cjs +209 -0
  27. package/dist/clients/upload.cjs.map +1 -0
  28. package/dist/clients/upload.d.ts +8 -4
  29. package/dist/clients/upload.js +183 -0
  30. package/dist/clients/upload.js.map +1 -0
  31. package/dist/drivers/download_blob/blob_key.cjs +34 -0
  32. package/dist/drivers/download_blob/blob_key.cjs.map +1 -0
  33. package/dist/drivers/download_blob/blob_key.d.ts +0 -1
  34. package/dist/drivers/download_blob/blob_key.js +12 -0
  35. package/dist/drivers/download_blob/blob_key.js.map +1 -0
  36. package/dist/drivers/download_blob/download_blob.cjs +442 -0
  37. package/dist/drivers/download_blob/download_blob.cjs.map +1 -0
  38. package/dist/drivers/download_blob/download_blob.d.ts +0 -1
  39. package/dist/drivers/download_blob/download_blob.js +417 -0
  40. package/dist/drivers/download_blob/download_blob.js.map +1 -0
  41. package/dist/drivers/download_blob/download_blob_task.cjs +170 -0
  42. package/dist/drivers/download_blob/download_blob_task.cjs.map +1 -0
  43. package/dist/drivers/download_blob/download_blob_task.d.ts +0 -1
  44. package/dist/drivers/download_blob/download_blob_task.js +146 -0
  45. package/dist/drivers/download_blob/download_blob_task.js.map +1 -0
  46. package/dist/drivers/download_blob/sparse_cache/cache.cjs +202 -0
  47. package/dist/drivers/download_blob/sparse_cache/cache.cjs.map +1 -0
  48. package/dist/drivers/download_blob/sparse_cache/cache.d.ts +0 -1
  49. package/dist/drivers/download_blob/sparse_cache/cache.js +197 -0
  50. package/dist/drivers/download_blob/sparse_cache/cache.js.map +1 -0
  51. package/dist/drivers/download_blob/sparse_cache/file.cjs +61 -0
  52. package/dist/drivers/download_blob/sparse_cache/file.cjs.map +1 -0
  53. package/dist/drivers/download_blob/sparse_cache/file.d.ts +0 -1
  54. package/dist/drivers/download_blob/sparse_cache/file.js +39 -0
  55. package/dist/drivers/download_blob/sparse_cache/file.js.map +1 -0
  56. package/dist/drivers/download_blob/sparse_cache/ranges.cjs +104 -0
  57. package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map +1 -0
  58. package/dist/drivers/download_blob/sparse_cache/ranges.d.ts +0 -1
  59. package/dist/drivers/download_blob/sparse_cache/ranges.js +76 -0
  60. package/dist/drivers/download_blob/sparse_cache/ranges.js.map +1 -0
  61. package/dist/drivers/download_blob_url/driver.cjs +169 -0
  62. package/dist/drivers/download_blob_url/driver.cjs.map +1 -0
  63. package/dist/drivers/download_blob_url/driver.d.ts +0 -1
  64. package/dist/drivers/download_blob_url/driver.js +148 -0
  65. package/dist/drivers/download_blob_url/driver.js.map +1 -0
  66. package/dist/drivers/download_blob_url/driver_id.cjs +9 -0
  67. package/dist/drivers/download_blob_url/driver_id.cjs.map +1 -0
  68. package/dist/drivers/download_blob_url/driver_id.d.ts +0 -1
  69. package/dist/drivers/download_blob_url/driver_id.js +7 -0
  70. package/dist/drivers/download_blob_url/driver_id.js.map +1 -0
  71. package/dist/drivers/download_blob_url/snapshot.cjs +18 -0
  72. package/dist/drivers/download_blob_url/snapshot.cjs.map +1 -0
  73. package/dist/drivers/download_blob_url/snapshot.d.ts +2 -3
  74. package/dist/drivers/download_blob_url/snapshot.js +15 -0
  75. package/dist/drivers/download_blob_url/snapshot.js.map +1 -0
  76. package/dist/drivers/download_blob_url/task.cjs +209 -0
  77. package/dist/drivers/download_blob_url/task.cjs.map +1 -0
  78. package/dist/drivers/download_blob_url/task.d.ts +0 -1
  79. package/dist/drivers/download_blob_url/task.js +184 -0
  80. package/dist/drivers/download_blob_url/task.js.map +1 -0
  81. package/dist/drivers/download_blob_url/url.d.ts +1 -1
  82. package/dist/drivers/download_url/driver.cjs +149 -0
  83. package/dist/drivers/download_url/driver.cjs.map +1 -0
  84. package/dist/drivers/download_url/driver.d.ts +0 -1
  85. package/dist/drivers/download_url/driver.js +128 -0
  86. package/dist/drivers/download_url/driver.js.map +1 -0
  87. package/dist/drivers/download_url/task.cjs +150 -0
  88. package/dist/drivers/download_url/task.cjs.map +1 -0
  89. package/dist/drivers/download_url/task.d.ts +0 -1
  90. package/dist/drivers/download_url/task.js +124 -0
  91. package/dist/drivers/download_url/task.js.map +1 -0
  92. package/dist/drivers/helpers/download_local_handle.cjs +26 -0
  93. package/dist/drivers/helpers/download_local_handle.cjs.map +1 -0
  94. package/dist/drivers/helpers/download_local_handle.d.ts +0 -1
  95. package/dist/drivers/helpers/download_local_handle.js +22 -0
  96. package/dist/drivers/helpers/download_local_handle.js.map +1 -0
  97. package/dist/drivers/helpers/download_remote_handle.cjs +36 -0
  98. package/dist/drivers/helpers/download_remote_handle.cjs.map +1 -0
  99. package/dist/drivers/helpers/download_remote_handle.d.ts +0 -1
  100. package/dist/drivers/helpers/download_remote_handle.js +32 -0
  101. package/dist/drivers/helpers/download_remote_handle.js.map +1 -0
  102. package/dist/drivers/helpers/files_cache.cjs +68 -0
  103. package/dist/drivers/helpers/files_cache.cjs.map +1 -0
  104. package/dist/drivers/helpers/files_cache.d.ts +0 -1
  105. package/dist/drivers/helpers/files_cache.js +66 -0
  106. package/dist/drivers/helpers/files_cache.js.map +1 -0
  107. package/dist/drivers/helpers/helpers.cjs +34 -0
  108. package/dist/drivers/helpers/helpers.cjs.map +1 -0
  109. package/dist/drivers/helpers/helpers.d.ts +0 -1
  110. package/dist/drivers/helpers/helpers.js +31 -0
  111. package/dist/drivers/helpers/helpers.js.map +1 -0
  112. package/dist/drivers/helpers/logs_handle.cjs +50 -0
  113. package/dist/drivers/helpers/logs_handle.cjs.map +1 -0
  114. package/dist/drivers/helpers/logs_handle.d.ts +0 -1
  115. package/dist/drivers/helpers/logs_handle.js +43 -0
  116. package/dist/drivers/helpers/logs_handle.js.map +1 -0
  117. package/dist/drivers/helpers/ls_remote_import_handle.cjs +34 -0
  118. package/dist/drivers/helpers/ls_remote_import_handle.cjs.map +1 -0
  119. package/dist/drivers/helpers/ls_remote_import_handle.d.ts +0 -1
  120. package/dist/drivers/helpers/ls_remote_import_handle.js +29 -0
  121. package/dist/drivers/helpers/ls_remote_import_handle.js.map +1 -0
  122. package/dist/drivers/helpers/ls_storage_entry.cjs +64 -0
  123. package/dist/drivers/helpers/ls_storage_entry.cjs.map +1 -0
  124. package/dist/drivers/helpers/ls_storage_entry.d.ts +0 -1
  125. package/dist/drivers/helpers/ls_storage_entry.js +58 -0
  126. package/dist/drivers/helpers/ls_storage_entry.js.map +1 -0
  127. package/dist/drivers/helpers/polling_ops.d.ts +0 -1
  128. package/dist/drivers/helpers/read_file.cjs +54 -0
  129. package/dist/drivers/helpers/read_file.cjs.map +1 -0
  130. package/dist/drivers/helpers/read_file.d.ts +0 -1
  131. package/dist/drivers/helpers/read_file.js +33 -0
  132. package/dist/drivers/helpers/read_file.js.map +1 -0
  133. package/dist/drivers/helpers/test_helpers.d.ts +0 -1
  134. package/dist/drivers/logs.cjs +118 -0
  135. package/dist/drivers/logs.cjs.map +1 -0
  136. package/dist/drivers/logs.d.ts +0 -1
  137. package/dist/drivers/logs.js +116 -0
  138. package/dist/drivers/logs.js.map +1 -0
  139. package/dist/drivers/logs_stream.cjs +238 -0
  140. package/dist/drivers/logs_stream.cjs.map +1 -0
  141. package/dist/drivers/logs_stream.d.ts +0 -1
  142. package/dist/drivers/logs_stream.js +236 -0
  143. package/dist/drivers/logs_stream.js.map +1 -0
  144. package/dist/drivers/ls.cjs +236 -0
  145. package/dist/drivers/ls.cjs.map +1 -0
  146. package/dist/drivers/ls.d.ts +0 -1
  147. package/dist/drivers/ls.js +214 -0
  148. package/dist/drivers/ls.js.map +1 -0
  149. package/dist/drivers/types.cjs +72 -0
  150. package/dist/drivers/types.cjs.map +1 -0
  151. package/dist/drivers/types.d.ts +4 -5
  152. package/dist/drivers/types.js +64 -0
  153. package/dist/drivers/types.js.map +1 -0
  154. package/dist/drivers/upload.cjs +154 -0
  155. package/dist/drivers/upload.cjs.map +1 -0
  156. package/dist/drivers/upload.d.ts +0 -1
  157. package/dist/drivers/upload.js +151 -0
  158. package/dist/drivers/upload.js.map +1 -0
  159. package/dist/drivers/upload_task.cjs +297 -0
  160. package/dist/drivers/upload_task.cjs.map +1 -0
  161. package/dist/drivers/upload_task.d.ts +2 -3
  162. package/dist/drivers/upload_task.js +289 -0
  163. package/dist/drivers/upload_task.js.map +1 -0
  164. package/dist/drivers/urls/url.cjs +59 -0
  165. package/dist/drivers/urls/url.cjs.map +1 -0
  166. package/dist/drivers/urls/url.d.ts +0 -1
  167. package/dist/drivers/urls/url.js +54 -0
  168. package/dist/drivers/urls/url.js.map +1 -0
  169. package/dist/drivers/virtual_storages.cjs +53 -0
  170. package/dist/drivers/virtual_storages.cjs.map +1 -0
  171. package/dist/drivers/virtual_storages.d.ts +0 -1
  172. package/dist/drivers/virtual_storages.js +51 -0
  173. package/dist/drivers/virtual_storages.js.map +1 -0
  174. package/dist/helpers/download.cjs +63 -0
  175. package/dist/helpers/download.cjs.map +1 -0
  176. package/dist/helpers/download.d.ts +0 -1
  177. package/dist/helpers/download.js +60 -0
  178. package/dist/helpers/download.js.map +1 -0
  179. package/dist/helpers/validate.cjs +12 -0
  180. package/dist/helpers/validate.cjs.map +1 -0
  181. package/dist/helpers/validate.d.ts +0 -1
  182. package/dist/helpers/validate.js +10 -0
  183. package/dist/helpers/validate.js.map +1 -0
  184. package/dist/index.cjs +73 -0
  185. package/dist/index.cjs.map +1 -0
  186. package/dist/index.d.ts +0 -1
  187. package/dist/index.js +19 -2
  188. package/dist/index.js.map +1 -1
  189. package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts +0 -1
  190. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs +261 -0
  191. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs.map +1 -0
  192. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs +31 -0
  193. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs.map +1 -0
  194. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +3 -5
  195. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js +29 -0
  196. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js.map +1 -0
  197. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +0 -1
  198. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js +256 -0
  199. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js.map +1 -0
  200. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs +301 -0
  201. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs.map +1 -0
  202. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs +34 -0
  203. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs.map +1 -0
  204. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts +3 -5
  205. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js +32 -0
  206. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js.map +1 -0
  207. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +0 -1
  208. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js +296 -0
  209. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js.map +1 -0
  210. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs +410 -0
  211. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs.map +1 -0
  212. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs +38 -0
  213. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs.map +1 -0
  214. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +3 -5
  215. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js +36 -0
  216. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js.map +1 -0
  217. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +0 -1
  218. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js +403 -0
  219. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js.map +1 -0
  220. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs +486 -0
  221. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs.map +1 -0
  222. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs +86 -0
  223. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs.map +1 -0
  224. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +3 -5
  225. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js +84 -0
  226. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js.map +1 -0
  227. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts +0 -1
  228. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js +478 -0
  229. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js.map +1 -0
  230. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs +758 -0
  231. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs.map +1 -0
  232. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs +71 -0
  233. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs.map +1 -0
  234. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts +3 -5
  235. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js +69 -0
  236. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js.map +1 -0
  237. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts +36 -1
  238. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js +747 -0
  239. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js.map +1 -0
  240. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts +3 -5
  241. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts +0 -1
  242. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts +0 -1
  243. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts +0 -1
  244. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts +0 -1
  245. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts +0 -1
  246. package/dist/proto/google/api/http.d.ts +0 -1
  247. package/dist/proto/google/protobuf/any.d.ts +0 -1
  248. package/dist/proto/google/protobuf/descriptor.d.ts +0 -1
  249. package/dist/proto/google/protobuf/duration.cjs +105 -0
  250. package/dist/proto/google/protobuf/duration.cjs.map +1 -0
  251. package/dist/proto/google/protobuf/duration.d.ts +0 -1
  252. package/dist/proto/google/protobuf/duration.js +103 -0
  253. package/dist/proto/google/protobuf/duration.js.map +1 -0
  254. package/dist/proto/google/protobuf/empty.d.ts +0 -1
  255. package/dist/proto/google/protobuf/struct.d.ts +0 -1
  256. package/dist/proto/google/protobuf/timestamp.cjs +133 -0
  257. package/dist/proto/google/protobuf/timestamp.cjs.map +1 -0
  258. package/dist/proto/google/protobuf/timestamp.d.ts +0 -1
  259. package/dist/proto/google/protobuf/timestamp.js +131 -0
  260. package/dist/proto/google/protobuf/timestamp.js.map +1 -0
  261. package/dist/proto/google/protobuf/wrappers.d.ts +0 -1
  262. package/dist/test_env.d.ts +0 -1
  263. package/package.json +17 -15
  264. package/src/clients/upload.ts +38 -14
  265. package/src/drivers/upload_task.ts +10 -3
  266. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +69 -1
  267. package/dist/clients/constructors.d.ts.map +0 -1
  268. package/dist/clients/download.d.ts.map +0 -1
  269. package/dist/clients/logs.d.ts.map +0 -1
  270. package/dist/clients/ls_api.d.ts.map +0 -1
  271. package/dist/clients/progress.d.ts.map +0 -1
  272. package/dist/clients/upload.d.ts.map +0 -1
  273. package/dist/drivers/download_blob/blob_key.d.ts.map +0 -1
  274. package/dist/drivers/download_blob/download_blob.d.ts.map +0 -1
  275. package/dist/drivers/download_blob/download_blob_task.d.ts.map +0 -1
  276. package/dist/drivers/download_blob/sparse_cache/cache.d.ts.map +0 -1
  277. package/dist/drivers/download_blob/sparse_cache/file.d.ts.map +0 -1
  278. package/dist/drivers/download_blob/sparse_cache/ranges.d.ts.map +0 -1
  279. package/dist/drivers/download_blob_url/driver.d.ts.map +0 -1
  280. package/dist/drivers/download_blob_url/driver_id.d.ts.map +0 -1
  281. package/dist/drivers/download_blob_url/snapshot.d.ts.map +0 -1
  282. package/dist/drivers/download_blob_url/task.d.ts.map +0 -1
  283. package/dist/drivers/download_blob_url/url.d.ts.map +0 -1
  284. package/dist/drivers/download_url/driver.d.ts.map +0 -1
  285. package/dist/drivers/download_url/task.d.ts.map +0 -1
  286. package/dist/drivers/helpers/download_local_handle.d.ts.map +0 -1
  287. package/dist/drivers/helpers/download_remote_handle.d.ts.map +0 -1
  288. package/dist/drivers/helpers/files_cache.d.ts.map +0 -1
  289. package/dist/drivers/helpers/helpers.d.ts.map +0 -1
  290. package/dist/drivers/helpers/logs_handle.d.ts.map +0 -1
  291. package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +0 -1
  292. package/dist/drivers/helpers/ls_storage_entry.d.ts.map +0 -1
  293. package/dist/drivers/helpers/polling_ops.d.ts.map +0 -1
  294. package/dist/drivers/helpers/read_file.d.ts.map +0 -1
  295. package/dist/drivers/helpers/test_helpers.d.ts.map +0 -1
  296. package/dist/drivers/logs.d.ts.map +0 -1
  297. package/dist/drivers/logs_stream.d.ts.map +0 -1
  298. package/dist/drivers/ls.d.ts.map +0 -1
  299. package/dist/drivers/types.d.ts.map +0 -1
  300. package/dist/drivers/upload.d.ts.map +0 -1
  301. package/dist/drivers/upload_task.d.ts.map +0 -1
  302. package/dist/drivers/urls/url.d.ts.map +0 -1
  303. package/dist/drivers/virtual_storages.d.ts.map +0 -1
  304. package/dist/helpers/download.d.ts.map +0 -1
  305. package/dist/helpers/validate.d.ts.map +0 -1
  306. package/dist/index.d.ts.map +0 -1
  307. package/dist/index.mjs +0 -4892
  308. package/dist/index.mjs.map +0 -1
  309. package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts.map +0 -1
  310. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +0 -1
  311. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +0 -1
  312. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts.map +0 -1
  313. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +0 -1
  314. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +0 -1
  315. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +0 -1
  316. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +0 -1
  317. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +0 -1
  318. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +0 -1
  319. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +0 -1
  320. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts.map +0 -1
  321. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts.map +0 -1
  322. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts.map +0 -1
  323. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts.map +0 -1
  324. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts.map +0 -1
  325. package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts.map +0 -1
  326. package/dist/proto/google/api/http.d.ts.map +0 -1
  327. package/dist/proto/google/protobuf/any.d.ts.map +0 -1
  328. package/dist/proto/google/protobuf/descriptor.d.ts.map +0 -1
  329. package/dist/proto/google/protobuf/duration.d.ts.map +0 -1
  330. package/dist/proto/google/protobuf/empty.d.ts.map +0 -1
  331. package/dist/proto/google/protobuf/struct.d.ts.map +0 -1
  332. package/dist/proto/google/protobuf/timestamp.d.ts.map +0 -1
  333. package/dist/proto/google/protobuf/wrappers.d.ts.map +0 -1
  334. package/dist/test_env.d.ts.map +0 -1
@@ -0,0 +1,150 @@
1
+ 'use strict';
2
+
3
+ var computable = require('@milaboratories/computable');
4
+ var tsHelpers = require('@milaboratories/ts-helpers');
5
+ var fsp = require('node:fs/promises');
6
+ var path = require('node:path');
7
+ var node_stream = require('node:stream');
8
+ var zlib = require('node:zlib');
9
+ var tar = require('tar-fs');
10
+ var download = require('../../helpers/download.cjs');
11
+ var url = require('../urls/url.cjs');
12
+
13
+ function _interopNamespaceDefault(e) {
14
+ var n = Object.create(null);
15
+ if (e) {
16
+ Object.keys(e).forEach(function (k) {
17
+ if (k !== 'default') {
18
+ var d = Object.getOwnPropertyDescriptor(e, k);
19
+ Object.defineProperty(n, k, d.get ? d : {
20
+ enumerable: true,
21
+ get: function () { return e[k]; }
22
+ });
23
+ }
24
+ });
25
+ }
26
+ n.default = e;
27
+ return Object.freeze(n);
28
+ }
29
+
30
+ var fsp__namespace = /*#__PURE__*/_interopNamespaceDefault(fsp);
31
+ var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
32
+ var zlib__namespace = /*#__PURE__*/_interopNamespaceDefault(zlib);
33
+ var tar__namespace = /*#__PURE__*/_interopNamespaceDefault(tar);
34
+
35
+ /** Downloads and extracts an archive to a directory. */
36
+ class DownloadByUrlTask {
37
+ logger;
38
+ path;
39
+ url;
40
+ signer;
41
+ saveDir;
42
+ counter = new tsHelpers.CallersCounter();
43
+ change = new computable.ChangeSource();
44
+ signalCtl = new AbortController();
45
+ error;
46
+ done = false;
47
+ size = 0;
48
+ constructor(logger, path, url, signer, saveDir) {
49
+ this.logger = logger;
50
+ this.path = path;
51
+ this.url = url;
52
+ this.signer = signer;
53
+ this.saveDir = saveDir;
54
+ }
55
+ info() {
56
+ return {
57
+ url: this.url.toString(),
58
+ path: this.path,
59
+ done: this.done,
60
+ size: this.size,
61
+ error: this.error,
62
+ };
63
+ }
64
+ attach(w, callerId) {
65
+ this.counter.inc(callerId);
66
+ if (!this.done)
67
+ this.change.attachWatcher(w);
68
+ }
69
+ async download(clientDownload, withGunzip) {
70
+ try {
71
+ const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
72
+ this.setDone(size);
73
+ this.change.markChanged(`download of ${this.url} finished`);
74
+ }
75
+ catch (e) {
76
+ if (e instanceof URLAborted || e instanceof download.NetworkError400) {
77
+ this.setError(e);
78
+ this.change.markChanged(`download of ${this.url} failed`);
79
+ // Just in case we were half-way extracting an archive.
80
+ await rmRFDir(this.path);
81
+ return;
82
+ }
83
+ throw e;
84
+ }
85
+ }
86
+ async downloadAndUntar(clientDownload, withGunzip, signal) {
87
+ await tsHelpers.ensureDirExists(path__namespace.dirname(this.path));
88
+ if (await tsHelpers.fileExists(this.path)) {
89
+ return await dirSize(this.path);
90
+ }
91
+ const size = await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
92
+ let processedContent = content;
93
+ if (withGunzip) {
94
+ const gunzip = node_stream.Transform.toWeb(zlib__namespace.createGunzip());
95
+ processedContent = content.pipeThrough(gunzip, { signal });
96
+ }
97
+ await tsHelpers.createPathAtomically(this.logger, this.path, async (fPath) => {
98
+ await fsp__namespace.mkdir(fPath); // throws if a directory already exists.
99
+ const untar = node_stream.Writable.toWeb(tar__namespace.extract(fPath));
100
+ await processedContent.pipeTo(untar, { signal });
101
+ });
102
+ return size;
103
+ });
104
+ return size;
105
+ }
106
+ getUrl() {
107
+ if (this.done)
108
+ return {
109
+ url: url.newBlockUIURL(this.signer, this.saveDir, tsHelpers.notEmpty(this.path))
110
+ };
111
+ if (this.error)
112
+ return { error: this.error };
113
+ return undefined;
114
+ }
115
+ setDone(size) {
116
+ this.done = true;
117
+ this.size = size;
118
+ }
119
+ setError(e) {
120
+ this.error = String(e);
121
+ }
122
+ abort(reason) {
123
+ this.signalCtl.abort(new URLAborted(reason));
124
+ }
125
+ }
126
+ /** Throws when a downloading aborts. */
127
+ class URLAborted extends Error {
128
+ name = 'URLAborted';
129
+ }
130
+ /** Gets a directory size by calculating sizes recursively. */
131
+ async function dirSize(dir) {
132
+ const files = await fsp__namespace.readdir(dir, { withFileTypes: true });
133
+ const sizes = await Promise.all(files.map(async (file) => {
134
+ const fPath = path__namespace.join(dir, file.name);
135
+ if (file.isDirectory())
136
+ return await dirSize(fPath);
137
+ const stat = await fsp__namespace.stat(fPath);
138
+ return stat.size;
139
+ }));
140
+ return sizes.reduce((sum, size) => sum + size, 0);
141
+ }
142
+ /** Do rm -rf on dir. */
143
+ async function rmRFDir(path) {
144
+ await fsp__namespace.rm(path, { recursive: true, force: true });
145
+ }
146
+
147
+ exports.DownloadByUrlTask = DownloadByUrlTask;
148
+ exports.URLAborted = URLAborted;
149
+ exports.rmRFDir = rmRFDir;
150
+ //# sourceMappingURL=task.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string | undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted || e instanceof NetworkError400) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult | undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":["CallersCounter","ChangeSource","NetworkError400","ensureDirExists","path","fileExists","Transform","zlib","createPathAtomically","fsp","Writable","tar","newBlockUIURL","notEmpty"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,IAAI,CAAC,YAAYC,wBAAe,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAGC,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAEC,iBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAEC,kBAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMJ,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGL,eAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMK,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;;;"}
@@ -37,4 +37,3 @@ export declare class URLAborted extends Error {
37
37
  }
38
38
  /** Do rm -rf on dir. */
39
39
  export declare function rmRFDir(path: string): Promise<void>;
40
- //# sourceMappingURL=task.d.ts.map
@@ -0,0 +1,124 @@
1
+ import { ChangeSource } from '@milaboratories/computable';
2
+ import { CallersCounter, ensureDirExists, fileExists, createPathAtomically, notEmpty } from '@milaboratories/ts-helpers';
3
+ import * as fsp from 'node:fs/promises';
4
+ import * as path from 'node:path';
5
+ import { Transform, Writable } from 'node:stream';
6
+ import * as zlib from 'node:zlib';
7
+ import * as tar from 'tar-fs';
8
+ import { NetworkError400 } from '../../helpers/download.js';
9
+ import { newBlockUIURL } from '../urls/url.js';
10
+
11
+ /** Downloads and extracts an archive to a directory. */
12
+ class DownloadByUrlTask {
13
+ logger;
14
+ path;
15
+ url;
16
+ signer;
17
+ saveDir;
18
+ counter = new CallersCounter();
19
+ change = new ChangeSource();
20
+ signalCtl = new AbortController();
21
+ error;
22
+ done = false;
23
+ size = 0;
24
+ constructor(logger, path, url, signer, saveDir) {
25
+ this.logger = logger;
26
+ this.path = path;
27
+ this.url = url;
28
+ this.signer = signer;
29
+ this.saveDir = saveDir;
30
+ }
31
+ info() {
32
+ return {
33
+ url: this.url.toString(),
34
+ path: this.path,
35
+ done: this.done,
36
+ size: this.size,
37
+ error: this.error,
38
+ };
39
+ }
40
+ attach(w, callerId) {
41
+ this.counter.inc(callerId);
42
+ if (!this.done)
43
+ this.change.attachWatcher(w);
44
+ }
45
+ async download(clientDownload, withGunzip) {
46
+ try {
47
+ const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
48
+ this.setDone(size);
49
+ this.change.markChanged(`download of ${this.url} finished`);
50
+ }
51
+ catch (e) {
52
+ if (e instanceof URLAborted || e instanceof NetworkError400) {
53
+ this.setError(e);
54
+ this.change.markChanged(`download of ${this.url} failed`);
55
+ // Just in case we were half-way extracting an archive.
56
+ await rmRFDir(this.path);
57
+ return;
58
+ }
59
+ throw e;
60
+ }
61
+ }
62
+ async downloadAndUntar(clientDownload, withGunzip, signal) {
63
+ await ensureDirExists(path.dirname(this.path));
64
+ if (await fileExists(this.path)) {
65
+ return await dirSize(this.path);
66
+ }
67
+ const size = await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
68
+ let processedContent = content;
69
+ if (withGunzip) {
70
+ const gunzip = Transform.toWeb(zlib.createGunzip());
71
+ processedContent = content.pipeThrough(gunzip, { signal });
72
+ }
73
+ await createPathAtomically(this.logger, this.path, async (fPath) => {
74
+ await fsp.mkdir(fPath); // throws if a directory already exists.
75
+ const untar = Writable.toWeb(tar.extract(fPath));
76
+ await processedContent.pipeTo(untar, { signal });
77
+ });
78
+ return size;
79
+ });
80
+ return size;
81
+ }
82
+ getUrl() {
83
+ if (this.done)
84
+ return {
85
+ url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))
86
+ };
87
+ if (this.error)
88
+ return { error: this.error };
89
+ return undefined;
90
+ }
91
+ setDone(size) {
92
+ this.done = true;
93
+ this.size = size;
94
+ }
95
+ setError(e) {
96
+ this.error = String(e);
97
+ }
98
+ abort(reason) {
99
+ this.signalCtl.abort(new URLAborted(reason));
100
+ }
101
+ }
102
+ /** Throws when a downloading aborts. */
103
+ class URLAborted extends Error {
104
+ name = 'URLAborted';
105
+ }
106
+ /** Gets a directory size by calculating sizes recursively. */
107
+ async function dirSize(dir) {
108
+ const files = await fsp.readdir(dir, { withFileTypes: true });
109
+ const sizes = await Promise.all(files.map(async (file) => {
110
+ const fPath = path.join(dir, file.name);
111
+ if (file.isDirectory())
112
+ return await dirSize(fPath);
113
+ const stat = await fsp.stat(fPath);
114
+ return stat.size;
115
+ }));
116
+ return sizes.reduce((sum, size) => sum + size, 0);
117
+ }
118
+ /** Do rm -rf on dir. */
119
+ async function rmRFDir(path) {
120
+ await fsp.rm(path, { recursive: true, force: true });
121
+ }
122
+
123
+ export { DownloadByUrlTask, URLAborted, rmRFDir };
124
+ //# sourceMappingURL=task.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task.js","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string | undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted || e instanceof NetworkError400) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult | undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":[],"mappings":";;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,IAAI,CAAC,YAAY,eAAe,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;"}
@@ -0,0 +1,26 @@
1
+ 'use strict';
2
+
3
+ /** Handle of locally downloaded blob. This handle is issued only after the
4
+ * blob's content is downloaded locally, and ready for quick access. */
5
+ // https://regex101.com/r/kfnBVX/1
6
+ const localHandleRegex = /^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
7
+ function newLocalHandle(path, signer) {
8
+ return `blob+local://download/${path}#${signer.sign(path)}`;
9
+ }
10
+ function isLocalBlobHandle(handle) {
11
+ return Boolean(handle.match(localHandleRegex));
12
+ }
13
+ function parseLocalHandle(handle, signer) {
14
+ const parsed = handle.match(localHandleRegex);
15
+ if (parsed === null) {
16
+ throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);
17
+ }
18
+ const { path, signature } = parsed.groups;
19
+ signer.verify(path, signature, `Signature verification failed for: ${handle}`);
20
+ return { path, signature };
21
+ }
22
+
23
+ exports.isLocalBlobHandle = isLocalBlobHandle;
24
+ exports.newLocalHandle = newLocalHandle;
25
+ exports.parseLocalHandle = parseLocalHandle;
26
+ //# sourceMappingURL=download_local_handle.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"download_local_handle.cjs","sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from '@milaboratories/pl-model-common';\nimport type { Signer } from '@milaboratories/ts-helpers';\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"names":[],"mappings":";;AAAA;AACuE;AAKvE;AACA,MAAM,gBAAgB,GAAG,0DAA0D;AAE7E,SAAU,cAAc,CAAC,IAAY,EAAE,MAAc,EAAA;IACzD,OAAO,CAAA,sBAAA,EAAyB,IAAI,CAAA,CAAA,EAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAqB;AAChF;AAEM,SAAU,iBAAiB,CAAC,MAAc,EAAA;IAC9C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAChD;AAEM,SAAU,gBAAgB,CAAC,MAAuB,EAAE,MAAc,EAAA;IACtE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;AAE7C,IAAA,IAAI,MAAM,KAAK,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,CAAA,2BAAA,EAA8B,MAAM,CAAA,WAAA,EAAc,MAAM,CAAA,CAAE,CAAC;IAC7E;IAEA,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,MAAO;IAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA,mCAAA,EAAsC,MAAM,CAAA,CAAE,CAAC;AAE9E,IAAA,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;AAC5B;;;;;;"}
@@ -6,4 +6,3 @@ export declare function parseLocalHandle(handle: LocalBlobHandle, signer: Signer
6
6
  path: string;
7
7
  signature: string;
8
8
  };
9
- //# sourceMappingURL=download_local_handle.d.ts.map
@@ -0,0 +1,22 @@
1
+ /** Handle of locally downloaded blob. This handle is issued only after the
2
+ * blob's content is downloaded locally, and ready for quick access. */
3
+ // https://regex101.com/r/kfnBVX/1
4
+ const localHandleRegex = /^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
5
+ function newLocalHandle(path, signer) {
6
+ return `blob+local://download/${path}#${signer.sign(path)}`;
7
+ }
8
+ function isLocalBlobHandle(handle) {
9
+ return Boolean(handle.match(localHandleRegex));
10
+ }
11
+ function parseLocalHandle(handle, signer) {
12
+ const parsed = handle.match(localHandleRegex);
13
+ if (parsed === null) {
14
+ throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);
15
+ }
16
+ const { path, signature } = parsed.groups;
17
+ signer.verify(path, signature, `Signature verification failed for: ${handle}`);
18
+ return { path, signature };
19
+ }
20
+
21
+ export { isLocalBlobHandle, newLocalHandle, parseLocalHandle };
22
+ //# sourceMappingURL=download_local_handle.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"download_local_handle.js","sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from '@milaboratories/pl-model-common';\nimport type { Signer } from '@milaboratories/ts-helpers';\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"names":[],"mappings":"AAAA;AACuE;AAKvE;AACA,MAAM,gBAAgB,GAAG,0DAA0D;AAE7E,SAAU,cAAc,CAAC,IAAY,EAAE,MAAc,EAAA;IACzD,OAAO,CAAA,sBAAA,EAAyB,IAAI,CAAA,CAAA,EAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAqB;AAChF;AAEM,SAAU,iBAAiB,CAAC,MAAc,EAAA;IAC9C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAChD;AAEM,SAAU,gBAAgB,CAAC,MAAuB,EAAE,MAAc,EAAA;IACtE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;AAE7C,IAAA,IAAI,MAAM,KAAK,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,CAAA,2BAAA,EAA8B,MAAM,CAAA,WAAA,EAAc,MAAM,CAAA,CAAE,CAAC;IAC7E;IAEA,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,MAAO;IAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA,mCAAA,EAAsC,MAAM,CAAA,CAAE,CAAC;AAE9E,IAAA,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;AAC5B;;;;"}
@@ -0,0 +1,36 @@
1
+ 'use strict';
2
+
3
+ var plClient = require('@milaboratories/pl-client');
4
+ var types = require('../types.cjs');
5
+
6
+ /** Handle of remote blob. This handle is issued as soon as the data becomes
7
+ * available on the remote server. */
8
+ // https://regex101.com/r/Q4YdTa/5
9
+ const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
10
+ function newRemoteHandle(rInfo, signer) {
11
+ let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${types.getSize(rInfo)}`;
12
+ return `blob+remote://download/${content}#${signer.sign(content)}`;
13
+ }
14
+ function isRemoteBlobHandle(handle) {
15
+ return Boolean(handle.match(remoteHandleRegex));
16
+ }
17
+ function parseRemoteHandle(handle, signer) {
18
+ const parsed = handle.match(remoteHandleRegex);
19
+ if (parsed === null) {
20
+ throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
21
+ }
22
+ const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups;
23
+ signer.verify(content, signature, `Signature verification failed for ${handle}`);
24
+ return {
25
+ info: {
26
+ id: plClient.bigintToResourceId(BigInt(resourceId)),
27
+ type: { name: resourceType, version: resourceVersion },
28
+ },
29
+ size: Number(size),
30
+ };
31
+ }
32
+
33
+ exports.isRemoteBlobHandle = isRemoteBlobHandle;
34
+ exports.newRemoteHandle = newRemoteHandle;
35
+ exports.parseRemoteHandle = parseRemoteHandle;
36
+ //# sourceMappingURL=download_remote_handle.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"download_remote_handle.cjs","sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from '@milaboratories/ts-helpers';\nimport type { OnDemandBlobResourceSnapshot } from '../types';\nimport type { RemoteBlobHandle } from '@milaboratories/pl-model-common';\nimport { bigintToResourceId } from '@milaboratories/pl-client';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\nimport { getSize } from '../types';\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex\n = /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(handle: RemoteBlobHandle, signer: Signer): {\n info: ResourceInfo;\n size: number;\n } {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n return {\n info:{\n id: bigintToResourceId(BigInt(resourceId)),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"names":["getSize","bigintToResourceId"],"mappings":";;;;;AAAA;AACqC;AASrC;AACA,MAAM,iBAAiB,GACnB,4IAA4I;AAE1I,SAAU,eAAe,CAC7B,KAAmC,EACnC,MAAc,EAAA;IAEd,IAAI,OAAO,GAAG,CAAA,EAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAA,CAAA,EAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,CAAA,EAAIA,aAAO,CAAC,KAAK,CAAC,CAAA,CAAE;IAE9F,OAAO,CAAA,uBAAA,EAA0B,OAAO,CAAA,CAAA,EAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAsB;AACxF;AAEM,SAAU,kBAAkB,CAAC,MAAc,EAAA;IAC/C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;AACjD;AAEM,SAAU,iBAAiB,CAAC,MAAwB,EAAE,MAAc,EAAA;IAIxE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;AAC9C,IAAA,IAAI,MAAM,KAAK,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,CAAA,4BAAA,EAA+B,MAAM,CAAA,WAAA,EAAc,MAAM,CAAA,CAAE,CAAC;IAC9E;AAEA,IAAA,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,MAAO;IAE9F,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,CAAA,kCAAA,EAAqC,MAAM,CAAA,CAAE,CAAC;IAEhF,OAAO;AACL,QAAA,IAAI,EAAC;AACH,YAAA,EAAE,EAAEC,2BAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE;AACvD,SAAA;AACD,QAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;KACnB;AACH;;;;;;"}
@@ -8,4 +8,3 @@ export declare function parseRemoteHandle(handle: RemoteBlobHandle, signer: Sign
8
8
  info: ResourceInfo;
9
9
  size: number;
10
10
  };
11
- //# sourceMappingURL=download_remote_handle.d.ts.map
@@ -0,0 +1,32 @@
1
+ import { bigintToResourceId } from '@milaboratories/pl-client';
2
+ import { getSize } from '../types.js';
3
+
4
+ /** Handle of remote blob. This handle is issued as soon as the data becomes
5
+ * available on the remote server. */
6
+ // https://regex101.com/r/Q4YdTa/5
7
+ const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
8
+ function newRemoteHandle(rInfo, signer) {
9
+ let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;
10
+ return `blob+remote://download/${content}#${signer.sign(content)}`;
11
+ }
12
+ function isRemoteBlobHandle(handle) {
13
+ return Boolean(handle.match(remoteHandleRegex));
14
+ }
15
+ function parseRemoteHandle(handle, signer) {
16
+ const parsed = handle.match(remoteHandleRegex);
17
+ if (parsed === null) {
18
+ throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
19
+ }
20
+ const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups;
21
+ signer.verify(content, signature, `Signature verification failed for ${handle}`);
22
+ return {
23
+ info: {
24
+ id: bigintToResourceId(BigInt(resourceId)),
25
+ type: { name: resourceType, version: resourceVersion },
26
+ },
27
+ size: Number(size),
28
+ };
29
+ }
30
+
31
+ export { isRemoteBlobHandle, newRemoteHandle, parseRemoteHandle };
32
+ //# sourceMappingURL=download_remote_handle.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"download_remote_handle.js","sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from '@milaboratories/ts-helpers';\nimport type { OnDemandBlobResourceSnapshot } from '../types';\nimport type { RemoteBlobHandle } from '@milaboratories/pl-model-common';\nimport { bigintToResourceId } from '@milaboratories/pl-client';\nimport { ResourceInfo } from '@milaboratories/pl-tree';\nimport { getSize } from '../types';\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex\n = /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(handle: RemoteBlobHandle, signer: Signer): {\n info: ResourceInfo;\n size: number;\n } {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n return {\n info:{\n id: bigintToResourceId(BigInt(resourceId)),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"names":[],"mappings":";;;AAAA;AACqC;AASrC;AACA,MAAM,iBAAiB,GACnB,4IAA4I;AAE1I,SAAU,eAAe,CAC7B,KAAmC,EACnC,MAAc,EAAA;IAEd,IAAI,OAAO,GAAG,CAAA,EAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAA,CAAA,EAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAA,CAAA,EAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,CAAA,EAAI,OAAO,CAAC,KAAK,CAAC,CAAA,CAAE;IAE9F,OAAO,CAAA,uBAAA,EAA0B,OAAO,CAAA,CAAA,EAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA,CAAsB;AACxF;AAEM,SAAU,kBAAkB,CAAC,MAAc,EAAA;IAC/C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;AACjD;AAEM,SAAU,iBAAiB,CAAC,MAAwB,EAAE,MAAc,EAAA;IAIxE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;AAC9C,IAAA,IAAI,MAAM,KAAK,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,CAAA,4BAAA,EAA+B,MAAM,CAAA,WAAA,EAAc,MAAM,CAAA,CAAE,CAAC;IAC9E;AAEA,IAAA,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,MAAO;IAE9F,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,CAAA,kCAAA,EAAqC,MAAM,CAAA,CAAE,CAAC;IAEhF,OAAO;AACL,QAAA,IAAI,EAAC;AACH,YAAA,EAAE,EAAE,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE;AACvD,SAAA;AACD,QAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;KACnB;AACH;;;;"}
@@ -0,0 +1,68 @@
1
+ 'use strict';
2
+
3
+ var tsHelpers = require('@milaboratories/ts-helpers');
4
+
5
+ /** Holds counters of how many callers need the file.
6
+ * If some counters become zero and a cache size exceeds a soft limit,
7
+ * remove not needed blobs one by one.
8
+ * If all the files are needed, do nothing. */
9
+ class FilesCache {
10
+ softSizeBytes;
11
+ cache = new Map();
12
+ totalSizeBytes = 0;
13
+ constructor(softSizeBytes) {
14
+ this.softSizeBytes = softSizeBytes;
15
+ }
16
+ existsFile(path) {
17
+ return this.cache.get(path) != undefined;
18
+ }
19
+ getFile(path, callerId) {
20
+ const file = this.cache.get(path);
21
+ if (file != undefined) {
22
+ file.counter.inc(callerId);
23
+ }
24
+ return file;
25
+ }
26
+ /** Decrements a counter in a cache and if we exceeds
27
+ * a soft limit, removes files with zero counters. */
28
+ removeFile(path, callerId) {
29
+ tsHelpers.mapGet(this.cache, path).counter.dec(callerId);
30
+ return this.toDelete();
31
+ }
32
+ /** Returns what results should be deleted to comply with the soft limit. */
33
+ toDelete() {
34
+ if (this.totalSizeBytes <= this.softSizeBytes)
35
+ return [];
36
+ const result = [];
37
+ let freedBytes = 0;
38
+ tsHelpers.mapEntries(this.cache)
39
+ .filter(([_, file]) => file.counter.isZero())
40
+ .forEach(([path, _]) => {
41
+ if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {
42
+ return;
43
+ }
44
+ const file = tsHelpers.mapGet(this.cache, path);
45
+ freedBytes += file.size;
46
+ result.push(file);
47
+ });
48
+ return result;
49
+ }
50
+ addCache(file, callerId) {
51
+ const created = this.cache.get(file.path) == undefined;
52
+ this.cache.set(file.path, file);
53
+ file.counter.inc(callerId);
54
+ if (file.size < 0) {
55
+ throw new Error(`empty sizeBytes: ${file}`);
56
+ }
57
+ if (created) {
58
+ this.totalSizeBytes += file.size;
59
+ }
60
+ }
61
+ removeCache(file) {
62
+ this.cache.delete(file.path);
63
+ this.totalSizeBytes -= file.size;
64
+ }
65
+ }
66
+
67
+ exports.FilesCache = FilesCache;
68
+ //# sourceMappingURL=files_cache.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"files_cache.cjs","sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from '@milaboratories/ts-helpers';\nimport { mapEntries, mapGet } from '@milaboratories/ts-helpers';\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"names":["mapGet","mapEntries"],"mappings":";;;;AAYA;;;AAG8C;MACjC,UAAU,CAAA;AAIQ,IAAA,aAAA;AAHrB,IAAA,KAAK,GAAqB,IAAI,GAAG,EAAE;IACnC,cAAc,GAAW,CAAC;AAElC,IAAA,WAAA,CAA6B,aAAqB,EAAA;QAArB,IAAA,CAAA,aAAa,GAAb,aAAa;IAAW;AAErD,IAAA,UAAU,CAAC,IAAc,EAAA;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;IAC1C;IAEA,OAAO,CAAC,IAAc,EAAE,QAAgB,EAAA;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACjC,QAAA,IAAI,IAAI,IAAI,SAAS,EAAE;AACrB,YAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC5B;AAEA,QAAA,OAAO,IAAI;IACb;AAEA;AACqD;IACrD,UAAU,CAAC,IAAc,EAAE,QAAgB,EAAA;AACzC,QAAAA,gBAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC9C,QAAA,OAAO,IAAI,CAAC,QAAQ,EAAE;IACxB;;IAGA,QAAQ,GAAA;AACN,QAAA,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,aAAa;AAAE,YAAA,OAAO,EAAE;QAExD,MAAM,MAAM,GAAQ,EAAE;QACtB,IAAI,UAAU,GAAG,CAAC;AAElB,QAAAC,oBAAU,CAAC,IAAI,CAAC,KAAK;AAClB,aAAA,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAc,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;aACxD,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAI;YACrB,IAAI,IAAI,CAAC,cAAc,GAAG,UAAU,IAAI,IAAI,CAAC,aAAa,EAAE;gBAC1D;YACF;YAEA,MAAM,IAAI,GAAGD,gBAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;AACrC,YAAA,UAAU,IAAI,IAAI,CAAC,IAAI;AACvB,YAAA,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnB,QAAA,CAAC,CAAC;AAEJ,QAAA,OAAO,MAAM;IACf;IAEA,QAAQ,CAAC,IAAO,EAAE,QAAgB,EAAA;AAChC,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS;QACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;AAC/B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAE1B,QAAA,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE;AACjB,YAAA,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAA,CAAE,CAAC;QAC7C;QAEA,IAAI,OAAO,EAAE;AACX,YAAA,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI;QAClC;IACF;AAEA,IAAA,WAAW,CAAC,IAAO,EAAA;QACjB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AAC5B,QAAA,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI;IAClC;AACD;;;;"}
@@ -26,4 +26,3 @@ export declare class FilesCache<T extends CachedFile> {
26
26
  removeCache(file: T): void;
27
27
  }
28
28
  export {};
29
- //# sourceMappingURL=files_cache.d.ts.map
@@ -0,0 +1,66 @@
1
+ import { mapGet, mapEntries } from '@milaboratories/ts-helpers';
2
+
3
+ /** Holds counters of how many callers need the file.
4
+ * If some counters become zero and a cache size exceeds a soft limit,
5
+ * remove not needed blobs one by one.
6
+ * If all the files are needed, do nothing. */
7
+ class FilesCache {
8
+ softSizeBytes;
9
+ cache = new Map();
10
+ totalSizeBytes = 0;
11
+ constructor(softSizeBytes) {
12
+ this.softSizeBytes = softSizeBytes;
13
+ }
14
+ existsFile(path) {
15
+ return this.cache.get(path) != undefined;
16
+ }
17
+ getFile(path, callerId) {
18
+ const file = this.cache.get(path);
19
+ if (file != undefined) {
20
+ file.counter.inc(callerId);
21
+ }
22
+ return file;
23
+ }
24
+ /** Decrements a counter in a cache and if we exceeds
25
+ * a soft limit, removes files with zero counters. */
26
+ removeFile(path, callerId) {
27
+ mapGet(this.cache, path).counter.dec(callerId);
28
+ return this.toDelete();
29
+ }
30
+ /** Returns what results should be deleted to comply with the soft limit. */
31
+ toDelete() {
32
+ if (this.totalSizeBytes <= this.softSizeBytes)
33
+ return [];
34
+ const result = [];
35
+ let freedBytes = 0;
36
+ mapEntries(this.cache)
37
+ .filter(([_, file]) => file.counter.isZero())
38
+ .forEach(([path, _]) => {
39
+ if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {
40
+ return;
41
+ }
42
+ const file = mapGet(this.cache, path);
43
+ freedBytes += file.size;
44
+ result.push(file);
45
+ });
46
+ return result;
47
+ }
48
+ addCache(file, callerId) {
49
+ const created = this.cache.get(file.path) == undefined;
50
+ this.cache.set(file.path, file);
51
+ file.counter.inc(callerId);
52
+ if (file.size < 0) {
53
+ throw new Error(`empty sizeBytes: ${file}`);
54
+ }
55
+ if (created) {
56
+ this.totalSizeBytes += file.size;
57
+ }
58
+ }
59
+ removeCache(file) {
60
+ this.cache.delete(file.path);
61
+ this.totalSizeBytes -= file.size;
62
+ }
63
+ }
64
+
65
+ export { FilesCache };
66
+ //# sourceMappingURL=files_cache.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"files_cache.js","sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from '@milaboratories/ts-helpers';\nimport { mapEntries, mapGet } from '@milaboratories/ts-helpers';\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"names":[],"mappings":";;AAYA;;;AAG8C;MACjC,UAAU,CAAA;AAIQ,IAAA,aAAA;AAHrB,IAAA,KAAK,GAAqB,IAAI,GAAG,EAAE;IACnC,cAAc,GAAW,CAAC;AAElC,IAAA,WAAA,CAA6B,aAAqB,EAAA;QAArB,IAAA,CAAA,aAAa,GAAb,aAAa;IAAW;AAErD,IAAA,UAAU,CAAC,IAAc,EAAA;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;IAC1C;IAEA,OAAO,CAAC,IAAc,EAAE,QAAgB,EAAA;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;AACjC,QAAA,IAAI,IAAI,IAAI,SAAS,EAAE;AACrB,YAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC5B;AAEA,QAAA,OAAO,IAAI;IACb;AAEA;AACqD;IACrD,UAAU,CAAC,IAAc,EAAE,QAAgB,EAAA;AACzC,QAAA,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC9C,QAAA,OAAO,IAAI,CAAC,QAAQ,EAAE;IACxB;;IAGA,QAAQ,GAAA;AACN,QAAA,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,aAAa;AAAE,YAAA,OAAO,EAAE;QAExD,MAAM,MAAM,GAAQ,EAAE;QACtB,IAAI,UAAU,GAAG,CAAC;AAElB,QAAA,UAAU,CAAC,IAAI,CAAC,KAAK;AAClB,aAAA,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAc,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;aACxD,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAI;YACrB,IAAI,IAAI,CAAC,cAAc,GAAG,UAAU,IAAI,IAAI,CAAC,aAAa,EAAE;gBAC1D;YACF;YAEA,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;AACrC,YAAA,UAAU,IAAI,IAAI,CAAC,IAAI;AACvB,YAAA,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnB,QAAA,CAAC,CAAC;AAEJ,QAAA,OAAO,MAAM;IACf;IAEA,QAAQ,CAAC,IAAO,EAAE,QAAgB,EAAA;AAChC,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS;QACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;AAC/B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAE1B,QAAA,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE;AACjB,YAAA,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAA,CAAE,CAAC;QAC7C;QAEA,IAAI,OAAO,EAAE;AACX,YAAA,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI;QAClC;IACF;AAEA,IAAA,WAAW,CAAC,IAAO,EAAA;QACjB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AAC5B,QAAA,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI;IAClC;AACD;;;;"}