@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +71 -2
  3. package/agent-meta/manifest.json +64 -0
  4. package/agent-meta/skills/file-streams/SKILL.md +183 -0
  5. package/agent-meta/skills/file-streams/parts/network.md +47 -0
  6. package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
  7. package/build/archive/consts.d.ts +16 -0
  8. package/build/archive/consts.d.ts.map +1 -0
  9. package/build/archive/consts.js +24 -0
  10. package/build/archive/consts.js.map +1 -0
  11. package/build/archive/formats.d.ts +4 -0
  12. package/build/archive/formats.d.ts.map +1 -0
  13. package/build/archive/formats.js +53 -0
  14. package/build/archive/formats.js.map +1 -0
  15. package/build/archive/guard.d.ts +5 -0
  16. package/build/archive/guard.d.ts.map +1 -0
  17. package/build/archive/guard.js +62 -0
  18. package/build/archive/guard.js.map +1 -0
  19. package/build/archive/ratio.d.ts +5 -0
  20. package/build/archive/ratio.d.ts.map +1 -0
  21. package/build/archive/ratio.js +30 -0
  22. package/build/archive/ratio.js.map +1 -0
  23. package/build/archive/tar.d.ts +4 -0
  24. package/build/archive/tar.d.ts.map +1 -0
  25. package/build/archive/tar.js +142 -0
  26. package/build/archive/tar.js.map +1 -0
  27. package/build/archive/types.d.ts +95 -0
  28. package/build/archive/types.d.ts.map +1 -0
  29. package/build/archive/types.js +2 -0
  30. package/build/archive/types.js.map +1 -0
  31. package/build/archive/types.local.d.ts +64 -0
  32. package/build/archive/types.local.d.ts.map +1 -0
  33. package/build/archive/types.local.js +2 -0
  34. package/build/archive/types.local.js.map +1 -0
  35. package/build/archive/zip.d.ts +4 -0
  36. package/build/archive/zip.d.ts.map +1 -0
  37. package/build/archive/zip.js +135 -0
  38. package/build/archive/zip.js.map +1 -0
  39. package/build/archive.d.ts +4 -0
  40. package/build/archive.d.ts.map +1 -0
  41. package/build/archive.js +265 -0
  42. package/build/archive.js.map +1 -0
  43. package/build/backed-workspace/types.d.ts +67 -0
  44. package/build/backed-workspace/types.d.ts.map +1 -0
  45. package/build/backed-workspace/types.js +2 -0
  46. package/build/backed-workspace/types.js.map +1 -0
  47. package/build/backed-workspace.d.ts +4 -0
  48. package/build/backed-workspace.d.ts.map +1 -0
  49. package/build/backed-workspace.js +63 -0
  50. package/build/backed-workspace.js.map +1 -0
  51. package/build/consts.d.ts +3 -0
  52. package/build/consts.d.ts.map +1 -0
  53. package/build/consts.js +3 -0
  54. package/build/consts.js.map +1 -0
  55. package/build/consts.local.d.ts +3 -0
  56. package/build/consts.local.d.ts.map +1 -0
  57. package/build/consts.local.js +3 -0
  58. package/build/consts.local.js.map +1 -0
  59. package/build/errors.d.ts +59 -0
  60. package/build/errors.d.ts.map +1 -0
  61. package/build/errors.js +105 -0
  62. package/build/errors.js.map +1 -0
  63. package/build/http/connector.d.ts +11 -0
  64. package/build/http/connector.d.ts.map +1 -0
  65. package/build/http/connector.js +124 -0
  66. package/build/http/connector.js.map +1 -0
  67. package/build/http/consts.d.ts +9 -0
  68. package/build/http/consts.d.ts.map +1 -0
  69. package/build/http/consts.js +8 -0
  70. package/build/http/consts.js.map +1 -0
  71. package/build/http/source.d.ts +5 -0
  72. package/build/http/source.d.ts.map +1 -0
  73. package/build/http/source.js +76 -0
  74. package/build/http/source.js.map +1 -0
  75. package/build/http/types.d.ts +67 -0
  76. package/build/http/types.d.ts.map +1 -0
  77. package/build/http/types.js +2 -0
  78. package/build/http/types.js.map +1 -0
  79. package/build/http/types.local.d.ts +21 -0
  80. package/build/http/types.local.d.ts.map +1 -0
  81. package/build/http/types.local.js +2 -0
  82. package/build/http/types.local.js.map +1 -0
  83. package/build/http.d.ts +4 -0
  84. package/build/http.d.ts.map +1 -0
  85. package/build/http.js +248 -0
  86. package/build/http.js.map +1 -0
  87. package/build/index.d.ts +53 -0
  88. package/build/index.d.ts.map +1 -0
  89. package/build/index.js +33 -0
  90. package/build/index.js.map +1 -0
  91. package/build/network/consts.d.ts +12 -0
  92. package/build/network/consts.d.ts.map +1 -0
  93. package/build/network/consts.js +20 -0
  94. package/build/network/consts.js.map +1 -0
  95. package/build/network/types.d.ts +43 -0
  96. package/build/network/types.d.ts.map +1 -0
  97. package/build/network/types.js +2 -0
  98. package/build/network/types.js.map +1 -0
  99. package/build/network.d.ts +4 -0
  100. package/build/network.d.ts.map +1 -0
  101. package/build/network.js +165 -0
  102. package/build/network.js.map +1 -0
  103. package/build/proxy/consts.d.ts +7 -0
  104. package/build/proxy/consts.d.ts.map +1 -0
  105. package/build/proxy/consts.js +9 -0
  106. package/build/proxy/consts.js.map +1 -0
  107. package/build/proxy/endpoint.d.ts +4 -0
  108. package/build/proxy/endpoint.d.ts.map +1 -0
  109. package/build/proxy/endpoint.js +27 -0
  110. package/build/proxy/endpoint.js.map +1 -0
  111. package/build/proxy/http.d.ts +5 -0
  112. package/build/proxy/http.d.ts.map +1 -0
  113. package/build/proxy/http.js +95 -0
  114. package/build/proxy/http.js.map +1 -0
  115. package/build/proxy/router.d.ts +4 -0
  116. package/build/proxy/router.d.ts.map +1 -0
  117. package/build/proxy/router.js +85 -0
  118. package/build/proxy/router.js.map +1 -0
  119. package/build/proxy/socks.d.ts +5 -0
  120. package/build/proxy/socks.d.ts.map +1 -0
  121. package/build/proxy/socks.js +44 -0
  122. package/build/proxy/socks.js.map +1 -0
  123. package/build/proxy/types.d.ts +99 -0
  124. package/build/proxy/types.d.ts.map +1 -0
  125. package/build/proxy/types.js +2 -0
  126. package/build/proxy/types.js.map +1 -0
  127. package/build/resolvers/google-drive/consts.d.ts +10 -0
  128. package/build/resolvers/google-drive/consts.d.ts.map +1 -0
  129. package/build/resolvers/google-drive/consts.js +16 -0
  130. package/build/resolvers/google-drive/consts.js.map +1 -0
  131. package/build/resolvers/google-drive/types.d.ts +21 -0
  132. package/build/resolvers/google-drive/types.d.ts.map +1 -0
  133. package/build/resolvers/google-drive/types.js +2 -0
  134. package/build/resolvers/google-drive/types.js.map +1 -0
  135. package/build/resolvers/google-drive.d.ts +10 -0
  136. package/build/resolvers/google-drive.d.ts.map +1 -0
  137. package/build/resolvers/google-drive.js +130 -0
  138. package/build/resolvers/google-drive.js.map +1 -0
  139. package/build/resolvers/types.d.ts +39 -0
  140. package/build/resolvers/types.d.ts.map +1 -0
  141. package/build/resolvers/types.js +2 -0
  142. package/build/resolvers/types.js.map +1 -0
  143. package/build/s3/consts.d.ts +9 -0
  144. package/build/s3/consts.d.ts.map +1 -0
  145. package/build/s3/consts.js +9 -0
  146. package/build/s3/consts.js.map +1 -0
  147. package/build/s3/prefix.d.ts +4 -0
  148. package/build/s3/prefix.d.ts.map +1 -0
  149. package/build/s3/prefix.js +87 -0
  150. package/build/s3/prefix.js.map +1 -0
  151. package/build/s3/sink.d.ts +6 -0
  152. package/build/s3/sink.d.ts.map +1 -0
  153. package/build/s3/sink.js +30 -0
  154. package/build/s3/sink.js.map +1 -0
  155. package/build/s3/source.d.ts +5 -0
  156. package/build/s3/source.d.ts.map +1 -0
  157. package/build/s3/source.js +81 -0
  158. package/build/s3/source.js.map +1 -0
  159. package/build/s3/types.d.ts +66 -0
  160. package/build/s3/types.d.ts.map +1 -0
  161. package/build/s3/types.js +2 -0
  162. package/build/s3/types.js.map +1 -0
  163. package/build/s3/types.local.d.ts +43 -0
  164. package/build/s3/types.local.d.ts.map +1 -0
  165. package/build/s3/types.local.js +2 -0
  166. package/build/s3/types.local.js.map +1 -0
  167. package/build/s3/upload.d.ts +4 -0
  168. package/build/s3/upload.d.ts.map +1 -0
  169. package/build/s3/upload.js +118 -0
  170. package/build/s3/upload.js.map +1 -0
  171. package/build/s3.d.ts +5 -0
  172. package/build/s3.d.ts.map +1 -0
  173. package/build/s3.js +68 -0
  174. package/build/s3.js.map +1 -0
  175. package/build/service/types.d.ts +66 -0
  176. package/build/service/types.d.ts.map +1 -0
  177. package/build/service/types.js +2 -0
  178. package/build/service/types.js.map +1 -0
  179. package/build/service/types.local.d.ts +11 -0
  180. package/build/service/types.local.d.ts.map +1 -0
  181. package/build/service/types.local.js +2 -0
  182. package/build/service/types.local.js.map +1 -0
  183. package/build/service.d.ts +6 -0
  184. package/build/service.d.ts.map +1 -0
  185. package/build/service.js +73 -0
  186. package/build/service.js.map +1 -0
  187. package/build/sinks/buffer/consts.d.ts +2 -0
  188. package/build/sinks/buffer/consts.d.ts.map +1 -0
  189. package/build/sinks/buffer/consts.js +2 -0
  190. package/build/sinks/buffer/consts.js.map +1 -0
  191. package/build/sinks/buffer/types.d.ts +11 -0
  192. package/build/sinks/buffer/types.d.ts.map +1 -0
  193. package/build/sinks/buffer/types.js +2 -0
  194. package/build/sinks/buffer/types.js.map +1 -0
  195. package/build/sinks/buffer.d.ts +4 -0
  196. package/build/sinks/buffer.d.ts.map +1 -0
  197. package/build/sinks/buffer.js +13 -0
  198. package/build/sinks/buffer.js.map +1 -0
  199. package/build/sinks/directory/types.d.ts +7 -0
  200. package/build/sinks/directory/types.d.ts.map +1 -0
  201. package/build/sinks/directory/types.js +2 -0
  202. package/build/sinks/directory/types.js.map +1 -0
  203. package/build/sinks/directory.d.ts +8 -0
  204. package/build/sinks/directory.d.ts.map +1 -0
  205. package/build/sinks/directory.js +43 -0
  206. package/build/sinks/directory.js.map +1 -0
  207. package/build/sinks/file.d.ts +5 -0
  208. package/build/sinks/file.d.ts.map +1 -0
  209. package/build/sinks/file.js +46 -0
  210. package/build/sinks/file.js.map +1 -0
  211. package/build/sinks/types.d.ts +11 -0
  212. package/build/sinks/types.d.ts.map +1 -0
  213. package/build/sinks/types.js +2 -0
  214. package/build/sinks/types.js.map +1 -0
  215. package/build/sinks/workload/consts.d.ts +5 -0
  216. package/build/sinks/workload/consts.d.ts.map +1 -0
  217. package/build/sinks/workload/consts.js +5 -0
  218. package/build/sinks/workload/consts.js.map +1 -0
  219. package/build/sinks/workload/types.d.ts +17 -0
  220. package/build/sinks/workload/types.d.ts.map +1 -0
  221. package/build/sinks/workload/types.js +2 -0
  222. package/build/sinks/workload/types.js.map +1 -0
  223. package/build/sinks/workload.d.ts +8 -0
  224. package/build/sinks/workload.d.ts.map +1 -0
  225. package/build/sinks/workload.js +51 -0
  226. package/build/sinks/workload.js.map +1 -0
  227. package/build/sources/buffer.d.ts +5 -0
  228. package/build/sources/buffer.d.ts.map +1 -0
  229. package/build/sources/buffer.js +20 -0
  230. package/build/sources/buffer.js.map +1 -0
  231. package/build/sources/file.d.ts +5 -0
  232. package/build/sources/file.d.ts.map +1 -0
  233. package/build/sources/file.js +50 -0
  234. package/build/sources/file.js.map +1 -0
  235. package/build/sources/stream.d.ts +5 -0
  236. package/build/sources/stream.d.ts.map +1 -0
  237. package/build/sources/stream.js +19 -0
  238. package/build/sources/stream.js.map +1 -0
  239. package/build/sources/types.d.ts +10 -0
  240. package/build/sources/types.d.ts.map +1 -0
  241. package/build/sources/types.js +2 -0
  242. package/build/sources/types.js.map +1 -0
  243. package/build/sources/workload/types.d.ts +29 -0
  244. package/build/sources/workload/types.d.ts.map +1 -0
  245. package/build/sources/workload/types.js +2 -0
  246. package/build/sources/workload/types.js.map +1 -0
  247. package/build/sources/workload.d.ts +7 -0
  248. package/build/sources/workload.d.ts.map +1 -0
  249. package/build/sources/workload.js +67 -0
  250. package/build/sources/workload.js.map +1 -0
  251. package/build/transfer/consts.d.ts +5 -0
  252. package/build/transfer/consts.d.ts.map +1 -0
  253. package/build/transfer/consts.js +8 -0
  254. package/build/transfer/consts.js.map +1 -0
  255. package/build/transfer/types.d.ts +49 -0
  256. package/build/transfer/types.d.ts.map +1 -0
  257. package/build/transfer/types.js +2 -0
  258. package/build/transfer/types.js.map +1 -0
  259. package/build/transfer.d.ts +4 -0
  260. package/build/transfer.d.ts.map +1 -0
  261. package/build/transfer.js +75 -0
  262. package/build/transfer.js.map +1 -0
  263. package/build/types.d.ts +99 -0
  264. package/build/types.d.ts.map +1 -0
  265. package/build/types.js +2 -0
  266. package/build/types.js.map +1 -0
  267. package/build/utils/names/consts.d.ts +5 -0
  268. package/build/utils/names/consts.d.ts.map +1 -0
  269. package/build/utils/names/consts.js +5 -0
  270. package/build/utils/names/consts.js.map +1 -0
  271. package/build/utils/names/types.d.ts +13 -0
  272. package/build/utils/names/types.d.ts.map +1 -0
  273. package/build/utils/names/types.js +2 -0
  274. package/build/utils/names/types.js.map +1 -0
  275. package/build/utils/names.d.ts +4 -0
  276. package/build/utils/names.d.ts.map +1 -0
  277. package/build/utils/names.js +60 -0
  278. package/build/utils/names.js.map +1 -0
  279. package/build/utils/paths/consts.d.ts +3 -0
  280. package/build/utils/paths/consts.d.ts.map +1 -0
  281. package/build/utils/paths/consts.js +2 -0
  282. package/build/utils/paths/consts.js.map +1 -0
  283. package/build/utils/paths/types.d.ts +20 -0
  284. package/build/utils/paths/types.d.ts.map +1 -0
  285. package/build/utils/paths/types.js +2 -0
  286. package/build/utils/paths/types.js.map +1 -0
  287. package/build/utils/paths.d.ts +4 -0
  288. package/build/utils/paths.d.ts.map +1 -0
  289. package/build/utils/paths.js +67 -0
  290. package/build/utils/paths.js.map +1 -0
  291. package/build/utils/range/types.d.ts +11 -0
  292. package/build/utils/range/types.d.ts.map +1 -0
  293. package/build/utils/range/types.js +2 -0
  294. package/build/utils/range/types.js.map +1 -0
  295. package/build/utils/range.d.ts +4 -0
  296. package/build/utils/range.d.ts.map +1 -0
  297. package/build/utils/range.js +64 -0
  298. package/build/utils/range.js.map +1 -0
  299. package/build/utils/redact/types.d.ts +16 -0
  300. package/build/utils/redact/types.d.ts.map +1 -0
  301. package/build/utils/redact/types.js +2 -0
  302. package/build/utils/redact/types.js.map +1 -0
  303. package/build/utils/redact.d.ts +4 -0
  304. package/build/utils/redact.d.ts.map +1 -0
  305. package/build/utils/redact.js +18 -0
  306. package/build/utils/redact.js.map +1 -0
  307. package/build/utils/signals/types.d.ts +17 -0
  308. package/build/utils/signals/types.d.ts.map +1 -0
  309. package/build/utils/signals/types.js +2 -0
  310. package/build/utils/signals/types.js.map +1 -0
  311. package/build/utils/signals.d.ts +4 -0
  312. package/build/utils/signals.d.ts.map +1 -0
  313. package/build/utils/signals.js +52 -0
  314. package/build/utils/signals.js.map +1 -0
  315. package/build/utils/streams/consts.d.ts +3 -0
  316. package/build/utils/streams/consts.d.ts.map +1 -0
  317. package/build/utils/streams/consts.js +3 -0
  318. package/build/utils/streams/consts.js.map +1 -0
  319. package/build/utils/streams/types.d.ts +59 -0
  320. package/build/utils/streams/types.d.ts.map +1 -0
  321. package/build/utils/streams/types.js +2 -0
  322. package/build/utils/streams/types.js.map +1 -0
  323. package/build/utils/streams.d.ts +4 -0
  324. package/build/utils/streams.d.ts.map +1 -0
  325. package/build/utils/streams.js +167 -0
  326. package/build/utils/streams.js.map +1 -0
  327. package/build/utils/tree/types.d.ts +6 -0
  328. package/build/utils/tree/types.d.ts.map +1 -0
  329. package/build/utils/tree/types.js +2 -0
  330. package/build/utils/tree/types.js.map +1 -0
  331. package/build/utils/tree.d.ts +4 -0
  332. package/build/utils/tree.d.ts.map +1 -0
  333. package/build/utils/tree.js +14 -0
  334. package/build/utils/tree.js.map +1 -0
  335. package/build/workspace/consts.d.ts +12 -0
  336. package/build/workspace/consts.d.ts.map +1 -0
  337. package/build/workspace/consts.js +12 -0
  338. package/build/workspace/consts.js.map +1 -0
  339. package/build/workspace/consts.local.d.ts +3 -0
  340. package/build/workspace/consts.local.d.ts.map +1 -0
  341. package/build/workspace/consts.local.js +3 -0
  342. package/build/workspace/consts.local.js.map +1 -0
  343. package/build/workspace/fs.d.ts +4 -0
  344. package/build/workspace/fs.d.ts.map +1 -0
  345. package/build/workspace/fs.js +45 -0
  346. package/build/workspace/fs.js.map +1 -0
  347. package/build/workspace/scope.d.ts +6 -0
  348. package/build/workspace/scope.d.ts.map +1 -0
  349. package/build/workspace/scope.js +180 -0
  350. package/build/workspace/scope.js.map +1 -0
  351. package/build/workspace/types.d.ts +66 -0
  352. package/build/workspace/types.d.ts.map +1 -0
  353. package/build/workspace/types.js +2 -0
  354. package/build/workspace/types.js.map +1 -0
  355. package/build/workspace/types.local.d.ts +20 -0
  356. package/build/workspace/types.local.d.ts.map +1 -0
  357. package/build/workspace/types.local.js +2 -0
  358. package/build/workspace/types.local.js.map +1 -0
  359. package/build/workspace.d.ts +7 -0
  360. package/build/workspace.d.ts.map +1 -0
  361. package/build/workspace.js +87 -0
  362. package/build/workspace.js.map +1 -0
  363. package/package.json +61 -4
  364. package/src/archive/consts.ts +28 -0
  365. package/src/archive/formats.ts +60 -0
  366. package/src/archive/guard.ts +68 -0
  367. package/src/archive/ratio.ts +35 -0
  368. package/src/archive/tar.ts +147 -0
  369. package/src/archive/types.local.ts +72 -0
  370. package/src/archive/types.ts +107 -0
  371. package/src/archive/zip.ts +142 -0
  372. package/src/archive.ts +298 -0
  373. package/src/backed-workspace/types.ts +72 -0
  374. package/src/backed-workspace.ts +85 -0
  375. package/src/consts.local.ts +2 -0
  376. package/src/consts.ts +2 -0
  377. package/src/errors.ts +128 -0
  378. package/src/http/connector.ts +136 -0
  379. package/src/http/consts.ts +13 -0
  380. package/src/http/source.ts +89 -0
  381. package/src/http/types.local.ts +23 -0
  382. package/src/http/types.ts +74 -0
  383. package/src/http.ts +276 -0
  384. package/src/index.ts +65 -0
  385. package/src/network/consts.ts +22 -0
  386. package/src/network/types.ts +45 -0
  387. package/src/network.ts +179 -0
  388. package/src/proxy/consts.ts +11 -0
  389. package/src/proxy/endpoint.ts +28 -0
  390. package/src/proxy/http.ts +108 -0
  391. package/src/proxy/router.ts +93 -0
  392. package/src/proxy/socks.ts +48 -0
  393. package/src/proxy/types.ts +114 -0
  394. package/src/resolvers/google-drive/consts.ts +22 -0
  395. package/src/resolvers/google-drive/types.ts +22 -0
  396. package/src/resolvers/google-drive.ts +153 -0
  397. package/src/resolvers/types.ts +45 -0
  398. package/src/s3/consts.ts +12 -0
  399. package/src/s3/prefix.ts +100 -0
  400. package/src/s3/sink.ts +36 -0
  401. package/src/s3/source.ts +90 -0
  402. package/src/s3/types.local.ts +49 -0
  403. package/src/s3/types.ts +70 -0
  404. package/src/s3/upload.ts +130 -0
  405. package/src/s3.ts +83 -0
  406. package/src/service/types.local.ts +9 -0
  407. package/src/service/types.ts +71 -0
  408. package/src/service.ts +97 -0
  409. package/src/sinks/buffer/consts.ts +1 -0
  410. package/src/sinks/buffer/types.ts +12 -0
  411. package/src/sinks/buffer.ts +18 -0
  412. package/src/sinks/directory/types.ts +7 -0
  413. package/src/sinks/directory.ts +49 -0
  414. package/src/sinks/file.ts +52 -0
  415. package/src/sinks/types.ts +12 -0
  416. package/src/sinks/workload/consts.ts +5 -0
  417. package/src/sinks/workload/types.ts +17 -0
  418. package/src/sinks/workload.ts +59 -0
  419. package/src/sources/buffer.ts +27 -0
  420. package/src/sources/file.ts +57 -0
  421. package/src/sources/stream.ts +26 -0
  422. package/src/sources/types.ts +12 -0
  423. package/src/sources/workload/types.ts +31 -0
  424. package/src/sources/workload.ts +79 -0
  425. package/src/transfer/consts.ts +10 -0
  426. package/src/transfer/types.ts +55 -0
  427. package/src/transfer.ts +89 -0
  428. package/src/types.ts +106 -0
  429. package/src/utils/names/consts.ts +5 -0
  430. package/src/utils/names/types.ts +12 -0
  431. package/src/utils/names.ts +70 -0
  432. package/src/utils/paths/consts.ts +3 -0
  433. package/src/utils/paths/types.ts +20 -0
  434. package/src/utils/paths.ts +75 -0
  435. package/src/utils/range/types.ts +11 -0
  436. package/src/utils/range.ts +80 -0
  437. package/src/utils/redact/types.ts +16 -0
  438. package/src/utils/redact.ts +23 -0
  439. package/src/utils/signals/types.ts +17 -0
  440. package/src/utils/signals.ts +63 -0
  441. package/src/utils/streams/consts.ts +2 -0
  442. package/src/utils/streams/types.ts +65 -0
  443. package/src/utils/streams.ts +188 -0
  444. package/src/utils/tree/types.ts +6 -0
  445. package/src/utils/tree.ts +18 -0
  446. package/src/workspace/consts.local.ts +2 -0
  447. package/src/workspace/consts.ts +14 -0
  448. package/src/workspace/fs.ts +48 -0
  449. package/src/workspace/scope.ts +196 -0
  450. package/src/workspace/types.local.ts +21 -0
  451. package/src/workspace/types.ts +71 -0
  452. package/src/workspace.ts +102 -0
  453. package/tests/archive.spec.ts +134 -0
  454. package/tests/backed.spec.ts +147 -0
  455. package/tests/context.ts +114 -0
  456. package/tests/fixtures.ts +190 -0
  457. package/tests/google-drive.spec.ts +76 -0
  458. package/tests/harness/drive.ts +92 -0
  459. package/tests/harness/http-proxy.ts +96 -0
  460. package/tests/harness/origin.ts +134 -0
  461. package/tests/harness/socks5.ts +135 -0
  462. package/tests/harness/workload.ts +118 -0
  463. package/tests/http.spec.ts +101 -0
  464. package/tests/inspector.spec.ts +40 -0
  465. package/tests/live.spec.ts +27 -0
  466. package/tests/network.spec.ts +79 -0
  467. package/tests/proxy.spec.ts +149 -0
  468. package/tests/s3.spec.ts +133 -0
  469. package/tests/service.spec.ts +77 -0
  470. package/tests/sources-sinks.spec.ts +96 -0
  471. package/tests/transfer.spec.ts +99 -0
  472. package/tests/tsconfig.json +12 -0
  473. package/tests/workload.spec.ts +140 -0
  474. package/tests/workspace.spec.ts +158 -0
  475. package/tsconfig.json +16 -0
@@ -0,0 +1,22 @@
1
+ /** The format each Google editor file exports as. */
2
+ export interface GoogleExportFormats {
3
+ document?: 'docx' | 'pdf' | 'odt' | 'rtf' | 'txt' | 'epub' | 'md' | 'html'
4
+ spreadsheet?: 'xlsx' | 'ods' | 'pdf' | 'csv' | 'tsv'
5
+ presentation?: 'pptx' | 'odp' | 'pdf' | 'txt'
6
+ drawing?: 'png' | 'svg' | 'pdf' | 'jpg'
7
+ }
8
+
9
+ /** Where Google answers; tests point these at a stand-in. */
10
+ export interface GoogleDriveOrigins {
11
+ drive?: string
12
+ usercontent?: string
13
+ docs?: string
14
+ accounts?: string
15
+ }
16
+
17
+ export interface GoogleDriveResolverOptions {
18
+ exports?: GoogleExportFormats
19
+ origins?: GoogleDriveOrigins
20
+ /** Bytes of an interstitial page read. Default 256 KiB. */
21
+ maxPageBytes?: number
22
+ }
@@ -0,0 +1,153 @@
1
+ import { logger } from '@owlmeans/log'
2
+ import { ResolverError } from '../errors.js'
3
+ import type { ResolverBodyReader, ResolverResponseHead, ResolverStep, UrlResolver } from './types.js'
4
+ import {
5
+ DEFAULT_DRIVE_PAGE_BYTES, DEFAULT_EXPORT_FORMATS, DEFAULT_GOOGLE_ORIGINS, DRIVE_ID_PATTERN, GOOGLE_DRIVE_RESOLVER, MAX_DRIVE_STEPS,
6
+ } from './google-drive/consts.js'
7
+ import type { GoogleDriveResolverOptions } from './google-drive/types.js'
8
+
9
+ const log = logger('file-streams:drive')
10
+
11
+ /**
12
+ * Google Drive and Docs links to the bytes behind them: share links become the usercontent
13
+ * download, the "can't scan for viruses" page is followed through its own form, and editor files
14
+ * become export URLs in the configured formats. Folders, quotas and non-public files fail.
15
+ */
16
+ export const createGoogleDriveResolver = (options: GoogleDriveResolverOptions = {}): UrlResolver => {
17
+ const origins = { ...DEFAULT_GOOGLE_ORIGINS, ...options.origins }
18
+ const drive = new URL(origins.drive).origin
19
+ const usercontent = new URL(origins.usercontent).origin
20
+ const docs = new URL(origins.docs).origin
21
+ const accounts = new URL(origins.accounts).origin
22
+ const formats = { ...DEFAULT_EXPORT_FORMATS, ...options.exports }
23
+ const maxPageBytes = options.maxPageBytes ?? DEFAULT_DRIVE_PAGE_BYTES
24
+
25
+ const owns = (url: URL): boolean => url.origin === drive || url.origin === usercontent || url.origin === docs
26
+
27
+ const matches = owns
28
+
29
+ const fail = (reason: string): ResolverError => new ResolverError(`${GOOGLE_DRIVE_RESOLVER}:${reason}`)
30
+
31
+ const exportOf = (url: URL): URL | undefined => {
32
+ const match = /^\/(document|spreadsheets|presentation|drawings)\/d\/([^/]+)/.exec(url.pathname)
33
+ if (match == null || !DRIVE_ID_PATTERN.test(match[2])) {
34
+ return undefined
35
+ }
36
+ const [, type, id] = match
37
+ switch (type) {
38
+ case 'document':
39
+ return new URL(`/document/d/${id}/export?format=${formats.document}`, docs)
40
+ case 'spreadsheets': {
41
+ const exported = new URL(`/spreadsheets/d/${id}/export?format=${formats.spreadsheet}`, docs)
42
+ const gid = url.searchParams.get('gid') ?? /gid=(\d+)/.exec(url.hash)?.[1]
43
+ if (gid != null && (formats.spreadsheet === 'csv' || formats.spreadsheet === 'tsv')) {
44
+ exported.searchParams.set('gid', gid)
45
+ }
46
+ return exported
47
+ }
48
+ case 'presentation':
49
+ return new URL(`/presentation/d/${id}/export/${formats.presentation}`, docs)
50
+ default:
51
+ return new URL(`/drawings/d/${id}/export/${formats.drawing}`, docs)
52
+ }
53
+ }
54
+
55
+ const rewrite = (url: URL): URL => {
56
+ if (url.origin === docs) {
57
+ const exported = exportOf(url)
58
+ if (exported != null) {
59
+ return exported
60
+ }
61
+ }
62
+ if (url.origin === drive) {
63
+ if (/\/drive\/(u\/\d+\/)?folders\//.test(url.pathname) || url.pathname.startsWith('/embeddedfolderview')) {
64
+ throw fail('folder')
65
+ }
66
+ const id = /\/file\/d\/([^/]+)/.exec(url.pathname)?.[1] ?? url.searchParams.get('id')
67
+ if (id == null || !DRIVE_ID_PATTERN.test(id)) {
68
+ return url
69
+ }
70
+ const download = new URL('/download', usercontent)
71
+ download.searchParams.set('id', id)
72
+ download.searchParams.set('export', 'download')
73
+ const resourceKey = url.searchParams.get('resourcekey')
74
+ if (resourceKey != null) {
75
+ download.searchParams.set('resourcekey', resourceKey)
76
+ }
77
+ return download
78
+ }
79
+
80
+ return url
81
+ }
82
+
83
+ const decodeHtml = (value: string): string => value
84
+ .replace(/&amp;/g, '&').replace(/&quot;/g, '"').replace(/&#39;|&#x27;/g, "'").replace(/&lt;/g, '<').replace(/&gt;/g, '>')
85
+
86
+ const attribute = (tag: string, name: string): string | undefined =>
87
+ new RegExp(`\\s${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(tag)?.slice(2).find(value => value != null)
88
+
89
+ const headerOf = (head: ResolverResponseHead, name: string): string[] => {
90
+ const value = head.headers[name]
91
+ return value == null ? [] : Array.isArray(value) ? value : [value]
92
+ }
93
+
94
+ const inspect = async (head: ResolverResponseHead, readBody: ResolverBodyReader): Promise<ResolverStep> => {
95
+ if (head.url.origin === accounts) {
96
+ return { action: 'fail', reason: 'not-public' }
97
+ }
98
+ const type = headerOf(head, 'content-type')[0]?.toLowerCase() ?? ''
99
+ const disposition = headerOf(head, 'content-disposition')[0]?.toLowerCase() ?? ''
100
+ if (disposition.includes('attachment') || !type.includes('text/html')) {
101
+ return { action: 'accept' }
102
+ }
103
+
104
+ const page = await readBody(maxPageBytes)
105
+ const formTag = /<form\b[^>]*\bid\s*=\s*["']download-form["'][^>]*>/i.exec(page)
106
+ if (formTag != null) {
107
+ const action = attribute(formTag[0], 'action')
108
+ if (action != null) {
109
+ const next = new URL(decodeHtml(action), head.url)
110
+ if (!owns(next)) {
111
+ return { action: 'fail', reason: 'foreign-form' }
112
+ }
113
+ const body = page.slice(formTag.index, page.indexOf('</form>', formTag.index) === -1 ? undefined : page.indexOf('</form>', formTag.index))
114
+ for (const input of body.match(/<input\b[^>]*>/gi) ?? []) {
115
+ const name = attribute(input, 'name')
116
+ if (name != null && (attribute(input, 'type') ?? '').toLowerCase() === 'hidden') {
117
+ next.searchParams.set(name, decodeHtml(attribute(input, 'value') ?? ''))
118
+ }
119
+ }
120
+ log.debug('Following the Drive confirmation form')
121
+ return { action: 'follow', url: next }
122
+ }
123
+ }
124
+
125
+ const legacy = /href\s*=\s*"(\/uc\?export=download[^"]*confirm=[^"]+)"/i.exec(page)
126
+ if (legacy != null) {
127
+ return { action: 'follow', url: new URL(decodeHtml(legacy[1]), head.url) }
128
+ }
129
+
130
+ const warning = headerOf(head, 'set-cookie').map(cookie => /^(download_warning[^=]*)=([^;]+)/.exec(cookie)).find(match => match != null)
131
+ if (warning != null) {
132
+ const next = new URL(head.url)
133
+ next.searchParams.set('confirm', warning[2])
134
+ return { action: 'follow', url: next, headers: { cookie: `${warning[1]}=${warning[2]}` } }
135
+ }
136
+
137
+ if (/quota|too many users have viewed or downloaded/i.test(page)) {
138
+ return { action: 'fail', reason: 'quota' }
139
+ }
140
+ if (/too large to be exported/i.test(page)) {
141
+ return { action: 'fail', reason: 'export-too-large' }
142
+ }
143
+ if (/accounts\.google\.com\/(ServiceLogin|v3\/signin)|you need access|request access/i.test(page)) {
144
+ return { action: 'fail', reason: 'not-public' }
145
+ }
146
+
147
+ return { action: 'fail', reason: 'unrecognized' }
148
+ }
149
+
150
+ return { name: GOOGLE_DRIVE_RESOLVER, maxSteps: MAX_DRIVE_STEPS, matches, rewrite, inspect, owns }
151
+ }
152
+
153
+ export const googleDriveResolver = createGoogleDriveResolver()
@@ -0,0 +1,45 @@
1
+ /** The head of a response a resolver inspects. */
2
+ export interface ResolverResponseHead {
3
+ url: URL
4
+ status: number
5
+ headers: Record<string, string | string[] | undefined>
6
+ }
7
+
8
+ export interface ResolverAccept {
9
+ action: 'accept'
10
+ }
11
+
12
+ export interface ResolverFollow {
13
+ action: 'follow'
14
+ url: URL
15
+ /** Only `cookie` is honoured, and only towards the resolver's own origins. */
16
+ headers?: Record<string, string>
17
+ }
18
+
19
+ export interface ResolverFail {
20
+ action: 'fail'
21
+ reason: string
22
+ }
23
+
24
+ /** What a resolver decides about a response. */
25
+ // Three distinct steps a discriminated union names best.
26
+ export type ResolverStep = ResolverAccept | ResolverFollow | ResolverFail
27
+
28
+ /** Reads at most `maxBytes` of the body as text; the bytes are replayed to the caller on `accept`. */
29
+ export interface ResolverBodyReader {
30
+ (maxBytes?: number): Promise<string>
31
+ }
32
+
33
+ /** Turns a page or share link into the bytes behind it (Google Drive, Docs exports). */
34
+ export interface UrlResolver {
35
+ readonly name: string
36
+ /** Follow-ups one download may take. */
37
+ readonly maxSteps: number
38
+ matches: (url: URL) => boolean
39
+ /** The URL that serves the bytes; throws `ResolverError` for links that cannot (folders). */
40
+ rewrite: (url: URL) => URL
41
+ /** Decides on a final (post-redirect) response before its body reaches the caller. */
42
+ inspect: (head: ResolverResponseHead, readBody: ResolverBodyReader) => Promise<ResolverStep>
43
+ /** Whether a follow-up URL belongs to this resolver (cookies are sent only there). */
44
+ owns: (url: URL) => boolean
45
+ }
@@ -0,0 +1,12 @@
1
+ export const DEFAULT_PART_SIZE = 8 * 1024 * 1024
2
+
3
+ /** S3's smallest part (all but the last). */
4
+ export const MIN_PART_SIZE = 5 * 1024 * 1024
5
+
6
+ /** S3's most parts per upload. */
7
+ export const MAX_PARTS = 10_000
8
+
9
+ export const DEFAULT_UPLOAD_CONCURRENCY = 2
10
+
11
+ /** S3's most keys per DeleteObjects request. */
12
+ export const S3_DELETE_BATCH_SIZE = 1000
@@ -0,0 +1,100 @@
1
+ import { createHash } from 'node:crypto'
2
+ import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
3
+ import type { ListObjectsV2CommandOutput } from '@aws-sdk/client-s3'
4
+ import { logger } from '@owlmeans/log'
5
+ import { S3TransferError } from '../errors.js'
6
+ import type { SignalOptions } from '../types.js'
7
+ import { S3_DELETE_BATCH_SIZE } from './consts.js'
8
+ import type { S3ListedObject } from './types.js'
9
+ import type { S3BuildHandler, S3Core, S3PrefixOps } from './types.local.js'
10
+
11
+ const log = logger('file-streams:s3')
12
+
13
+ /** Listing and deleting everything under one prefix of the bucket. */
14
+ export const makeS3PrefixOps = (core: S3Core): S3PrefixOps => {
15
+ const failure = (error: unknown, op: string, signal?: AbortSignal): unknown => {
16
+ if (signal?.aborted === true) {
17
+ return signal.reason
18
+ }
19
+ log.error('S3 prefix operation failed', error as Error)
20
+ return new S3TransferError(op)
21
+ }
22
+
23
+ // DeleteObjects requires a checksum. The SDK's flexible checksums (CRC32) are not understood by
24
+ // every S3-compatible store; Content-MD5 is, by all of them and by S3 itself — the SDK's documented
25
+ // MD5 fallback: the flexible checksum headers go, a Content-MD5 of the body takes their place.
26
+ const contentMd5 = (next: S3BuildHandler): S3BuildHandler => async args => {
27
+ const request = args.request as { headers?: Record<string, string>, body?: unknown }
28
+ if (request.headers != null && (typeof request.body === 'string' || request.body instanceof Uint8Array)) {
29
+ for (const header of Object.keys(request.headers)) {
30
+ if (/^x-amz-(sdk-)?checksum-/i.test(header)) {
31
+ delete request.headers[header]
32
+ }
33
+ }
34
+ request.headers['content-md5'] = createHash('md5').update(request.body).digest('base64')
35
+ }
36
+ return await next(args)
37
+ }
38
+
39
+ const withContentMd5 = (command: DeleteObjectsCommand): DeleteObjectsCommand => {
40
+ // A plain build handler: the SDK's middleware union cannot be inferred from it, hence the cast.
41
+ command.middlewareStack.addRelativeTo(contentMd5 as never, {
42
+ relation: 'after', toMiddleware: 'flexibleChecksumsMiddleware', name: 'owlmeansContentMd5',
43
+ })
44
+ return command
45
+ }
46
+
47
+ const list = async (prefix: string, options: SignalOptions = {}): Promise<S3ListedObject[]> => {
48
+ const base = `${core.keyOf(prefix)}/`
49
+ const items: S3ListedObject[] = []
50
+ let token: string | undefined
51
+ do {
52
+ let page: ListObjectsV2CommandOutput
53
+ try {
54
+ page = await core.client().send(
55
+ new ListObjectsV2Command({ Bucket: core.model.bucket, Prefix: base, ...(token != null ? { ContinuationToken: token } : {}) }),
56
+ { abortSignal: options.signal }
57
+ )
58
+ } catch (error) {
59
+ throw failure(error, 'list', options.signal)
60
+ }
61
+ for (const object of page.Contents ?? []) {
62
+ if (object.Key != null && object.Key.startsWith(base) && object.Key.length > base.length) {
63
+ items.push({
64
+ key: object.Key, path: object.Key.slice(base.length), size: object.Size ?? 0, etag: object.ETag, modifiedAt: object.LastModified,
65
+ })
66
+ }
67
+ }
68
+ token = page.IsTruncated === true ? page.NextContinuationToken : undefined
69
+ } while (token != null)
70
+
71
+ return items
72
+ }
73
+
74
+ const removePrefix = async (prefix: string, options: SignalOptions = {}): Promise<number> => {
75
+ const keys = (await list(prefix, options)).map(item => item.key)
76
+ let removed = 0
77
+ for (let index = 0; index < keys.length; index += S3_DELETE_BATCH_SIZE) {
78
+ const batch = keys.slice(index, index + S3_DELETE_BATCH_SIZE)
79
+ let errors
80
+ try {
81
+ const result = await core.client().send(withContentMd5(new DeleteObjectsCommand({
82
+ Bucket: core.model.bucket, Delete: { Objects: batch.map(Key => ({ Key })), Quiet: true },
83
+ })), { abortSignal: options.signal })
84
+ errors = result.Errors ?? []
85
+ } catch (error) {
86
+ throw failure(error, 'delete-objects', options.signal)
87
+ }
88
+ if (errors.length > 0) {
89
+ log.error('S3 refused deleting keys', { refused: errors.length, code: errors[0].Code })
90
+ throw new S3TransferError(`delete-objects:${errors[0].Code ?? 'error'}`)
91
+ }
92
+ removed += batch.length
93
+ }
94
+ log.debug('Prefix removed', { objects: removed })
95
+
96
+ return removed
97
+ }
98
+
99
+ return { list, removePrefix }
100
+ }
package/src/s3/sink.ts ADDED
@@ -0,0 +1,36 @@
1
+ import { EXTENSION_MIME } from '@owlmeans/file-inspector'
2
+ import type { FileSink, SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
3
+ import { nameUtils } from '../utils/names.js'
4
+ import { streamUtils } from '../utils/streams.js'
5
+ import type { S3SinkOptions } from './types.js'
6
+ import type { S3Core, S3Uploader } from './types.local.js'
7
+
8
+ /** One object as a sink: its type from the verdict, never inline when it carries scripts. */
9
+ export const makeS3Sink = (core: S3Core, uploader: S3Uploader, prefix: string, options: S3SinkOptions = {}): FileSink => {
10
+ const key = core.keyOf(prefix)
11
+
12
+ const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
13
+ const ext = nameUtils.extOf(prefix)
14
+ const contentType = options.contentType ?? input.verdict?.detected?.mime ?? input.info.mime
15
+ ?? (ext != null ? EXTENSION_MIME[ext] : undefined) ?? 'application/octet-stream'
16
+ const active = input.verdict?.details.active === true && options.allowActiveInline !== true
17
+ const disposition = active ? 'attachment' : options.disposition ?? 'inline'
18
+ const meter = streamUtils.meter({ maxBytes: writeOptions.maxBytes })
19
+ const stream = input.stream.pipe(meter.stream)
20
+ input.stream.once('error', error => meter.stream.destroy(error))
21
+
22
+ let result
23
+ try {
24
+ result = await uploader.upload({
25
+ key, stream, contentType, disposition, cacheControl: options.cacheControl, metadata: options.metadata, signal: writeOptions.signal,
26
+ })
27
+ } catch (error) {
28
+ streamUtils.discard(input.stream)
29
+ throw error
30
+ }
31
+
32
+ return { kind: 's3', location: core.location(key), bytes: result.bytes, url: core.model.urlOf(prefix), etag: result.etag, mime: contentType }
33
+ }
34
+
35
+ return { kind: 's3', write }
36
+ }
@@ -0,0 +1,90 @@
1
+ import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'
2
+ import type { Readable } from 'node:stream'
3
+ import { S3TransferError, SourceError } from '../errors.js'
4
+ import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader, SignalOptions } from '../types.js'
5
+ import { nameUtils } from '../utils/names.js'
6
+ import { streamUtils } from '../utils/streams.js'
7
+ import type { S3Core } from './types.local.js'
8
+
9
+ /** One object as a source: re-openable, with random access through Range and If-Match. */
10
+ export const makeS3Source = (core: S3Core, key: string): FileSource => {
11
+ let known: FileInfo | undefined
12
+
13
+ const statusOf = (error: unknown): number | undefined => (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
14
+
15
+ const failure = (error: unknown, op: string, signal?: AbortSignal): unknown => {
16
+ if (signal?.aborted === true) {
17
+ return signal.reason
18
+ }
19
+ const status = statusOf(error)
20
+ if (status === 404 || (error as Error).name === 'NoSuchKey' || (error as Error).name === 'NotFound') {
21
+ return new SourceError('not-found')
22
+ }
23
+ if (status === 412 || (error as Error).name === 'PreconditionFailed') {
24
+ return new SourceError('changed')
25
+ }
26
+ return new S3TransferError(op)
27
+ }
28
+
29
+ const describe = (head: { ContentLength?: number, ContentType?: string, ETag?: string, LastModified?: Date }): FileInfo => ({
30
+ origin: core.location(key),
31
+ name: nameUtils.fromKey(key),
32
+ size: head.ContentLength,
33
+ mime: head.ContentType?.split(';')[0].trim().toLowerCase(),
34
+ etag: head.ETag,
35
+ modifiedAt: head.LastModified,
36
+ })
37
+
38
+ const info = async (options: SignalOptions = {}): Promise<FileInfo> => {
39
+ if (known == null) {
40
+ try {
41
+ known = describe(await core.client().send(new HeadObjectCommand({ Bucket: core.model.bucket, Key: key }), { abortSignal: options.signal }))
42
+ } catch (error) {
43
+ throw failure(error, 'head', options.signal)
44
+ }
45
+ }
46
+ return known
47
+ }
48
+
49
+ const get = async (range: string | undefined, ifMatch: string | undefined, signal?: AbortSignal) => {
50
+ try {
51
+ return await core.client().send(new GetObjectCommand({
52
+ Bucket: core.model.bucket, Key: key, ...(range != null ? { Range: range } : {}), ...(ifMatch != null ? { IfMatch: ifMatch } : {}),
53
+ }), { abortSignal: signal })
54
+ } catch (error) {
55
+ throw failure(error, 'get', signal)
56
+ }
57
+ }
58
+
59
+ const open = async (options: OpenOptions = {}): Promise<OpenedFile> => {
60
+ const range = options.range != null ? `bytes=${options.range.start}-${options.range.end ?? ''}` : undefined
61
+ const response = await get(range, undefined, options.signal)
62
+ const opened: OpenedFile = { stream: response.Body as Readable, info: { ...describe(response), size: known?.size ?? response.ContentLength } }
63
+ const span = /bytes (\d+)-(\d+)\/(\d+)/.exec(response.ContentRange ?? '')
64
+ if (span != null) {
65
+ opened.range = { start: Number(span[1]), end: Number(span[2]), total: Number(span[3]) }
66
+ opened.info.size = Number(span[3])
67
+ }
68
+ return opened
69
+ }
70
+
71
+ const ranges = async (options: SignalOptions = {}): Promise<RangeReader> => {
72
+ const current = await info(options)
73
+ const size = current.size ?? 0
74
+ return {
75
+ size,
76
+ read: async (offset, length, signal) => {
77
+ const end = Math.min(offset + length, size) - 1
78
+ if (end < offset) {
79
+ return new Uint8Array(0)
80
+ }
81
+ const response = await get(`bytes=${offset}-${end}`, current.etag, signal)
82
+ return await streamUtils.collect(response.Body as Readable, end - offset + 1)
83
+ },
84
+ stream: async (start, end, signal) => (await get(`bytes=${start}-${end - 1}`, current.etag, signal)).Body as Readable,
85
+ close: async () => undefined,
86
+ }
87
+ }
88
+
89
+ return { kind: 's3', info, open, ranges }
90
+ }
@@ -0,0 +1,49 @@
1
+ import type { S3Client } from '@aws-sdk/client-s3'
2
+ import type { Readable } from 'node:stream'
3
+ import type { StorageBucketModel } from '@owlmeans/storage-resource'
4
+ import type { SignalOptions } from '../types.js'
5
+ import type { S3ListedObject } from './types.js'
6
+
7
+ /** What the bucket's parts share. */
8
+ export interface S3Core {
9
+ readonly model: StorageBucketModel
10
+ client: () => S3Client
11
+ keyOf: (prefix: string) => string
12
+ location: (key: string) => string
13
+ partSize: number
14
+ concurrency: number
15
+ }
16
+
17
+ export interface S3UploadRequest {
18
+ key: string
19
+ stream: Readable
20
+ contentType: string
21
+ disposition: string
22
+ cacheControl?: string
23
+ metadata?: Record<string, string>
24
+ signal?: AbortSignal
25
+ }
26
+
27
+ export interface S3UploadResult {
28
+ bytes: number
29
+ etag?: string
30
+ }
31
+
32
+ export interface S3Uploader {
33
+ /** One object from a stream of unknown length. */
34
+ upload: (request: S3UploadRequest) => Promise<S3UploadResult>
35
+ }
36
+
37
+ export interface S3PrefixOps {
38
+ list: (prefix: string, options?: SignalOptions) => Promise<S3ListedObject[]>
39
+ removePrefix: (prefix: string, options?: SignalOptions) => Promise<number>
40
+ }
41
+
42
+ /** The part of an SDK build-step handler's arguments the Content-MD5 middleware touches. */
43
+ export interface S3BuildArgs {
44
+ request: unknown
45
+ }
46
+
47
+ export interface S3BuildHandler {
48
+ (args: S3BuildArgs): Promise<unknown>
49
+ }
@@ -0,0 +1,70 @@
1
+ import type { StorageBucketModel } from '@owlmeans/storage-resource'
2
+ import type { StoredFileInstance } from '@owlmeans/storage-common'
3
+ import type { FileSink, FileSource, SignalOptions, SinkResult, TreeSink } from '../types.js'
4
+
5
+ export interface S3BucketOptions {
6
+ /** Multipart part size. Default 8 MiB, at least 5 MiB. */
7
+ partSize?: number
8
+ /** Parts in flight. Default 2. */
9
+ concurrency?: number
10
+ }
11
+
12
+ export interface S3SinkOptions {
13
+ /** Overrides the type taken from the verdict or the source. */
14
+ contentType?: string
15
+ /** Default `inline`; active svg/html is written as `attachment` unless allowed. */
16
+ disposition?: 'inline' | 'attachment'
17
+ allowActiveInline?: boolean
18
+ cacheControl?: string
19
+ metadata?: Record<string, string>
20
+ }
21
+
22
+ export interface S3ObjectInfo {
23
+ key: string
24
+ size: number
25
+ etag?: string
26
+ mime?: string
27
+ modifiedAt?: Date
28
+ }
29
+
30
+ /** One object of a prefix listing (a listing carries no media type). */
31
+ export interface S3ListedObject extends S3ObjectInfo {
32
+ /** The key relative to the listed prefix — `media/a.png` for `<basePrefix>/<prefix>/media/a.png`. */
33
+ path: string
34
+ }
35
+
36
+ /**
37
+ * One storage bucket as sources and sinks, addressed exactly as `@owlmeans/storage-resource`
38
+ * addresses it: a `prefix` lands at `<basePrefix>/<prefix>` and answers on the same public URL.
39
+ */
40
+ export interface S3Bucket {
41
+ readonly model: StorageBucketModel
42
+ /** The object key of a prefix (normalized; `..` and absolute prefixes refused). */
43
+ keyOf: (prefix: string) => string
44
+ /** The public URL of a prefix. */
45
+ urlOf: (prefix: string) => string
46
+ /** GetObject; random access through Range with If-Match. */
47
+ source: (prefix: string) => FileSource
48
+ /** PutObject when the body fits one part, a multipart upload (aborted on failure) otherwise. */
49
+ sink: (prefix: string, options?: S3SinkOptions) => FileSink
50
+ /** Many objects under one prefix. */
51
+ tree: (prefix: string, options?: S3SinkOptions) => TreeSink
52
+ head: (prefix: string, options?: SignalOptions) => Promise<S3ObjectInfo | null>
53
+ remove: (prefix: string, options?: SignalOptions) => Promise<void>
54
+ /**
55
+ * Every object under `<prefix>/` — ListObjectsV2, all pages — with its full `key` and its `path`
56
+ * relative to the prefix. An empty prefix is refused (`PathError('empty')`), never the whole bucket.
57
+ */
58
+ list: (prefix: string, options?: SignalOptions) => Promise<S3ListedObject[]>
59
+ /**
60
+ * Deletes every object under `<prefix>/` — DeleteObjects in batches of at most 1000 keys, each
61
+ * checksummed by Content-MD5 (the SDK's MD5 fallback, which S3-compatible stores accept) — and
62
+ * answers how many went.
63
+ * @throws {S3TransferError} `delete-objects:<code>` when the store refuses any key of a batch.
64
+ */
65
+ removePrefix: (prefix: string, options?: SignalOptions) => Promise<number>
66
+ /** A sink result as a storage-common file instance. */
67
+ instanceOf: (result: SinkResult, alias: string) => StoredFileInstance
68
+ /** Releases the client. */
69
+ close: () => void
70
+ }