@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +71 -2
  3. package/agent-meta/manifest.json +64 -0
  4. package/agent-meta/skills/file-streams/SKILL.md +183 -0
  5. package/agent-meta/skills/file-streams/parts/network.md +47 -0
  6. package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
  7. package/build/archive/consts.d.ts +16 -0
  8. package/build/archive/consts.d.ts.map +1 -0
  9. package/build/archive/consts.js +24 -0
  10. package/build/archive/consts.js.map +1 -0
  11. package/build/archive/formats.d.ts +4 -0
  12. package/build/archive/formats.d.ts.map +1 -0
  13. package/build/archive/formats.js +53 -0
  14. package/build/archive/formats.js.map +1 -0
  15. package/build/archive/guard.d.ts +5 -0
  16. package/build/archive/guard.d.ts.map +1 -0
  17. package/build/archive/guard.js +62 -0
  18. package/build/archive/guard.js.map +1 -0
  19. package/build/archive/ratio.d.ts +5 -0
  20. package/build/archive/ratio.d.ts.map +1 -0
  21. package/build/archive/ratio.js +30 -0
  22. package/build/archive/ratio.js.map +1 -0
  23. package/build/archive/tar.d.ts +4 -0
  24. package/build/archive/tar.d.ts.map +1 -0
  25. package/build/archive/tar.js +142 -0
  26. package/build/archive/tar.js.map +1 -0
  27. package/build/archive/types.d.ts +95 -0
  28. package/build/archive/types.d.ts.map +1 -0
  29. package/build/archive/types.js +2 -0
  30. package/build/archive/types.js.map +1 -0
  31. package/build/archive/types.local.d.ts +64 -0
  32. package/build/archive/types.local.d.ts.map +1 -0
  33. package/build/archive/types.local.js +2 -0
  34. package/build/archive/types.local.js.map +1 -0
  35. package/build/archive/zip.d.ts +4 -0
  36. package/build/archive/zip.d.ts.map +1 -0
  37. package/build/archive/zip.js +135 -0
  38. package/build/archive/zip.js.map +1 -0
  39. package/build/archive.d.ts +4 -0
  40. package/build/archive.d.ts.map +1 -0
  41. package/build/archive.js +265 -0
  42. package/build/archive.js.map +1 -0
  43. package/build/backed-workspace/types.d.ts +67 -0
  44. package/build/backed-workspace/types.d.ts.map +1 -0
  45. package/build/backed-workspace/types.js +2 -0
  46. package/build/backed-workspace/types.js.map +1 -0
  47. package/build/backed-workspace.d.ts +4 -0
  48. package/build/backed-workspace.d.ts.map +1 -0
  49. package/build/backed-workspace.js +63 -0
  50. package/build/backed-workspace.js.map +1 -0
  51. package/build/consts.d.ts +3 -0
  52. package/build/consts.d.ts.map +1 -0
  53. package/build/consts.js +3 -0
  54. package/build/consts.js.map +1 -0
  55. package/build/consts.local.d.ts +3 -0
  56. package/build/consts.local.d.ts.map +1 -0
  57. package/build/consts.local.js +3 -0
  58. package/build/consts.local.js.map +1 -0
  59. package/build/errors.d.ts +59 -0
  60. package/build/errors.d.ts.map +1 -0
  61. package/build/errors.js +105 -0
  62. package/build/errors.js.map +1 -0
  63. package/build/http/connector.d.ts +11 -0
  64. package/build/http/connector.d.ts.map +1 -0
  65. package/build/http/connector.js +124 -0
  66. package/build/http/connector.js.map +1 -0
  67. package/build/http/consts.d.ts +9 -0
  68. package/build/http/consts.d.ts.map +1 -0
  69. package/build/http/consts.js +8 -0
  70. package/build/http/consts.js.map +1 -0
  71. package/build/http/source.d.ts +5 -0
  72. package/build/http/source.d.ts.map +1 -0
  73. package/build/http/source.js +76 -0
  74. package/build/http/source.js.map +1 -0
  75. package/build/http/types.d.ts +67 -0
  76. package/build/http/types.d.ts.map +1 -0
  77. package/build/http/types.js +2 -0
  78. package/build/http/types.js.map +1 -0
  79. package/build/http/types.local.d.ts +21 -0
  80. package/build/http/types.local.d.ts.map +1 -0
  81. package/build/http/types.local.js +2 -0
  82. package/build/http/types.local.js.map +1 -0
  83. package/build/http.d.ts +4 -0
  84. package/build/http.d.ts.map +1 -0
  85. package/build/http.js +248 -0
  86. package/build/http.js.map +1 -0
  87. package/build/index.d.ts +53 -0
  88. package/build/index.d.ts.map +1 -0
  89. package/build/index.js +33 -0
  90. package/build/index.js.map +1 -0
  91. package/build/network/consts.d.ts +12 -0
  92. package/build/network/consts.d.ts.map +1 -0
  93. package/build/network/consts.js +20 -0
  94. package/build/network/consts.js.map +1 -0
  95. package/build/network/types.d.ts +43 -0
  96. package/build/network/types.d.ts.map +1 -0
  97. package/build/network/types.js +2 -0
  98. package/build/network/types.js.map +1 -0
  99. package/build/network.d.ts +4 -0
  100. package/build/network.d.ts.map +1 -0
  101. package/build/network.js +165 -0
  102. package/build/network.js.map +1 -0
  103. package/build/proxy/consts.d.ts +7 -0
  104. package/build/proxy/consts.d.ts.map +1 -0
  105. package/build/proxy/consts.js +9 -0
  106. package/build/proxy/consts.js.map +1 -0
  107. package/build/proxy/endpoint.d.ts +4 -0
  108. package/build/proxy/endpoint.d.ts.map +1 -0
  109. package/build/proxy/endpoint.js +27 -0
  110. package/build/proxy/endpoint.js.map +1 -0
  111. package/build/proxy/http.d.ts +5 -0
  112. package/build/proxy/http.d.ts.map +1 -0
  113. package/build/proxy/http.js +95 -0
  114. package/build/proxy/http.js.map +1 -0
  115. package/build/proxy/router.d.ts +4 -0
  116. package/build/proxy/router.d.ts.map +1 -0
  117. package/build/proxy/router.js +85 -0
  118. package/build/proxy/router.js.map +1 -0
  119. package/build/proxy/socks.d.ts +5 -0
  120. package/build/proxy/socks.d.ts.map +1 -0
  121. package/build/proxy/socks.js +44 -0
  122. package/build/proxy/socks.js.map +1 -0
  123. package/build/proxy/types.d.ts +99 -0
  124. package/build/proxy/types.d.ts.map +1 -0
  125. package/build/proxy/types.js +2 -0
  126. package/build/proxy/types.js.map +1 -0
  127. package/build/resolvers/google-drive/consts.d.ts +10 -0
  128. package/build/resolvers/google-drive/consts.d.ts.map +1 -0
  129. package/build/resolvers/google-drive/consts.js +16 -0
  130. package/build/resolvers/google-drive/consts.js.map +1 -0
  131. package/build/resolvers/google-drive/types.d.ts +21 -0
  132. package/build/resolvers/google-drive/types.d.ts.map +1 -0
  133. package/build/resolvers/google-drive/types.js +2 -0
  134. package/build/resolvers/google-drive/types.js.map +1 -0
  135. package/build/resolvers/google-drive.d.ts +10 -0
  136. package/build/resolvers/google-drive.d.ts.map +1 -0
  137. package/build/resolvers/google-drive.js +130 -0
  138. package/build/resolvers/google-drive.js.map +1 -0
  139. package/build/resolvers/types.d.ts +39 -0
  140. package/build/resolvers/types.d.ts.map +1 -0
  141. package/build/resolvers/types.js +2 -0
  142. package/build/resolvers/types.js.map +1 -0
  143. package/build/s3/consts.d.ts +9 -0
  144. package/build/s3/consts.d.ts.map +1 -0
  145. package/build/s3/consts.js +9 -0
  146. package/build/s3/consts.js.map +1 -0
  147. package/build/s3/prefix.d.ts +4 -0
  148. package/build/s3/prefix.d.ts.map +1 -0
  149. package/build/s3/prefix.js +87 -0
  150. package/build/s3/prefix.js.map +1 -0
  151. package/build/s3/sink.d.ts +6 -0
  152. package/build/s3/sink.d.ts.map +1 -0
  153. package/build/s3/sink.js +30 -0
  154. package/build/s3/sink.js.map +1 -0
  155. package/build/s3/source.d.ts +5 -0
  156. package/build/s3/source.d.ts.map +1 -0
  157. package/build/s3/source.js +81 -0
  158. package/build/s3/source.js.map +1 -0
  159. package/build/s3/types.d.ts +66 -0
  160. package/build/s3/types.d.ts.map +1 -0
  161. package/build/s3/types.js +2 -0
  162. package/build/s3/types.js.map +1 -0
  163. package/build/s3/types.local.d.ts +43 -0
  164. package/build/s3/types.local.d.ts.map +1 -0
  165. package/build/s3/types.local.js +2 -0
  166. package/build/s3/types.local.js.map +1 -0
  167. package/build/s3/upload.d.ts +4 -0
  168. package/build/s3/upload.d.ts.map +1 -0
  169. package/build/s3/upload.js +118 -0
  170. package/build/s3/upload.js.map +1 -0
  171. package/build/s3.d.ts +5 -0
  172. package/build/s3.d.ts.map +1 -0
  173. package/build/s3.js +68 -0
  174. package/build/s3.js.map +1 -0
  175. package/build/service/types.d.ts +66 -0
  176. package/build/service/types.d.ts.map +1 -0
  177. package/build/service/types.js +2 -0
  178. package/build/service/types.js.map +1 -0
  179. package/build/service/types.local.d.ts +11 -0
  180. package/build/service/types.local.d.ts.map +1 -0
  181. package/build/service/types.local.js +2 -0
  182. package/build/service/types.local.js.map +1 -0
  183. package/build/service.d.ts +6 -0
  184. package/build/service.d.ts.map +1 -0
  185. package/build/service.js +73 -0
  186. package/build/service.js.map +1 -0
  187. package/build/sinks/buffer/consts.d.ts +2 -0
  188. package/build/sinks/buffer/consts.d.ts.map +1 -0
  189. package/build/sinks/buffer/consts.js +2 -0
  190. package/build/sinks/buffer/consts.js.map +1 -0
  191. package/build/sinks/buffer/types.d.ts +11 -0
  192. package/build/sinks/buffer/types.d.ts.map +1 -0
  193. package/build/sinks/buffer/types.js +2 -0
  194. package/build/sinks/buffer/types.js.map +1 -0
  195. package/build/sinks/buffer.d.ts +4 -0
  196. package/build/sinks/buffer.d.ts.map +1 -0
  197. package/build/sinks/buffer.js +13 -0
  198. package/build/sinks/buffer.js.map +1 -0
  199. package/build/sinks/directory/types.d.ts +7 -0
  200. package/build/sinks/directory/types.d.ts.map +1 -0
  201. package/build/sinks/directory/types.js +2 -0
  202. package/build/sinks/directory/types.js.map +1 -0
  203. package/build/sinks/directory.d.ts +8 -0
  204. package/build/sinks/directory.d.ts.map +1 -0
  205. package/build/sinks/directory.js +43 -0
  206. package/build/sinks/directory.js.map +1 -0
  207. package/build/sinks/file.d.ts +5 -0
  208. package/build/sinks/file.d.ts.map +1 -0
  209. package/build/sinks/file.js +46 -0
  210. package/build/sinks/file.js.map +1 -0
  211. package/build/sinks/types.d.ts +11 -0
  212. package/build/sinks/types.d.ts.map +1 -0
  213. package/build/sinks/types.js +2 -0
  214. package/build/sinks/types.js.map +1 -0
  215. package/build/sinks/workload/consts.d.ts +5 -0
  216. package/build/sinks/workload/consts.d.ts.map +1 -0
  217. package/build/sinks/workload/consts.js +5 -0
  218. package/build/sinks/workload/consts.js.map +1 -0
  219. package/build/sinks/workload/types.d.ts +17 -0
  220. package/build/sinks/workload/types.d.ts.map +1 -0
  221. package/build/sinks/workload/types.js +2 -0
  222. package/build/sinks/workload/types.js.map +1 -0
  223. package/build/sinks/workload.d.ts +8 -0
  224. package/build/sinks/workload.d.ts.map +1 -0
  225. package/build/sinks/workload.js +51 -0
  226. package/build/sinks/workload.js.map +1 -0
  227. package/build/sources/buffer.d.ts +5 -0
  228. package/build/sources/buffer.d.ts.map +1 -0
  229. package/build/sources/buffer.js +20 -0
  230. package/build/sources/buffer.js.map +1 -0
  231. package/build/sources/file.d.ts +5 -0
  232. package/build/sources/file.d.ts.map +1 -0
  233. package/build/sources/file.js +50 -0
  234. package/build/sources/file.js.map +1 -0
  235. package/build/sources/stream.d.ts +5 -0
  236. package/build/sources/stream.d.ts.map +1 -0
  237. package/build/sources/stream.js +19 -0
  238. package/build/sources/stream.js.map +1 -0
  239. package/build/sources/types.d.ts +10 -0
  240. package/build/sources/types.d.ts.map +1 -0
  241. package/build/sources/types.js +2 -0
  242. package/build/sources/types.js.map +1 -0
  243. package/build/sources/workload/types.d.ts +29 -0
  244. package/build/sources/workload/types.d.ts.map +1 -0
  245. package/build/sources/workload/types.js +2 -0
  246. package/build/sources/workload/types.js.map +1 -0
  247. package/build/sources/workload.d.ts +7 -0
  248. package/build/sources/workload.d.ts.map +1 -0
  249. package/build/sources/workload.js +67 -0
  250. package/build/sources/workload.js.map +1 -0
  251. package/build/transfer/consts.d.ts +5 -0
  252. package/build/transfer/consts.d.ts.map +1 -0
  253. package/build/transfer/consts.js +8 -0
  254. package/build/transfer/consts.js.map +1 -0
  255. package/build/transfer/types.d.ts +49 -0
  256. package/build/transfer/types.d.ts.map +1 -0
  257. package/build/transfer/types.js +2 -0
  258. package/build/transfer/types.js.map +1 -0
  259. package/build/transfer.d.ts +4 -0
  260. package/build/transfer.d.ts.map +1 -0
  261. package/build/transfer.js +75 -0
  262. package/build/transfer.js.map +1 -0
  263. package/build/types.d.ts +99 -0
  264. package/build/types.d.ts.map +1 -0
  265. package/build/types.js +2 -0
  266. package/build/types.js.map +1 -0
  267. package/build/utils/names/consts.d.ts +5 -0
  268. package/build/utils/names/consts.d.ts.map +1 -0
  269. package/build/utils/names/consts.js +5 -0
  270. package/build/utils/names/consts.js.map +1 -0
  271. package/build/utils/names/types.d.ts +13 -0
  272. package/build/utils/names/types.d.ts.map +1 -0
  273. package/build/utils/names/types.js +2 -0
  274. package/build/utils/names/types.js.map +1 -0
  275. package/build/utils/names.d.ts +4 -0
  276. package/build/utils/names.d.ts.map +1 -0
  277. package/build/utils/names.js +60 -0
  278. package/build/utils/names.js.map +1 -0
  279. package/build/utils/paths/consts.d.ts +3 -0
  280. package/build/utils/paths/consts.d.ts.map +1 -0
  281. package/build/utils/paths/consts.js +2 -0
  282. package/build/utils/paths/consts.js.map +1 -0
  283. package/build/utils/paths/types.d.ts +20 -0
  284. package/build/utils/paths/types.d.ts.map +1 -0
  285. package/build/utils/paths/types.js +2 -0
  286. package/build/utils/paths/types.js.map +1 -0
  287. package/build/utils/paths.d.ts +4 -0
  288. package/build/utils/paths.d.ts.map +1 -0
  289. package/build/utils/paths.js +67 -0
  290. package/build/utils/paths.js.map +1 -0
  291. package/build/utils/range/types.d.ts +11 -0
  292. package/build/utils/range/types.d.ts.map +1 -0
  293. package/build/utils/range/types.js +2 -0
  294. package/build/utils/range/types.js.map +1 -0
  295. package/build/utils/range.d.ts +4 -0
  296. package/build/utils/range.d.ts.map +1 -0
  297. package/build/utils/range.js +64 -0
  298. package/build/utils/range.js.map +1 -0
  299. package/build/utils/redact/types.d.ts +16 -0
  300. package/build/utils/redact/types.d.ts.map +1 -0
  301. package/build/utils/redact/types.js +2 -0
  302. package/build/utils/redact/types.js.map +1 -0
  303. package/build/utils/redact.d.ts +4 -0
  304. package/build/utils/redact.d.ts.map +1 -0
  305. package/build/utils/redact.js +18 -0
  306. package/build/utils/redact.js.map +1 -0
  307. package/build/utils/signals/types.d.ts +17 -0
  308. package/build/utils/signals/types.d.ts.map +1 -0
  309. package/build/utils/signals/types.js +2 -0
  310. package/build/utils/signals/types.js.map +1 -0
  311. package/build/utils/signals.d.ts +4 -0
  312. package/build/utils/signals.d.ts.map +1 -0
  313. package/build/utils/signals.js +52 -0
  314. package/build/utils/signals.js.map +1 -0
  315. package/build/utils/streams/consts.d.ts +3 -0
  316. package/build/utils/streams/consts.d.ts.map +1 -0
  317. package/build/utils/streams/consts.js +3 -0
  318. package/build/utils/streams/consts.js.map +1 -0
  319. package/build/utils/streams/types.d.ts +59 -0
  320. package/build/utils/streams/types.d.ts.map +1 -0
  321. package/build/utils/streams/types.js +2 -0
  322. package/build/utils/streams/types.js.map +1 -0
  323. package/build/utils/streams.d.ts +4 -0
  324. package/build/utils/streams.d.ts.map +1 -0
  325. package/build/utils/streams.js +167 -0
  326. package/build/utils/streams.js.map +1 -0
  327. package/build/utils/tree/types.d.ts +6 -0
  328. package/build/utils/tree/types.d.ts.map +1 -0
  329. package/build/utils/tree/types.js +2 -0
  330. package/build/utils/tree/types.js.map +1 -0
  331. package/build/utils/tree.d.ts +4 -0
  332. package/build/utils/tree.d.ts.map +1 -0
  333. package/build/utils/tree.js +14 -0
  334. package/build/utils/tree.js.map +1 -0
  335. package/build/workspace/consts.d.ts +12 -0
  336. package/build/workspace/consts.d.ts.map +1 -0
  337. package/build/workspace/consts.js +12 -0
  338. package/build/workspace/consts.js.map +1 -0
  339. package/build/workspace/consts.local.d.ts +3 -0
  340. package/build/workspace/consts.local.d.ts.map +1 -0
  341. package/build/workspace/consts.local.js +3 -0
  342. package/build/workspace/consts.local.js.map +1 -0
  343. package/build/workspace/fs.d.ts +4 -0
  344. package/build/workspace/fs.d.ts.map +1 -0
  345. package/build/workspace/fs.js +45 -0
  346. package/build/workspace/fs.js.map +1 -0
  347. package/build/workspace/scope.d.ts +6 -0
  348. package/build/workspace/scope.d.ts.map +1 -0
  349. package/build/workspace/scope.js +180 -0
  350. package/build/workspace/scope.js.map +1 -0
  351. package/build/workspace/types.d.ts +66 -0
  352. package/build/workspace/types.d.ts.map +1 -0
  353. package/build/workspace/types.js +2 -0
  354. package/build/workspace/types.js.map +1 -0
  355. package/build/workspace/types.local.d.ts +20 -0
  356. package/build/workspace/types.local.d.ts.map +1 -0
  357. package/build/workspace/types.local.js +2 -0
  358. package/build/workspace/types.local.js.map +1 -0
  359. package/build/workspace.d.ts +7 -0
  360. package/build/workspace.d.ts.map +1 -0
  361. package/build/workspace.js +87 -0
  362. package/build/workspace.js.map +1 -0
  363. package/package.json +61 -4
  364. package/src/archive/consts.ts +28 -0
  365. package/src/archive/formats.ts +60 -0
  366. package/src/archive/guard.ts +68 -0
  367. package/src/archive/ratio.ts +35 -0
  368. package/src/archive/tar.ts +147 -0
  369. package/src/archive/types.local.ts +72 -0
  370. package/src/archive/types.ts +107 -0
  371. package/src/archive/zip.ts +142 -0
  372. package/src/archive.ts +298 -0
  373. package/src/backed-workspace/types.ts +72 -0
  374. package/src/backed-workspace.ts +85 -0
  375. package/src/consts.local.ts +2 -0
  376. package/src/consts.ts +2 -0
  377. package/src/errors.ts +128 -0
  378. package/src/http/connector.ts +136 -0
  379. package/src/http/consts.ts +13 -0
  380. package/src/http/source.ts +89 -0
  381. package/src/http/types.local.ts +23 -0
  382. package/src/http/types.ts +74 -0
  383. package/src/http.ts +276 -0
  384. package/src/index.ts +65 -0
  385. package/src/network/consts.ts +22 -0
  386. package/src/network/types.ts +45 -0
  387. package/src/network.ts +179 -0
  388. package/src/proxy/consts.ts +11 -0
  389. package/src/proxy/endpoint.ts +28 -0
  390. package/src/proxy/http.ts +108 -0
  391. package/src/proxy/router.ts +93 -0
  392. package/src/proxy/socks.ts +48 -0
  393. package/src/proxy/types.ts +114 -0
  394. package/src/resolvers/google-drive/consts.ts +22 -0
  395. package/src/resolvers/google-drive/types.ts +22 -0
  396. package/src/resolvers/google-drive.ts +153 -0
  397. package/src/resolvers/types.ts +45 -0
  398. package/src/s3/consts.ts +12 -0
  399. package/src/s3/prefix.ts +100 -0
  400. package/src/s3/sink.ts +36 -0
  401. package/src/s3/source.ts +90 -0
  402. package/src/s3/types.local.ts +49 -0
  403. package/src/s3/types.ts +70 -0
  404. package/src/s3/upload.ts +130 -0
  405. package/src/s3.ts +83 -0
  406. package/src/service/types.local.ts +9 -0
  407. package/src/service/types.ts +71 -0
  408. package/src/service.ts +97 -0
  409. package/src/sinks/buffer/consts.ts +1 -0
  410. package/src/sinks/buffer/types.ts +12 -0
  411. package/src/sinks/buffer.ts +18 -0
  412. package/src/sinks/directory/types.ts +7 -0
  413. package/src/sinks/directory.ts +49 -0
  414. package/src/sinks/file.ts +52 -0
  415. package/src/sinks/types.ts +12 -0
  416. package/src/sinks/workload/consts.ts +5 -0
  417. package/src/sinks/workload/types.ts +17 -0
  418. package/src/sinks/workload.ts +59 -0
  419. package/src/sources/buffer.ts +27 -0
  420. package/src/sources/file.ts +57 -0
  421. package/src/sources/stream.ts +26 -0
  422. package/src/sources/types.ts +12 -0
  423. package/src/sources/workload/types.ts +31 -0
  424. package/src/sources/workload.ts +79 -0
  425. package/src/transfer/consts.ts +10 -0
  426. package/src/transfer/types.ts +55 -0
  427. package/src/transfer.ts +89 -0
  428. package/src/types.ts +106 -0
  429. package/src/utils/names/consts.ts +5 -0
  430. package/src/utils/names/types.ts +12 -0
  431. package/src/utils/names.ts +70 -0
  432. package/src/utils/paths/consts.ts +3 -0
  433. package/src/utils/paths/types.ts +20 -0
  434. package/src/utils/paths.ts +75 -0
  435. package/src/utils/range/types.ts +11 -0
  436. package/src/utils/range.ts +80 -0
  437. package/src/utils/redact/types.ts +16 -0
  438. package/src/utils/redact.ts +23 -0
  439. package/src/utils/signals/types.ts +17 -0
  440. package/src/utils/signals.ts +63 -0
  441. package/src/utils/streams/consts.ts +2 -0
  442. package/src/utils/streams/types.ts +65 -0
  443. package/src/utils/streams.ts +188 -0
  444. package/src/utils/tree/types.ts +6 -0
  445. package/src/utils/tree.ts +18 -0
  446. package/src/workspace/consts.local.ts +2 -0
  447. package/src/workspace/consts.ts +14 -0
  448. package/src/workspace/fs.ts +48 -0
  449. package/src/workspace/scope.ts +196 -0
  450. package/src/workspace/types.local.ts +21 -0
  451. package/src/workspace/types.ts +71 -0
  452. package/src/workspace.ts +102 -0
  453. package/tests/archive.spec.ts +134 -0
  454. package/tests/backed.spec.ts +147 -0
  455. package/tests/context.ts +114 -0
  456. package/tests/fixtures.ts +190 -0
  457. package/tests/google-drive.spec.ts +76 -0
  458. package/tests/harness/drive.ts +92 -0
  459. package/tests/harness/http-proxy.ts +96 -0
  460. package/tests/harness/origin.ts +134 -0
  461. package/tests/harness/socks5.ts +135 -0
  462. package/tests/harness/workload.ts +118 -0
  463. package/tests/http.spec.ts +101 -0
  464. package/tests/inspector.spec.ts +40 -0
  465. package/tests/live.spec.ts +27 -0
  466. package/tests/network.spec.ts +79 -0
  467. package/tests/proxy.spec.ts +149 -0
  468. package/tests/s3.spec.ts +133 -0
  469. package/tests/service.spec.ts +77 -0
  470. package/tests/sources-sinks.spec.ts +96 -0
  471. package/tests/transfer.spec.ts +99 -0
  472. package/tests/tsconfig.json +12 -0
  473. package/tests/workload.spec.ts +140 -0
  474. package/tests/workspace.spec.ts +158 -0
  475. package/tsconfig.json +16 -0
@@ -0,0 +1,130 @@
1
+ import {
2
+ AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, UploadPartCommand,
3
+ } from '@aws-sdk/client-s3'
4
+ import type { CompletedPart, ObjectCannedACL } from '@aws-sdk/client-s3'
5
+ import type { Readable } from 'node:stream'
6
+ import { logger } from '@owlmeans/log'
7
+ import { LimitError, S3TransferError } from '../errors.js'
8
+ import { MAX_PARTS } from './consts.js'
9
+ import type { S3Core, S3Uploader, S3UploadRequest, S3UploadResult } from './types.local.js'
10
+
11
+ const log = logger('file-streams:s3')
12
+
13
+ /** Uploads of unknown length: one PutObject when the body fits a part, a multipart upload otherwise. */
14
+ export const makeS3Uploader = (core: S3Core): S3Uploader => {
15
+ const parts = async function * (stream: Readable, size: number): AsyncGenerator<Buffer> {
16
+ let pending: Buffer[] = []
17
+ let length = 0
18
+ for await (const chunk of stream) {
19
+ let buffer = chunk as Buffer
20
+ while (buffer.length > 0) {
21
+ const take = Math.min(size - length, buffer.length)
22
+ pending.push(buffer.subarray(0, take))
23
+ length += take
24
+ buffer = buffer.subarray(take)
25
+ if (length === size) {
26
+ yield Buffer.concat(pending, length)
27
+ pending = []
28
+ length = 0
29
+ }
30
+ }
31
+ }
32
+ if (length > 0) {
33
+ yield Buffer.concat(pending, length)
34
+ }
35
+ }
36
+
37
+ const acl = (): { ACL?: ObjectCannedACL } => core.model.acl != null ? { ACL: core.model.acl as ObjectCannedACL } : {}
38
+
39
+ const upload = async (request: S3UploadRequest): Promise<S3UploadResult> => {
40
+ const client = core.client()
41
+ const base = {
42
+ Bucket: core.model.bucket,
43
+ Key: request.key,
44
+ ContentType: request.contentType,
45
+ ContentDisposition: request.disposition,
46
+ ...(request.cacheControl != null ? { CacheControl: request.cacheControl } : {}),
47
+ ...(request.metadata != null ? { Metadata: request.metadata } : {}),
48
+ ...acl(),
49
+ }
50
+ const iterator = parts(request.stream, core.partSize)
51
+ const first = await iterator.next()
52
+ const firstPart = first.done === true ? Buffer.alloc(0) : first.value
53
+ const second = first.done === true ? first : await iterator.next()
54
+
55
+ if (second.done === true) {
56
+ try {
57
+ const result = await client.send(new PutObjectCommand({ ...base, Body: firstPart, ContentLength: firstPart.length }), { abortSignal: request.signal })
58
+ return { bytes: firstPart.length, etag: result.ETag }
59
+ } catch (error) {
60
+ if (request.signal?.aborted === true) {
61
+ throw request.signal.reason
62
+ }
63
+ log.error('S3 upload failed', error as Error)
64
+ throw new S3TransferError('put')
65
+ }
66
+ }
67
+
68
+ let uploadId: string | undefined
69
+ try {
70
+ const created = await client.send(new CreateMultipartUploadCommand(base), { abortSignal: request.signal })
71
+ uploadId = created.UploadId
72
+ const completed: CompletedPart[] = []
73
+ const inFlight = new Set<Promise<void>>()
74
+ let bytes = 0
75
+ let number = 0
76
+
77
+ const send = (body: Buffer): void => {
78
+ const partNumber = ++number
79
+ if (partNumber > MAX_PARTS) {
80
+ throw new LimitError('s3-parts')
81
+ }
82
+ bytes += body.length
83
+ const pending = client.send(new UploadPartCommand({
84
+ Bucket: core.model.bucket, Key: request.key, UploadId: uploadId, PartNumber: partNumber, Body: body, ContentLength: body.length,
85
+ }), { abortSignal: request.signal }).then(result => {
86
+ completed.push({ ETag: result.ETag, PartNumber: partNumber })
87
+ })
88
+ const tracked = pending.finally(() => inFlight.delete(tracked))
89
+ inFlight.add(tracked)
90
+ }
91
+
92
+ send(firstPart)
93
+ send(second.value)
94
+ for (;;) {
95
+ while (inFlight.size >= core.concurrency) {
96
+ await Promise.race(inFlight)
97
+ }
98
+ const next = await iterator.next()
99
+ if (next.done === true) {
100
+ break
101
+ }
102
+ send(next.value)
103
+ }
104
+ await Promise.all(inFlight)
105
+ completed.sort((a, b) => (a.PartNumber ?? 0) - (b.PartNumber ?? 0))
106
+ const result = await client.send(new CompleteMultipartUploadCommand({
107
+ Bucket: core.model.bucket, Key: request.key, UploadId: uploadId, MultipartUpload: { Parts: completed },
108
+ }), { abortSignal: request.signal })
109
+ log.debug('Multipart upload completed', { parts: completed.length, bytes })
110
+
111
+ return { bytes, etag: result.ETag }
112
+ } catch (error) {
113
+ if (uploadId != null) {
114
+ await client.send(new AbortMultipartUploadCommand({ Bucket: core.model.bucket, Key: request.key, UploadId: uploadId }))
115
+ .catch(abortError => log.warn('Multipart abort failed', abortError as Error))
116
+ }
117
+ await iterator.return(undefined).catch(() => undefined)
118
+ if (request.signal?.aborted === true) {
119
+ throw request.signal.reason
120
+ }
121
+ if (error instanceof LimitError || error instanceof S3TransferError) {
122
+ throw error
123
+ }
124
+ log.error('S3 multipart upload failed', error as Error)
125
+ throw new S3TransferError('multipart')
126
+ }
127
+ }
128
+
129
+ return { upload }
130
+ }
package/src/s3.ts ADDED
@@ -0,0 +1,83 @@
1
+ import { DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'
2
+ import type { S3Client } from '@aws-sdk/client-s3'
3
+ import { makeStorageBucketModel } from '@owlmeans/storage-resource'
4
+ import type { StorageBucketModel, StorageConfig } from '@owlmeans/storage-resource'
5
+ import type { StoredFileInstance } from '@owlmeans/storage-common'
6
+ import { S3TransferError } from './errors.js'
7
+ import { DEFAULT_PART_SIZE, DEFAULT_UPLOAD_CONCURRENCY, MIN_PART_SIZE } from './s3/consts.js'
8
+ import { makeS3PrefixOps } from './s3/prefix.js'
9
+ import { makeS3Sink } from './s3/sink.js'
10
+ import { makeS3Source } from './s3/source.js'
11
+ import type { S3Bucket, S3BucketOptions, S3ObjectInfo, S3SinkOptions } from './s3/types.js'
12
+ import type { S3Core } from './s3/types.local.js'
13
+ import { makeS3Uploader } from './s3/upload.js'
14
+ import type { FileSink, FileSource, SignalOptions, SinkResult, TreeSink } from './types.js'
15
+ import { pathUtils } from './utils/paths.js'
16
+
17
+ /** A storage bucket — a `cfg.storageBuckets` entry or its model — as sources and sinks. */
18
+ export const createS3Bucket = (config: StorageBucketModel | StorageConfig, options: S3BucketOptions = {}): S3Bucket => {
19
+ const model = 'createClient' in config ? config : makeStorageBucketModel(config)
20
+ let created: S3Client | undefined
21
+ const client = (): S3Client => {
22
+ created ??= model.createClient()
23
+ return created
24
+ }
25
+
26
+ const keyOf = (prefix: string): string => model.keyOf(pathUtils.entry(prefix))
27
+
28
+ const core: S3Core = {
29
+ model,
30
+ client,
31
+ keyOf,
32
+ location: key => `s3://${model.bucket}/${key}`,
33
+ partSize: Math.max(options.partSize ?? DEFAULT_PART_SIZE, MIN_PART_SIZE),
34
+ concurrency: Math.max(1, options.concurrency ?? DEFAULT_UPLOAD_CONCURRENCY),
35
+ }
36
+ const uploader = makeS3Uploader(core)
37
+ const { list, removePrefix } = makeS3PrefixOps(core)
38
+
39
+ const urlOf = (prefix: string): string => model.urlOf(pathUtils.entry(prefix))
40
+
41
+ const source = (prefix: string): FileSource => makeS3Source(core, keyOf(prefix))
42
+
43
+ const sink = (prefix: string, sinkOptions?: S3SinkOptions): FileSink => makeS3Sink(core, uploader, pathUtils.entry(prefix), sinkOptions)
44
+
45
+ const tree = (prefix: string, sinkOptions?: S3SinkOptions): TreeSink => {
46
+ const base = pathUtils.entry(prefix)
47
+ return {
48
+ kind: 's3',
49
+ entry: path => sink(`${base}/${pathUtils.entry(path)}`, sinkOptions),
50
+ directory: async () => undefined,
51
+ }
52
+ }
53
+
54
+ const head = async (prefix: string, headOptions: SignalOptions = {}): Promise<S3ObjectInfo | null> => {
55
+ const key = keyOf(prefix)
56
+ try {
57
+ const result = await client().send(new HeadObjectCommand({ Bucket: model.bucket, Key: key }), { abortSignal: headOptions.signal })
58
+ return { key, size: result.ContentLength ?? 0, etag: result.ETag, mime: result.ContentType, modifiedAt: result.LastModified }
59
+ } catch (error) {
60
+ if ((error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode === 404 || (error as Error).name === 'NotFound') {
61
+ return null
62
+ }
63
+ throw new S3TransferError('head')
64
+ }
65
+ }
66
+
67
+ const remove = async (prefix: string, removeOptions: SignalOptions = {}): Promise<void> => {
68
+ try {
69
+ await client().send(new DeleteObjectCommand({ Bucket: model.bucket, Key: keyOf(prefix) }), { abortSignal: removeOptions.signal })
70
+ } catch {
71
+ throw new S3TransferError('delete')
72
+ }
73
+ }
74
+
75
+ const instanceOf = (result: SinkResult, alias: string): StoredFileInstance => ({ size: result.bytes, alias, url: result.url ?? '' })
76
+
77
+ const close = (): void => {
78
+ created?.destroy()
79
+ created = undefined
80
+ }
81
+
82
+ return { model, keyOf, urlOf, source, sink, tree, head, remove, list, removePrefix, instanceOf, close }
83
+ }
@@ -0,0 +1,9 @@
1
+ import type { BasicConfig, BasicContext } from '@owlmeans/context'
2
+ import type { StorageConfig } from '@owlmeans/storage-resource'
3
+ import type { FileStreamsConfigAppend } from './types.js'
4
+
5
+ export interface Config extends BasicConfig, FileStreamsConfigAppend {
6
+ storageBuckets?: { [key: string]: StorageConfig }
7
+ }
8
+
9
+ export interface Context<C extends Config = Config> extends BasicContext<C> {}
@@ -0,0 +1,71 @@
1
+ import type { InitializedService } from '@owlmeans/context'
2
+ import type { FileInspector, FileInspectorOptions } from '@owlmeans/file-inspector'
3
+ import type { ArchiveHelper, ArchiveLimits } from '../archive/types.js'
4
+ import type { BackedWorkspace } from '../backed-workspace/types.js'
5
+ import type { HttpClient } from '../http/types.js'
6
+ import type { AddressResolver, NetworkPolicyOptions } from '../network/types.js'
7
+ import type { ProxyCredentialsProvider, ProxyPlugin, ProxyRouterOptions, ProxyStrategy } from '../proxy/types.js'
8
+ import type { UrlResolver } from '../resolvers/types.js'
9
+ import type { S3Bucket, S3BucketOptions } from '../s3/types.js'
10
+ import type { TransferHelper, TransferLimits } from '../transfer/types.js'
11
+ import type { Workspace, WorkspaceOptions } from '../workspace/types.js'
12
+
13
+ /** A proxy in config; string leaves may be secret files (`/etc/…`), which server contexts resolve. */
14
+ export interface FileStreamsProxyConfig {
15
+ /** `http://host:3128`, `socks5h://host:1080`. */
16
+ url: string
17
+ username?: string
18
+ password?: string
19
+ id?: string
20
+ }
21
+
22
+ /** `cfg.fileStreams`. */
23
+ export interface FileStreamsConfig {
24
+ limits?: Partial<TransferLimits>
25
+ network?: Omit<NetworkPolicyOptions, 'resolver'>
26
+ proxies?: FileStreamsProxyConfig[]
27
+ proxyStrategy?: ProxyStrategy
28
+ userAgent?: string
29
+ /**
30
+ * The temporary workspace: `root`, folder and file modes. A server context reads every string leaf
31
+ * starting with `/` as a secret file, so give `root` relative to `os.tmpdir()` (default
32
+ * `owlmeans-workspace`), or as a file that holds an absolute path.
33
+ */
34
+ workspace?: WorkspaceOptions
35
+ }
36
+
37
+ export interface FileStreamsBackedOptions {
38
+ /** Objects land under `<prefix>/` inside the bucket's base prefix. */
39
+ prefix?: string
40
+ }
41
+
42
+ export interface FileStreamsConfigAppend {
43
+ fileStreams?: FileStreamsConfig
44
+ }
45
+
46
+ /** What config cannot carry: callbacks and plugins. */
47
+ export interface FileStreamsServiceOptions {
48
+ /** Replaces `cfg.fileStreams.proxies` — a list or a provider by key and URL. */
49
+ proxies?: ProxyRouterOptions['proxies']
50
+ proxyCredentials?: ProxyCredentialsProvider
51
+ proxyPlugins?: ProxyPlugin[]
52
+ resolvers?: UrlResolver[]
53
+ resolver?: AddressResolver
54
+ bucketOptions?: S3BucketOptions
55
+ inspector?: FileInspectorOptions
56
+ archiveLimits?: Partial<ArchiveLimits>
57
+ }
58
+
59
+ /** The package's objects configured from the context. */
60
+ export interface FileStreamsService extends InitializedService {
61
+ http: () => HttpClient
62
+ /** The bucket of a `cfg.storageBuckets` entry, addressed as storage-resource addresses it. */
63
+ bucket: (configKey: string) => S3Bucket
64
+ transfer: () => TransferHelper
65
+ inspector: () => FileInspector
66
+ archives: () => ArchiveHelper
67
+ /** The workspace configured by `cfg.fileStreams.workspace`. */
68
+ workspace: () => Workspace
69
+ /** The workspace mirrored to the bucket of a `cfg.storageBuckets` entry (one object per key and prefix). */
70
+ backed: (bucketConfigKey: string, options?: FileStreamsBackedOptions) => BackedWorkspace
71
+ }
package/src/service.ts ADDED
@@ -0,0 +1,97 @@
1
+ import { createService, memoHelper } from '@owlmeans/context'
2
+ import type { BasicConfig, BasicContext } from '@owlmeans/context'
3
+ import { FilePropertyError } from '@owlmeans/storage-common'
4
+ import { createFileInspector } from '@owlmeans/file-inspector'
5
+ import { createArchiveHelper } from './archive.js'
6
+ import { createBackedWorkspace } from './backed-workspace.js'
7
+ import type { BackedWorkspace } from './backed-workspace/types.js'
8
+ import { FILE_STREAMS_SERVICE } from './consts.js'
9
+ import { createHttpClient } from './http.js'
10
+ import { parseProxyEndpoint } from './proxy/endpoint.js'
11
+ import type { ProxyEndpoint } from './proxy/types.js'
12
+ import { createS3Bucket } from './s3.js'
13
+ import type { S3Bucket } from './s3/types.js'
14
+ import type {
15
+ FileStreamsBackedOptions, FileStreamsProxyConfig, FileStreamsService, FileStreamsServiceOptions,
16
+ } from './service/types.js'
17
+ import type { Config, Context } from './service/types.local.js'
18
+ import { createTransferHelper } from './transfer.js'
19
+ import { createWorkspace } from './workspace.js'
20
+
21
+ export const createFileStreamsService = (
22
+ alias: string = FILE_STREAMS_SERVICE, options: FileStreamsServiceOptions = {}
23
+ ): FileStreamsService => {
24
+ const buckets = new Map<string, S3Bucket>()
25
+ const backedWorkspaces = new Map<string, BackedWorkspace>()
26
+
27
+ const configOf = (): Config => service.assertCtx<Config, Context>(alias).cfg
28
+
29
+ const endpointOf = (proxy: FileStreamsProxyConfig): ProxyEndpoint => {
30
+ const endpoint = parseProxyEndpoint(proxy.url)
31
+ return {
32
+ ...endpoint,
33
+ ...(proxy.username != null ? { username: proxy.username, password: proxy.password ?? '' } : {}),
34
+ ...(proxy.id != null ? { id: proxy.id } : {}),
35
+ }
36
+ }
37
+
38
+ const http = memoHelper.once(() => {
39
+ const config = configOf().fileStreams ?? {}
40
+ return createHttpClient({
41
+ network: { ...config.network, resolver: options.resolver },
42
+ proxies: {
43
+ proxies: options.proxies ?? config.proxies?.map(endpointOf),
44
+ strategy: config.proxyStrategy,
45
+ credentials: options.proxyCredentials,
46
+ plugins: options.proxyPlugins,
47
+ },
48
+ resolvers: options.resolvers,
49
+ userAgent: config.userAgent,
50
+ })
51
+ })
52
+
53
+ const inspector = memoHelper.once(() => createFileInspector(options.inspector))
54
+
55
+ const transfer = memoHelper.once(() => createTransferHelper({ inspector: inspector(), limits: configOf().fileStreams?.limits }))
56
+
57
+ const archives = memoHelper.once(() => createArchiveHelper({ inspector: inspector(), limits: options.archiveLimits }))
58
+
59
+ const bucket = (configKey: string): S3Bucket => {
60
+ let known = buckets.get(configKey)
61
+ if (known == null) {
62
+ const entry = configOf().storageBuckets?.[configKey]
63
+ if (entry == null) {
64
+ throw new FilePropertyError(`storage-config:${configKey}`)
65
+ }
66
+ known = createS3Bucket(entry, options.bucketOptions)
67
+ buckets.set(configKey, known)
68
+ }
69
+ return known
70
+ }
71
+
72
+ const workspace = memoHelper.once(() => createWorkspace(configOf().fileStreams?.workspace))
73
+
74
+ const backed = (bucketConfigKey: string, backedOptions: FileStreamsBackedOptions = {}): BackedWorkspace => {
75
+ const key = `${bucketConfigKey}\u0000${backedOptions.prefix ?? ''}`
76
+ let known = backedWorkspaces.get(key)
77
+ if (known == null) {
78
+ known = createBackedWorkspace({ workspace: workspace(), bucket: bucket(bucketConfigKey), prefix: backedOptions.prefix, transfer: transfer() })
79
+ backedWorkspaces.set(key, known)
80
+ }
81
+ return known
82
+ }
83
+
84
+ const service: FileStreamsService = createService<FileStreamsService>(alias, {
85
+ http, bucket, transfer, inspector, archives, workspace, backed,
86
+ })
87
+
88
+ return service
89
+ }
90
+
91
+ /** Registers the file-streams service on a context. */
92
+ export const appendFileStreamsService = <C extends BasicConfig, T extends BasicContext<C>>(
93
+ ctx: T, alias: string = FILE_STREAMS_SERVICE, options?: FileStreamsServiceOptions
94
+ ): T => {
95
+ ctx.registerService(createFileStreamsService(alias, options))
96
+ return ctx
97
+ }
@@ -0,0 +1 @@
1
+ export const DEFAULT_BUFFER_SINK_BYTES = 64 * 1024 * 1024
@@ -0,0 +1,12 @@
1
+ import type { FileSink } from '../../types.js'
2
+
3
+ export interface BufferSinkOptions {
4
+ /** Ceiling on collected bytes. Default 64 MiB. */
5
+ maxBytes?: number
6
+ }
7
+
8
+ /** A sink that keeps what it receives in memory. */
9
+ export interface BufferSink extends FileSink {
10
+ /** The bytes of the last write. */
11
+ bytes: () => Uint8Array
12
+ }
@@ -0,0 +1,18 @@
1
+ import type { SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
2
+ import { streamUtils } from '../utils/streams.js'
3
+ import { DEFAULT_BUFFER_SINK_BYTES } from './buffer/consts.js'
4
+ import type { BufferSink, BufferSinkOptions } from './buffer/types.js'
5
+
6
+ /** Collects a file in memory, refusing past its ceiling. */
7
+ export const createBufferSink = (options: BufferSinkOptions = {}): BufferSink => {
8
+ let collected: Uint8Array = new Uint8Array(0)
9
+
10
+ const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
11
+ const ceiling = Math.min(options.maxBytes ?? DEFAULT_BUFFER_SINK_BYTES, writeOptions.maxBytes ?? Infinity)
12
+ collected = await streamUtils.collect(input.stream, ceiling)
13
+
14
+ return { kind: 'buffer', location: 'buffer', bytes: collected.length, mime: input.verdict?.detected?.mime ?? input.info.mime }
15
+ }
16
+
17
+ return { kind: 'buffer', write, bytes: () => collected }
18
+ }
@@ -0,0 +1,7 @@
1
+ import type { FileSinkOptions } from '../types.js'
2
+ import type { PathLimits } from '../../utils/paths/types.js'
3
+
4
+ export interface DirectorySinkOptions extends FileSinkOptions, Partial<PathLimits> {
5
+ /** Mode of created directories. Default `0o755`. */
6
+ dirMode?: number
7
+ }
@@ -0,0 +1,49 @@
1
+ import { mkdir } from 'node:fs/promises'
2
+ import { resolve } from 'node:path'
3
+ import { PathError } from '../errors.js'
4
+ import type { FileSink, SignalOptions, TreeSink } from '../types.js'
5
+ import { pathUtils } from '../utils/paths.js'
6
+ import { createFileSink } from './file.js'
7
+ import type { DirectorySinkOptions } from './directory/types.js'
8
+
9
+ /**
10
+ * A directory every entry is confined to: normalized relative paths only, never through an
11
+ * existing symlink, never the same entry (or a case-fold twin) twice.
12
+ */
13
+ export const createDirectorySink = (root: string, options: DirectorySinkOptions = {}): TreeSink => {
14
+ const base = resolve(root)
15
+ const seen = new Set<string>()
16
+
17
+ const entry = (path: string): FileSink => {
18
+ const relative = pathUtils.entry(path, options)
19
+ const target = pathUtils.join(base, relative)
20
+ const folded = relative.toLowerCase()
21
+ if (seen.has(folded)) {
22
+ throw new PathError('duplicate')
23
+ }
24
+ const sink = createFileSink(target, options)
25
+
26
+ return {
27
+ kind: 'file',
28
+ write: async (input, writeOptions) => {
29
+ if (seen.has(folded)) {
30
+ throw new PathError('duplicate')
31
+ }
32
+ await mkdir(base, { recursive: true, mode: options.dirMode ?? 0o755 })
33
+ await pathUtils.assertNoLinks(base, target)
34
+ const result = await sink.write(input, writeOptions)
35
+ seen.add(folded)
36
+ return result
37
+ },
38
+ }
39
+ }
40
+
41
+ const directory = async (path: string, _options?: SignalOptions): Promise<void> => {
42
+ const target = pathUtils.join(base, pathUtils.entry(path, options))
43
+ await mkdir(base, { recursive: true, mode: options.dirMode ?? 0o755 })
44
+ await pathUtils.assertNoLinks(base, target)
45
+ await mkdir(target, { recursive: true, mode: options.dirMode ?? 0o755 })
46
+ }
47
+
48
+ return { kind: 'directory', entry, directory }
49
+ }
@@ -0,0 +1,52 @@
1
+ import { randomBytes } from 'node:crypto'
2
+ import { createWriteStream } from 'node:fs'
3
+ import { chmod, link, mkdir, rename, rm } from 'node:fs/promises'
4
+ import { basename, dirname, join } from 'node:path'
5
+ import { pipeline } from 'node:stream/promises'
6
+ import { logger } from '@owlmeans/log'
7
+ import { PathError } from '../errors.js'
8
+ import type { FileSink, SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
9
+ import { streamUtils } from '../utils/streams.js'
10
+ import type { FileSinkOptions } from './types.js'
11
+
12
+ const log = logger('file-streams:file')
13
+
14
+ /** One file on disk, written to a private temporary file and committed atomically. */
15
+ export const createFileSink = (path: string, options: FileSinkOptions = {}): FileSink => {
16
+ const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
17
+ const dir = dirname(path)
18
+ if (options.mkdirs !== false) {
19
+ await mkdir(dir, { recursive: true, mode: 0o755 })
20
+ }
21
+ const temporary = join(dir, `.${basename(path)}.${randomBytes(6).toString('hex')}.tmp`)
22
+ const meter = streamUtils.meter({ maxBytes: writeOptions.maxBytes })
23
+
24
+ try {
25
+ await pipeline(
26
+ input.stream, meter.stream, createWriteStream(temporary, { mode: 0o600, flags: 'wx', flush: true }),
27
+ { signal: writeOptions.signal }
28
+ )
29
+ await chmod(temporary, options.mode ?? 0o644)
30
+ if (options.onConflict === 'overwrite') {
31
+ await rename(temporary, path)
32
+ } else {
33
+ try {
34
+ await link(temporary, path)
35
+ } catch (error) {
36
+ throw (error as NodeJS.ErrnoException).code === 'EEXIST' ? new PathError('exists') : error
37
+ }
38
+ }
39
+ } catch (error) {
40
+ streamUtils.discard(input.stream)
41
+ throw error
42
+ } finally {
43
+ await rm(temporary, { force: true })
44
+ }
45
+
46
+ log.debug('File written', { bytes: meter.bytes() })
47
+
48
+ return { kind: 'file', location: path, bytes: meter.bytes(), mime: input.verdict?.detected?.mime ?? input.info.mime }
49
+ }
50
+
51
+ return { kind: 'file', write }
52
+ }
@@ -0,0 +1,12 @@
1
+ /** What a file sink does when its target already exists. */
2
+ // Two behaviours, nothing else to say about them — a union.
3
+ export type ConflictPolicy = 'error' | 'overwrite'
4
+
5
+ export interface FileSinkOptions {
6
+ /** Default `error` (`PathError('exists')`). */
7
+ onConflict?: ConflictPolicy
8
+ /** File mode of the committed file. Default `0o644`. */
9
+ mode?: number
10
+ /** Create missing parent directories. Default `true`. */
11
+ mkdirs?: boolean
12
+ }
@@ -0,0 +1,5 @@
1
+ /** The publisher refuses a `writeBinaryFile` payload longer than this many base64 characters. */
2
+ export const WORKLOAD_MAX_BASE64_CHARS = 16 * 1024 * 1024
3
+
4
+ /** The largest file whose base64 fits the publisher's ceiling (12 MiB). */
5
+ export const WORKLOAD_MAX_FILE_BYTES = WORKLOAD_MAX_BASE64_CHARS / 4 * 3
@@ -0,0 +1,17 @@
1
+ /**
2
+ * The one binary write a Viable ephemeral workload accepts (its publisher's `writeBinaryFile`
3
+ * command). viable-agent's `FileHelper` satisfies it as is — pass the helper itself, never a
4
+ * destructured method.
5
+ */
6
+ export interface WorkloadWriter {
7
+ writeBinaryFile: (filePath: string, base64: string) => Promise<void>
8
+ }
9
+
10
+ export interface WorkloadSinkOptions {
11
+ /** Folder inside the workload's project, e.g. `sources/web/public/media`. */
12
+ root?: string
13
+ /** Per-file ceiling; never above (and by default) `WORKLOAD_MAX_FILE_BYTES`. */
14
+ maxFileBytes?: number
15
+ /** Retries of a failed write — the write is atomic on the workload, so a retry is safe. Default 1. */
16
+ retries?: number
17
+ }
@@ -0,0 +1,59 @@
1
+ import { logger } from '@owlmeans/log'
2
+ import { LimitError, WorkloadError } from '../errors.js'
3
+ import type { FileSink, SinkInput, SinkResult, SinkWriteOptions, TreeSink } from '../types.js'
4
+ import { pathUtils } from '../utils/paths.js'
5
+ import { streamUtils } from '../utils/streams.js'
6
+ import { WORKLOAD_MAX_FILE_BYTES } from './workload/consts.js'
7
+ import type { WorkloadSinkOptions, WorkloadWriter } from './workload/types.js'
8
+
9
+ const log = logger('file-streams:workload')
10
+
11
+ /**
12
+ * Files written into a Viable ephemeral workload through its writer: one atomic write per file,
13
+ * each at most 12 MiB (the publisher's base64 ceiling) or a lower `maxFileBytes`.
14
+ */
15
+ export const makeWorkloadSink = (writer: WorkloadWriter, options: WorkloadSinkOptions = {}): TreeSink => {
16
+ const ceiling = Math.min(options.maxFileBytes ?? WORKLOAD_MAX_FILE_BYTES, WORKLOAD_MAX_FILE_BYTES)
17
+ const root = options.root != null && options.root.replace(/^\/+|\/+$/g, '') !== '' ? pathUtils.entry(options.root) : ''
18
+ const retries = options.retries ?? 1
19
+
20
+ const entry = (path: string): FileSink => {
21
+ const filePath = root === '' ? pathUtils.entry(path) : `${root}/${pathUtils.entry(path)}`
22
+
23
+ const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
24
+ const limit = Math.min(ceiling, writeOptions.maxBytes ?? Infinity)
25
+ if (input.info.size != null && input.info.size > limit) {
26
+ streamUtils.discard(input.stream)
27
+ throw new WorkloadError('too-large')
28
+ }
29
+ let bytes: Buffer
30
+ try {
31
+ bytes = await streamUtils.collect(input.stream, limit)
32
+ } catch (error) {
33
+ throw error instanceof LimitError ? new WorkloadError('too-large') : error
34
+ }
35
+ const base64 = bytes.toString('base64')
36
+
37
+ for (let attempt = 0; ; attempt++) {
38
+ writeOptions.signal?.throwIfAborted()
39
+ try {
40
+ await writer.writeBinaryFile(filePath, base64)
41
+ break
42
+ } catch (error) {
43
+ if (attempt >= retries) {
44
+ log.warn('Workload write failed', { bytes: bytes.length, attempts: attempt + 1 })
45
+ throw new WorkloadError('write-failed')
46
+ }
47
+ }
48
+ }
49
+
50
+ return { kind: 'workload', location: filePath, bytes: bytes.length, mime: input.verdict?.detected?.mime ?? input.info.mime }
51
+ }
52
+
53
+ return { kind: 'workload', write }
54
+ }
55
+
56
+ const directory = async (): Promise<void> => undefined
57
+
58
+ return { kind: 'workload', entry, directory }
59
+ }
@@ -0,0 +1,27 @@
1
+ import { Readable } from 'node:stream'
2
+ import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader } from '../types.js'
3
+ import { nameUtils } from '../utils/names.js'
4
+ import { rangeUtils } from '../utils/range.js'
5
+ import type { SourceClaims } from './types.js'
6
+
7
+ /** Bytes in memory — a screenshot, a generated file: re-openable, with random access. */
8
+ export const createBufferSource = (bytes: Uint8Array, claims: SourceClaims = {}): FileSource => {
9
+ const known: FileInfo = { origin: 'buffer', name: nameUtils.sanitize(claims.name), size: bytes.length, mime: claims.mime }
10
+
11
+ const info = async (): Promise<FileInfo> => known
12
+
13
+ const open = async (options: OpenOptions = {}): Promise<OpenedFile> => {
14
+ if (options.range == null) {
15
+ return { stream: Readable.from([Buffer.from(bytes)], { objectMode: false }), info: known }
16
+ }
17
+ const start = Math.min(options.range.start, bytes.length)
18
+ const end = Math.min(options.range.end ?? bytes.length - 1, bytes.length - 1)
19
+ const slice = Buffer.from(bytes.subarray(start, end + 1))
20
+
21
+ return { stream: Readable.from(slice.length > 0 ? [slice] : [], { objectMode: false }), info: known, range: { start, end, total: bytes.length } }
22
+ }
23
+
24
+ const ranges = async (): Promise<RangeReader> => rangeUtils.ofBuffer(bytes)
25
+
26
+ return { kind: 'buffer', info, open, ranges }
27
+ }