@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +71 -2
  3. package/agent-meta/manifest.json +64 -0
  4. package/agent-meta/skills/file-streams/SKILL.md +183 -0
  5. package/agent-meta/skills/file-streams/parts/network.md +47 -0
  6. package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
  7. package/build/archive/consts.d.ts +16 -0
  8. package/build/archive/consts.d.ts.map +1 -0
  9. package/build/archive/consts.js +24 -0
  10. package/build/archive/consts.js.map +1 -0
  11. package/build/archive/formats.d.ts +4 -0
  12. package/build/archive/formats.d.ts.map +1 -0
  13. package/build/archive/formats.js +53 -0
  14. package/build/archive/formats.js.map +1 -0
  15. package/build/archive/guard.d.ts +5 -0
  16. package/build/archive/guard.d.ts.map +1 -0
  17. package/build/archive/guard.js +62 -0
  18. package/build/archive/guard.js.map +1 -0
  19. package/build/archive/ratio.d.ts +5 -0
  20. package/build/archive/ratio.d.ts.map +1 -0
  21. package/build/archive/ratio.js +30 -0
  22. package/build/archive/ratio.js.map +1 -0
  23. package/build/archive/tar.d.ts +4 -0
  24. package/build/archive/tar.d.ts.map +1 -0
  25. package/build/archive/tar.js +142 -0
  26. package/build/archive/tar.js.map +1 -0
  27. package/build/archive/types.d.ts +95 -0
  28. package/build/archive/types.d.ts.map +1 -0
  29. package/build/archive/types.js +2 -0
  30. package/build/archive/types.js.map +1 -0
  31. package/build/archive/types.local.d.ts +64 -0
  32. package/build/archive/types.local.d.ts.map +1 -0
  33. package/build/archive/types.local.js +2 -0
  34. package/build/archive/types.local.js.map +1 -0
  35. package/build/archive/zip.d.ts +4 -0
  36. package/build/archive/zip.d.ts.map +1 -0
  37. package/build/archive/zip.js +135 -0
  38. package/build/archive/zip.js.map +1 -0
  39. package/build/archive.d.ts +4 -0
  40. package/build/archive.d.ts.map +1 -0
  41. package/build/archive.js +265 -0
  42. package/build/archive.js.map +1 -0
  43. package/build/backed-workspace/types.d.ts +67 -0
  44. package/build/backed-workspace/types.d.ts.map +1 -0
  45. package/build/backed-workspace/types.js +2 -0
  46. package/build/backed-workspace/types.js.map +1 -0
  47. package/build/backed-workspace.d.ts +4 -0
  48. package/build/backed-workspace.d.ts.map +1 -0
  49. package/build/backed-workspace.js +63 -0
  50. package/build/backed-workspace.js.map +1 -0
  51. package/build/consts.d.ts +3 -0
  52. package/build/consts.d.ts.map +1 -0
  53. package/build/consts.js +3 -0
  54. package/build/consts.js.map +1 -0
  55. package/build/consts.local.d.ts +3 -0
  56. package/build/consts.local.d.ts.map +1 -0
  57. package/build/consts.local.js +3 -0
  58. package/build/consts.local.js.map +1 -0
  59. package/build/errors.d.ts +59 -0
  60. package/build/errors.d.ts.map +1 -0
  61. package/build/errors.js +105 -0
  62. package/build/errors.js.map +1 -0
  63. package/build/http/connector.d.ts +11 -0
  64. package/build/http/connector.d.ts.map +1 -0
  65. package/build/http/connector.js +124 -0
  66. package/build/http/connector.js.map +1 -0
  67. package/build/http/consts.d.ts +9 -0
  68. package/build/http/consts.d.ts.map +1 -0
  69. package/build/http/consts.js +8 -0
  70. package/build/http/consts.js.map +1 -0
  71. package/build/http/source.d.ts +5 -0
  72. package/build/http/source.d.ts.map +1 -0
  73. package/build/http/source.js +76 -0
  74. package/build/http/source.js.map +1 -0
  75. package/build/http/types.d.ts +67 -0
  76. package/build/http/types.d.ts.map +1 -0
  77. package/build/http/types.js +2 -0
  78. package/build/http/types.js.map +1 -0
  79. package/build/http/types.local.d.ts +21 -0
  80. package/build/http/types.local.d.ts.map +1 -0
  81. package/build/http/types.local.js +2 -0
  82. package/build/http/types.local.js.map +1 -0
  83. package/build/http.d.ts +4 -0
  84. package/build/http.d.ts.map +1 -0
  85. package/build/http.js +248 -0
  86. package/build/http.js.map +1 -0
  87. package/build/index.d.ts +53 -0
  88. package/build/index.d.ts.map +1 -0
  89. package/build/index.js +33 -0
  90. package/build/index.js.map +1 -0
  91. package/build/network/consts.d.ts +12 -0
  92. package/build/network/consts.d.ts.map +1 -0
  93. package/build/network/consts.js +20 -0
  94. package/build/network/consts.js.map +1 -0
  95. package/build/network/types.d.ts +43 -0
  96. package/build/network/types.d.ts.map +1 -0
  97. package/build/network/types.js +2 -0
  98. package/build/network/types.js.map +1 -0
  99. package/build/network.d.ts +4 -0
  100. package/build/network.d.ts.map +1 -0
  101. package/build/network.js +165 -0
  102. package/build/network.js.map +1 -0
  103. package/build/proxy/consts.d.ts +7 -0
  104. package/build/proxy/consts.d.ts.map +1 -0
  105. package/build/proxy/consts.js +9 -0
  106. package/build/proxy/consts.js.map +1 -0
  107. package/build/proxy/endpoint.d.ts +4 -0
  108. package/build/proxy/endpoint.d.ts.map +1 -0
  109. package/build/proxy/endpoint.js +27 -0
  110. package/build/proxy/endpoint.js.map +1 -0
  111. package/build/proxy/http.d.ts +5 -0
  112. package/build/proxy/http.d.ts.map +1 -0
  113. package/build/proxy/http.js +95 -0
  114. package/build/proxy/http.js.map +1 -0
  115. package/build/proxy/router.d.ts +4 -0
  116. package/build/proxy/router.d.ts.map +1 -0
  117. package/build/proxy/router.js +85 -0
  118. package/build/proxy/router.js.map +1 -0
  119. package/build/proxy/socks.d.ts +5 -0
  120. package/build/proxy/socks.d.ts.map +1 -0
  121. package/build/proxy/socks.js +44 -0
  122. package/build/proxy/socks.js.map +1 -0
  123. package/build/proxy/types.d.ts +99 -0
  124. package/build/proxy/types.d.ts.map +1 -0
  125. package/build/proxy/types.js +2 -0
  126. package/build/proxy/types.js.map +1 -0
  127. package/build/resolvers/google-drive/consts.d.ts +10 -0
  128. package/build/resolvers/google-drive/consts.d.ts.map +1 -0
  129. package/build/resolvers/google-drive/consts.js +16 -0
  130. package/build/resolvers/google-drive/consts.js.map +1 -0
  131. package/build/resolvers/google-drive/types.d.ts +21 -0
  132. package/build/resolvers/google-drive/types.d.ts.map +1 -0
  133. package/build/resolvers/google-drive/types.js +2 -0
  134. package/build/resolvers/google-drive/types.js.map +1 -0
  135. package/build/resolvers/google-drive.d.ts +10 -0
  136. package/build/resolvers/google-drive.d.ts.map +1 -0
  137. package/build/resolvers/google-drive.js +130 -0
  138. package/build/resolvers/google-drive.js.map +1 -0
  139. package/build/resolvers/types.d.ts +39 -0
  140. package/build/resolvers/types.d.ts.map +1 -0
  141. package/build/resolvers/types.js +2 -0
  142. package/build/resolvers/types.js.map +1 -0
  143. package/build/s3/consts.d.ts +9 -0
  144. package/build/s3/consts.d.ts.map +1 -0
  145. package/build/s3/consts.js +9 -0
  146. package/build/s3/consts.js.map +1 -0
  147. package/build/s3/prefix.d.ts +4 -0
  148. package/build/s3/prefix.d.ts.map +1 -0
  149. package/build/s3/prefix.js +87 -0
  150. package/build/s3/prefix.js.map +1 -0
  151. package/build/s3/sink.d.ts +6 -0
  152. package/build/s3/sink.d.ts.map +1 -0
  153. package/build/s3/sink.js +30 -0
  154. package/build/s3/sink.js.map +1 -0
  155. package/build/s3/source.d.ts +5 -0
  156. package/build/s3/source.d.ts.map +1 -0
  157. package/build/s3/source.js +81 -0
  158. package/build/s3/source.js.map +1 -0
  159. package/build/s3/types.d.ts +66 -0
  160. package/build/s3/types.d.ts.map +1 -0
  161. package/build/s3/types.js +2 -0
  162. package/build/s3/types.js.map +1 -0
  163. package/build/s3/types.local.d.ts +43 -0
  164. package/build/s3/types.local.d.ts.map +1 -0
  165. package/build/s3/types.local.js +2 -0
  166. package/build/s3/types.local.js.map +1 -0
  167. package/build/s3/upload.d.ts +4 -0
  168. package/build/s3/upload.d.ts.map +1 -0
  169. package/build/s3/upload.js +118 -0
  170. package/build/s3/upload.js.map +1 -0
  171. package/build/s3.d.ts +5 -0
  172. package/build/s3.d.ts.map +1 -0
  173. package/build/s3.js +68 -0
  174. package/build/s3.js.map +1 -0
  175. package/build/service/types.d.ts +66 -0
  176. package/build/service/types.d.ts.map +1 -0
  177. package/build/service/types.js +2 -0
  178. package/build/service/types.js.map +1 -0
  179. package/build/service/types.local.d.ts +11 -0
  180. package/build/service/types.local.d.ts.map +1 -0
  181. package/build/service/types.local.js +2 -0
  182. package/build/service/types.local.js.map +1 -0
  183. package/build/service.d.ts +6 -0
  184. package/build/service.d.ts.map +1 -0
  185. package/build/service.js +73 -0
  186. package/build/service.js.map +1 -0
  187. package/build/sinks/buffer/consts.d.ts +2 -0
  188. package/build/sinks/buffer/consts.d.ts.map +1 -0
  189. package/build/sinks/buffer/consts.js +2 -0
  190. package/build/sinks/buffer/consts.js.map +1 -0
  191. package/build/sinks/buffer/types.d.ts +11 -0
  192. package/build/sinks/buffer/types.d.ts.map +1 -0
  193. package/build/sinks/buffer/types.js +2 -0
  194. package/build/sinks/buffer/types.js.map +1 -0
  195. package/build/sinks/buffer.d.ts +4 -0
  196. package/build/sinks/buffer.d.ts.map +1 -0
  197. package/build/sinks/buffer.js +13 -0
  198. package/build/sinks/buffer.js.map +1 -0
  199. package/build/sinks/directory/types.d.ts +7 -0
  200. package/build/sinks/directory/types.d.ts.map +1 -0
  201. package/build/sinks/directory/types.js +2 -0
  202. package/build/sinks/directory/types.js.map +1 -0
  203. package/build/sinks/directory.d.ts +8 -0
  204. package/build/sinks/directory.d.ts.map +1 -0
  205. package/build/sinks/directory.js +43 -0
  206. package/build/sinks/directory.js.map +1 -0
  207. package/build/sinks/file.d.ts +5 -0
  208. package/build/sinks/file.d.ts.map +1 -0
  209. package/build/sinks/file.js +46 -0
  210. package/build/sinks/file.js.map +1 -0
  211. package/build/sinks/types.d.ts +11 -0
  212. package/build/sinks/types.d.ts.map +1 -0
  213. package/build/sinks/types.js +2 -0
  214. package/build/sinks/types.js.map +1 -0
  215. package/build/sinks/workload/consts.d.ts +5 -0
  216. package/build/sinks/workload/consts.d.ts.map +1 -0
  217. package/build/sinks/workload/consts.js +5 -0
  218. package/build/sinks/workload/consts.js.map +1 -0
  219. package/build/sinks/workload/types.d.ts +17 -0
  220. package/build/sinks/workload/types.d.ts.map +1 -0
  221. package/build/sinks/workload/types.js +2 -0
  222. package/build/sinks/workload/types.js.map +1 -0
  223. package/build/sinks/workload.d.ts +8 -0
  224. package/build/sinks/workload.d.ts.map +1 -0
  225. package/build/sinks/workload.js +51 -0
  226. package/build/sinks/workload.js.map +1 -0
  227. package/build/sources/buffer.d.ts +5 -0
  228. package/build/sources/buffer.d.ts.map +1 -0
  229. package/build/sources/buffer.js +20 -0
  230. package/build/sources/buffer.js.map +1 -0
  231. package/build/sources/file.d.ts +5 -0
  232. package/build/sources/file.d.ts.map +1 -0
  233. package/build/sources/file.js +50 -0
  234. package/build/sources/file.js.map +1 -0
  235. package/build/sources/stream.d.ts +5 -0
  236. package/build/sources/stream.d.ts.map +1 -0
  237. package/build/sources/stream.js +19 -0
  238. package/build/sources/stream.js.map +1 -0
  239. package/build/sources/types.d.ts +10 -0
  240. package/build/sources/types.d.ts.map +1 -0
  241. package/build/sources/types.js +2 -0
  242. package/build/sources/types.js.map +1 -0
  243. package/build/sources/workload/types.d.ts +29 -0
  244. package/build/sources/workload/types.d.ts.map +1 -0
  245. package/build/sources/workload/types.js +2 -0
  246. package/build/sources/workload/types.js.map +1 -0
  247. package/build/sources/workload.d.ts +7 -0
  248. package/build/sources/workload.d.ts.map +1 -0
  249. package/build/sources/workload.js +67 -0
  250. package/build/sources/workload.js.map +1 -0
  251. package/build/transfer/consts.d.ts +5 -0
  252. package/build/transfer/consts.d.ts.map +1 -0
  253. package/build/transfer/consts.js +8 -0
  254. package/build/transfer/consts.js.map +1 -0
  255. package/build/transfer/types.d.ts +49 -0
  256. package/build/transfer/types.d.ts.map +1 -0
  257. package/build/transfer/types.js +2 -0
  258. package/build/transfer/types.js.map +1 -0
  259. package/build/transfer.d.ts +4 -0
  260. package/build/transfer.d.ts.map +1 -0
  261. package/build/transfer.js +75 -0
  262. package/build/transfer.js.map +1 -0
  263. package/build/types.d.ts +99 -0
  264. package/build/types.d.ts.map +1 -0
  265. package/build/types.js +2 -0
  266. package/build/types.js.map +1 -0
  267. package/build/utils/names/consts.d.ts +5 -0
  268. package/build/utils/names/consts.d.ts.map +1 -0
  269. package/build/utils/names/consts.js +5 -0
  270. package/build/utils/names/consts.js.map +1 -0
  271. package/build/utils/names/types.d.ts +13 -0
  272. package/build/utils/names/types.d.ts.map +1 -0
  273. package/build/utils/names/types.js +2 -0
  274. package/build/utils/names/types.js.map +1 -0
  275. package/build/utils/names.d.ts +4 -0
  276. package/build/utils/names.d.ts.map +1 -0
  277. package/build/utils/names.js +60 -0
  278. package/build/utils/names.js.map +1 -0
  279. package/build/utils/paths/consts.d.ts +3 -0
  280. package/build/utils/paths/consts.d.ts.map +1 -0
  281. package/build/utils/paths/consts.js +2 -0
  282. package/build/utils/paths/consts.js.map +1 -0
  283. package/build/utils/paths/types.d.ts +20 -0
  284. package/build/utils/paths/types.d.ts.map +1 -0
  285. package/build/utils/paths/types.js +2 -0
  286. package/build/utils/paths/types.js.map +1 -0
  287. package/build/utils/paths.d.ts +4 -0
  288. package/build/utils/paths.d.ts.map +1 -0
  289. package/build/utils/paths.js +67 -0
  290. package/build/utils/paths.js.map +1 -0
  291. package/build/utils/range/types.d.ts +11 -0
  292. package/build/utils/range/types.d.ts.map +1 -0
  293. package/build/utils/range/types.js +2 -0
  294. package/build/utils/range/types.js.map +1 -0
  295. package/build/utils/range.d.ts +4 -0
  296. package/build/utils/range.d.ts.map +1 -0
  297. package/build/utils/range.js +64 -0
  298. package/build/utils/range.js.map +1 -0
  299. package/build/utils/redact/types.d.ts +16 -0
  300. package/build/utils/redact/types.d.ts.map +1 -0
  301. package/build/utils/redact/types.js +2 -0
  302. package/build/utils/redact/types.js.map +1 -0
  303. package/build/utils/redact.d.ts +4 -0
  304. package/build/utils/redact.d.ts.map +1 -0
  305. package/build/utils/redact.js +18 -0
  306. package/build/utils/redact.js.map +1 -0
  307. package/build/utils/signals/types.d.ts +17 -0
  308. package/build/utils/signals/types.d.ts.map +1 -0
  309. package/build/utils/signals/types.js +2 -0
  310. package/build/utils/signals/types.js.map +1 -0
  311. package/build/utils/signals.d.ts +4 -0
  312. package/build/utils/signals.d.ts.map +1 -0
  313. package/build/utils/signals.js +52 -0
  314. package/build/utils/signals.js.map +1 -0
  315. package/build/utils/streams/consts.d.ts +3 -0
  316. package/build/utils/streams/consts.d.ts.map +1 -0
  317. package/build/utils/streams/consts.js +3 -0
  318. package/build/utils/streams/consts.js.map +1 -0
  319. package/build/utils/streams/types.d.ts +59 -0
  320. package/build/utils/streams/types.d.ts.map +1 -0
  321. package/build/utils/streams/types.js +2 -0
  322. package/build/utils/streams/types.js.map +1 -0
  323. package/build/utils/streams.d.ts +4 -0
  324. package/build/utils/streams.d.ts.map +1 -0
  325. package/build/utils/streams.js +167 -0
  326. package/build/utils/streams.js.map +1 -0
  327. package/build/utils/tree/types.d.ts +6 -0
  328. package/build/utils/tree/types.d.ts.map +1 -0
  329. package/build/utils/tree/types.js +2 -0
  330. package/build/utils/tree/types.js.map +1 -0
  331. package/build/utils/tree.d.ts +4 -0
  332. package/build/utils/tree.d.ts.map +1 -0
  333. package/build/utils/tree.js +14 -0
  334. package/build/utils/tree.js.map +1 -0
  335. package/build/workspace/consts.d.ts +12 -0
  336. package/build/workspace/consts.d.ts.map +1 -0
  337. package/build/workspace/consts.js +12 -0
  338. package/build/workspace/consts.js.map +1 -0
  339. package/build/workspace/consts.local.d.ts +3 -0
  340. package/build/workspace/consts.local.d.ts.map +1 -0
  341. package/build/workspace/consts.local.js +3 -0
  342. package/build/workspace/consts.local.js.map +1 -0
  343. package/build/workspace/fs.d.ts +4 -0
  344. package/build/workspace/fs.d.ts.map +1 -0
  345. package/build/workspace/fs.js +45 -0
  346. package/build/workspace/fs.js.map +1 -0
  347. package/build/workspace/scope.d.ts +6 -0
  348. package/build/workspace/scope.d.ts.map +1 -0
  349. package/build/workspace/scope.js +180 -0
  350. package/build/workspace/scope.js.map +1 -0
  351. package/build/workspace/types.d.ts +66 -0
  352. package/build/workspace/types.d.ts.map +1 -0
  353. package/build/workspace/types.js +2 -0
  354. package/build/workspace/types.js.map +1 -0
  355. package/build/workspace/types.local.d.ts +20 -0
  356. package/build/workspace/types.local.d.ts.map +1 -0
  357. package/build/workspace/types.local.js +2 -0
  358. package/build/workspace/types.local.js.map +1 -0
  359. package/build/workspace.d.ts +7 -0
  360. package/build/workspace.d.ts.map +1 -0
  361. package/build/workspace.js +87 -0
  362. package/build/workspace.js.map +1 -0
  363. package/package.json +61 -4
  364. package/src/archive/consts.ts +28 -0
  365. package/src/archive/formats.ts +60 -0
  366. package/src/archive/guard.ts +68 -0
  367. package/src/archive/ratio.ts +35 -0
  368. package/src/archive/tar.ts +147 -0
  369. package/src/archive/types.local.ts +72 -0
  370. package/src/archive/types.ts +107 -0
  371. package/src/archive/zip.ts +142 -0
  372. package/src/archive.ts +298 -0
  373. package/src/backed-workspace/types.ts +72 -0
  374. package/src/backed-workspace.ts +85 -0
  375. package/src/consts.local.ts +2 -0
  376. package/src/consts.ts +2 -0
  377. package/src/errors.ts +128 -0
  378. package/src/http/connector.ts +136 -0
  379. package/src/http/consts.ts +13 -0
  380. package/src/http/source.ts +89 -0
  381. package/src/http/types.local.ts +23 -0
  382. package/src/http/types.ts +74 -0
  383. package/src/http.ts +276 -0
  384. package/src/index.ts +65 -0
  385. package/src/network/consts.ts +22 -0
  386. package/src/network/types.ts +45 -0
  387. package/src/network.ts +179 -0
  388. package/src/proxy/consts.ts +11 -0
  389. package/src/proxy/endpoint.ts +28 -0
  390. package/src/proxy/http.ts +108 -0
  391. package/src/proxy/router.ts +93 -0
  392. package/src/proxy/socks.ts +48 -0
  393. package/src/proxy/types.ts +114 -0
  394. package/src/resolvers/google-drive/consts.ts +22 -0
  395. package/src/resolvers/google-drive/types.ts +22 -0
  396. package/src/resolvers/google-drive.ts +153 -0
  397. package/src/resolvers/types.ts +45 -0
  398. package/src/s3/consts.ts +12 -0
  399. package/src/s3/prefix.ts +100 -0
  400. package/src/s3/sink.ts +36 -0
  401. package/src/s3/source.ts +90 -0
  402. package/src/s3/types.local.ts +49 -0
  403. package/src/s3/types.ts +70 -0
  404. package/src/s3/upload.ts +130 -0
  405. package/src/s3.ts +83 -0
  406. package/src/service/types.local.ts +9 -0
  407. package/src/service/types.ts +71 -0
  408. package/src/service.ts +97 -0
  409. package/src/sinks/buffer/consts.ts +1 -0
  410. package/src/sinks/buffer/types.ts +12 -0
  411. package/src/sinks/buffer.ts +18 -0
  412. package/src/sinks/directory/types.ts +7 -0
  413. package/src/sinks/directory.ts +49 -0
  414. package/src/sinks/file.ts +52 -0
  415. package/src/sinks/types.ts +12 -0
  416. package/src/sinks/workload/consts.ts +5 -0
  417. package/src/sinks/workload/types.ts +17 -0
  418. package/src/sinks/workload.ts +59 -0
  419. package/src/sources/buffer.ts +27 -0
  420. package/src/sources/file.ts +57 -0
  421. package/src/sources/stream.ts +26 -0
  422. package/src/sources/types.ts +12 -0
  423. package/src/sources/workload/types.ts +31 -0
  424. package/src/sources/workload.ts +79 -0
  425. package/src/transfer/consts.ts +10 -0
  426. package/src/transfer/types.ts +55 -0
  427. package/src/transfer.ts +89 -0
  428. package/src/types.ts +106 -0
  429. package/src/utils/names/consts.ts +5 -0
  430. package/src/utils/names/types.ts +12 -0
  431. package/src/utils/names.ts +70 -0
  432. package/src/utils/paths/consts.ts +3 -0
  433. package/src/utils/paths/types.ts +20 -0
  434. package/src/utils/paths.ts +75 -0
  435. package/src/utils/range/types.ts +11 -0
  436. package/src/utils/range.ts +80 -0
  437. package/src/utils/redact/types.ts +16 -0
  438. package/src/utils/redact.ts +23 -0
  439. package/src/utils/signals/types.ts +17 -0
  440. package/src/utils/signals.ts +63 -0
  441. package/src/utils/streams/consts.ts +2 -0
  442. package/src/utils/streams/types.ts +65 -0
  443. package/src/utils/streams.ts +188 -0
  444. package/src/utils/tree/types.ts +6 -0
  445. package/src/utils/tree.ts +18 -0
  446. package/src/workspace/consts.local.ts +2 -0
  447. package/src/workspace/consts.ts +14 -0
  448. package/src/workspace/fs.ts +48 -0
  449. package/src/workspace/scope.ts +196 -0
  450. package/src/workspace/types.local.ts +21 -0
  451. package/src/workspace/types.ts +71 -0
  452. package/src/workspace.ts +102 -0
  453. package/tests/archive.spec.ts +134 -0
  454. package/tests/backed.spec.ts +147 -0
  455. package/tests/context.ts +114 -0
  456. package/tests/fixtures.ts +190 -0
  457. package/tests/google-drive.spec.ts +76 -0
  458. package/tests/harness/drive.ts +92 -0
  459. package/tests/harness/http-proxy.ts +96 -0
  460. package/tests/harness/origin.ts +134 -0
  461. package/tests/harness/socks5.ts +135 -0
  462. package/tests/harness/workload.ts +118 -0
  463. package/tests/http.spec.ts +101 -0
  464. package/tests/inspector.spec.ts +40 -0
  465. package/tests/live.spec.ts +27 -0
  466. package/tests/network.spec.ts +79 -0
  467. package/tests/proxy.spec.ts +149 -0
  468. package/tests/s3.spec.ts +133 -0
  469. package/tests/service.spec.ts +77 -0
  470. package/tests/sources-sinks.spec.ts +96 -0
  471. package/tests/transfer.spec.ts +99 -0
  472. package/tests/tsconfig.json +12 -0
  473. package/tests/workload.spec.ts +140 -0
  474. package/tests/workspace.spec.ts +158 -0
  475. package/tsconfig.json +16 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 OwlMeans Common — Fullstack typescript framework
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,72 @@
1
- # Temporary Holding Version
1
+ # @owlmeans/file-streams
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Online files as Node streams, for OwlMeans servers and agents (Node and Bun):
4
+
5
+ - **Sources** — a URL (under a connect-time SSRF policy, through HTTP/SOCKS proxy plugins, Google
6
+ Drive links resolved), an S3 object, a file, bytes in memory, a foreign stream.
7
+ - **Sinks** — an atomic file, a confined directory tree, an S3 object or prefix (addressed exactly
8
+ like `@owlmeans/storage-resource`), a buffer, a Viable ephemeral workload (which is also a source).
9
+ - **Workspaces** — a service's temporary files, one confined folder per organization / project /
10
+ purpose, optionally mirrored to an S3 prefix under the same nesting so a restarted pod restores
11
+ them; whole S3 prefixes listed and deleted.
12
+ - **Archives** — zip, tar, tar.gz and gz: detect, list, extract, pack — with traversal, link, bomb and
13
+ overlap guards.
14
+ - **Verification** — what a file really is (images, audio, video, pdf, office documents including
15
+ legacy doc/xls/ppt, csv, text, source code), compared with what its name and media type claim,
16
+ from its head or a few range reads — never a full download.
17
+
18
+ ## Installation
19
+
20
+ ```bash
21
+ bun add @owlmeans/file-streams@^0.2.0-alpha.0
22
+ ```
23
+
24
+ ## Usage
25
+
26
+ ```typescript
27
+ import { createDirectorySink, createS3Bucket, FileKind, fileInspector, httpClient, transferHelper } from '@owlmeans/file-streams'
28
+
29
+ // Download an image into a folder — refused before a byte is written unless it is an image
30
+ await transferHelper.copyInto(httpClient.source('https://example.com/hero.png'), createDirectorySink('/work/assets'), {
31
+ verify: { allow: [FileKind.Image] },
32
+ })
33
+
34
+ // Check a remote file without downloading it
35
+ const verdict = await fileInspector.probe(httpClient.source('https://example.com/report.pdf'))
36
+ verdict.status // 'match' | 'mismatch' | 'inconclusive'
37
+ verdict.detected // { ext: 'pdf', mime: 'application/pdf', kind: 'pdf', … }
38
+
39
+ // Store it in the bucket a cfg.storageBuckets entry names
40
+ const bucket = createS3Bucket(cfg.storageBuckets.media)
41
+ await transferHelper.copy(httpClient.source('https://example.com/report.pdf'), bucket.sink('reports/q3.pdf'))
42
+ ```
43
+
44
+ ```typescript
45
+ import { createBackedWorkspace, createBufferSource, createS3Bucket, createWorkspace } from '@owlmeans/file-streams'
46
+
47
+ // Keep an upload in the service's tmp folder and in the bucket, under <org>/<project>/<purpose>
48
+ const backed = createBackedWorkspace({ workspace: createWorkspace(), bucket: createS3Bucket(cfg.storageBuckets.projects) })
49
+ const files = backed.scope({ organization: 'acme', project: 'slot-12', purpose: 'new-project' })
50
+ await files.put('brand/logo.png', createBufferSource(png, { name: 'logo.png' }))
51
+ await files.restore() // after a restart: download what the tmp folder lost
52
+ await files.drop() // the folder and every object under the scope's prefix
53
+ ```
54
+
55
+ Proxies, the Viable workload destination and source, workspaces, archive limits, the context service
56
+ and every default are described in the package skill (`file-streams`).
57
+
58
+ <!-- owlmeans:agent-guidance:start -->
59
+ ## Agent guidance
60
+
61
+ This package ships embedded agent skills under `agent-meta/`. After installing your
62
+ `@owlmeans/*` packages, run the OwlMeans agent-skills installer to place them into
63
+ your project's skill store (`.agents/skills/`):
64
+
65
+ ```sh
66
+ npx @owlmeans/agent-skills@^0.2.0-alpha.2
67
+ ```
68
+
69
+ The embedded files are version-matched to this package release. Do not edit them
70
+ directly — they are regenerated on each publish. To contribute guidance edits,
71
+ open a PR against the source monorepo.
72
+ <!-- owlmeans:agent-guidance:end -->
@@ -0,0 +1,64 @@
1
+ {
2
+ "schemaVersion": 3,
3
+ "package": "@owlmeans/file-streams",
4
+ "version": "0.2.0-alpha.0",
5
+ "generatedAt": "2026-11-11T15:44:02.969Z",
6
+ "canonicalRepo": "https://github.com/owlmeans/common",
7
+ "entries": [
8
+ {
9
+ "kind": "skill",
10
+ "name": "file-streams",
11
+ "category": "package-specific",
12
+ "file": "skills/file-streams/SKILL.md",
13
+ "canonicalPath": ".agents/skills/file-streams/SKILL.md",
14
+ "parts": [
15
+ {
16
+ "name": "network",
17
+ "file": "skills/file-streams/parts/network.md",
18
+ "description": "What the HTTP client may reach — the connect-time network policy, redirects and sensitive headers, the HTTP and SOCKS proxy plugins and their pinning, why the client is node:http over its own socket under Bun, and the Google Drive and Docs resolver.",
19
+ "purposes": [
20
+ "config",
21
+ "ops"
22
+ ],
23
+ "mentions": [
24
+ "createNetworkPolicy",
25
+ "allowCidrs",
26
+ "allowPrivate",
27
+ "createProxyRouter",
28
+ "createHttpProxyPlugin",
29
+ "createSocksProxyPlugin",
30
+ "trustProxyDns",
31
+ "createGoogleDriveResolver",
32
+ "googleDriveResolver",
33
+ "ResolverError",
34
+ "NetworkPolicyError"
35
+ ]
36
+ },
37
+ {
38
+ "name": "workspaces",
39
+ "file": "skills/file-streams/parts/workspaces.md",
40
+ "description": "Scoped temporary workspaces nested by organization/project/purpose, workspaces mirrored to a bucket (put, restore, evict, drop), bucket prefix listing and removal, reading files back from a Viable workload, and the service's workspace config.",
41
+ "purposes": [
42
+ "data",
43
+ "config",
44
+ "ops"
45
+ ],
46
+ "mentions": [
47
+ "createWorkspace",
48
+ "createBackedWorkspace",
49
+ "makeWorkloadSource",
50
+ "WorkloadReader",
51
+ "readBinaryFile",
52
+ "removePrefix",
53
+ "fileScopeHelper",
54
+ "FileScope",
55
+ "WorkspaceScope",
56
+ "BackedScope",
57
+ "BackedWorkspace",
58
+ "WORKSPACE_SCOPE_MARKER"
59
+ ]
60
+ }
61
+ ]
62
+ }
63
+ ]
64
+ }
@@ -0,0 +1,183 @@
1
+ ---
2
+ name: file-streams
3
+ description: "How to use @owlmeans/file-streams — online files as Node streams: URL downloads under a connect-time SSRF policy, HTTP and SOCKS proxy plugins (a list or a callback by key and URL), the Google Drive resolver, S3 buckets addressed exactly like storage-resource (with prefix listing and removal), folders, buffers and Viable ephemeral workloads (writes and reads); scoped temporary workspaces nested by organization/project/purpose and mirrored to a bucket for restore after a restart; zip/tar/tar.gz/gz with traversal and bomb guards; file-type verification (office, pdf, csv, media, images, source code) from a head or a few range reads. Auto-invoked when downloading a file, moving bytes between HTTP, S3, disk and a workload, keeping uploaded files in a service tmp folder backed by S3, extracting or packing an archive, checking that a file is what it claims, or configuring outbound proxies."
4
+ user-invocable: false
5
+ ---
6
+ <!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
7
+
8
+ # @owlmeans/file-streams
9
+
10
+ **Layer:** Infra
11
+ **Install:** `"@owlmeans/file-streams": "^0.2.0-alpha.0"` in `dependencies`
12
+
13
+ Server-side (Node and Bun). Every byte moves as a Node `Readable` — the shape storage-resource's
14
+ `StoredRecord.stream`, the AWS SDK, zlib and the archive codecs share. A web stream, an async
15
+ iterable or a `Uint8Array` is accepted where a source is built.
16
+
17
+ ## Key Exports
18
+
19
+ | Export | What it is |
20
+ |---|---|
21
+ | `FileSource` / `FileSink` / `TreeSink` | Where bytes come from (`info`, `open`, `ranges`), where one file goes, where many go |
22
+ | `httpClient` / `createHttpClient(opts)` | Requests under the network policy, through proxies, redirects and resolvers; `.source(url)` is a re-openable, Range-capable source |
23
+ | `createFileSource`, `createBufferSource`, `createStreamSource` | A file on disk, bytes in memory (a screenshot), a stream opened elsewhere (one-shot) |
24
+ | `createFileSink`, `createDirectorySink`, `createBufferSink` | Atomic file write, a confined directory tree, an in-memory collector |
25
+ | `createS3Bucket(entry or model, opts)` | A `cfg.storageBuckets` entry as `source`, `sink`, `tree`, `head`, `remove`, `list`, `removePrefix`, `instanceOf` |
26
+ | `makeWorkloadSink(writer, { root })` / `makeWorkloadSource(reader, { root })` | A Viable ephemeral workload as a tree, through its `writeBinaryFile` port; its files as sources, through `readBinaryFile` |
27
+ | `createWorkspace({ root?, dirMode?, fileMode? })` | A service's temporary files, one confined folder per `FileScope` (`scope`, `scopes`, `prune`) |
28
+ | `createBackedWorkspace({ workspace, bucket, prefix? })` | Scopes mirrored to a bucket under the same nesting: `put`, `restore`, `list`, `remove`, `evict`, `drop` |
29
+ | `fileScopeHelper`, `FileScope`, `FileScopeSchema`, `FileScopeError` | Re-exported from `@owlmeans/storage-common`: the `<organization>/<project?>/<purpose>` nesting |
30
+ | `transferHelper.copy(source, sink, opts)` / `.copyInto(source, tree, opts)` | One file moved under limits, its head verified before the sink sees a byte |
31
+ | `fileInspector.probe(source)` / `.inspect(bytes, claims)` / `.enforce(verdict, policy)` | What a file really is, against what it claims — re-exported from `@owlmeans/file-inspector` |
32
+ | `archiveHelper.extract / list / pack / gzip / gunzip / detect` | zip, tar, tar.gz and gz, detected from bytes |
33
+ | `createProxyRouter`, `createHttpProxyPlugin`, `createSocksProxyPlugin` | Proxy selection and the built-in protocol plugins |
34
+ | `createGoogleDriveResolver(opts)` / `googleDriveResolver` | Drive and Docs links to the bytes behind them (on by default in `createHttpClient`) |
35
+ | `createNetworkPolicy(opts)` | What may be reached; every connection goes through it |
36
+ | `appendFileStreamsService(ctx, alias?, opts?)` | The objects configured from `cfg.fileStreams` and `cfg.storageBuckets` |
37
+ | `FileKind`, `ArchiveFormat`, `WORKLOAD_MAX_FILE_BYTES`, `S3_DELETE_BATCH_SIZE`, `WORKSPACE_SCOPE_MARKER` | Enums, ceilings, the reserved marker name |
38
+ | `NetworkPolicyError`, `HttpError` (`.status`), `ProxyError`, `LimitError`, `SourceError`, `PathError`, `ArchiveError`, `ResolverError`, `WorkloadError` | Failures, by message code (`file-streams:<area>:<reason>`) |
39
+ | `FileMimicryError`, `FileKindError` (storage `FileTypeError`, from file-inspector), `S3TransferError` (storage `StorageApiError`) | Type refusals and S3 failures a storage-resource catch site already handles |
40
+
41
+ ## Usage
42
+
43
+ ```typescript
44
+ import {
45
+ archiveHelper, createBufferSource, createDirectorySink, createHttpClient, createS3Bucket, FileKind,
46
+ fileInspector, httpClient, makeWorkloadSink, transferHelper,
47
+ } from '@owlmeans/file-streams'
48
+
49
+ // An image from the internet into a folder, refused unless it is an image
50
+ await transferHelper.copyInto(httpClient.source(url), createDirectorySink(workDir), { verify: { allow: [FileKind.Image] } })
51
+
52
+ // A screenshot (a Buffer) into the bucket a storage entry names — same key and URL storage-resource uses
53
+ const bucket = createS3Bucket(cfg.storageBuckets.media)
54
+ const { sink } = await transferHelper.copy(createBufferSource(png, { name: 'shot.png' }), bucket.sink(`${entityId}/shots/${id}.png`))
55
+ sink.url // === storage-resource's URL for that prefix
56
+ bucket.instanceOf(sink, 'original') // a storage-common StoredFileInstance
57
+
58
+ // A media archive into a folder or a workload; entries that cannot be written are skipped
59
+ await archiveHelper.extract(httpClient.source(archiveUrl), createDirectorySink(dir), { onEntryError: 'skip' })
60
+
61
+ // What a remote file is, without downloading it (a few range reads, at most 1 MiB)
62
+ const verdict = await fileInspector.probe(httpClient.source(url))
63
+
64
+ // Proxies: a list (round-robin, random, failover, by-key) or a callback by key and URL
65
+ const pooled = createHttpClient({ proxies: { proxies: ['socks5h://u:p@egress-1:1080', 'http://egress-2:3128'], strategy: 'by-key' } })
66
+ const vaulted = createHttpClient({ proxies: { proxies: async ({ key, url }) => vault.proxyFor(key, url.hostname) } })
67
+ await pooled.request(url, { proxyKey: entityId })
68
+ ```
69
+
70
+ `copy` reads the head of the very stream it moves (no second download), verifies it, and replays
71
+ it into the sink. Verification is on by default: a kind mismatch is refused, dangerous content (an
72
+ executable under another claim, a script under a passive one, a PDF with active content in the head
73
+ sample) is always refused, `verify.refuse` refuses kinds outright, an inconclusive verdict passes
74
+ unless `refuseInconclusive`; `verify: false` turns it off.
75
+
76
+ ## The Viable workload
77
+
78
+ A workload accepts bytes only as the publisher's signed `writeBinaryFile { filePath, base64 }`
79
+ command; the agent reaches it through viable-agent's `FileHelper`, which satisfies
80
+ `WorkloadWriter` as is (pass the helper, never a destructured method):
81
+
82
+ ```typescript
83
+ const media = makeWorkloadSink(fileHelper, { root: 'sources/web/public/media' })
84
+ await transferHelper.copy(httpClient.source(imageUrl), media.entry('hero.png'), { verify: { allow: [FileKind.Image] } })
85
+ await archiveHelper.extract(httpClient.source(zipUrl), media, { onEntryError: 'skip' })
86
+ ```
87
+
88
+ Each file is one atomic write of at most 12 MiB (`WORKLOAD_MAX_FILE_BYTES`, the publisher's 16 MiB
89
+ base64 limit); `maxFileBytes` lowers it, and an oversized file is refused before any request
90
+ (`WorkloadError('too-large')`). Larger files need a product route (chunked writes, or the workload
91
+ pulling a URL itself — it has public egress and can run this package).
92
+
93
+ Reads go the other way through `makeWorkloadSource(reader, { root })` (`readBinaryFile`, same
94
+ ceiling): the [workspaces](parts/workspaces.md) part.
95
+
96
+ ## Workspaces
97
+
98
+ `createWorkspace().scope({ organization, project?, purpose })` is the folder
99
+ `<root>/<organization>/<project?>/<purpose>` (segments by `fileScopeHelper`); a backed workspace
100
+ mirrors every scope to `<basePrefix>/<prefix?>/<scope path>/<path>` in a bucket, and the bucket is the
101
+ authority — a pod restart loses the tmp folder, `restore()` brings the scope back. Paths are confined
102
+ (no `..`, no absolute paths, no existing symlink from the workspace root, the marker name reserved);
103
+ a relative `root` lives under `os.tmpdir()`. Full rules, overlap and prune semantics, prefix
104
+ operations: the [workspaces](parts/workspaces.md) part.
105
+
106
+ ## Detection
107
+
108
+ Verification is `@owlmeans/file-inspector` (its `file-inspector` skill: verdicts, policies, kinds,
109
+ active PDF, probes). This package re-exports its inspector, kinds, tables, verdict types and the two
110
+ refusals unchanged, and every source here (`FileSource`) is a file-inspector `ProbeSource` as is: a
111
+ probe seeks through `ranges()` when the origin has them and reads the head of `open()` otherwise. The
112
+ S3 sink takes the Content-Type from the verdict and writes `attachment` for active svg/html.
113
+
114
+ ## Archives
115
+
116
+ zip is read from its central directory (yauzl) — through range reads when the source has them, else
117
+ from a private temporary spool — and written as a stream (yazl); tar and tar.gz stream through
118
+ tar-stream (an entry is drained, never destroyed: destroying one destroys the archive); gz is zlib.
119
+ The format comes from the bytes. Refused: absolute paths, `..`, drive letters, control and bidi
120
+ characters, overlong or deep paths, duplicate and case-fold-twin entries, encrypted entries,
121
+ overlapping zip entries, an expansion ratio above 200 (past 1 MiB), more entries or bytes than the
122
+ limits. Links and devices are skipped (or refused with `links: 'refuse'`), never created. Nested
123
+ archives stay files unless `limits.maxDepth` > 1. Each entry commits atomically; a refusal leaves the
124
+ entries already written.
125
+
126
+ ## Defaults
127
+
128
+ | Area | Default |
129
+ |---|---|
130
+ | Transfer | 512 MiB · 10 min total · 30 s idle |
131
+ | HTTP | connect 10 s · headers 30 s · idle 30 s · 5 redirects · `Accept-Encoding: identity` |
132
+ | Probe | 1 MiB budget · 64 KiB head sample |
133
+ | Archives | 10 000 entries · 512 MiB per entry · 2 GiB total · ratio 200 · depth 1 |
134
+ | S3 | 8 MiB parts (≥ 5 MiB), 2 in flight; one PutObject when the body fits a part; aborted on failure |
135
+ | Workload | 12 MiB per file (`WORKLOAD_MAX_FILE_BYTES`), one retry, writes and reads |
136
+ | Workspace | `<os.tmpdir()>/owlmeans-workspace`, folders `0o700`, files `0o600` |
137
+ | S3 prefixes | ListObjectsV2 all pages; DeleteObjects ≤ 1000 keys, checksummed by Content-MD5 |
138
+
139
+ ## Service
140
+
141
+ `appendFileStreamsService(ctx)` registers `file-streams`: `http()`, `transfer()`, `inspector()`,
142
+ `archives()`, `bucket(configKey)`, `workspace()` and `backed(configKey, { prefix? })` (one object per
143
+ key and prefix, over the configured workspace and transfer helper), built lazily from
144
+ `cfg.fileStreams` (`limits`, `network`, `proxies: [{ url, username?, password? }]`, `proxyStrategy`,
145
+ `userAgent`, `workspace: { root?, dirMode?, fileMode? }`) and `cfg.storageBuckets`. Proxy entries are
146
+ objects so a server context resolves their secret-file leaves; callbacks and plugins come through the
147
+ options argument. A server context reads EVERY string leaf starting with `/` as a file, so
148
+ `workspace.root` is given relative to `os.tmpdir()` (or as a file holding an absolute path).
149
+
150
+ ## External docs
151
+
152
+ - https://github.com/thejoshwolfe/yauzl — central-directory zip reading, `fromRandomAccessReader` (override `read` and `_readStreamForRange`), `validateEntrySizes`, automatic `validateFileName` (v3)
153
+ - https://github.com/mafintosh/tar-stream — v3 ships streamx typings; destroying an entry destroys the extract, so entries are drained
154
+
155
+ ## Tests
156
+
157
+ Category C. Offline specs run against local stand-ins (`tests/harness/`: an origin with Range,
158
+ redirects and stalls, a CONNECT/forward proxy, a SOCKS5 server, a fake Drive, a workload
159
+ `/command/` receiver answering `writeBinaryFile` and `readBinaryFile` with the real limits) and
160
+ code-built fixtures; `workspace.spec.ts` runs offline over a temporary root; `s3.spec.ts` and
161
+ `backed.spec.ts` need the S3 gate (path-style for a non-AWS endpoint, objects under a random base
162
+ prefix, deleted after); `live.spec.ts` needs `FILE_STREAMS_LIVE=1` (and `FILE_STREAMS_LIVE_DRIVE_URL`).
163
+
164
+ ## Depends On
165
+
166
+ - `@owlmeans/file-inspector` (detection, re-exported), `@owlmeans/storage-resource` (the bucket
167
+ model), `@owlmeans/storage-common` (errors, instances), `@owlmeans/context`, `@owlmeans/error`,
168
+ `@owlmeans/log`
169
+ - `@aws-sdk/client-s3`, `socks`, `tar-stream`, `yauzl`, `yazl`
170
+
171
+ ## Related
172
+
173
+ - [[file-inspector]] — what a file is: verdicts, policies, kinds, active PDF, safe names
174
+ - [[storage-resource]] — the upload resource and the bucket model
175
+ - [[storage-common]] — stored-file types and errors, `FileScope` and `fileScopeHelper`
176
+ - [[logging]] — scopes `file-streams:<area>`; URLs are logged redacted, contents never
177
+
178
+ ## Parts
179
+
180
+ | Part | Load it when |
181
+ |---|---|
182
+ | [network](parts/network.md) | changing what may be reached, routing through proxies, or downloading Google Drive and Docs links |
183
+ | [workspaces](parts/workspaces.md) | keeping files in a service tmp folder by scope, mirroring them to a bucket, restoring after a restart, listing or deleting a bucket prefix, or reading files back from a workload |
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: network
3
+ description: What the HTTP client may reach — the connect-time network policy, redirects and sensitive headers, the HTTP and SOCKS proxy plugins and their pinning, why the client is node:http over its own socket under Bun, and the Google Drive and Docs resolver.
4
+ metadata:
5
+ purposes: config, ops
6
+ mentions: createNetworkPolicy, allowCidrs, allowPrivate, createProxyRouter, createHttpProxyPlugin, createSocksProxyPlugin, trustProxyDns, createGoogleDriveResolver, googleDriveResolver, ResolverError, NetworkPolicyError
7
+ ---
8
+ <!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
9
+
10
+ # @owlmeans/file-streams — Network policy, proxies and Google Drive
11
+
12
+ ## Network policy and proxies
13
+
14
+ - Loopback, private, link-local, CGNAT, multicast, reserved and documentation ranges are refused,
15
+ and the IPv4 inside a mapped, NAT64, 6to4 or Teredo address is checked too; internal names
16
+ (`localhost`, `.local`, `.internal`, `.svc`, `.cluster.local`, `.home.arpa`) are refused before DNS.
17
+ - The check is pinned to the connect: a direct socket goes to the address just resolved and
18
+ checked, a proxy tunnel carries that address (TLS keeps the host name for SNI and certificates).
19
+ Every redirect hop is checked again; https → http is refused; `Authorization`, `Cookie`,
20
+ `Proxy-Authorization` and `sensitiveHeaders` never cross origins.
21
+ - A local stand-in or an in-cluster origin needs `network: { allowCidrs: [...] }` (or
22
+ `allowPrivate`). S3 traffic through the SDK is not policed — its endpoint is operator config.
23
+ - Built-in plugins: `http`/`https` endpoints (CONNECT for https targets, absolute-form forward for
24
+ plain http — which a proxy resolves itself, so pinning holds only through CONNECT:
25
+ `createHttpProxyPlugin({ httpTargets: 'connect' })`), and `socks4`/`socks4a`/`socks5`/`socks5h`.
26
+ Remote-DNS endpoints (`socks5h`, `socks4a`) still pre-check the name locally and are refused when
27
+ it does not resolve unless `trustProxyDns`.
28
+ - Bun's own `fetch({ proxy })` speaks only http/https proxies and undici dispatchers are unreliable
29
+ under Bun, so the client is `node:http(s)` over a socket it connects itself — Bun 1.4.2 honours
30
+ the `createConnection` option, an Agent's `createConnection`, the `lookup` option and TLS over an
31
+ existing socket.
32
+
33
+ ## Google Drive
34
+
35
+ Share links (`/file/d/<id>/…`, `open?id=`, `uc?id=`) become `drive.usercontent.google.com/download`;
36
+ the "can't scan this file for viruses" page is followed through its own `download-form` (action must
37
+ be a Google origin; hidden `confirm`/`uuid` carried), then the legacy `confirm=` link, then a
38
+ `download_warning` cookie. Docs, Sheets, Slides and Drawings links become export URLs (docx, xlsx,
39
+ pptx, png by default; csv/tsv keep the `gid`). Folders, quota pages, sign-in redirects and forms
40
+ posting elsewhere fail as `ResolverError('google-drive:<reason>')`. The page flow is the one gdown
41
+ documents; it is reverse-engineered, so `FILE_STREAMS_LIVE_DRIVE_URL` exercises it against Google.
42
+
43
+ ## External docs
44
+
45
+ - https://bun.com/docs/runtime/networking/fetch — Bun `fetch({ proxy })` accepts http/https proxy URLs only (no SOCKS) (Bun 1.4)
46
+ - https://undici.nodejs.org/api/Socks5ProxyAgent — undici's SOCKS5 agent is experimental; dispatchers are not relied on under Bun
47
+ - https://docs.rs/modde-sources/latest/src/modde_sources/gdrive/mod.rs.html — the usercontent download host and the virus-scan form flow (`confirm`, `uuid`) public Drive downloads take; no official Google documentation exists for it
@@ -0,0 +1,129 @@
1
+ ---
2
+ name: workspaces
3
+ description: Scoped temporary workspaces nested by organization/project/purpose, workspaces mirrored to a bucket (put, restore, evict, drop), bucket prefix listing and removal, reading files back from a Viable workload, and the service's workspace config.
4
+ metadata:
5
+ purposes: data, config, ops
6
+ mentions: createWorkspace, createBackedWorkspace, makeWorkloadSource, WorkloadReader, readBinaryFile, removePrefix, fileScopeHelper, FileScope, WorkspaceScope, BackedScope, BackedWorkspace, WORKSPACE_SCOPE_MARKER
7
+ ---
8
+ <!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
9
+
10
+ # @owlmeans/file-streams — Workspaces, backed workspaces, prefixes and workload reads
11
+
12
+ ## Scopes
13
+
14
+ A `FileScope` (`@owlmeans/storage-common`, re-exported here) is `{ organization, project?, purpose }`:
15
+ the organization's `entitySlug`, an optional project id or slug, the task (`new-project`, `upload`).
16
+ Each is one segment — 1–128 characters of `[A-Za-z0-9][A-Za-z0-9._-]*`, so never `.` or `..` —
17
+ and `fileScopeHelper.pathOf(scope)` nests them as `org/project/purpose` or `org/purpose`
18
+ (`parse` reads a path back, `assert` throws `FileScopeError('<field>')`). Claim a scope once, store
19
+ it with the records it serves and read it back; never recompute it from names that can change.
20
+
21
+ ## Workspace
22
+
23
+ ```typescript
24
+ import { createWorkspace, transferHelper, archiveHelper, httpClient, FileKind } from '@owlmeans/file-streams'
25
+
26
+ const workspace = createWorkspace() // <os.tmpdir()>/owlmeans-workspace
27
+ const files = workspace.scope({ organization: entitySlug, project: slotId, purpose: 'new-project' })
28
+ await transferHelper.copy(httpClient.source(url), files.entry('brand/logo.png'), { verify: { allow: [FileKind.Image] } })
29
+ await archiveHelper.extract(source, files.tree(), { onEntryError: 'skip' })
30
+ await files.list() // [{ path: 'brand/logo.png', size, modifiedAt }, …]
31
+ await files.drop() // the folder and the parents it leaves empty
32
+ await workspace.prune({ olderThanMs: 24 * 3600_000 })
33
+ ```
34
+
35
+ - **Root**: absolute, or relative to `os.tmpdir()` (which `TMPDIR` moves) — never to the cwd.
36
+ Folders are created `0o700`, files written `0o600` (`dirMode`, `fileMode`). Nothing survives a pod
37
+ restart; that is what a backed workspace is for.
38
+ - **Scope folders**: `workspace.scope(scope)` does no I/O. The first write creates
39
+ `<root>/<scope path>` and its marker `.owlmeans-scope` (`WORKSPACE_SCOPE_MARKER`, the scope as
40
+ JSON); `scopes(organization?)` answers the marked folders, sorted by path.
41
+ - **Confinement**: every path goes through the package's path rules (`\` → `/`, NFC; refused:
42
+ `traversal`, `absolute`, `invalid-char`, `too-long`, `too-deep`, `empty`), a segment named like the
43
+ marker is `reserved`, and any existing symlink from the workspace root down to the target is
44
+ refused (`PathError('symlink')`) — checked before and after folders are created, and on every
45
+ `source` call. `list()` neither follows nor shows links and hides in-flight temporary files.
46
+ - **Writes**: `entry(path, { onConflict })` (default `error` → `PathError('exists')`) and
47
+ `tree({ onConflict })` — a confined directory sink for archives, refusing case-fold twins within
48
+ one tree. Each file commits atomically. `source(path, claims?)` is re-openable with random access.
49
+ - **Reads and removal**: `has(path)` is true for a regular file only; `remove(path)` takes a file or a
50
+ folder and the parents it empties (the scope folder stays), `false` when absent; `drop()` removes
51
+ the scope folder and every parent it empties, up to — never including — the root.
52
+ - **Overlap**: `org/<purpose>` contains `org/<project>/<purpose>` when a project segment equals a
53
+ purpose. The scope on disk first keeps the folder; the other's first write throws
54
+ `PathError('scope-overlap')`. A bucket cannot see this (it has no markers), so purposes never equal
55
+ a project segment.
56
+ - **Prune** drops whole scopes whose newest file, scope folder and marker are all older than
57
+ `olderThanMs` (never single files, so a scope is never half there), answers how many and logs it at
58
+ info. Writes to different scopes are independent; a drop that empties a shared parent while a
59
+ sibling scope is being created is retried once.
60
+
61
+ ## Backed workspace
62
+
63
+ ```typescript
64
+ import { createBackedWorkspace, createS3Bucket, createWorkspace } from '@owlmeans/file-streams'
65
+
66
+ const backed = createBackedWorkspace({ workspace: createWorkspace(), bucket: createS3Bucket(cfg.storageBuckets.projects), prefix: 'materials' })
67
+ const files = backed.scope(scope)
68
+ const { local, remote, transfer } = await files.put('brand/logo.png', source, { verify: { refuse: [FileKind.Executable] } })
69
+ files.key('brand/logo.png') // '<basePrefix>/materials/<org>/<project>/<purpose>/brand/logo.png'
70
+ files.prefixOf('brand/logo.png') // the same without the base prefix — what bucket.source/sink/head/urlOf take
71
+ const { restored, kept } = await files.restore() // after a restart: the bucket is the authority
72
+ await files.drop() // local folder + every object under the scope
73
+ ```
74
+
75
+ - **Layout**: an object lands at `<basePrefix>/<prefix?>/<scope path>/<path>` — the same nesting as
76
+ the folder. Give the bucket entry `acl: false` (or a private bucket): the files are people's uploads,
77
+ and storage-resource's default ACL is `public-read`.
78
+ - **`put(path, source, options)`** copies the source into the local entry through the transfer helper
79
+ (`verify`, `limits`, `hash`, `onProgress` pass through; `onConflict` defaults to `overwrite`, like the
80
+ bucket), then uploads the LOCAL file, with the Content-Type and disposition of the copy's verdict
81
+ (`options.sink` sets cache control or metadata for every object). A failed upload removes the local
82
+ copy and rethrows, so everything local is backed. A refused verification writes nothing anywhere.
83
+ - **`restore()`** lists the scope's prefix and downloads every object that is missing locally or has
84
+ another size — byte-identical, unverified (it was verified on the way in); the rest are `kept`.
85
+ Local files the bucket does not name are left alone; folder placeholder keys (ending `/`) are skipped.
86
+ - **`list()`** is the remote listing with paths relative to the scope; **`remove(path)`** deletes the
87
+ local file or folder, the object and anything under `<path>/`; **`evict()`** drops the local copy
88
+ only; **`drop()`** drops the local copy and every object under the scope.
89
+
90
+ ## Bucket prefixes
91
+
92
+ - `bucket.list(prefix)` answers every object under `<prefix>/` (ListObjectsV2, all pages — a sibling
93
+ `prefix-x` is not included) as `{ key, path, size, etag, modifiedAt }`, `path` relative to the
94
+ prefix; a listing carries no media type. An empty prefix is refused (`PathError('empty')`), so no call
95
+ ever addresses the whole base prefix.
96
+ - `bucket.removePrefix(prefix)` lists, then deletes in DeleteObjects batches of at most 1000 keys
97
+ (`S3_DELETE_BATCH_SIZE`) and answers the count. DeleteObjects requires a checksum: the request carries
98
+ Content-MD5 in place of the SDK's flexible checksum headers (the SDK's own MD5 fallback), which S3 and
99
+ stores that predate flexible checksums both accept; every other operation keeps the client's
100
+ `WHEN_REQUIRED` checksums. A key the store refuses throws `S3TransferError('delete-objects:<code>')`;
101
+ removal is not atomic, and running it again is safe.
102
+
103
+ ## Workload reads
104
+
105
+ ```typescript
106
+ const materials = makeWorkloadSource(fileHelper, { root: 'docs/materials' }) // WorkloadReader: readBinaryFile(filePath) → base64
107
+ await transferHelper.copy(materials.entry('logo.png'), files.entry('logo.png'), { verify: { allow: [FileKind.Image] } })
108
+ ```
109
+
110
+ The publisher's `readBinaryFile { filePath } → { base64 }` command, reached through viable-agent's
111
+ `FileHelper` (pass the helper itself). Each `open` is one read of the whole file; `info()` reads
112
+ nothing and knows the size once a read has happened; `ranges()` serves the bytes of one read, so
113
+ probes and zip extraction work. Files above 12 MiB (`WORKLOAD_MAX_FILE_BYTES`) or a lower
114
+ `maxFileBytes` are refused before decoding (`WorkloadError('too-large')`); a failed read is retried
115
+ once (`retries`), then `WorkloadError('read-failed')`.
116
+
117
+ ## Service
118
+
119
+ `cfg.fileStreams.workspace: { root?, dirMode?, fileMode? }` → `service.workspace()`;
120
+ `service.backed(storageKey, { prefix? })` is one backed workspace per storage entry and prefix, over
121
+ that workspace and the service's transfer helper (its configured limits). A server context reads
122
+ every string leaf that starts with `/` as a secret file, so `root` is relative to `os.tmpdir()`
123
+ (`root: 'agent-workspace'`), or a file that holds an absolute path.
124
+
125
+ ## External docs
126
+
127
+ - https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/MD5_FALLBACK.md — since v3.729 the SDK checksums with CRC32; DeleteObjects is the one operation that requires a checksum, and the documented fallback strips the flexible checksum headers and adds Content-MD5 after `flexibleChecksumsMiddleware`
128
+ - https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html — at most 1000 keys per request; Content-MD5 or an `x-amz-checksum-*` header is required
129
+ - https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html — at most 1000 keys per page, `IsTruncated` + `NextContinuationToken`
@@ -0,0 +1,16 @@
1
+ import type { ArchiveLimits } from './types.js';
2
+ /** The archive formats read and written; detected from bytes, never from names. */
3
+ export declare enum ArchiveFormat {
4
+ Zip = "zip",
5
+ Tar = "tar",
6
+ TarGz = "tar.gz",
7
+ Gzip = "gz"
8
+ }
9
+ export declare const DEFAULT_ARCHIVE_LIMITS: Readonly<ArchiveLimits>;
10
+ /** Expansion ratios are judged only past this many produced bytes. */
11
+ export declare const RATIO_FLOOR_BYTES: number;
12
+ /** A zip local file header without its name and extra field. */
13
+ export declare const ZIP_LOCAL_HEADER_BYTES = 30;
14
+ /** Names that mark a nested archive. */
15
+ export declare const NESTED_ARCHIVE_EXTENSIONS: readonly string[];
16
+ //# sourceMappingURL=consts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../src/archive/consts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,mFAAmF;AACnF,oBAAY,aAAa;IACvB,GAAG,QAAQ;IACX,GAAG,QAAQ;IACX,KAAK,WAAW;IAChB,IAAI,OAAO;CACZ;AAED,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,aAAa,CAQzD,CAAA;AAEF,sEAAsE;AACtE,eAAO,MAAM,iBAAiB,QAAc,CAAA;AAE5C,gEAAgE;AAChE,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC,wCAAwC;AACxC,eAAO,MAAM,yBAAyB,EAAE,SAAS,MAAM,EAAyD,CAAA"}
@@ -0,0 +1,24 @@
1
+ /** The archive formats read and written; detected from bytes, never from names. */
2
+ export var ArchiveFormat;
3
+ (function (ArchiveFormat) {
4
+ ArchiveFormat["Zip"] = "zip";
5
+ ArchiveFormat["Tar"] = "tar";
6
+ ArchiveFormat["TarGz"] = "tar.gz";
7
+ ArchiveFormat["Gzip"] = "gz";
8
+ })(ArchiveFormat || (ArchiveFormat = {}));
9
+ export const DEFAULT_ARCHIVE_LIMITS = Object.freeze({
10
+ maxEntries: 10_000,
11
+ maxEntryBytes: 512 * 1024 * 1024,
12
+ maxTotalBytes: 2 * 1024 * 1024 * 1024,
13
+ maxRatio: 200,
14
+ maxDepth: 1,
15
+ maxPathBytes: 1024,
16
+ maxPathSegments: 64,
17
+ });
18
+ /** Expansion ratios are judged only past this many produced bytes. */
19
+ export const RATIO_FLOOR_BYTES = 1024 * 1024;
20
+ /** A zip local file header without its name and extra field. */
21
+ export const ZIP_LOCAL_HEADER_BYTES = 30;
22
+ /** Names that mark a nested archive. */
23
+ export const NESTED_ARCHIVE_EXTENSIONS = Object.freeze(['zip', 'tar', 'tar.gz', 'tgz', 'gz']);
24
+ //# sourceMappingURL=consts.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/archive/consts.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,MAAM,CAAN,IAAY,aAKX;AALD,WAAY,aAAa;IACvB,4BAAW,CAAA;IACX,4BAAW,CAAA;IACX,iCAAgB,CAAA;IAChB,4BAAW,CAAA;AACb,CAAC,EALW,aAAa,KAAb,aAAa,QAKxB;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA4B,MAAM,CAAC,MAAM,CAAC;IAC3E,UAAU,EAAE,MAAM;IAClB,aAAa,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI;IAChC,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;IACrC,QAAQ,EAAE,GAAG;IACb,QAAQ,EAAE,CAAC;IACX,YAAY,EAAE,IAAI;IAClB,eAAe,EAAE,EAAE;CACpB,CAAC,CAAA;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,IAAI,CAAA;AAE5C,gEAAgE;AAChE,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAExC,wCAAwC;AACxC,MAAM,CAAC,MAAM,yBAAyB,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAA"}
@@ -0,0 +1,4 @@
1
+ import type { FormatSniffer } from './types.local.js';
2
+ /** Archive formats told from leading bytes. */
3
+ export declare const createFormatSniffer: () => FormatSniffer;
4
+ //# sourceMappingURL=formats.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"formats.d.ts","sourceRoot":"","sources":["../../src/archive/formats.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErD,+CAA+C;AAC/C,eAAO,MAAM,mBAAmB,QAAO,aAqDtC,CAAA"}
@@ -0,0 +1,53 @@
1
+ import { gunzipSync } from 'node:zlib';
2
+ import { constants } from 'node:zlib';
3
+ import { ArchiveFormat } from './consts.js';
4
+ /** Archive formats told from leading bytes. */
5
+ export const createFormatSniffer = () => {
6
+ const isTarHeader = (block) => {
7
+ if (block.length < 512) {
8
+ return false;
9
+ }
10
+ if (Buffer.from(block.subarray(257, 262)).toString('latin1') === 'ustar') {
11
+ return true;
12
+ }
13
+ const stored = parseInt(Buffer.from(block.subarray(148, 156)).toString('latin1').replace(/\0.*$/, '').trim(), 8);
14
+ if (Number.isNaN(stored)) {
15
+ return false;
16
+ }
17
+ let sum = 0;
18
+ for (let index = 0; index < 512; index++) {
19
+ sum += index >= 148 && index < 156 ? 0x20 : block[index];
20
+ }
21
+ return sum === stored;
22
+ };
23
+ const inflateHead = (head) => {
24
+ try {
25
+ return gunzipSync(head, { finishFlush: constants.Z_SYNC_FLUSH, maxOutputLength: 64 * 1024 });
26
+ }
27
+ catch {
28
+ return new Uint8Array(0);
29
+ }
30
+ };
31
+ const formatOf = (head) => {
32
+ if (head[0] === 0x50 && head[1] === 0x4b && [0x0304, 0x0506, 0x0708].includes((head[2] << 8) | head[3])) {
33
+ return ArchiveFormat.Zip;
34
+ }
35
+ if (head[0] === 0x1f && head[1] === 0x8b) {
36
+ return isTarHeader(inflateHead(head)) ? ArchiveFormat.TarGz : ArchiveFormat.Gzip;
37
+ }
38
+ return isTarHeader(head) ? ArchiveFormat.Tar : null;
39
+ };
40
+ const gzipName = (head) => {
41
+ if (head[0] !== 0x1f || head[1] !== 0x8b || (head[3] & 0x08) === 0) {
42
+ return undefined;
43
+ }
44
+ let offset = 10;
45
+ if ((head[3] & 0x04) !== 0) {
46
+ offset += 2 + (head[10] | (head[11] << 8));
47
+ }
48
+ const end = head.indexOf(0, offset);
49
+ return end > offset ? Buffer.from(head.subarray(offset, end)).toString('latin1') : undefined;
50
+ };
51
+ return { formatOf, gzipName };
52
+ };
53
+ //# sourceMappingURL=formats.js.map