@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +71 -2
  3. package/agent-meta/manifest.json +64 -0
  4. package/agent-meta/skills/file-streams/SKILL.md +183 -0
  5. package/agent-meta/skills/file-streams/parts/network.md +47 -0
  6. package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
  7. package/build/archive/consts.d.ts +16 -0
  8. package/build/archive/consts.d.ts.map +1 -0
  9. package/build/archive/consts.js +24 -0
  10. package/build/archive/consts.js.map +1 -0
  11. package/build/archive/formats.d.ts +4 -0
  12. package/build/archive/formats.d.ts.map +1 -0
  13. package/build/archive/formats.js +53 -0
  14. package/build/archive/formats.js.map +1 -0
  15. package/build/archive/guard.d.ts +5 -0
  16. package/build/archive/guard.d.ts.map +1 -0
  17. package/build/archive/guard.js +62 -0
  18. package/build/archive/guard.js.map +1 -0
  19. package/build/archive/ratio.d.ts +5 -0
  20. package/build/archive/ratio.d.ts.map +1 -0
  21. package/build/archive/ratio.js +30 -0
  22. package/build/archive/ratio.js.map +1 -0
  23. package/build/archive/tar.d.ts +4 -0
  24. package/build/archive/tar.d.ts.map +1 -0
  25. package/build/archive/tar.js +142 -0
  26. package/build/archive/tar.js.map +1 -0
  27. package/build/archive/types.d.ts +95 -0
  28. package/build/archive/types.d.ts.map +1 -0
  29. package/build/archive/types.js +2 -0
  30. package/build/archive/types.js.map +1 -0
  31. package/build/archive/types.local.d.ts +64 -0
  32. package/build/archive/types.local.d.ts.map +1 -0
  33. package/build/archive/types.local.js +2 -0
  34. package/build/archive/types.local.js.map +1 -0
  35. package/build/archive/zip.d.ts +4 -0
  36. package/build/archive/zip.d.ts.map +1 -0
  37. package/build/archive/zip.js +135 -0
  38. package/build/archive/zip.js.map +1 -0
  39. package/build/archive.d.ts +4 -0
  40. package/build/archive.d.ts.map +1 -0
  41. package/build/archive.js +265 -0
  42. package/build/archive.js.map +1 -0
  43. package/build/backed-workspace/types.d.ts +67 -0
  44. package/build/backed-workspace/types.d.ts.map +1 -0
  45. package/build/backed-workspace/types.js +2 -0
  46. package/build/backed-workspace/types.js.map +1 -0
  47. package/build/backed-workspace.d.ts +4 -0
  48. package/build/backed-workspace.d.ts.map +1 -0
  49. package/build/backed-workspace.js +63 -0
  50. package/build/backed-workspace.js.map +1 -0
  51. package/build/consts.d.ts +3 -0
  52. package/build/consts.d.ts.map +1 -0
  53. package/build/consts.js +3 -0
  54. package/build/consts.js.map +1 -0
  55. package/build/consts.local.d.ts +3 -0
  56. package/build/consts.local.d.ts.map +1 -0
  57. package/build/consts.local.js +3 -0
  58. package/build/consts.local.js.map +1 -0
  59. package/build/errors.d.ts +59 -0
  60. package/build/errors.d.ts.map +1 -0
  61. package/build/errors.js +105 -0
  62. package/build/errors.js.map +1 -0
  63. package/build/http/connector.d.ts +11 -0
  64. package/build/http/connector.d.ts.map +1 -0
  65. package/build/http/connector.js +124 -0
  66. package/build/http/connector.js.map +1 -0
  67. package/build/http/consts.d.ts +9 -0
  68. package/build/http/consts.d.ts.map +1 -0
  69. package/build/http/consts.js +8 -0
  70. package/build/http/consts.js.map +1 -0
  71. package/build/http/source.d.ts +5 -0
  72. package/build/http/source.d.ts.map +1 -0
  73. package/build/http/source.js +76 -0
  74. package/build/http/source.js.map +1 -0
  75. package/build/http/types.d.ts +67 -0
  76. package/build/http/types.d.ts.map +1 -0
  77. package/build/http/types.js +2 -0
  78. package/build/http/types.js.map +1 -0
  79. package/build/http/types.local.d.ts +21 -0
  80. package/build/http/types.local.d.ts.map +1 -0
  81. package/build/http/types.local.js +2 -0
  82. package/build/http/types.local.js.map +1 -0
  83. package/build/http.d.ts +4 -0
  84. package/build/http.d.ts.map +1 -0
  85. package/build/http.js +248 -0
  86. package/build/http.js.map +1 -0
  87. package/build/index.d.ts +53 -0
  88. package/build/index.d.ts.map +1 -0
  89. package/build/index.js +33 -0
  90. package/build/index.js.map +1 -0
  91. package/build/network/consts.d.ts +12 -0
  92. package/build/network/consts.d.ts.map +1 -0
  93. package/build/network/consts.js +20 -0
  94. package/build/network/consts.js.map +1 -0
  95. package/build/network/types.d.ts +43 -0
  96. package/build/network/types.d.ts.map +1 -0
  97. package/build/network/types.js +2 -0
  98. package/build/network/types.js.map +1 -0
  99. package/build/network.d.ts +4 -0
  100. package/build/network.d.ts.map +1 -0
  101. package/build/network.js +165 -0
  102. package/build/network.js.map +1 -0
  103. package/build/proxy/consts.d.ts +7 -0
  104. package/build/proxy/consts.d.ts.map +1 -0
  105. package/build/proxy/consts.js +9 -0
  106. package/build/proxy/consts.js.map +1 -0
  107. package/build/proxy/endpoint.d.ts +4 -0
  108. package/build/proxy/endpoint.d.ts.map +1 -0
  109. package/build/proxy/endpoint.js +27 -0
  110. package/build/proxy/endpoint.js.map +1 -0
  111. package/build/proxy/http.d.ts +5 -0
  112. package/build/proxy/http.d.ts.map +1 -0
  113. package/build/proxy/http.js +95 -0
  114. package/build/proxy/http.js.map +1 -0
  115. package/build/proxy/router.d.ts +4 -0
  116. package/build/proxy/router.d.ts.map +1 -0
  117. package/build/proxy/router.js +85 -0
  118. package/build/proxy/router.js.map +1 -0
  119. package/build/proxy/socks.d.ts +5 -0
  120. package/build/proxy/socks.d.ts.map +1 -0
  121. package/build/proxy/socks.js +44 -0
  122. package/build/proxy/socks.js.map +1 -0
  123. package/build/proxy/types.d.ts +99 -0
  124. package/build/proxy/types.d.ts.map +1 -0
  125. package/build/proxy/types.js +2 -0
  126. package/build/proxy/types.js.map +1 -0
  127. package/build/resolvers/google-drive/consts.d.ts +10 -0
  128. package/build/resolvers/google-drive/consts.d.ts.map +1 -0
  129. package/build/resolvers/google-drive/consts.js +16 -0
  130. package/build/resolvers/google-drive/consts.js.map +1 -0
  131. package/build/resolvers/google-drive/types.d.ts +21 -0
  132. package/build/resolvers/google-drive/types.d.ts.map +1 -0
  133. package/build/resolvers/google-drive/types.js +2 -0
  134. package/build/resolvers/google-drive/types.js.map +1 -0
  135. package/build/resolvers/google-drive.d.ts +10 -0
  136. package/build/resolvers/google-drive.d.ts.map +1 -0
  137. package/build/resolvers/google-drive.js +130 -0
  138. package/build/resolvers/google-drive.js.map +1 -0
  139. package/build/resolvers/types.d.ts +39 -0
  140. package/build/resolvers/types.d.ts.map +1 -0
  141. package/build/resolvers/types.js +2 -0
  142. package/build/resolvers/types.js.map +1 -0
  143. package/build/s3/consts.d.ts +9 -0
  144. package/build/s3/consts.d.ts.map +1 -0
  145. package/build/s3/consts.js +9 -0
  146. package/build/s3/consts.js.map +1 -0
  147. package/build/s3/prefix.d.ts +4 -0
  148. package/build/s3/prefix.d.ts.map +1 -0
  149. package/build/s3/prefix.js +87 -0
  150. package/build/s3/prefix.js.map +1 -0
  151. package/build/s3/sink.d.ts +6 -0
  152. package/build/s3/sink.d.ts.map +1 -0
  153. package/build/s3/sink.js +30 -0
  154. package/build/s3/sink.js.map +1 -0
  155. package/build/s3/source.d.ts +5 -0
  156. package/build/s3/source.d.ts.map +1 -0
  157. package/build/s3/source.js +81 -0
  158. package/build/s3/source.js.map +1 -0
  159. package/build/s3/types.d.ts +66 -0
  160. package/build/s3/types.d.ts.map +1 -0
  161. package/build/s3/types.js +2 -0
  162. package/build/s3/types.js.map +1 -0
  163. package/build/s3/types.local.d.ts +43 -0
  164. package/build/s3/types.local.d.ts.map +1 -0
  165. package/build/s3/types.local.js +2 -0
  166. package/build/s3/types.local.js.map +1 -0
  167. package/build/s3/upload.d.ts +4 -0
  168. package/build/s3/upload.d.ts.map +1 -0
  169. package/build/s3/upload.js +118 -0
  170. package/build/s3/upload.js.map +1 -0
  171. package/build/s3.d.ts +5 -0
  172. package/build/s3.d.ts.map +1 -0
  173. package/build/s3.js +68 -0
  174. package/build/s3.js.map +1 -0
  175. package/build/service/types.d.ts +66 -0
  176. package/build/service/types.d.ts.map +1 -0
  177. package/build/service/types.js +2 -0
  178. package/build/service/types.js.map +1 -0
  179. package/build/service/types.local.d.ts +11 -0
  180. package/build/service/types.local.d.ts.map +1 -0
  181. package/build/service/types.local.js +2 -0
  182. package/build/service/types.local.js.map +1 -0
  183. package/build/service.d.ts +6 -0
  184. package/build/service.d.ts.map +1 -0
  185. package/build/service.js +73 -0
  186. package/build/service.js.map +1 -0
  187. package/build/sinks/buffer/consts.d.ts +2 -0
  188. package/build/sinks/buffer/consts.d.ts.map +1 -0
  189. package/build/sinks/buffer/consts.js +2 -0
  190. package/build/sinks/buffer/consts.js.map +1 -0
  191. package/build/sinks/buffer/types.d.ts +11 -0
  192. package/build/sinks/buffer/types.d.ts.map +1 -0
  193. package/build/sinks/buffer/types.js +2 -0
  194. package/build/sinks/buffer/types.js.map +1 -0
  195. package/build/sinks/buffer.d.ts +4 -0
  196. package/build/sinks/buffer.d.ts.map +1 -0
  197. package/build/sinks/buffer.js +13 -0
  198. package/build/sinks/buffer.js.map +1 -0
  199. package/build/sinks/directory/types.d.ts +7 -0
  200. package/build/sinks/directory/types.d.ts.map +1 -0
  201. package/build/sinks/directory/types.js +2 -0
  202. package/build/sinks/directory/types.js.map +1 -0
  203. package/build/sinks/directory.d.ts +8 -0
  204. package/build/sinks/directory.d.ts.map +1 -0
  205. package/build/sinks/directory.js +43 -0
  206. package/build/sinks/directory.js.map +1 -0
  207. package/build/sinks/file.d.ts +5 -0
  208. package/build/sinks/file.d.ts.map +1 -0
  209. package/build/sinks/file.js +46 -0
  210. package/build/sinks/file.js.map +1 -0
  211. package/build/sinks/types.d.ts +11 -0
  212. package/build/sinks/types.d.ts.map +1 -0
  213. package/build/sinks/types.js +2 -0
  214. package/build/sinks/types.js.map +1 -0
  215. package/build/sinks/workload/consts.d.ts +5 -0
  216. package/build/sinks/workload/consts.d.ts.map +1 -0
  217. package/build/sinks/workload/consts.js +5 -0
  218. package/build/sinks/workload/consts.js.map +1 -0
  219. package/build/sinks/workload/types.d.ts +17 -0
  220. package/build/sinks/workload/types.d.ts.map +1 -0
  221. package/build/sinks/workload/types.js +2 -0
  222. package/build/sinks/workload/types.js.map +1 -0
  223. package/build/sinks/workload.d.ts +8 -0
  224. package/build/sinks/workload.d.ts.map +1 -0
  225. package/build/sinks/workload.js +51 -0
  226. package/build/sinks/workload.js.map +1 -0
  227. package/build/sources/buffer.d.ts +5 -0
  228. package/build/sources/buffer.d.ts.map +1 -0
  229. package/build/sources/buffer.js +20 -0
  230. package/build/sources/buffer.js.map +1 -0
  231. package/build/sources/file.d.ts +5 -0
  232. package/build/sources/file.d.ts.map +1 -0
  233. package/build/sources/file.js +50 -0
  234. package/build/sources/file.js.map +1 -0
  235. package/build/sources/stream.d.ts +5 -0
  236. package/build/sources/stream.d.ts.map +1 -0
  237. package/build/sources/stream.js +19 -0
  238. package/build/sources/stream.js.map +1 -0
  239. package/build/sources/types.d.ts +10 -0
  240. package/build/sources/types.d.ts.map +1 -0
  241. package/build/sources/types.js +2 -0
  242. package/build/sources/types.js.map +1 -0
  243. package/build/sources/workload/types.d.ts +29 -0
  244. package/build/sources/workload/types.d.ts.map +1 -0
  245. package/build/sources/workload/types.js +2 -0
  246. package/build/sources/workload/types.js.map +1 -0
  247. package/build/sources/workload.d.ts +7 -0
  248. package/build/sources/workload.d.ts.map +1 -0
  249. package/build/sources/workload.js +67 -0
  250. package/build/sources/workload.js.map +1 -0
  251. package/build/transfer/consts.d.ts +5 -0
  252. package/build/transfer/consts.d.ts.map +1 -0
  253. package/build/transfer/consts.js +8 -0
  254. package/build/transfer/consts.js.map +1 -0
  255. package/build/transfer/types.d.ts +49 -0
  256. package/build/transfer/types.d.ts.map +1 -0
  257. package/build/transfer/types.js +2 -0
  258. package/build/transfer/types.js.map +1 -0
  259. package/build/transfer.d.ts +4 -0
  260. package/build/transfer.d.ts.map +1 -0
  261. package/build/transfer.js +75 -0
  262. package/build/transfer.js.map +1 -0
  263. package/build/types.d.ts +99 -0
  264. package/build/types.d.ts.map +1 -0
  265. package/build/types.js +2 -0
  266. package/build/types.js.map +1 -0
  267. package/build/utils/names/consts.d.ts +5 -0
  268. package/build/utils/names/consts.d.ts.map +1 -0
  269. package/build/utils/names/consts.js +5 -0
  270. package/build/utils/names/consts.js.map +1 -0
  271. package/build/utils/names/types.d.ts +13 -0
  272. package/build/utils/names/types.d.ts.map +1 -0
  273. package/build/utils/names/types.js +2 -0
  274. package/build/utils/names/types.js.map +1 -0
  275. package/build/utils/names.d.ts +4 -0
  276. package/build/utils/names.d.ts.map +1 -0
  277. package/build/utils/names.js +60 -0
  278. package/build/utils/names.js.map +1 -0
  279. package/build/utils/paths/consts.d.ts +3 -0
  280. package/build/utils/paths/consts.d.ts.map +1 -0
  281. package/build/utils/paths/consts.js +2 -0
  282. package/build/utils/paths/consts.js.map +1 -0
  283. package/build/utils/paths/types.d.ts +20 -0
  284. package/build/utils/paths/types.d.ts.map +1 -0
  285. package/build/utils/paths/types.js +2 -0
  286. package/build/utils/paths/types.js.map +1 -0
  287. package/build/utils/paths.d.ts +4 -0
  288. package/build/utils/paths.d.ts.map +1 -0
  289. package/build/utils/paths.js +67 -0
  290. package/build/utils/paths.js.map +1 -0
  291. package/build/utils/range/types.d.ts +11 -0
  292. package/build/utils/range/types.d.ts.map +1 -0
  293. package/build/utils/range/types.js +2 -0
  294. package/build/utils/range/types.js.map +1 -0
  295. package/build/utils/range.d.ts +4 -0
  296. package/build/utils/range.d.ts.map +1 -0
  297. package/build/utils/range.js +64 -0
  298. package/build/utils/range.js.map +1 -0
  299. package/build/utils/redact/types.d.ts +16 -0
  300. package/build/utils/redact/types.d.ts.map +1 -0
  301. package/build/utils/redact/types.js +2 -0
  302. package/build/utils/redact/types.js.map +1 -0
  303. package/build/utils/redact.d.ts +4 -0
  304. package/build/utils/redact.d.ts.map +1 -0
  305. package/build/utils/redact.js +18 -0
  306. package/build/utils/redact.js.map +1 -0
  307. package/build/utils/signals/types.d.ts +17 -0
  308. package/build/utils/signals/types.d.ts.map +1 -0
  309. package/build/utils/signals/types.js +2 -0
  310. package/build/utils/signals/types.js.map +1 -0
  311. package/build/utils/signals.d.ts +4 -0
  312. package/build/utils/signals.d.ts.map +1 -0
  313. package/build/utils/signals.js +52 -0
  314. package/build/utils/signals.js.map +1 -0
  315. package/build/utils/streams/consts.d.ts +3 -0
  316. package/build/utils/streams/consts.d.ts.map +1 -0
  317. package/build/utils/streams/consts.js +3 -0
  318. package/build/utils/streams/consts.js.map +1 -0
  319. package/build/utils/streams/types.d.ts +59 -0
  320. package/build/utils/streams/types.d.ts.map +1 -0
  321. package/build/utils/streams/types.js +2 -0
  322. package/build/utils/streams/types.js.map +1 -0
  323. package/build/utils/streams.d.ts +4 -0
  324. package/build/utils/streams.d.ts.map +1 -0
  325. package/build/utils/streams.js +167 -0
  326. package/build/utils/streams.js.map +1 -0
  327. package/build/utils/tree/types.d.ts +6 -0
  328. package/build/utils/tree/types.d.ts.map +1 -0
  329. package/build/utils/tree/types.js +2 -0
  330. package/build/utils/tree/types.js.map +1 -0
  331. package/build/utils/tree.d.ts +4 -0
  332. package/build/utils/tree.d.ts.map +1 -0
  333. package/build/utils/tree.js +14 -0
  334. package/build/utils/tree.js.map +1 -0
  335. package/build/workspace/consts.d.ts +12 -0
  336. package/build/workspace/consts.d.ts.map +1 -0
  337. package/build/workspace/consts.js +12 -0
  338. package/build/workspace/consts.js.map +1 -0
  339. package/build/workspace/consts.local.d.ts +3 -0
  340. package/build/workspace/consts.local.d.ts.map +1 -0
  341. package/build/workspace/consts.local.js +3 -0
  342. package/build/workspace/consts.local.js.map +1 -0
  343. package/build/workspace/fs.d.ts +4 -0
  344. package/build/workspace/fs.d.ts.map +1 -0
  345. package/build/workspace/fs.js +45 -0
  346. package/build/workspace/fs.js.map +1 -0
  347. package/build/workspace/scope.d.ts +6 -0
  348. package/build/workspace/scope.d.ts.map +1 -0
  349. package/build/workspace/scope.js +180 -0
  350. package/build/workspace/scope.js.map +1 -0
  351. package/build/workspace/types.d.ts +66 -0
  352. package/build/workspace/types.d.ts.map +1 -0
  353. package/build/workspace/types.js +2 -0
  354. package/build/workspace/types.js.map +1 -0
  355. package/build/workspace/types.local.d.ts +20 -0
  356. package/build/workspace/types.local.d.ts.map +1 -0
  357. package/build/workspace/types.local.js +2 -0
  358. package/build/workspace/types.local.js.map +1 -0
  359. package/build/workspace.d.ts +7 -0
  360. package/build/workspace.d.ts.map +1 -0
  361. package/build/workspace.js +87 -0
  362. package/build/workspace.js.map +1 -0
  363. package/package.json +61 -4
  364. package/src/archive/consts.ts +28 -0
  365. package/src/archive/formats.ts +60 -0
  366. package/src/archive/guard.ts +68 -0
  367. package/src/archive/ratio.ts +35 -0
  368. package/src/archive/tar.ts +147 -0
  369. package/src/archive/types.local.ts +72 -0
  370. package/src/archive/types.ts +107 -0
  371. package/src/archive/zip.ts +142 -0
  372. package/src/archive.ts +298 -0
  373. package/src/backed-workspace/types.ts +72 -0
  374. package/src/backed-workspace.ts +85 -0
  375. package/src/consts.local.ts +2 -0
  376. package/src/consts.ts +2 -0
  377. package/src/errors.ts +128 -0
  378. package/src/http/connector.ts +136 -0
  379. package/src/http/consts.ts +13 -0
  380. package/src/http/source.ts +89 -0
  381. package/src/http/types.local.ts +23 -0
  382. package/src/http/types.ts +74 -0
  383. package/src/http.ts +276 -0
  384. package/src/index.ts +65 -0
  385. package/src/network/consts.ts +22 -0
  386. package/src/network/types.ts +45 -0
  387. package/src/network.ts +179 -0
  388. package/src/proxy/consts.ts +11 -0
  389. package/src/proxy/endpoint.ts +28 -0
  390. package/src/proxy/http.ts +108 -0
  391. package/src/proxy/router.ts +93 -0
  392. package/src/proxy/socks.ts +48 -0
  393. package/src/proxy/types.ts +114 -0
  394. package/src/resolvers/google-drive/consts.ts +22 -0
  395. package/src/resolvers/google-drive/types.ts +22 -0
  396. package/src/resolvers/google-drive.ts +153 -0
  397. package/src/resolvers/types.ts +45 -0
  398. package/src/s3/consts.ts +12 -0
  399. package/src/s3/prefix.ts +100 -0
  400. package/src/s3/sink.ts +36 -0
  401. package/src/s3/source.ts +90 -0
  402. package/src/s3/types.local.ts +49 -0
  403. package/src/s3/types.ts +70 -0
  404. package/src/s3/upload.ts +130 -0
  405. package/src/s3.ts +83 -0
  406. package/src/service/types.local.ts +9 -0
  407. package/src/service/types.ts +71 -0
  408. package/src/service.ts +97 -0
  409. package/src/sinks/buffer/consts.ts +1 -0
  410. package/src/sinks/buffer/types.ts +12 -0
  411. package/src/sinks/buffer.ts +18 -0
  412. package/src/sinks/directory/types.ts +7 -0
  413. package/src/sinks/directory.ts +49 -0
  414. package/src/sinks/file.ts +52 -0
  415. package/src/sinks/types.ts +12 -0
  416. package/src/sinks/workload/consts.ts +5 -0
  417. package/src/sinks/workload/types.ts +17 -0
  418. package/src/sinks/workload.ts +59 -0
  419. package/src/sources/buffer.ts +27 -0
  420. package/src/sources/file.ts +57 -0
  421. package/src/sources/stream.ts +26 -0
  422. package/src/sources/types.ts +12 -0
  423. package/src/sources/workload/types.ts +31 -0
  424. package/src/sources/workload.ts +79 -0
  425. package/src/transfer/consts.ts +10 -0
  426. package/src/transfer/types.ts +55 -0
  427. package/src/transfer.ts +89 -0
  428. package/src/types.ts +106 -0
  429. package/src/utils/names/consts.ts +5 -0
  430. package/src/utils/names/types.ts +12 -0
  431. package/src/utils/names.ts +70 -0
  432. package/src/utils/paths/consts.ts +3 -0
  433. package/src/utils/paths/types.ts +20 -0
  434. package/src/utils/paths.ts +75 -0
  435. package/src/utils/range/types.ts +11 -0
  436. package/src/utils/range.ts +80 -0
  437. package/src/utils/redact/types.ts +16 -0
  438. package/src/utils/redact.ts +23 -0
  439. package/src/utils/signals/types.ts +17 -0
  440. package/src/utils/signals.ts +63 -0
  441. package/src/utils/streams/consts.ts +2 -0
  442. package/src/utils/streams/types.ts +65 -0
  443. package/src/utils/streams.ts +188 -0
  444. package/src/utils/tree/types.ts +6 -0
  445. package/src/utils/tree.ts +18 -0
  446. package/src/workspace/consts.local.ts +2 -0
  447. package/src/workspace/consts.ts +14 -0
  448. package/src/workspace/fs.ts +48 -0
  449. package/src/workspace/scope.ts +196 -0
  450. package/src/workspace/types.local.ts +21 -0
  451. package/src/workspace/types.ts +71 -0
  452. package/src/workspace.ts +102 -0
  453. package/tests/archive.spec.ts +134 -0
  454. package/tests/backed.spec.ts +147 -0
  455. package/tests/context.ts +114 -0
  456. package/tests/fixtures.ts +190 -0
  457. package/tests/google-drive.spec.ts +76 -0
  458. package/tests/harness/drive.ts +92 -0
  459. package/tests/harness/http-proxy.ts +96 -0
  460. package/tests/harness/origin.ts +134 -0
  461. package/tests/harness/socks5.ts +135 -0
  462. package/tests/harness/workload.ts +118 -0
  463. package/tests/http.spec.ts +101 -0
  464. package/tests/inspector.spec.ts +40 -0
  465. package/tests/live.spec.ts +27 -0
  466. package/tests/network.spec.ts +79 -0
  467. package/tests/proxy.spec.ts +149 -0
  468. package/tests/s3.spec.ts +133 -0
  469. package/tests/service.spec.ts +77 -0
  470. package/tests/sources-sinks.spec.ts +96 -0
  471. package/tests/transfer.spec.ts +99 -0
  472. package/tests/tsconfig.json +12 -0
  473. package/tests/workload.spec.ts +140 -0
  474. package/tests/workspace.spec.ts +158 -0
  475. package/tsconfig.json +16 -0
@@ -0,0 +1,136 @@
1
+ import { connect as connectTcp, isIP } from 'node:net'
2
+ import type { Socket } from 'node:net'
3
+ import type { Duplex } from 'node:stream'
4
+ import { connect as connectTls } from 'node:tls'
5
+ import { logger } from '@owlmeans/log'
6
+ import { HttpError, NetworkPolicyError, ProxyError } from '../errors.js'
7
+ import type { NetworkPolicy, ResolvedAddress } from '../network/types.js'
8
+ import type { ProxyRouter } from '../proxy/types.js'
9
+ import { redactUtils } from '../utils/redact.js'
10
+ import { signalUtils } from '../utils/signals.js'
11
+ import type { HttpTimeouts } from './types.js'
12
+ import type { Connection, HttpConnector } from './types.local.js'
13
+
14
+ const log = logger('file-streams:http')
15
+
16
+ /**
17
+ * Connections the policy approved: a direct socket goes to an address that was resolved and
18
+ * checked a moment before (no second lookup to rebind), a proxied one carries that address into
19
+ * the tunnel; TLS is layered on top with the host name for SNI and certificate checks.
20
+ */
21
+ export const makeHttpConnector = (
22
+ policy: NetworkPolicy, router: ProxyRouter, timeouts: HttpTimeouts, ca?: string | string[]
23
+ ): HttpConnector => {
24
+ const tcp = (address: ResolvedAddress, port: number, signal?: AbortSignal): Promise<Socket> => new Promise((resolve, reject) => {
25
+ const socket = connectTcp({ host: address.address, port, family: address.family })
26
+ const timer = setTimeout(() => fail(new HttpError('connect-timeout')), timeouts.connectMs)
27
+ const release = signalUtils.onAbort(signal, reason => fail(reason as Error))
28
+ function fail (error: Error): void {
29
+ clearTimeout(timer)
30
+ release()
31
+ socket.destroy()
32
+ reject(error)
33
+ }
34
+ socket.once('connect', () => {
35
+ clearTimeout(timer)
36
+ release()
37
+ socket.removeAllListeners('error')
38
+ resolve(socket)
39
+ })
40
+ socket.once('error', () => fail(new HttpError('connect')))
41
+ })
42
+
43
+ const direct = async (addresses: ResolvedAddress[], port: number, signal?: AbortSignal): Promise<Socket> => {
44
+ let last: unknown
45
+ for (const address of addresses) {
46
+ try {
47
+ return await tcp(address, port, signal)
48
+ } catch (error) {
49
+ if (signal?.aborted === true) {
50
+ throw error
51
+ }
52
+ last = error
53
+ }
54
+ }
55
+ throw last ?? new HttpError('connect')
56
+ }
57
+
58
+ const tls = (socket: Duplex, hostname: string, signal?: AbortSignal): Promise<Duplex> => new Promise((resolve, reject) => {
59
+ const literal = isIP(hostname) !== 0
60
+ const secure = connectTls({
61
+ socket, host: hostname, servername: literal ? undefined : hostname, ALPNProtocols: ['http/1.1'], minVersion: 'TLSv1.2', ca,
62
+ })
63
+ const timer = setTimeout(() => fail(new HttpError('tls-timeout')), timeouts.connectMs)
64
+ const release = signalUtils.onAbort(signal, reason => fail(reason as Error))
65
+ function fail (error: Error): void {
66
+ clearTimeout(timer)
67
+ release()
68
+ secure.destroy()
69
+ socket.destroy()
70
+ reject(error)
71
+ }
72
+ secure.once('secureConnect', () => {
73
+ clearTimeout(timer)
74
+ release()
75
+ secure.removeAllListeners('error')
76
+ resolve(secure)
77
+ })
78
+ secure.once('error', error => fail(error instanceof HttpError ? error : new HttpError(`tls:${(error as NodeJS.ErrnoException).code ?? 'failed'}`)))
79
+ })
80
+
81
+ const proxied = async (url: URL, hostname: string, port: number, secure: boolean, proxyKey: string | undefined, signal?: AbortSignal): Promise<Connection | null> => {
82
+ let attempts = 0
83
+ for (;;) {
84
+ const selection = await router.select({ key: proxyKey, url })
85
+ if (selection == null) {
86
+ return null
87
+ }
88
+ const { endpoint, plugin } = selection
89
+ let host = hostname
90
+ if (plugin.remoteDns(endpoint)) {
91
+ try {
92
+ await policy.resolve(hostname, signal)
93
+ } catch (error) {
94
+ if (!(error instanceof NetworkPolicyError) || error.message !== 'file-streams:policy:unresolvable' || !policy.trustProxyDns) {
95
+ throw error
96
+ }
97
+ }
98
+ } else {
99
+ host = (await policy.resolve(hostname, signal))[0].address
100
+ }
101
+ try {
102
+ const connection = await plugin.connect(endpoint, { host, port, hostname, secure }, { signal, timeoutMs: timeouts.connectMs })
103
+ router.report(endpoint, true)
104
+ log.debug('Connected through a proxy', { proxy: redactUtils.proxy(endpoint), mode: connection.mode })
105
+ return { socket: connection.socket, mode: connection.mode, headers: connection.headers }
106
+ } catch (error) {
107
+ if (signal?.aborted === true) {
108
+ throw error
109
+ }
110
+ router.report(endpoint, false)
111
+ if (++attempts > router.retries) {
112
+ throw error instanceof ProxyError ? error : new ProxyError('connect')
113
+ }
114
+ }
115
+ }
116
+ }
117
+
118
+ const connect = async (url: URL, proxyKey: string | undefined, signal?: AbortSignal): Promise<Connection> => {
119
+ const secure = url.protocol === 'https:'
120
+ const hostname = url.hostname.replace(/^\[|\]$/g, '')
121
+ const port = url.port !== '' ? Number(url.port) : secure ? 443 : 80
122
+
123
+ let connection = await proxied(url, hostname, port, secure, proxyKey, signal)
124
+ if (connection == null) {
125
+ const addresses = await policy.resolve(hostname, signal)
126
+ connection = { socket: await direct(addresses, port, signal), mode: 'direct' }
127
+ }
128
+ if (secure && connection.mode !== 'forward') {
129
+ connection = { ...connection, socket: await tls(connection.socket, hostname, signal) }
130
+ }
131
+
132
+ return connection
133
+ }
134
+
135
+ return { connect }
136
+ }
@@ -0,0 +1,13 @@
1
+ import type { HttpTimeouts } from './types.js'
2
+
3
+ export const DEFAULT_HTTP_TIMEOUTS: Readonly<HttpTimeouts> = Object.freeze({ connectMs: 10_000, headersMs: 30_000, idleMs: 30_000 })
4
+
5
+ export const DEFAULT_MAX_REDIRECTS = 5
6
+
7
+ export const REDIRECT_STATUSES: readonly number[] = Object.freeze([301, 302, 303, 307, 308])
8
+
9
+ /** Headers never carried to another origin. */
10
+ export const ALWAYS_SENSITIVE_HEADERS: readonly string[] = Object.freeze(['authorization', 'cookie', 'proxy-authorization'])
11
+
12
+ /** Bytes a resolver may read from a page by default. */
13
+ export const DEFAULT_RESOLVER_PAGE_BYTES = 256 * 1024
@@ -0,0 +1,89 @@
1
+ import { SourceError } from '../errors.js'
2
+ import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader, SignalOptions } from '../types.js'
3
+ import { nameUtils } from '../utils/names.js'
4
+ import { streamUtils } from '../utils/streams.js'
5
+ import type { HttpClient, HttpRequestOptions, HttpSourceOptions } from './types.js'
6
+ import type { KnownHttpFile } from './types.local.js'
7
+
8
+ /** A URL as a source: re-openable; Range-capable when the server answers ranges. */
9
+ export const makeHttpSource = (client: HttpClient, url: URL, options: HttpSourceOptions = {}): FileSource => {
10
+ let known: Promise<KnownHttpFile> | undefined
11
+
12
+ const base = (signal?: AbortSignal): HttpRequestOptions => ({ headers: options.headers, proxyKey: options.proxyKey, raw: options.raw, signal })
13
+
14
+ const named = (info: FileInfo): FileInfo => ({ ...info, name: nameUtils.sanitize(options.name) ?? info.name })
15
+
16
+ const discover = async (signal?: AbortSignal): Promise<KnownHttpFile> => {
17
+ const response = await client.request(url, { ...base(signal), range: { start: 0, end: 0 } })
18
+ streamUtils.discard(response.body)
19
+ const etag = response.info.etag
20
+ return {
21
+ info: named(response.info),
22
+ rangeable: response.status === 206 && response.info.size != null,
23
+ final: response.url,
24
+ validator: etag != null && !etag.startsWith('W/') ? etag : undefined,
25
+ }
26
+ }
27
+
28
+ const learn = async (signal?: AbortSignal): Promise<KnownHttpFile> => {
29
+ known ??= discover(signal)
30
+ try {
31
+ return await known
32
+ } catch (error) {
33
+ known = undefined
34
+ throw error
35
+ }
36
+ }
37
+
38
+ const info = async (infoOptions: SignalOptions = {}): Promise<FileInfo> => (await learn(infoOptions.signal)).info
39
+
40
+ const open = async (openOptions: OpenOptions = {}): Promise<OpenedFile> => {
41
+ const response = await client.request(url, { ...base(openOptions.signal), range: openOptions.range })
42
+ if (openOptions.range != null && response.status !== 206) {
43
+ streamUtils.discard(response.body)
44
+ throw new SourceError('range-unsupported')
45
+ }
46
+ const opened: OpenedFile = { stream: response.body, info: named(response.info) }
47
+ const span = /bytes (\d+)-(\d+)\/(\d+)/.exec(String(response.headers['content-range'] ?? ''))
48
+ if (span != null) {
49
+ opened.range = { start: Number(span[1]), end: Number(span[2]), total: Number(span[3]) }
50
+ }
51
+
52
+ return opened
53
+ }
54
+
55
+ const ranges = async (rangeOptions: SignalOptions = {}): Promise<RangeReader | null> => {
56
+ const current = await learn(rangeOptions.signal)
57
+ if (!current.rangeable || current.info.size == null) {
58
+ return null
59
+ }
60
+ const size = current.info.size
61
+
62
+ const fetchRange = async (start: number, end: number, signal?: AbortSignal) => {
63
+ const response = await client.request(current.final, {
64
+ ...base(signal), raw: true, range: { start, end }, ifRange: current.validator,
65
+ })
66
+ if (response.status !== 206) {
67
+ streamUtils.discard(response.body)
68
+ throw new SourceError('changed')
69
+ }
70
+ return response
71
+ }
72
+
73
+ return {
74
+ size,
75
+ read: async (offset, length, signal) => {
76
+ const end = Math.min(offset + length, size) - 1
77
+ if (end < offset) {
78
+ return new Uint8Array(0)
79
+ }
80
+ const response = await fetchRange(offset, end, signal)
81
+ return await streamUtils.collect(response.body, end - offset + 1)
82
+ },
83
+ stream: async (start, end, signal) => (await fetchRange(start, end - 1, signal)).body,
84
+ close: async () => undefined,
85
+ }
86
+ }
87
+
88
+ return { kind: 'http', info, open, ranges }
89
+ }
@@ -0,0 +1,23 @@
1
+ import type { Duplex } from 'node:stream'
2
+ import type { FileInfo } from '../types.js'
3
+
4
+ export interface Connection {
5
+ socket: Duplex
6
+ /** `direct` and `tunnel` carry origin-form requests; `forward` absolute-form ones. */
7
+ mode: 'direct' | 'tunnel' | 'forward'
8
+ headers?: Record<string, string>
9
+ }
10
+
11
+ export interface HttpConnector {
12
+ /** A connected (and, for https, TLS-wrapped) socket to the URL's origin. */
13
+ connect: (url: URL, proxyKey: string | undefined, signal?: AbortSignal) => Promise<Connection>
14
+ }
15
+
16
+ /** What a source learned from its first response. */
17
+ export interface KnownHttpFile {
18
+ info: FileInfo
19
+ rangeable: boolean
20
+ /** The URL that served the bytes, after redirects and resolvers. */
21
+ final: URL
22
+ validator?: string
23
+ }
@@ -0,0 +1,74 @@
1
+ import type { Readable } from 'node:stream'
2
+ import type { NetworkPolicy, NetworkPolicyOptions } from '../network/types.js'
3
+ import type { ProxyRouter, ProxyRouterOptions } from '../proxy/types.js'
4
+ import type { UrlResolver } from '../resolvers/types.js'
5
+ import type { ByteRange, FileInfo, FileSource, SignalOptions } from '../types.js'
6
+
7
+ export interface HttpTimeouts {
8
+ /** TCP (and proxy handshake) connect. Default 10 000 ms. */
9
+ connectMs: number
10
+ /** From the request to the response head. Default 30 000 ms. */
11
+ headersMs: number
12
+ /** Longest silence while the body flows. Default 30 000 ms. */
13
+ idleMs: number
14
+ }
15
+
16
+ export interface HttpClientOptions {
17
+ network?: NetworkPolicy | NetworkPolicyOptions
18
+ proxies?: ProxyRouter | ProxyRouterOptions
19
+ /** Default `[googleDriveResolver]`; `[]` turns resolving off. */
20
+ resolvers?: UrlResolver[]
21
+ timeouts?: Partial<HttpTimeouts>
22
+ /** Default 5. */
23
+ maxRedirects?: number
24
+ userAgent?: string
25
+ /** Sent with every request. */
26
+ headers?: Record<string, string>
27
+ /** Dropped on a cross-origin redirect, beside `authorization`, `cookie` and `proxy-authorization`. */
28
+ sensitiveHeaders?: string[]
29
+ /** Extra CAs (verification is never turned off). */
30
+ ca?: string | string[]
31
+ }
32
+
33
+ /** The two methods a download issues. */
34
+ // A closed pair of HTTP verbs — a union.
35
+ export type HttpMethod = 'GET' | 'HEAD'
36
+
37
+ export interface HttpRequestOptions extends SignalOptions {
38
+ method?: HttpMethod
39
+ headers?: Record<string, string>
40
+ range?: ByteRange
41
+ /** Routing key for `by-key` pools and proxy providers. */
42
+ proxyKey?: string
43
+ /** Skip the URL resolvers. */
44
+ raw?: boolean
45
+ /** Send `If-Range` with this strong validator. */
46
+ ifRange?: string
47
+ }
48
+
49
+ export interface HttpResponse {
50
+ /** The final URL; redact before logging. */
51
+ url: URL
52
+ status: number
53
+ headers: Record<string, string | string[] | undefined>
54
+ /** The body, transport-decoded (never a gzip file itself), idle-guarded, length-checked. */
55
+ body: Readable
56
+ info: FileInfo
57
+ redirects: number
58
+ }
59
+
60
+ export interface HttpSourceOptions {
61
+ headers?: Record<string, string>
62
+ proxyKey?: string
63
+ raw?: boolean
64
+ /** The file name to claim, over what the response says. */
65
+ name?: string
66
+ }
67
+
68
+ /** Downloads under the network policy, through proxies, redirects and resolvers. */
69
+ export interface HttpClient {
70
+ /** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')`. */
71
+ request: (url: string | URL, options?: HttpRequestOptions) => Promise<HttpResponse>
72
+ /** A re-openable source; Range-capable when the server answers 206 with Content-Range. */
73
+ source: (url: string | URL, options?: HttpSourceOptions) => FileSource
74
+ }
package/src/http.ts ADDED
@@ -0,0 +1,276 @@
1
+ import { request as httpRequest } from 'node:http'
2
+ import type { IncomingMessage, RequestOptions } from 'node:http'
3
+ import { request as httpsRequest } from 'node:https'
4
+ import type { Socket } from 'node:net'
5
+ import { pipeline, Transform } from 'node:stream'
6
+ import type { Readable } from 'node:stream'
7
+ import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'
8
+ import { logger } from '@owlmeans/log'
9
+ import { NON_INFORMATIVE_MIMES } from '@owlmeans/file-inspector'
10
+ import { HttpError, ProxyError, ResolverError } from './errors.js'
11
+ import { makeHttpConnector } from './http/connector.js'
12
+ import { ALWAYS_SENSITIVE_HEADERS, DEFAULT_HTTP_TIMEOUTS, DEFAULT_MAX_REDIRECTS, DEFAULT_RESOLVER_PAGE_BYTES, REDIRECT_STATUSES } from './http/consts.js'
13
+ import { makeHttpSource } from './http/source.js'
14
+ import type { HttpClient, HttpClientOptions, HttpRequestOptions, HttpResponse } from './http/types.js'
15
+ import type { Connection } from './http/types.local.js'
16
+ import { createNetworkPolicy } from './network.js'
17
+ import type { NetworkPolicy } from './network/types.js'
18
+ import { createProxyRouter } from './proxy/router.js'
19
+ import type { ProxyRouter } from './proxy/types.js'
20
+ import { googleDriveResolver } from './resolvers/google-drive.js'
21
+ import type { UrlResolver } from './resolvers/types.js'
22
+ import type { FileInfo } from './types.js'
23
+ import { nameUtils } from './utils/names.js'
24
+ import { redactUtils } from './utils/redact.js'
25
+ import { signalUtils } from './utils/signals.js'
26
+ import { streamUtils } from './utils/streams.js'
27
+ import type { PeekedStream } from './utils/streams/types.js'
28
+ import { PACKAGE_USER_AGENT } from './consts.local.js'
29
+
30
+ const log = logger('file-streams:http')
31
+
32
+ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient => {
33
+ const policy: NetworkPolicy = options.network != null && 'assertUrl' in options.network
34
+ ? options.network
35
+ : createNetworkPolicy(options.network)
36
+ const router: ProxyRouter = options.proxies != null && 'select' in options.proxies && typeof options.proxies.select === 'function'
37
+ ? options.proxies as ProxyRouter
38
+ : createProxyRouter(options.proxies as Parameters<typeof createProxyRouter>[0])
39
+ const resolvers: UrlResolver[] = options.resolvers ?? [googleDriveResolver]
40
+ const timeouts = { ...DEFAULT_HTTP_TIMEOUTS, ...options.timeouts }
41
+ const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS
42
+ const sensitive = [...ALWAYS_SENSITIVE_HEADERS, ...(options.sensitiveHeaders ?? []).map(name => name.toLowerCase())]
43
+ const connector = makeHttpConnector(policy, router, timeouts, options.ca)
44
+
45
+ const header = (headers: IncomingMessage['headers'], name: string): string | undefined => {
46
+ const value = headers[name]
47
+ return Array.isArray(value) ? value[0] : value
48
+ }
49
+
50
+ const mimeOf = (value: string | undefined): string | undefined => {
51
+ const mime = value?.split(';')[0].trim().toLowerCase()
52
+ return mime == null || mime === '' || NON_INFORMATIVE_MIMES.includes(mime) ? undefined : mime
53
+ }
54
+
55
+ const isGzipFile = (url: URL, response: IncomingMessage): boolean => {
56
+ const mime = mimeOf(header(response.headers, 'content-type'))
57
+ return mime === 'application/gzip' || mime === 'application/x-gzip' || /\.(t?gz)$/i.test(url.pathname)
58
+ }
59
+
60
+ const infoOf = (url: URL, response: IncomingMessage, decoded: boolean): FileInfo => {
61
+ const status = response.statusCode ?? 0
62
+ let size: number | undefined
63
+ if (status === 206) {
64
+ const total = /\/(\d+)\s*$/.exec(header(response.headers, 'content-range') ?? '')?.[1]
65
+ size = total != null ? Number(total) : undefined
66
+ } else if (!decoded) {
67
+ const length = header(response.headers, 'content-length')
68
+ size = length != null && /^\d+$/.test(length) ? Number(length) : undefined
69
+ }
70
+ const modified = header(response.headers, 'last-modified')
71
+ const info: FileInfo = {
72
+ origin: redactUtils.url(url),
73
+ name: nameUtils.fromDisposition(header(response.headers, 'content-disposition')) ?? nameUtils.fromUrl(url),
74
+ size,
75
+ mime: mimeOf(header(response.headers, 'content-type')),
76
+ etag: header(response.headers, 'etag'),
77
+ }
78
+ if (modified != null && !Number.isNaN(Date.parse(modified))) {
79
+ info.modifiedAt = new Date(modified)
80
+ }
81
+
82
+ return info
83
+ }
84
+
85
+ const send = (url: URL, method: string, headers: Record<string, string>, connection: Connection, signal?: AbortSignal): Promise<IncomingMessage> =>
86
+ new Promise((resolve, reject) => {
87
+ const secure = url.protocol === 'https:'
88
+ const forward = connection.mode === 'forward'
89
+ const requestOptions: RequestOptions = {
90
+ method,
91
+ host: url.hostname.replace(/^\[|\]$/g, ''),
92
+ port: url.port !== '' ? Number(url.port) : secure ? 443 : 80,
93
+ path: forward ? url.href : `${url.pathname}${url.search}`,
94
+ headers: { ...headers, ...connection.headers },
95
+ createConnection: (() => connection.socket as Socket) as unknown as RequestOptions['createConnection'],
96
+ }
97
+ const request = (secure ? httpsRequest : httpRequest)(requestOptions)
98
+ const timer = setTimeout(() => request.destroy(new HttpError('headers-timeout')), timeouts.headersMs)
99
+ const release = signalUtils.onAbort(signal, reason => request.destroy(reason as Error))
100
+ request.once('response', response => {
101
+ clearTimeout(timer)
102
+ release()
103
+ resolve(response)
104
+ })
105
+ request.once('error', error => {
106
+ clearTimeout(timer)
107
+ release()
108
+ connection.socket.destroy()
109
+ reject(error instanceof HttpError || signal?.aborted === true ? error : new HttpError('request'))
110
+ })
111
+ request.end()
112
+ })
113
+
114
+ const bodyOf = (url: URL, response: IncomingMessage): { body: Readable, decoded: boolean } => {
115
+ const encoding = header(response.headers, 'content-encoding')?.trim().toLowerCase()
116
+ const stages: NodeJS.ReadWriteStream[] = []
117
+ let decoded = false
118
+ if (encoding != null && encoding !== '' && encoding !== 'identity' && !isGzipFile(url, response)) {
119
+ decoded = true
120
+ if (encoding === 'gzip' || encoding === 'x-gzip') {
121
+ stages.push(createGunzip())
122
+ } else if (encoding === 'deflate') {
123
+ stages.push(createInflate())
124
+ } else if (encoding === 'br') {
125
+ stages.push(createBrotliDecompress())
126
+ } else {
127
+ response.destroy()
128
+ throw new HttpError('encoding')
129
+ }
130
+ } else {
131
+ const length = header(response.headers, 'content-length')
132
+ if (length != null && /^\d+$/.test(length)) {
133
+ const expected = Number(length)
134
+ let seen = 0
135
+ stages.push(new Transform({
136
+ transform: (chunk: Buffer, _encoding, callback) => {
137
+ seen += chunk.length
138
+ callback(null, chunk)
139
+ },
140
+ flush: callback => callback(seen < expected ? new HttpError('truncated') : null),
141
+ }))
142
+ }
143
+ }
144
+ stages.push(streamUtils.meter({ idleTimeoutMs: timeouts.idleMs }).stream)
145
+ response.once('aborted', () => response.destroy(new HttpError('truncated')))
146
+ const body = (pipeline as unknown as (...streams: unknown[]) => Readable)(response, ...stages, () => undefined)
147
+
148
+ return { body, decoded }
149
+ }
150
+
151
+ const request = async (input: string | URL, requestOptions: HttpRequestOptions = {}): Promise<HttpResponse> => {
152
+ const signal = requestOptions.signal
153
+ const method = requestOptions.method ?? 'GET'
154
+ let current = new URL(input)
155
+ policy.assertUrl(current)
156
+
157
+ const resolver = requestOptions.raw === true ? undefined : resolvers.find(candidate => candidate.matches(current))
158
+ if (resolver != null && requestOptions.range != null) {
159
+ // A resolver reads pages whole: resolve first, then ask the bytes' own URL for the range.
160
+ const resolved = await request(current, { ...requestOptions, range: undefined, ifRange: undefined })
161
+ streamUtils.discard(resolved.body)
162
+ return await request(resolved.url, { ...requestOptions, raw: true })
163
+ }
164
+ if (resolver != null) {
165
+ current = resolver.rewrite(current)
166
+ policy.assertUrl(current)
167
+ }
168
+
169
+ const headers: Record<string, string> = {
170
+ 'user-agent': options.userAgent ?? PACKAGE_USER_AGENT,
171
+ accept: '*/*',
172
+ 'accept-encoding': 'identity',
173
+ connection: 'close',
174
+ ...lowerKeys(options.headers),
175
+ ...lowerKeys(requestOptions.headers),
176
+ }
177
+ if (requestOptions.range != null) {
178
+ headers.range = `bytes=${requestOptions.range.start}-${requestOptions.range.end ?? ''}`
179
+ if (requestOptions.ifRange != null) {
180
+ headers['if-range'] = requestOptions.ifRange
181
+ }
182
+ }
183
+
184
+ let redirects = 0
185
+ let steps = 0
186
+ let cookie: string | undefined
187
+ for (;;) {
188
+ signalUtils.throwIfAborted(signal)
189
+ const connection = await connector.connect(current, requestOptions.proxyKey, signal)
190
+ const outgoing: Record<string, string> = { ...headers, host: current.host }
191
+ if (cookie != null && resolver?.owns(current) === true) {
192
+ outgoing.cookie = cookie
193
+ }
194
+ const response = await send(current, method, outgoing, connection, signal)
195
+ const status = response.statusCode ?? 0
196
+ const location = header(response.headers, 'location')
197
+
198
+ if (REDIRECT_STATUSES.includes(status) && location != null) {
199
+ response.resume()
200
+ response.destroy()
201
+ if (++redirects > maxRedirects) {
202
+ throw new HttpError('redirects')
203
+ }
204
+ let next: URL
205
+ try {
206
+ next = new URL(location, current)
207
+ } catch {
208
+ throw new HttpError('location')
209
+ }
210
+ policy.assertUrl(next, current)
211
+ if (next.origin !== current.origin) {
212
+ sensitive.forEach(name => { delete headers[name] })
213
+ }
214
+ current = next
215
+ continue
216
+ }
217
+ if (status < 200 || status >= 300) {
218
+ response.resume()
219
+ response.destroy()
220
+ if (status === 407 && connection.mode === 'forward') {
221
+ throw new ProxyError('auth-failed')
222
+ }
223
+ throw new HttpError(`status:${status}`)
224
+ }
225
+
226
+ let { body, decoded } = method === 'HEAD'
227
+ ? { body: response, decoded: false }
228
+ : bodyOf(current, response)
229
+
230
+ if (resolver != null && method === 'GET') {
231
+ let peeked: PeekedStream | undefined
232
+ const readBody = async (maxBytes: number = DEFAULT_RESOLVER_PAGE_BYTES): Promise<string> => {
233
+ peeked ??= await streamUtils.peek(body, maxBytes)
234
+ return Buffer.from(peeked.head.subarray(0, maxBytes)).toString('utf8')
235
+ }
236
+ const step = await resolver.inspect({ url: current, status, headers: response.headers }, readBody)
237
+ if (peeked != null) {
238
+ body = peeked.stream
239
+ }
240
+ if (step.action === 'follow') {
241
+ streamUtils.discard(body)
242
+ if (++steps > resolver.maxSteps) {
243
+ throw new ResolverError(`${resolver.name}:steps`)
244
+ }
245
+ policy.assertUrl(step.url, current)
246
+ if (!resolver.owns(step.url)) {
247
+ throw new ResolverError(`${resolver.name}:foreign-follow`)
248
+ }
249
+ cookie = step.headers?.cookie ?? cookie
250
+ current = step.url
251
+ continue
252
+ }
253
+ if (step.action === 'fail') {
254
+ streamUtils.discard(body)
255
+ throw new ResolverError(`${resolver.name}:${step.reason}`)
256
+ }
257
+ }
258
+
259
+ log.debug('Response', { url: redactUtils.url(current), status, redirects })
260
+
261
+ return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects }
262
+ }
263
+ }
264
+
265
+ const lowerKeys = (headers: Record<string, string> | undefined): Record<string, string> =>
266
+ Object.fromEntries(Object.entries(headers ?? {}).map(([name, value]) => [name.toLowerCase(), value]))
267
+
268
+ const client: HttpClient = {
269
+ request,
270
+ source: (url, sourceOptions) => makeHttpSource(client, new URL(url), sourceOptions),
271
+ }
272
+
273
+ return client
274
+ }
275
+
276
+ export const httpClient = createHttpClient()
package/src/index.ts ADDED
@@ -0,0 +1,65 @@
1
+ export type * from './types.js'
2
+ export type * from './archive/types.js'
3
+ export type * from './http/types.js'
4
+ export type * from './network/types.js'
5
+ export type * from './proxy/types.js'
6
+ export type * from './resolvers/types.js'
7
+ export type * from './resolvers/google-drive/types.js'
8
+ export type * from './s3/types.js'
9
+ export type * from './sources/types.js'
10
+ export type * from './sinks/types.js'
11
+ export type * from './sinks/directory/types.js'
12
+ export type * from './sinks/buffer/types.js'
13
+ export type * from './sinks/workload/types.js'
14
+ export type * from './sources/workload/types.js'
15
+ export type * from './workspace/types.js'
16
+ export type * from './backed-workspace/types.js'
17
+ export type * from './transfer/types.js'
18
+ export type * from './service/types.js'
19
+
20
+ export * from './consts.js'
21
+ export * from './archive/consts.js'
22
+ export * from './network/consts.js'
23
+ export * from './resolvers/google-drive/consts.js'
24
+ export * from './s3/consts.js'
25
+ export * from './sinks/workload/consts.js'
26
+ export * from './transfer/consts.js'
27
+ export * from './workspace/consts.js'
28
+ export * from './errors.js'
29
+
30
+ export * from './archive.js'
31
+ export * from './network.js'
32
+ export * from './http.js'
33
+ export * from './proxy/endpoint.js'
34
+ export * from './proxy/router.js'
35
+ export * from './proxy/http.js'
36
+ export * from './proxy/socks.js'
37
+ export * from './resolvers/google-drive.js'
38
+ export * from './transfer.js'
39
+ export * from './s3.js'
40
+ export * from './sources/file.js'
41
+ export * from './sources/buffer.js'
42
+ export * from './sources/stream.js'
43
+ export * from './sinks/file.js'
44
+ export * from './sinks/directory.js'
45
+ export * from './sinks/buffer.js'
46
+ export * from './sinks/workload.js'
47
+ export * from './sources/workload.js'
48
+ export * from './workspace.js'
49
+ export * from './backed-workspace.js'
50
+ export * from './service.js'
51
+
52
+ // Detection lives in @owlmeans/file-inspector; these names stay importable from here.
53
+ export type {
54
+ DeclaredType, DetectedType, FileClaims, FileDetails, FileInspector, FileInspectorOptions, FileVerdict, InspectOptions, ProbeOptions,
55
+ VerdictStatus, VerifyPolicy,
56
+ } from '@owlmeans/file-inspector'
57
+ export {
58
+ CODE_LANGUAGE_BY_EXT, CONTAINER_EXTENSIONS, createFileInspector, DEFAULT_PROBE_BYTES, DEFAULT_SAMPLE_BYTES, EXTENSION_GROUPS, EXTENSION_MIME,
59
+ FileKind, FileKindError, FileMimicryError, fileInspector, KIND_BY_MIME, KIND_BY_MIME_PREFIX, MIME_ALIASES, NON_INFORMATIVE_MIMES,
60
+ OFFICE_KINDS, PASSIVE_KINDS, PDF_TRAILER_BYTES, SHEBANG_LANGUAGES, TEXT_FAMILY,
61
+ } from '@owlmeans/file-inspector'
62
+
63
+ // Scopes are storage-common's vocabulary; workspaces are addressed by them.
64
+ export type { FileScope, FileScopeHelper } from '@owlmeans/storage-common'
65
+ export { FileScopeError, FileScopeSchema, fileScopeHelper } from '@owlmeans/storage-common'