@minca/core 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (420) hide show
  1. package/README.md +2 -2
  2. package/dist/{session → hosting}/hosting.d.ts +2 -8
  3. package/dist/hosting/hosting.d.ts.map +1 -0
  4. package/dist/{session → hosting}/hosting.js +3 -3
  5. package/dist/hosting/hosting.js.map +1 -0
  6. package/dist/identity/abilities.d.ts +0 -2
  7. package/dist/identity/abilities.d.ts.map +1 -1
  8. package/dist/identity/abilities.js +2 -2
  9. package/dist/identity/abilities.js.map +1 -1
  10. package/dist/identity/account-vault.d.ts +0 -2
  11. package/dist/identity/account-vault.d.ts.map +1 -1
  12. package/dist/identity/account-vault.js +1 -1
  13. package/dist/identity/account-vault.js.map +1 -1
  14. package/dist/identity/crypto-p256.d.ts +1 -1
  15. package/dist/identity/crypto-p256.d.ts.map +1 -1
  16. package/dist/identity/ucan.d.ts +2 -1
  17. package/dist/identity/ucan.d.ts.map +1 -1
  18. package/dist/identity/ucan.js +1 -1
  19. package/dist/identity/ucan.js.map +1 -1
  20. package/dist/inbox/inbox.d.ts +0 -21
  21. package/dist/inbox/inbox.d.ts.map +1 -1
  22. package/dist/inbox/inbox.js +4 -4
  23. package/dist/inbox/inbox.js.map +1 -1
  24. package/dist/inbox/index.d.ts +4 -5
  25. package/dist/inbox/index.d.ts.map +1 -1
  26. package/dist/inbox/index.js +2 -3
  27. package/dist/inbox/index.js.map +1 -1
  28. package/dist/inbox/names.d.ts +0 -4
  29. package/dist/inbox/names.d.ts.map +1 -1
  30. package/dist/inbox/names.js +2 -2
  31. package/dist/inbox/names.js.map +1 -1
  32. package/dist/index.d.ts +50 -84
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +38 -60
  35. package/dist/index.js.map +1 -1
  36. package/dist/network/mesh.d.ts.map +1 -1
  37. package/dist/network/mesh.js +23 -3
  38. package/dist/network/mesh.js.map +1 -1
  39. package/dist/network/network-manager.d.ts +5 -0
  40. package/dist/network/network-manager.d.ts.map +1 -1
  41. package/dist/network/network-manager.js +3 -0
  42. package/dist/network/network-manager.js.map +1 -1
  43. package/dist/network/peer-auth.d.ts +17 -5
  44. package/dist/network/peer-auth.d.ts.map +1 -1
  45. package/dist/network/peer-auth.js +19 -9
  46. package/dist/network/peer-auth.js.map +1 -1
  47. package/dist/network/rendezvous.d.ts +2 -0
  48. package/dist/network/rendezvous.d.ts.map +1 -1
  49. package/dist/network/rendezvous.js +5 -0
  50. package/dist/network/rendezvous.js.map +1 -1
  51. package/dist/network/rtc-transport.d.ts.map +1 -1
  52. package/dist/network/rtc-transport.js +9 -3
  53. package/dist/network/rtc-transport.js.map +1 -1
  54. package/dist/network/transport.d.ts +12 -0
  55. package/dist/network/transport.d.ts.map +1 -1
  56. package/dist/network/ws-transport.d.ts +1 -1
  57. package/dist/network/ws-transport.d.ts.map +1 -1
  58. package/dist/network/ws-transport.js +14 -4
  59. package/dist/network/ws-transport.js.map +1 -1
  60. package/dist/node/carrier.d.ts +1 -33
  61. package/dist/node/carrier.d.ts.map +1 -1
  62. package/dist/node/carrier.js +38 -68
  63. package/dist/node/carrier.js.map +1 -1
  64. package/dist/node/index.d.ts +5 -16
  65. package/dist/node/index.d.ts.map +1 -1
  66. package/dist/node/index.js +2 -9
  67. package/dist/node/index.js.map +1 -1
  68. package/dist/node/node-carriers.d.ts.map +1 -1
  69. package/dist/node/node-carriers.js +21 -52
  70. package/dist/node/node-carriers.js.map +1 -1
  71. package/dist/node/node-context.d.ts +0 -2
  72. package/dist/node/node-context.d.ts.map +1 -1
  73. package/dist/node/node-context.js.map +1 -1
  74. package/dist/node/node-hosting.d.ts.map +1 -1
  75. package/dist/node/node-hosting.js +14 -5
  76. package/dist/node/node-hosting.js.map +1 -1
  77. package/dist/node/node-records.js +2 -2
  78. package/dist/node/node-records.js.map +1 -1
  79. package/dist/node/node.d.ts.map +1 -1
  80. package/dist/node/node.js +152 -36
  81. package/dist/node/node.js.map +1 -1
  82. package/dist/node/space-checks.d.ts +3 -1
  83. package/dist/node/space-checks.d.ts.map +1 -1
  84. package/dist/node/space-checks.js +7 -3
  85. package/dist/node/space-checks.js.map +1 -1
  86. package/dist/node/space-holding.d.ts +5 -0
  87. package/dist/node/space-holding.d.ts.map +1 -1
  88. package/dist/node/space-holding.js +90 -5
  89. package/dist/node/space-holding.js.map +1 -1
  90. package/dist/node/space-lines.d.ts +3 -0
  91. package/dist/node/space-lines.d.ts.map +1 -1
  92. package/dist/node/space-lines.js +13 -1
  93. package/dist/node/space-lines.js.map +1 -1
  94. package/dist/node/space-membership.js +4 -4
  95. package/dist/node/space-membership.js.map +1 -1
  96. package/dist/node/space-mirrors.d.ts +1 -1
  97. package/dist/node/space-mirrors.d.ts.map +1 -1
  98. package/dist/node/space-mirrors.js +17 -5
  99. package/dist/node/space-mirrors.js.map +1 -1
  100. package/dist/node/space-peers.d.ts +7 -2
  101. package/dist/node/space-peers.d.ts.map +1 -1
  102. package/dist/node/space-peers.js +44 -10
  103. package/dist/node/space-peers.js.map +1 -1
  104. package/dist/node/space-runtime.d.ts +7 -3
  105. package/dist/node/space-runtime.d.ts.map +1 -1
  106. package/dist/node/space-runtime.js +32 -10
  107. package/dist/node/space-runtime.js.map +1 -1
  108. package/dist/node/space-standing.d.ts +1 -1
  109. package/dist/node/space-standing.d.ts.map +1 -1
  110. package/dist/node/space-standing.js +23 -42
  111. package/dist/node/space-standing.js.map +1 -1
  112. package/dist/node/space-warrants.js +1 -1
  113. package/dist/node/space-warrants.js.map +1 -1
  114. package/dist/node/space-writing.d.ts.map +1 -1
  115. package/dist/node/space-writing.js +21 -32
  116. package/dist/node/space-writing.js.map +1 -1
  117. package/dist/node/stores.d.ts +9 -57
  118. package/dist/node/stores.d.ts.map +1 -1
  119. package/dist/node/stores.js +1 -72
  120. package/dist/node/stores.js.map +1 -1
  121. package/dist/node/types.d.ts +19 -8
  122. package/dist/node/types.d.ts.map +1 -1
  123. package/dist/node/writer-keys.d.ts +15 -14
  124. package/dist/node/writer-keys.d.ts.map +1 -1
  125. package/dist/node/writer-keys.js +3 -115
  126. package/dist/node/writer-keys.js.map +1 -1
  127. package/dist/privacy/direct.d.ts +0 -7
  128. package/dist/privacy/direct.d.ts.map +1 -1
  129. package/dist/privacy/direct.js +2 -2
  130. package/dist/privacy/direct.js.map +1 -1
  131. package/dist/query/filter.d.ts +0 -2
  132. package/dist/query/filter.d.ts.map +1 -1
  133. package/dist/query/filter.js +1 -1
  134. package/dist/query/filter.js.map +1 -1
  135. package/dist/query/types.d.ts +2 -6
  136. package/dist/query/types.d.ts.map +1 -1
  137. package/dist/query/types.js.map +1 -1
  138. package/dist/records/key.d.ts +18 -19
  139. package/dist/records/key.d.ts.map +1 -1
  140. package/dist/records/key.js +55 -43
  141. package/dist/records/key.js.map +1 -1
  142. package/dist/records/links.js +1 -1
  143. package/dist/records/rules.d.ts +1 -1
  144. package/dist/records/rules.d.ts.map +1 -1
  145. package/dist/records/rules.js +6 -4
  146. package/dist/records/rules.js.map +1 -1
  147. package/dist/schemas/direct.d.ts +70 -0
  148. package/dist/schemas/direct.d.ts.map +1 -0
  149. package/dist/schemas/direct.js +39 -0
  150. package/dist/schemas/direct.js.map +1 -0
  151. package/dist/schemas/index.d.ts +10 -73
  152. package/dist/schemas/index.d.ts.map +1 -1
  153. package/dist/schemas/index.js +8 -66
  154. package/dist/schemas/index.js.map +1 -1
  155. package/dist/space/account-registry.d.ts +0 -3
  156. package/dist/space/account-registry.d.ts.map +1 -1
  157. package/dist/space/account-registry.js.map +1 -1
  158. package/dist/space/own-automations.d.ts +51 -95
  159. package/dist/space/own-automations.d.ts.map +1 -1
  160. package/dist/space/own-automations.js +100 -295
  161. package/dist/space/own-automations.js.map +1 -1
  162. package/dist/space/pass.d.ts +8 -3
  163. package/dist/space/pass.d.ts.map +1 -1
  164. package/dist/space/pass.js +4 -3
  165. package/dist/space/pass.js.map +1 -1
  166. package/dist/space/space-manager.d.ts +1 -3
  167. package/dist/space/space-manager.d.ts.map +1 -1
  168. package/dist/space/space-manager.js +5 -7
  169. package/dist/space/space-manager.js.map +1 -1
  170. package/dist/storage/body-cache.d.ts +29 -0
  171. package/dist/storage/body-cache.d.ts.map +1 -0
  172. package/dist/storage/body-cache.js +64 -0
  173. package/dist/storage/body-cache.js.map +1 -0
  174. package/dist/storage/encrypted-adapter.d.ts +0 -5
  175. package/dist/storage/encrypted-adapter.d.ts.map +1 -1
  176. package/dist/storage/encrypted-adapter.js +1 -1
  177. package/dist/storage/encrypted-adapter.js.map +1 -1
  178. package/dist/storage/mirror.d.ts.map +1 -1
  179. package/dist/storage/mirror.js +35 -16
  180. package/dist/storage/mirror.js.map +1 -1
  181. package/dist/storage/segment.d.ts +6 -2
  182. package/dist/storage/segment.d.ts.map +1 -1
  183. package/dist/storage/segment.js +8 -4
  184. package/dist/storage/segment.js.map +1 -1
  185. package/dist/storage/shared-store.d.ts +32 -0
  186. package/dist/storage/shared-store.d.ts.map +1 -0
  187. package/dist/storage/{folder-reconcile.js → shared-store.js} +15 -15
  188. package/dist/storage/shared-store.js.map +1 -0
  189. package/dist/storage/storage-provider.d.ts +19 -9
  190. package/dist/storage/storage-provider.d.ts.map +1 -1
  191. package/dist/storage/storage-provider.js +155 -62
  192. package/dist/storage/storage-provider.js.map +1 -1
  193. package/dist/sync/sync-engine.d.ts +17 -4
  194. package/dist/sync/sync-engine.d.ts.map +1 -1
  195. package/dist/sync/sync-engine.js +298 -84
  196. package/dist/sync/sync-engine.js.map +1 -1
  197. package/dist/sync/sync-messages.d.ts +5 -1
  198. package/dist/sync/sync-messages.d.ts.map +1 -1
  199. package/dist/sync/sync-messages.js +1 -1
  200. package/dist/sync/sync-messages.js.map +1 -1
  201. package/dist/types.d.ts +19 -7
  202. package/dist/types.d.ts.map +1 -1
  203. package/dist/types.js +4 -1
  204. package/dist/types.js.map +1 -1
  205. package/dist/utils/index.d.ts +5 -5
  206. package/dist/utils/index.d.ts.map +1 -1
  207. package/dist/utils/index.js +5 -5
  208. package/dist/utils/index.js.map +1 -1
  209. package/dist/utils/step.d.ts +8 -0
  210. package/dist/utils/step.d.ts.map +1 -0
  211. package/dist/utils/step.js +15 -0
  212. package/dist/utils/step.js.map +1 -0
  213. package/docs/README.md +4 -3
  214. package/docs/actions.md +41 -2
  215. package/docs/agents.md +13 -10
  216. package/docs/automations.md +16 -48
  217. package/docs/blueprints-and-guides.md +2 -2
  218. package/docs/bots.md +3 -3
  219. package/docs/building-an-app.md +2 -2
  220. package/docs/collections.md +2 -4
  221. package/docs/computed.md +1 -2
  222. package/docs/node.md +108 -40
  223. package/docs/proposing-collections.md +3 -3
  224. package/docs/query-format.md +6 -5
  225. package/docs/records-and-queries.md +12 -13
  226. package/docs/sign-in.md +16 -16
  227. package/docs/spaces.md +0 -5
  228. package/docs/standard-library.md +6 -8
  229. package/docs/storage.md +114 -22
  230. package/docs/usernames.md +1 -1
  231. package/package.json +15 -25
  232. package/dist/identity/account-store.d.ts +0 -83
  233. package/dist/identity/account-store.d.ts.map +0 -1
  234. package/dist/identity/account-store.js +0 -234
  235. package/dist/identity/account-store.js.map +0 -1
  236. package/dist/identity/device-key.d.ts +0 -26
  237. package/dist/identity/device-key.d.ts.map +0 -1
  238. package/dist/identity/device-key.js +0 -48
  239. package/dist/identity/device-key.js.map +0 -1
  240. package/dist/identity/idb.d.ts +0 -19
  241. package/dist/identity/idb.d.ts.map +0 -1
  242. package/dist/identity/idb.js +0 -56
  243. package/dist/identity/idb.js.map +0 -1
  244. package/dist/node/actions.d.ts +0 -85
  245. package/dist/node/actions.d.ts.map +0 -1
  246. package/dist/node/actions.js +0 -921
  247. package/dist/node/actions.js.map +0 -1
  248. package/dist/node/host.d.ts +0 -187
  249. package/dist/node/host.d.ts.map +0 -1
  250. package/dist/node/host.js +0 -818
  251. package/dist/node/host.js.map +0 -1
  252. package/dist/node/usage.d.ts +0 -32
  253. package/dist/node/usage.d.ts.map +0 -1
  254. package/dist/node/usage.js +0 -217
  255. package/dist/node/usage.js.map +0 -1
  256. package/dist/node/watch-notifications.d.ts +0 -25
  257. package/dist/node/watch-notifications.d.ts.map +0 -1
  258. package/dist/node/watch-notifications.js +0 -166
  259. package/dist/node/watch-notifications.js.map +0 -1
  260. package/dist/node/worker.d.ts +0 -83
  261. package/dist/node/worker.d.ts.map +0 -1
  262. package/dist/node/worker.js +0 -305
  263. package/dist/node/worker.js.map +0 -1
  264. package/dist/node-worker.d.ts +0 -2
  265. package/dist/node-worker.d.ts.map +0 -1
  266. package/dist/node-worker.js +0 -16
  267. package/dist/node-worker.js.map +0 -1
  268. package/dist/schemas/automations.d.ts +0 -390
  269. package/dist/schemas/automations.d.ts.map +0 -1
  270. package/dist/schemas/automations.js +0 -619
  271. package/dist/schemas/automations.js.map +0 -1
  272. package/dist/schemas/blueprints.d.ts +0 -122
  273. package/dist/schemas/blueprints.d.ts.map +0 -1
  274. package/dist/schemas/blueprints.js +0 -337
  275. package/dist/schemas/blueprints.js.map +0 -1
  276. package/dist/schemas/choosing.d.ts +0 -19
  277. package/dist/schemas/choosing.d.ts.map +0 -1
  278. package/dist/schemas/choosing.js +0 -69
  279. package/dist/schemas/choosing.js.map +0 -1
  280. package/dist/schemas/connectors.d.ts +0 -202
  281. package/dist/schemas/connectors.d.ts.map +0 -1
  282. package/dist/schemas/connectors.js +0 -140
  283. package/dist/schemas/connectors.js.map +0 -1
  284. package/dist/schemas/cron.d.ts +0 -14
  285. package/dist/schemas/cron.d.ts.map +0 -1
  286. package/dist/schemas/cron.js +0 -75
  287. package/dist/schemas/cron.js.map +0 -1
  288. package/dist/schemas/geo.d.ts +0 -63
  289. package/dist/schemas/geo.d.ts.map +0 -1
  290. package/dist/schemas/geo.js +0 -131
  291. package/dist/schemas/geo.js.map +0 -1
  292. package/dist/schemas/guides.d.ts +0 -26
  293. package/dist/schemas/guides.d.ts.map +0 -1
  294. package/dist/schemas/guides.js +0 -33
  295. package/dist/schemas/guides.js.map +0 -1
  296. package/dist/schemas/library/annotations.d.ts +0 -616
  297. package/dist/schemas/library/annotations.d.ts.map +0 -1
  298. package/dist/schemas/library/annotations.js +0 -156
  299. package/dist/schemas/library/annotations.js.map +0 -1
  300. package/dist/schemas/library/collections.d.ts +0 -211
  301. package/dist/schemas/library/collections.d.ts.map +0 -1
  302. package/dist/schemas/library/collections.js +0 -57
  303. package/dist/schemas/library/collections.js.map +0 -1
  304. package/dist/schemas/library/community.d.ts +0 -1136
  305. package/dist/schemas/library/community.d.ts.map +0 -1
  306. package/dist/schemas/library/community.js +0 -315
  307. package/dist/schemas/library/community.js.map +0 -1
  308. package/dist/schemas/library/life.d.ts +0 -1440
  309. package/dist/schemas/library/life.d.ts.map +0 -1
  310. package/dist/schemas/library/life.js +0 -202
  311. package/dist/schemas/library/life.js.map +0 -1
  312. package/dist/schemas/library/media.d.ts +0 -1002
  313. package/dist/schemas/library/media.d.ts.map +0 -1
  314. package/dist/schemas/library/media.js +0 -110
  315. package/dist/schemas/library/media.js.map +0 -1
  316. package/dist/schemas/library/money.d.ts +0 -1221
  317. package/dist/schemas/library/money.d.ts.map +0 -1
  318. package/dist/schemas/library/money.js +0 -168
  319. package/dist/schemas/library/money.js.map +0 -1
  320. package/dist/schemas/library/planning.d.ts +0 -1107
  321. package/dist/schemas/library/planning.d.ts.map +0 -1
  322. package/dist/schemas/library/planning.js +0 -214
  323. package/dist/schemas/library/planning.js.map +0 -1
  324. package/dist/schemas/library/publishing.d.ts +0 -1499
  325. package/dist/schemas/library/publishing.d.ts.map +0 -1
  326. package/dist/schemas/library/publishing.js +0 -285
  327. package/dist/schemas/library/publishing.js.map +0 -1
  328. package/dist/schemas/library/social.d.ts +0 -1257
  329. package/dist/schemas/library/social.d.ts.map +0 -1
  330. package/dist/schemas/library/social.js +0 -147
  331. package/dist/schemas/library/social.js.map +0 -1
  332. package/dist/schemas/standard.d.ts +0 -21
  333. package/dist/schemas/standard.d.ts.map +0 -1
  334. package/dist/schemas/standard.js +0 -39
  335. package/dist/schemas/standard.js.map +0 -1
  336. package/dist/session/account-hint.d.ts +0 -13
  337. package/dist/session/account-hint.d.ts.map +0 -1
  338. package/dist/session/account-hint.js +0 -54
  339. package/dist/session/account-hint.js.map +0 -1
  340. package/dist/session/auth.d.ts +0 -411
  341. package/dist/session/auth.d.ts.map +0 -1
  342. package/dist/session/auth.js +0 -1529
  343. package/dist/session/auth.js.map +0 -1
  344. package/dist/session/channel-room.d.ts +0 -26
  345. package/dist/session/channel-room.d.ts.map +0 -1
  346. package/dist/session/channel-room.js +0 -56
  347. package/dist/session/channel-room.js.map +0 -1
  348. package/dist/session/connect-code.d.ts +0 -127
  349. package/dist/session/connect-code.d.ts.map +0 -1
  350. package/dist/session/connect-code.js +0 -408
  351. package/dist/session/connect-code.js.map +0 -1
  352. package/dist/session/connect.d.ts +0 -386
  353. package/dist/session/connect.d.ts.map +0 -1
  354. package/dist/session/connect.js +0 -583
  355. package/dist/session/connect.js.map +0 -1
  356. package/dist/session/connection.d.ts +0 -135
  357. package/dist/session/connection.d.ts.map +0 -1
  358. package/dist/session/connection.js +0 -215
  359. package/dist/session/connection.js.map +0 -1
  360. package/dist/session/credentials.d.ts +0 -30
  361. package/dist/session/credentials.d.ts.map +0 -1
  362. package/dist/session/credentials.js +0 -63
  363. package/dist/session/credentials.js.map +0 -1
  364. package/dist/session/hosting.d.ts.map +0 -1
  365. package/dist/session/hosting.js.map +0 -1
  366. package/dist/session/index.d.ts +0 -29
  367. package/dist/session/index.d.ts.map +0 -1
  368. package/dist/session/index.js +0 -21
  369. package/dist/session/index.js.map +0 -1
  370. package/dist/session/pairing.d.ts +0 -97
  371. package/dist/session/pairing.d.ts.map +0 -1
  372. package/dist/session/pairing.js +0 -251
  373. package/dist/session/pairing.js.map +0 -1
  374. package/dist/session/password-strength.d.ts +0 -22
  375. package/dist/session/password-strength.d.ts.map +0 -1
  376. package/dist/session/password-strength.js +0 -130
  377. package/dist/session/password-strength.js.map +0 -1
  378. package/dist/session/places.d.ts +0 -71
  379. package/dist/session/places.d.ts.map +0 -1
  380. package/dist/session/places.js +0 -130
  381. package/dist/session/places.js.map +0 -1
  382. package/dist/session/popup.d.ts +0 -13
  383. package/dist/session/popup.d.ts.map +0 -1
  384. package/dist/session/popup.js +0 -23
  385. package/dist/session/popup.js.map +0 -1
  386. package/dist/session/stay-signed-in.d.ts +0 -49
  387. package/dist/session/stay-signed-in.d.ts.map +0 -1
  388. package/dist/session/stay-signed-in.js +0 -141
  389. package/dist/session/stay-signed-in.js.map +0 -1
  390. package/dist/storage/blob/directory.d.ts +0 -14
  391. package/dist/storage/blob/directory.d.ts.map +0 -1
  392. package/dist/storage/blob/directory.js +0 -87
  393. package/dist/storage/blob/directory.js.map +0 -1
  394. package/dist/storage/blob/memory.d.ts +0 -6
  395. package/dist/storage/blob/memory.d.ts.map +0 -1
  396. package/dist/storage/blob/memory.js +0 -20
  397. package/dist/storage/blob/memory.js.map +0 -1
  398. package/dist/storage/blob/s3.d.ts +0 -16
  399. package/dist/storage/blob/s3.d.ts.map +0 -1
  400. package/dist/storage/blob/s3.js +0 -91
  401. package/dist/storage/blob/s3.js.map +0 -1
  402. package/dist/storage/directory-access.d.ts +0 -36
  403. package/dist/storage/directory-access.d.ts.map +0 -1
  404. package/dist/storage/directory-access.js +0 -89
  405. package/dist/storage/directory-access.js.map +0 -1
  406. package/dist/storage/folder-adapter.d.ts +0 -94
  407. package/dist/storage/folder-adapter.d.ts.map +0 -1
  408. package/dist/storage/folder-adapter.js +0 -215
  409. package/dist/storage/folder-adapter.js.map +0 -1
  410. package/dist/storage/folder-reconcile.d.ts +0 -33
  411. package/dist/storage/folder-reconcile.d.ts.map +0 -1
  412. package/dist/storage/folder-reconcile.js.map +0 -1
  413. package/dist/storage/index.d.ts +0 -6
  414. package/dist/storage/index.d.ts.map +0 -1
  415. package/dist/storage/index.js +0 -5
  416. package/dist/storage/index.js.map +0 -1
  417. package/dist/storage/indexeddb-adapter.d.ts +0 -8
  418. package/dist/storage/indexeddb-adapter.d.ts.map +0 -1
  419. package/dist/storage/indexeddb-adapter.js +0 -103
  420. package/dist/storage/indexeddb-adapter.js.map +0 -1
@@ -18,18 +18,36 @@
18
18
  */
19
19
  import { getOrAdd } from '../utils/map.js';
20
20
  import { cidDigest, cidOfDigest } from '../utils/hash.js';
21
- import { base64UrlDecode, base64UrlEncode, bytesToHex } from '../utils/encoding.js';
21
+ import { base64UrlDecode, base64UrlEncode, bytesToHex, utf8Encode } from '../utils/encoding.js';
22
22
  import { createEmitter } from '../utils/events.js';
23
23
  import { createReconciler, fingerprintOf } from './negentropy.js';
24
24
  import { parseSyncMessage, SYNC_PROTOCOL_VERSION, } from './sync-messages.js';
25
- /** At most this many records wait for what they depend on; the oldest give way */
26
- const MAX_WAITING = 1000;
25
+ /**
26
+ * Records waiting for what they depend on, past which nothing more is asked
27
+ * for but what they wait on: a long history fills in from below instead of
28
+ * being dropped and asked for again. High, since below it histories arrive
29
+ * as fast as the peer sends them: pausing at 1,000 made 100 records of 50
30
+ * edits each settle in 30 s instead of 2.
31
+ */
32
+ const MAX_WAITING = 20_000;
33
+ /** Records held while what they wait on is fetched; past this the oldest is let go and asked for again later */
34
+ const MAX_HELD = 2 * MAX_WAITING;
27
35
  /** Refusals remembered, so reconciling doesn't ask a peer for the same bad version every round */
28
36
  const MAX_REFUSED = 10_000;
29
37
  /** Records asked for, or sent, in one message */
30
38
  const MAX_IDS_PER_REQUEST = 200;
39
+ /**
40
+ * Bytes of versions in one message, besides the count: a message of large
41
+ * records is held whole in memory at both ends, and a socket refuses one past
42
+ * its limit. One version larger than this still goes, alone.
43
+ */
44
+ const MAX_MESSAGE_BYTES = 1024 * 1024;
45
+ /** How often a sender waiting for its connection to drain looks again */
46
+ const DRAIN_POLL_MS = 25;
31
47
  /** `want`s in flight to one peer at a time; the rest queue */
32
48
  const MAX_WANTS_IN_FLIGHT = 4;
49
+ /** Answers to one peer's `want`s waiting for room, past which more are left unanswered */
50
+ const MAX_ANSWERS_WAITING = 4 * MAX_WANTS_IN_FLIGHT;
33
51
  /** Largest Negentropy message, in bytes before base64: well inside a data channel's limit */
34
52
  const FRAME_SIZE_LIMIT = 32_000;
35
53
  /** Collections one hello may name. Past this it is hostile, not big. */
@@ -50,36 +68,10 @@ function readHolds(value) {
50
68
  }
51
69
  /** Collection definitions first, then records by version: what others depend on comes first */
52
70
  const rank = (e) => (e?.collection === 'sys.collection' ? -1 : (e?.seq ?? 0));
53
- /**
54
- * Versions in messages of at most `size`, with every version of one record in
55
- * the same message where they fit. A message is taken in whole, so a record
56
- * arrives with the versions that changed or deleted it, never as it once was.
57
- */
58
- function byRecord(versions, size) {
59
- const records = new Map();
60
- for (const version of versions)
61
- getOrAdd(records, version.key, () => []).push(version);
62
- const messages = [];
63
- let message = [];
64
- for (const record of records.values()) {
65
- if (message.length + record.length > size && message.length > 0) {
66
- messages.push(message);
67
- message = [];
68
- }
69
- // A record with more versions than fit in one message goes over several.
70
- for (let i = 0; i < record.length; i += size) {
71
- const part = record.slice(i, i + size);
72
- if (message.length + part.length > size) {
73
- messages.push(message);
74
- message = [];
75
- }
76
- message.push(...part);
77
- }
78
- }
79
- if (message.length > 0)
80
- messages.push(message);
81
- return messages;
82
- }
71
+ /** The bytes a version takes in a message, as UTF-8 JSON */
72
+ const wireBytes = (version) => utf8Encode(JSON.stringify(version)).length;
73
+ /** Whether a message already holding `bytes` in `count` versions has no room for one of `size` */
74
+ const full = (count, bytes, size) => count >= MAX_IDS_PER_REQUEST || (count > 0 && bytes + size > MAX_MESSAGE_BYTES);
83
75
  /** Creates a sync engine orchestrator. */
84
76
  export function createSyncEngine(config) {
85
77
  const { storageProvider: storage, sendToPeer, heartbeatInterval = 30000, validate, self } = config;
@@ -91,8 +83,12 @@ export function createSyncEngine(config) {
91
83
  const states = new Map();
92
84
  /** Peers found level with this node, until it takes in or writes anything more */
93
85
  const level = new Set();
94
- /** Records that could not be judged yet, tried again whenever something new is admitted */
86
+ /** Records that could not be judged yet, until what they wait for is admitted */
95
87
  const waiting = new Map();
88
+ /** Waiting records by a version of their own chain they lack (`prev` or first): tried again once it is stored */
89
+ const waitingOn = new Map();
90
+ /** Waiting records that lack nothing of their own chain, so wait for something else: tried after every intake */
91
+ const waitingOther = new Set();
96
92
  /**
97
93
  * Refused versions, by peer and id, oldest first. An id doesn't cover the
98
94
  * signature, so a stranger's mangled copy must not stop us fetching the real one.
@@ -102,6 +98,11 @@ export function createSyncEngine(config) {
102
98
  const connections = new Map();
103
99
  /** Where each peer's messages are up to */
104
100
  const inOrder = new Map();
101
+ /** What is still being sent to each peer, so one session's versions go after another's */
102
+ const sending = new Map();
103
+ /** Answers to each peer's `want`s still waiting for room, in the order asked, and how many */
104
+ const answering = new Map();
105
+ const waitingAnswers = new Map();
105
106
  let lastConnection = 0;
106
107
  let intervalId = null;
107
108
  let nextId = 1;
@@ -112,7 +113,7 @@ export function createSyncEngine(config) {
112
113
  // ─── Taking in versions ────────────────────────────────────────────
113
114
  /** Stores a version from a peer if the gatekeeper allows it; one that must wait names what it waits for */
114
115
  const admit = async (peerId, version, intake) => {
115
- const verdict = validate ? await validate(version) : { valid: true };
116
+ const verdict = validate ? await validate(version, peerId) : { valid: true };
116
117
  if (!verdict.valid) {
117
118
  if (verdict.later)
118
119
  await holdBack(peerId, version, intake);
@@ -128,31 +129,71 @@ export function createSyncEngine(config) {
128
129
  };
129
130
  const holdBack = async (peerId, version, intake) => {
130
131
  waiting.set(version.id, { peerId, version });
131
- dropOldest(waiting, MAX_WAITING);
132
+ if (waiting.size > MAX_HELD)
133
+ letGoOldest();
132
134
  const dependsOn = version.seq > 0 ? [version.genesis, version.prev] : [];
135
+ const lacking = [];
133
136
  for (const needed of dependsOn) {
134
- if (typeof needed === 'string' && !(await storage.getVersion(needed)))
135
- intake.firsts.set(needed, version.collection);
137
+ if (typeof needed !== 'string' || (await storage.getVersion(needed)))
138
+ continue;
139
+ lacking.push(needed);
140
+ // Syncing shallow, the version before is not fetched: a keeper's copy of this one comes without it.
141
+ if (needed === version.prev && config.shallowWith?.(peerId))
142
+ continue;
143
+ intake.firsts.set(needed, version.collection);
136
144
  }
145
+ if (lacking.length === 0)
146
+ waitingOther.add(version.id);
147
+ for (const needed of lacking)
148
+ getOrAdd(waitingOn, needed, () => new Set()).add(version.id);
149
+ };
150
+ /** Lets the longest-waiting record go, still to be asked for: dropped and forgotten, a peer would look level without it */
151
+ const letGoOldest = () => {
152
+ const [id, held] = waiting.entries().next().value ?? [];
153
+ if (!id || !held)
154
+ return;
155
+ waiting.delete(id);
156
+ waitingOther.delete(id);
157
+ if (peers.has(held.peerId))
158
+ want(held.peerId, held.version.collection, [id]);
137
159
  };
138
160
  const refuse = (peerId, version, reason) => {
139
161
  refused.add(refusal(peerId, version.id));
140
162
  dropOldest(refused, MAX_REFUSED);
141
163
  emit('rejected', peerId, version, reason);
142
164
  };
143
- /** Tries every waiting version again until nothing more goes in; each sender hears what was stored */
165
+ /**
166
+ * Tries again what waited for the versions just stored, and what those let
167
+ * in in turn, then what waits for anything else; each sender hears what was
168
+ * stored. Each waiting record is tried once per version it lacked: trying
169
+ * every one on every arrival took a 1,000-edit history quadratic time.
170
+ */
144
171
  const retryWaiting = async (intake) => {
145
172
  const stored = new Map();
146
- let progressed = true;
147
- while (progressed && waiting.size > 0) {
148
- progressed = false;
149
- for (const [id, held] of [...waiting]) {
150
- waiting.delete(id);
151
- if (!(await admit(held.peerId, held.version, intake)))
152
- continue;
153
- progressed = true;
173
+ const tryAgain = async (id) => {
174
+ const held = waiting.get(id);
175
+ if (!held)
176
+ return;
177
+ waiting.delete(id);
178
+ waitingOther.delete(id);
179
+ if (await admit(held.peerId, held.version, intake))
154
180
  getOrAdd(stored, held.peerId, () => []).push(id);
181
+ };
182
+ let next = 0;
183
+ for (;;) {
184
+ // Stored versions are appended to `placed` as they go in, so this follows a chain to its end.
185
+ for (; next < intake.placed.length; next++) {
186
+ const placedId = intake.placed[next].id;
187
+ const dependents = waitingOn.get(placedId);
188
+ waitingOn.delete(placedId);
189
+ for (const id of dependents ?? [])
190
+ await tryAgain(id);
155
191
  }
192
+ const placedBefore = intake.placed.length;
193
+ for (const id of [...waitingOther])
194
+ await tryAgain(id);
195
+ if (intake.placed.length === placedBefore)
196
+ break;
156
197
  }
157
198
  for (const [peerId, ids] of stored)
158
199
  if (peers.has(peerId))
@@ -165,7 +206,9 @@ export function createSyncEngine(config) {
165
206
  const want = (peerId, collection, ids, ahead = false) => {
166
207
  const first = ahead || collection.startsWith('sys.');
167
208
  const state = stateOf(peerId);
168
- const entries = ids.filter((id) => first || !state.queuedIds.has(id)).map((id) => ({ id, collection }));
209
+ const entries = ids
210
+ .filter((id) => first || !state.queuedIds.has(id))
211
+ .map((id) => ({ id, collection, first }));
169
212
  for (const { id } of entries)
170
213
  state.queuedIds.add(id);
171
214
  if (first)
@@ -180,6 +223,8 @@ export function createSyncEngine(config) {
180
223
  if (!state)
181
224
  return;
182
225
  while (state.wants.size < MAX_WANTS_IN_FLIGHT && state.queuedIds.size > 0) {
226
+ if (waiting.size >= MAX_WAITING && !nextIsFirst(state))
227
+ break;
183
228
  const batch = takeBatch(state);
184
229
  if (!batch)
185
230
  break;
@@ -215,9 +260,13 @@ export function createSyncEngine(config) {
215
260
  /** Asks at once for the versions still waited on, unless asked, refused or already here */
216
261
  const askFirsts = async (peerId, firsts) => {
217
262
  const inFlight = askedOf(peerId);
263
+ const state = states.get(peerId);
264
+ // Coming anyway, a batch at a time: asking for each one alone walks a chain a link per round trip.
265
+ // Or here already, waiting in turn for the one below it: asked again, it comes again, and waits again.
266
+ const coming = (id, collection) => inFlight.has(id) || waiting.has(id) || !!state?.queuedIds.has(id) || !!state?.sessions.has(collection);
218
267
  const byCollection = new Map();
219
268
  for (const [id, collection] of firsts) {
220
- if (inFlight.has(id) || isRefused(peerId, id) || (await storage.getVersion(id)))
269
+ if (coming(id, collection) || isRefused(peerId, id) || (await storage.getVersion(id)))
221
270
  continue;
222
271
  getOrAdd(byCollection, collection, () => []).push(id);
223
272
  }
@@ -227,10 +276,12 @@ export function createSyncEngine(config) {
227
276
  /** Ids asked of a peer and not yet answered */
228
277
  const askedOf = (peerId) => new Set([...(states.get(peerId)?.wants.values() ?? [])].flatMap((w) => [...w.ids]));
229
278
  // ─── Hello ─────────────────────────────────────────────────────────
230
- /** A fingerprint of each collection held and kept, hex */
231
- const ourSums = async (holds) => {
279
+ /** Whether an exchange with a peer compares shallow sets: either side may say so */
280
+ const shallowWith = (peerId) => !!states.get(peerId)?.shallow || (config.shallowWith?.(peerId) ?? false);
281
+ /** A fingerprint of each collection held and kept, hex: of its shallow set in a shallow exchange */
282
+ const ourSums = async (holds, shallow = false) => {
232
283
  const out = {};
233
- for (const [collection, sum] of await storage.sums()) {
284
+ for (const [collection, sum] of await (shallow ? storage.shallowSums() : storage.sums())) {
234
285
  if (holdsCollection(holds, collection))
235
286
  out[collection] = bytesToHex(await fingerprintOf(sum));
236
287
  }
@@ -238,15 +289,18 @@ export function createSyncEngine(config) {
238
289
  };
239
290
  const hello = async (peerId, reply = false) => {
240
291
  const holds = ourHolds();
292
+ // Only this node's own wish: a peer that asked has its answer in kind, not the same ask back.
293
+ const askShallow = config.shallowWith?.(peerId) ?? false;
241
294
  send(peerId, {
242
295
  type: 'hello',
243
296
  holds: holds === 'all' ? 'all' : [...holds],
244
- sums: await ourSums(holds),
297
+ sums: await ourSums(holds, shallowWith(peerId)),
245
298
  ...(reply ? { reply } : {}),
299
+ ...(askShallow ? { shallow: true } : {}),
246
300
  });
247
301
  };
248
302
  const sayHello = (peerId, reply = false) => void hello(peerId, reply).catch(report);
249
- const onHello = async (peerId, theirs, theirHolds, reply) => {
303
+ const onHello = async (peerId, theirs, theirHolds, reply, theirShallow) => {
250
304
  if (typeof theirs !== 'object' || theirs === null)
251
305
  return;
252
306
  const names = Object.keys(theirs);
@@ -254,8 +308,9 @@ export function createSyncEngine(config) {
254
308
  return;
255
309
  const state = stateOf(peerId);
256
310
  state.holds = readHolds(theirHolds);
311
+ state.shallow = theirShallow;
257
312
  const holds = ourHolds();
258
- const ours = await ourSums(holds);
313
+ const ours = await ourSums(holds, shallowWith(peerId));
259
314
  // Every collection this node holds counts, even one it has nothing of yet.
260
315
  const named = new Set([...names, ...Object.keys(ours), ...(holds === 'all' ? [] : holds)]);
261
316
  const shared = [...named].filter((c) => holdsCollection(holds, c) && holdsCollection(state.holds, c));
@@ -263,21 +318,35 @@ export function createSyncEngine(config) {
263
318
  for (const c of shared)
264
319
  if (theirs[c] === ours[c])
265
320
  emit('level', peerId, c, state.holds === 'all');
266
- if (differing.length === 0)
321
+ if (differing.length === 0) {
322
+ // A shallow ask hears that it is level: no round of reconciling tells it so.
323
+ if (theirShallow && !reply)
324
+ await hello(peerId, true);
267
325
  return settleIfDone(peerId);
326
+ }
268
327
  if (self !== undefined && self > peerId) {
269
328
  // The other side starts. It may not have heard from us yet.
270
329
  if (!reply)
271
330
  await hello(peerId, true);
272
331
  return;
273
332
  }
274
- for (const collection of differing)
333
+ // Who may do what first, then the smallest: what a screen shows first is rarely a long log.
334
+ const sizes = await storage.sums();
335
+ const order = (c) => (c.startsWith('sys.') ? -1 : (sizes.get(c)?.count ?? 0));
336
+ for (const collection of differing.sort((a, b) => order(a) - order(b)))
275
337
  await begin(peerId, collection);
276
338
  };
277
339
  // ─── Reconciling, as initiator ─────────────────────────────────────
278
340
  const busy = (peerId) => {
279
341
  const state = states.get(peerId);
280
- return !!state && (state.sessions.size > 0 || state.wants.size > 0 || state.queuedIds.size > 0);
342
+ if (state &&
343
+ (state.sessions.size > 0 || state.wants.size > 0 || state.queuedIds.size > 0 || state.sending.size > 0))
344
+ return true;
345
+ // What it sent that waits for its chain isn't taken in yet.
346
+ for (const held of waiting.values())
347
+ if (held.peerId === peerId)
348
+ return true;
349
+ return false;
281
350
  };
282
351
  const settleIfDone = (peerId) => {
283
352
  if (busy(peerId))
@@ -290,6 +359,7 @@ export function createSyncEngine(config) {
290
359
  id: session.id,
291
360
  collection: session.collection,
292
361
  message: base64UrlEncode(message),
362
+ ...(session.shallow ? { shallow: true } : {}),
293
363
  });
294
364
  const begin = async (peerId, collection) => {
295
365
  if (typeof collection !== 'string')
@@ -300,14 +370,17 @@ export function createSyncEngine(config) {
300
370
  current.again = true;
301
371
  return;
302
372
  }
303
- const reconciler = createReconciler(await storage.items(collection), {
304
- initiator: true,
305
- frameSizeLimit: FRAME_SIZE_LIMIT,
306
- });
373
+ if (state.sending.has(collection)) {
374
+ state.beginAfter.add(collection);
375
+ return;
376
+ }
377
+ const shallow = shallowWith(peerId);
378
+ const reconciler = createReconciler(await (shallow ? storage.shallowItems(collection) : storage.items(collection)), { initiator: true, frameSizeLimit: FRAME_SIZE_LIMIT });
307
379
  const session = {
308
380
  id: nextId++,
309
381
  collection,
310
382
  reconciler,
383
+ shallow,
311
384
  rounds: 0,
312
385
  touchedAt: Date.now(),
313
386
  again: false,
@@ -324,7 +397,7 @@ export function createSyncEngine(config) {
324
397
  */
325
398
  const levelIfDone = (peerId, collection) => {
326
399
  const state = states.get(peerId);
327
- if (!state || state.sessions.has(collection))
400
+ if (!state || state.sessions.has(collection) || state.sending.has(collection))
328
401
  return;
329
402
  const wanting = [...state.wants.values()].some((w) => w.collection === collection);
330
403
  const queued = state.queued.some((q) => q.collection === collection && state.queuedIds.has(q.id));
@@ -342,13 +415,91 @@ export function createSyncEngine(config) {
342
415
  // Asked for before the session goes, so the peer never looks idle in between.
343
416
  want(peerId, session.collection, session.need.reverse());
344
417
  state.sessions.delete(session.collection);
345
- const found = await Promise.all(session.have.reverse().map((id) => storage.getVersion(id)));
346
- const have = found.filter((v) => v !== null);
347
- for (const versions of byRecord(have, MAX_IDS_PER_REQUEST))
348
- send(peerId, { type: 'versions', versions });
418
+ sendAll(peerId, state, session.collection, session.have.reverse());
349
419
  if (session.again)
350
420
  void begin(peerId, session.collection).catch(report);
351
421
  };
422
+ /**
423
+ * Sends a peer versions it lacks, in order, after whatever is still going
424
+ * to it. Off the peer's message queue, so taking in what the peer sends
425
+ * never waits on it taking in what this node sends.
426
+ */
427
+ const sendAll = (peerId, state, collection, ids) => {
428
+ if (ids.length === 0)
429
+ return;
430
+ const connection = connections.get(peerId);
431
+ state.sending.set(collection, (state.sending.get(collection) ?? 0) + 1);
432
+ const run = (sending.get(peerId) ?? Promise.resolve())
433
+ .then(() => sendPaced(peerId, connection, ids))
434
+ .catch(report);
435
+ sending.set(peerId, run);
436
+ void run.finally(() => {
437
+ if (sending.get(peerId) === run)
438
+ sending.delete(peerId);
439
+ // A peer that went, or came back on a new connection, starts afresh.
440
+ if (states.get(peerId) !== state)
441
+ return;
442
+ const left = (state.sending.get(collection) ?? 1) - 1;
443
+ if (left > 0)
444
+ return void state.sending.set(collection, left);
445
+ state.sending.delete(collection);
446
+ if (state.beginAfter.delete(collection))
447
+ return void begin(peerId, collection).catch(report);
448
+ levelIfDone(peerId, collection);
449
+ settleIfDone(peerId);
450
+ });
451
+ };
452
+ /**
453
+ * Reads versions from the store a message at a time, and waits for the
454
+ * connection to drain before each: what is held in memory at once is one
455
+ * message, however much the peer lacks. A record's versions go in the same
456
+ * message, in the place of its newest (spec 05 §6.3), so the store is read
457
+ * twice: once to learn which record each version is of, keeping only that,
458
+ * then a message's worth at a time to send.
459
+ */
460
+ const sendPaced = async (peerId, connection, ids) => {
461
+ const open = () => connections.get(peerId) === connection && peers.has(peerId);
462
+ const records = new Map();
463
+ for (const id of ids) {
464
+ if (!open())
465
+ return;
466
+ const version = await storage.getVersion(id);
467
+ if (version)
468
+ getOrAdd(records, version.key, () => []).push(id);
469
+ }
470
+ let message = [];
471
+ let bytes = 0;
472
+ const flush = async () => {
473
+ if (message.length === 0)
474
+ return;
475
+ if (await drained(peerId, connection))
476
+ send(peerId, { type: 'versions', versions: message });
477
+ message = [];
478
+ bytes = 0;
479
+ };
480
+ for (const record of records.values()) {
481
+ if (!open())
482
+ return;
483
+ const versions = [];
484
+ for (const id of record) {
485
+ const version = await storage.getVersion(id);
486
+ if (version)
487
+ versions.push({ version, size: wireBytes(version) });
488
+ }
489
+ const size = versions.reduce((sum, one) => sum + one.size, 0);
490
+ // A record that doesn't fit in what is left starts a message of its own.
491
+ if (message.length + versions.length > MAX_IDS_PER_REQUEST || full(message.length, bytes, size))
492
+ await flush();
493
+ // One with more than a message holds goes over several.
494
+ for (const one of versions) {
495
+ if (full(message.length, bytes, one.size))
496
+ await flush();
497
+ message.push(one.version);
498
+ bytes += one.size;
499
+ }
500
+ }
501
+ await flush();
502
+ };
352
503
  const onReconciled = async (peerId, id, message, held) => {
353
504
  const state = states.get(peerId);
354
505
  const session = state && [...state.sessions.values()].find((s) => s.id === id);
@@ -386,29 +537,71 @@ export function createSyncEngine(config) {
386
537
  settleIfDone(peerId);
387
538
  };
388
539
  // ─── Answering ─────────────────────────────────────────────────────
389
- const onReconcile = async (peerId, id, collection, message) => {
540
+ const onReconcile = async (peerId, id, collection, message, shallow) => {
390
541
  if (typeof collection !== 'string' || typeof message !== 'string')
391
542
  return;
392
543
  if (!holdsCollection(ourHolds(), collection)) {
393
544
  send(peerId, { type: 'reconciled', id, message: '', held: false });
394
545
  return;
395
546
  }
396
- const responder = createReconciler(await storage.items(collection), {
397
- initiator: false,
398
- frameSizeLimit: FRAME_SIZE_LIMIT,
399
- });
547
+ const responder = createReconciler(await (shallow ? storage.shallowItems(collection) : storage.items(collection)), { initiator: false, frameSizeLimit: FRAME_SIZE_LIMIT });
400
548
  const round = await responder.reconcile(base64UrlDecode(message));
401
549
  send(peerId, { type: 'reconciled', id, message: base64UrlEncode(round.message) });
402
550
  };
403
- const serveWant = async (peerId, id, ids) => {
404
- const versions = [];
405
- for (const v of ids.slice(0, MAX_IDS_PER_REQUEST)) {
406
- const version = typeof v === 'string' && cidDigest(v) ? await storage.getVersion(v) : null;
407
- if (version)
551
+ /**
552
+ * Answers a `want` once the connection has room, after the answers before
553
+ * it. Off the peer's message queue, like `sendAll`, and apart from it, so an
554
+ * answer never waits behind a large push. A peer that asks and doesn't read
555
+ * holds up only its own answers, and what waits is the ids it asked for.
556
+ */
557
+ const serveWant = (peerId, id, ids) => {
558
+ const connection = connections.get(peerId);
559
+ const asked = ids.slice(0, MAX_IDS_PER_REQUEST).filter((v) => typeof v === 'string' && !!cidDigest(v));
560
+ // An honest peer has 4 in flight. Past this many waiting, one that doesn't read is asking
561
+ // for memory, not versions: left unanswered, an honest asker gives up on it and asks again.
562
+ const queued = (waitingAnswers.get(peerId) ?? 0) + 1;
563
+ if (queued > MAX_ANSWERS_WAITING)
564
+ return;
565
+ waitingAnswers.set(peerId, queued);
566
+ const run = (answering.get(peerId) ?? Promise.resolve())
567
+ .then(async () => {
568
+ if (!(await drained(peerId, connection)))
569
+ return;
570
+ const versions = [];
571
+ let bytes = 0;
572
+ // In the order asked, until a message's worth: the asker asks again for what comes after.
573
+ for (const v of asked) {
574
+ const version = await storage.getVersion(v);
575
+ if (!version)
576
+ continue;
577
+ const size = wireBytes(version);
578
+ if (full(versions.length, bytes, size))
579
+ break;
408
580
  versions.push(version);
409
- }
410
- // Always answered, even empty: the asker counts replies to know it is done.
411
- send(peerId, { type: 'versions', id, versions });
581
+ bytes += size;
582
+ }
583
+ // Always answered, even empty: the asker counts replies to know it is done.
584
+ if (connections.get(peerId) === connection)
585
+ send(peerId, { type: 'versions', id, versions });
586
+ })
587
+ .catch(report);
588
+ answering.set(peerId, run);
589
+ void run.finally(() => {
590
+ if (answering.get(peerId) === run)
591
+ answering.delete(peerId);
592
+ const left = (waitingAnswers.get(peerId) ?? 1) - 1;
593
+ if (left > 0)
594
+ waitingAnswers.set(peerId, left);
595
+ else
596
+ waitingAnswers.delete(peerId);
597
+ });
598
+ };
599
+ /** Waits until the connection has under a message's worth left to send; false once it is gone */
600
+ const drained = async (peerId, connection) => {
601
+ const open = () => connections.get(peerId) === connection && peers.has(peerId);
602
+ while (open() && (config.backlog?.(peerId) ?? 0) > MAX_MESSAGE_BYTES)
603
+ await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS));
604
+ return open();
412
605
  };
413
606
  const onVersions = async (peerId, id, versions) => {
414
607
  // Sent unasked, because we lacked them: each still passes the gatekeeper.
@@ -418,8 +611,19 @@ export function createSyncEngine(config) {
418
611
  const asked = state?.wants.get(id);
419
612
  if (!state || !asked)
420
613
  return;
421
- await admitAll(peerId, versions.filter((v) => asked.ids.has(v?.id)));
614
+ const answered = versions.filter((v) => asked.ids.has(v?.id));
615
+ await admitAll(peerId, answered);
422
616
  state.wants.delete(id);
617
+ // An answer that stopped at a message's worth: what came after the last sent is asked for again,
618
+ // ahead of the rest, where it was.
619
+ // One that brought none is not, so a peer lacking them is asked once.
620
+ const order = [...asked.ids];
621
+ const sent = new Set(answered.map((v) => v.id));
622
+ let last = order.length - 1;
623
+ while (last >= 0 && !sent.has(order[last]))
624
+ last--;
625
+ if (last >= 0 && last < order.length - 1)
626
+ want(peerId, asked.collection, order.slice(last + 1), true);
423
627
  pump(peerId);
424
628
  levelIfDone(peerId, asked.collection);
425
629
  settleIfDone(peerId);
@@ -458,13 +662,13 @@ export function createSyncEngine(config) {
458
662
  return;
459
663
  switch (msg.type) {
460
664
  case 'hello':
461
- return await onHello(peerId, msg.sums, msg.holds, msg.reply === true);
665
+ return await onHello(peerId, msg.sums, msg.holds, msg.reply === true, msg.shallow === true);
462
666
  case 'reconcile':
463
- return await onReconcile(peerId, msg.id, msg.collection, msg.message);
667
+ return await onReconcile(peerId, msg.id, msg.collection, msg.message, msg.shallow === true);
464
668
  case 'reconciled':
465
669
  return await onReconciled(peerId, msg.id, msg.message, msg.held !== false);
466
670
  case 'want':
467
- return await serveWant(peerId, msg.id, Array.isArray(msg.ids) ? msg.ids : []);
671
+ return serveWant(peerId, msg.id, Array.isArray(msg.ids) ? msg.ids : []);
468
672
  case 'versions':
469
673
  return await onVersions(peerId, msg.id, Array.isArray(msg.versions) ? msg.versions : []);
470
674
  case 'push-update':
@@ -502,6 +706,8 @@ export function createSyncEngine(config) {
502
706
  clearInterval(intervalId);
503
707
  intervalId = null;
504
708
  states.clear();
709
+ // Every send still going ends: none of the connections it was for is current any more.
710
+ connections.clear();
505
711
  },
506
712
  handleMessage(peerId, message) {
507
713
  // One peer's messages run in the order they came: a hello must not
@@ -552,6 +758,11 @@ export function createSyncEngine(config) {
552
758
  };
553
759
  }
554
760
  /** Takes up to one want's worth of queued ids, all of one collection, off the front of the queue */
761
+ /** Whether the next id to ask for is one held records wait on */
762
+ function nextIsFirst(state) {
763
+ const next = state.queued.find((entry) => state.queuedIds.has(entry.id));
764
+ return next?.first ?? false;
765
+ }
555
766
  function takeBatch(state) {
556
767
  const ids = [];
557
768
  let collection = null;
@@ -576,6 +787,9 @@ const newPeerState = () => ({
576
787
  queued: [],
577
788
  queuedIds: new Set(),
578
789
  holds: 'all',
790
+ shallow: false,
791
+ sending: new Map(),
792
+ beginAfter: new Set(),
579
793
  });
580
794
  const refusal = (peerId, id) => `${peerId}\n${id}`;
581
795
  const abandoned = (peerId, why) => new Error(`Sync with ${peerId} abandoned: ${why}`);