@minca/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{session → hosting}/hosting.d.ts +2 -8
- package/dist/hosting/hosting.d.ts.map +1 -0
- package/dist/{session → hosting}/hosting.js +3 -3
- package/dist/hosting/hosting.js.map +1 -0
- package/dist/identity/abilities.d.ts +0 -2
- package/dist/identity/abilities.d.ts.map +1 -1
- package/dist/identity/abilities.js +2 -2
- package/dist/identity/abilities.js.map +1 -1
- package/dist/identity/account-vault.d.ts +0 -2
- package/dist/identity/account-vault.d.ts.map +1 -1
- package/dist/identity/account-vault.js +1 -1
- package/dist/identity/account-vault.js.map +1 -1
- package/dist/identity/crypto-p256.d.ts +1 -1
- package/dist/identity/crypto-p256.d.ts.map +1 -1
- package/dist/identity/ucan.d.ts +2 -1
- package/dist/identity/ucan.d.ts.map +1 -1
- package/dist/identity/ucan.js +1 -1
- package/dist/identity/ucan.js.map +1 -1
- package/dist/inbox/inbox.d.ts +0 -21
- package/dist/inbox/inbox.d.ts.map +1 -1
- package/dist/inbox/inbox.js +4 -4
- package/dist/inbox/inbox.js.map +1 -1
- package/dist/inbox/index.d.ts +4 -5
- package/dist/inbox/index.d.ts.map +1 -1
- package/dist/inbox/index.js +2 -3
- package/dist/inbox/index.js.map +1 -1
- package/dist/inbox/names.d.ts +0 -4
- package/dist/inbox/names.d.ts.map +1 -1
- package/dist/inbox/names.js +2 -2
- package/dist/inbox/names.js.map +1 -1
- package/dist/index.d.ts +50 -84
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -60
- package/dist/index.js.map +1 -1
- package/dist/network/mesh.d.ts.map +1 -1
- package/dist/network/mesh.js +23 -3
- package/dist/network/mesh.js.map +1 -1
- package/dist/network/network-manager.d.ts +5 -0
- package/dist/network/network-manager.d.ts.map +1 -1
- package/dist/network/network-manager.js +3 -0
- package/dist/network/network-manager.js.map +1 -1
- package/dist/network/peer-auth.d.ts +17 -5
- package/dist/network/peer-auth.d.ts.map +1 -1
- package/dist/network/peer-auth.js +19 -9
- package/dist/network/peer-auth.js.map +1 -1
- package/dist/network/rendezvous.d.ts +2 -0
- package/dist/network/rendezvous.d.ts.map +1 -1
- package/dist/network/rendezvous.js +5 -0
- package/dist/network/rendezvous.js.map +1 -1
- package/dist/network/rtc-transport.d.ts.map +1 -1
- package/dist/network/rtc-transport.js +9 -3
- package/dist/network/rtc-transport.js.map +1 -1
- package/dist/network/transport.d.ts +12 -0
- package/dist/network/transport.d.ts.map +1 -1
- package/dist/network/ws-transport.d.ts +1 -1
- package/dist/network/ws-transport.d.ts.map +1 -1
- package/dist/network/ws-transport.js +14 -4
- package/dist/network/ws-transport.js.map +1 -1
- package/dist/node/carrier.d.ts +1 -33
- package/dist/node/carrier.d.ts.map +1 -1
- package/dist/node/carrier.js +38 -68
- package/dist/node/carrier.js.map +1 -1
- package/dist/node/index.d.ts +5 -16
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +2 -9
- package/dist/node/index.js.map +1 -1
- package/dist/node/node-carriers.d.ts.map +1 -1
- package/dist/node/node-carriers.js +21 -52
- package/dist/node/node-carriers.js.map +1 -1
- package/dist/node/node-context.d.ts +0 -2
- package/dist/node/node-context.d.ts.map +1 -1
- package/dist/node/node-context.js.map +1 -1
- package/dist/node/node-hosting.d.ts.map +1 -1
- package/dist/node/node-hosting.js +14 -5
- package/dist/node/node-hosting.js.map +1 -1
- package/dist/node/node-records.js +2 -2
- package/dist/node/node-records.js.map +1 -1
- package/dist/node/node.d.ts.map +1 -1
- package/dist/node/node.js +152 -36
- package/dist/node/node.js.map +1 -1
- package/dist/node/space-checks.d.ts +3 -1
- package/dist/node/space-checks.d.ts.map +1 -1
- package/dist/node/space-checks.js +7 -3
- package/dist/node/space-checks.js.map +1 -1
- package/dist/node/space-holding.d.ts +5 -0
- package/dist/node/space-holding.d.ts.map +1 -1
- package/dist/node/space-holding.js +90 -5
- package/dist/node/space-holding.js.map +1 -1
- package/dist/node/space-lines.d.ts +3 -0
- package/dist/node/space-lines.d.ts.map +1 -1
- package/dist/node/space-lines.js +13 -1
- package/dist/node/space-lines.js.map +1 -1
- package/dist/node/space-membership.js +4 -4
- package/dist/node/space-membership.js.map +1 -1
- package/dist/node/space-mirrors.d.ts +1 -1
- package/dist/node/space-mirrors.d.ts.map +1 -1
- package/dist/node/space-mirrors.js +17 -5
- package/dist/node/space-mirrors.js.map +1 -1
- package/dist/node/space-peers.d.ts +7 -2
- package/dist/node/space-peers.d.ts.map +1 -1
- package/dist/node/space-peers.js +44 -10
- package/dist/node/space-peers.js.map +1 -1
- package/dist/node/space-runtime.d.ts +7 -3
- package/dist/node/space-runtime.d.ts.map +1 -1
- package/dist/node/space-runtime.js +32 -10
- package/dist/node/space-runtime.js.map +1 -1
- package/dist/node/space-standing.d.ts +1 -1
- package/dist/node/space-standing.d.ts.map +1 -1
- package/dist/node/space-standing.js +23 -42
- package/dist/node/space-standing.js.map +1 -1
- package/dist/node/space-warrants.js +1 -1
- package/dist/node/space-warrants.js.map +1 -1
- package/dist/node/space-writing.d.ts.map +1 -1
- package/dist/node/space-writing.js +21 -32
- package/dist/node/space-writing.js.map +1 -1
- package/dist/node/stores.d.ts +9 -57
- package/dist/node/stores.d.ts.map +1 -1
- package/dist/node/stores.js +1 -72
- package/dist/node/stores.js.map +1 -1
- package/dist/node/types.d.ts +19 -8
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/writer-keys.d.ts +15 -14
- package/dist/node/writer-keys.d.ts.map +1 -1
- package/dist/node/writer-keys.js +3 -115
- package/dist/node/writer-keys.js.map +1 -1
- package/dist/privacy/direct.d.ts +0 -7
- package/dist/privacy/direct.d.ts.map +1 -1
- package/dist/privacy/direct.js +2 -2
- package/dist/privacy/direct.js.map +1 -1
- package/dist/query/filter.d.ts +0 -2
- package/dist/query/filter.d.ts.map +1 -1
- package/dist/query/filter.js +1 -1
- package/dist/query/filter.js.map +1 -1
- package/dist/query/types.d.ts +2 -6
- package/dist/query/types.d.ts.map +1 -1
- package/dist/query/types.js.map +1 -1
- package/dist/records/key.d.ts +18 -19
- package/dist/records/key.d.ts.map +1 -1
- package/dist/records/key.js +55 -43
- package/dist/records/key.js.map +1 -1
- package/dist/records/links.js +1 -1
- package/dist/records/rules.d.ts +1 -1
- package/dist/records/rules.d.ts.map +1 -1
- package/dist/records/rules.js +6 -4
- package/dist/records/rules.js.map +1 -1
- package/dist/schemas/direct.d.ts +70 -0
- package/dist/schemas/direct.d.ts.map +1 -0
- package/dist/schemas/direct.js +39 -0
- package/dist/schemas/direct.js.map +1 -0
- package/dist/schemas/index.d.ts +10 -73
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +8 -66
- package/dist/schemas/index.js.map +1 -1
- package/dist/space/account-registry.d.ts +0 -3
- package/dist/space/account-registry.d.ts.map +1 -1
- package/dist/space/account-registry.js.map +1 -1
- package/dist/space/own-automations.d.ts +51 -95
- package/dist/space/own-automations.d.ts.map +1 -1
- package/dist/space/own-automations.js +100 -295
- package/dist/space/own-automations.js.map +1 -1
- package/dist/space/pass.d.ts +8 -3
- package/dist/space/pass.d.ts.map +1 -1
- package/dist/space/pass.js +4 -3
- package/dist/space/pass.js.map +1 -1
- package/dist/space/space-manager.d.ts +1 -3
- package/dist/space/space-manager.d.ts.map +1 -1
- package/dist/space/space-manager.js +5 -7
- package/dist/space/space-manager.js.map +1 -1
- package/dist/storage/body-cache.d.ts +29 -0
- package/dist/storage/body-cache.d.ts.map +1 -0
- package/dist/storage/body-cache.js +64 -0
- package/dist/storage/body-cache.js.map +1 -0
- package/dist/storage/encrypted-adapter.d.ts +0 -5
- package/dist/storage/encrypted-adapter.d.ts.map +1 -1
- package/dist/storage/encrypted-adapter.js +1 -1
- package/dist/storage/encrypted-adapter.js.map +1 -1
- package/dist/storage/mirror.d.ts.map +1 -1
- package/dist/storage/mirror.js +35 -16
- package/dist/storage/mirror.js.map +1 -1
- package/dist/storage/segment.d.ts +6 -2
- package/dist/storage/segment.d.ts.map +1 -1
- package/dist/storage/segment.js +8 -4
- package/dist/storage/segment.js.map +1 -1
- package/dist/storage/shared-store.d.ts +32 -0
- package/dist/storage/shared-store.d.ts.map +1 -0
- package/dist/storage/{folder-reconcile.js → shared-store.js} +15 -15
- package/dist/storage/shared-store.js.map +1 -0
- package/dist/storage/storage-provider.d.ts +19 -9
- package/dist/storage/storage-provider.d.ts.map +1 -1
- package/dist/storage/storage-provider.js +155 -62
- package/dist/storage/storage-provider.js.map +1 -1
- package/dist/sync/sync-engine.d.ts +17 -4
- package/dist/sync/sync-engine.d.ts.map +1 -1
- package/dist/sync/sync-engine.js +298 -84
- package/dist/sync/sync-engine.js.map +1 -1
- package/dist/sync/sync-messages.d.ts +5 -1
- package/dist/sync/sync-messages.d.ts.map +1 -1
- package/dist/sync/sync-messages.js +1 -1
- package/dist/sync/sync-messages.js.map +1 -1
- package/dist/types.d.ts +19 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +4 -1
- package/dist/types.js.map +1 -1
- package/dist/utils/index.d.ts +5 -5
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +5 -5
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/step.d.ts +8 -0
- package/dist/utils/step.d.ts.map +1 -0
- package/dist/utils/step.js +15 -0
- package/dist/utils/step.js.map +1 -0
- package/docs/README.md +4 -3
- package/docs/actions.md +41 -2
- package/docs/agents.md +13 -10
- package/docs/automations.md +16 -48
- package/docs/blueprints-and-guides.md +2 -2
- package/docs/bots.md +3 -3
- package/docs/building-an-app.md +2 -2
- package/docs/collections.md +2 -4
- package/docs/computed.md +1 -2
- package/docs/node.md +108 -40
- package/docs/proposing-collections.md +3 -3
- package/docs/query-format.md +6 -5
- package/docs/records-and-queries.md +12 -13
- package/docs/sign-in.md +16 -16
- package/docs/spaces.md +0 -5
- package/docs/standard-library.md +6 -8
- package/docs/storage.md +114 -22
- package/docs/usernames.md +1 -1
- package/package.json +15 -25
- package/dist/identity/account-store.d.ts +0 -83
- package/dist/identity/account-store.d.ts.map +0 -1
- package/dist/identity/account-store.js +0 -234
- package/dist/identity/account-store.js.map +0 -1
- package/dist/identity/device-key.d.ts +0 -26
- package/dist/identity/device-key.d.ts.map +0 -1
- package/dist/identity/device-key.js +0 -48
- package/dist/identity/device-key.js.map +0 -1
- package/dist/identity/idb.d.ts +0 -19
- package/dist/identity/idb.d.ts.map +0 -1
- package/dist/identity/idb.js +0 -56
- package/dist/identity/idb.js.map +0 -1
- package/dist/node/actions.d.ts +0 -85
- package/dist/node/actions.d.ts.map +0 -1
- package/dist/node/actions.js +0 -921
- package/dist/node/actions.js.map +0 -1
- package/dist/node/host.d.ts +0 -187
- package/dist/node/host.d.ts.map +0 -1
- package/dist/node/host.js +0 -818
- package/dist/node/host.js.map +0 -1
- package/dist/node/usage.d.ts +0 -32
- package/dist/node/usage.d.ts.map +0 -1
- package/dist/node/usage.js +0 -217
- package/dist/node/usage.js.map +0 -1
- package/dist/node/watch-notifications.d.ts +0 -25
- package/dist/node/watch-notifications.d.ts.map +0 -1
- package/dist/node/watch-notifications.js +0 -166
- package/dist/node/watch-notifications.js.map +0 -1
- package/dist/node/worker.d.ts +0 -83
- package/dist/node/worker.d.ts.map +0 -1
- package/dist/node/worker.js +0 -305
- package/dist/node/worker.js.map +0 -1
- package/dist/node-worker.d.ts +0 -2
- package/dist/node-worker.d.ts.map +0 -1
- package/dist/node-worker.js +0 -16
- package/dist/node-worker.js.map +0 -1
- package/dist/schemas/automations.d.ts +0 -390
- package/dist/schemas/automations.d.ts.map +0 -1
- package/dist/schemas/automations.js +0 -619
- package/dist/schemas/automations.js.map +0 -1
- package/dist/schemas/blueprints.d.ts +0 -122
- package/dist/schemas/blueprints.d.ts.map +0 -1
- package/dist/schemas/blueprints.js +0 -337
- package/dist/schemas/blueprints.js.map +0 -1
- package/dist/schemas/choosing.d.ts +0 -19
- package/dist/schemas/choosing.d.ts.map +0 -1
- package/dist/schemas/choosing.js +0 -69
- package/dist/schemas/choosing.js.map +0 -1
- package/dist/schemas/connectors.d.ts +0 -202
- package/dist/schemas/connectors.d.ts.map +0 -1
- package/dist/schemas/connectors.js +0 -140
- package/dist/schemas/connectors.js.map +0 -1
- package/dist/schemas/cron.d.ts +0 -14
- package/dist/schemas/cron.d.ts.map +0 -1
- package/dist/schemas/cron.js +0 -75
- package/dist/schemas/cron.js.map +0 -1
- package/dist/schemas/geo.d.ts +0 -63
- package/dist/schemas/geo.d.ts.map +0 -1
- package/dist/schemas/geo.js +0 -131
- package/dist/schemas/geo.js.map +0 -1
- package/dist/schemas/guides.d.ts +0 -26
- package/dist/schemas/guides.d.ts.map +0 -1
- package/dist/schemas/guides.js +0 -33
- package/dist/schemas/guides.js.map +0 -1
- package/dist/schemas/library/annotations.d.ts +0 -616
- package/dist/schemas/library/annotations.d.ts.map +0 -1
- package/dist/schemas/library/annotations.js +0 -156
- package/dist/schemas/library/annotations.js.map +0 -1
- package/dist/schemas/library/collections.d.ts +0 -211
- package/dist/schemas/library/collections.d.ts.map +0 -1
- package/dist/schemas/library/collections.js +0 -57
- package/dist/schemas/library/collections.js.map +0 -1
- package/dist/schemas/library/community.d.ts +0 -1136
- package/dist/schemas/library/community.d.ts.map +0 -1
- package/dist/schemas/library/community.js +0 -315
- package/dist/schemas/library/community.js.map +0 -1
- package/dist/schemas/library/life.d.ts +0 -1440
- package/dist/schemas/library/life.d.ts.map +0 -1
- package/dist/schemas/library/life.js +0 -202
- package/dist/schemas/library/life.js.map +0 -1
- package/dist/schemas/library/media.d.ts +0 -1002
- package/dist/schemas/library/media.d.ts.map +0 -1
- package/dist/schemas/library/media.js +0 -110
- package/dist/schemas/library/media.js.map +0 -1
- package/dist/schemas/library/money.d.ts +0 -1221
- package/dist/schemas/library/money.d.ts.map +0 -1
- package/dist/schemas/library/money.js +0 -168
- package/dist/schemas/library/money.js.map +0 -1
- package/dist/schemas/library/planning.d.ts +0 -1107
- package/dist/schemas/library/planning.d.ts.map +0 -1
- package/dist/schemas/library/planning.js +0 -214
- package/dist/schemas/library/planning.js.map +0 -1
- package/dist/schemas/library/publishing.d.ts +0 -1499
- package/dist/schemas/library/publishing.d.ts.map +0 -1
- package/dist/schemas/library/publishing.js +0 -285
- package/dist/schemas/library/publishing.js.map +0 -1
- package/dist/schemas/library/social.d.ts +0 -1257
- package/dist/schemas/library/social.d.ts.map +0 -1
- package/dist/schemas/library/social.js +0 -147
- package/dist/schemas/library/social.js.map +0 -1
- package/dist/schemas/standard.d.ts +0 -21
- package/dist/schemas/standard.d.ts.map +0 -1
- package/dist/schemas/standard.js +0 -39
- package/dist/schemas/standard.js.map +0 -1
- package/dist/session/account-hint.d.ts +0 -13
- package/dist/session/account-hint.d.ts.map +0 -1
- package/dist/session/account-hint.js +0 -54
- package/dist/session/account-hint.js.map +0 -1
- package/dist/session/auth.d.ts +0 -411
- package/dist/session/auth.d.ts.map +0 -1
- package/dist/session/auth.js +0 -1529
- package/dist/session/auth.js.map +0 -1
- package/dist/session/channel-room.d.ts +0 -26
- package/dist/session/channel-room.d.ts.map +0 -1
- package/dist/session/channel-room.js +0 -56
- package/dist/session/channel-room.js.map +0 -1
- package/dist/session/connect-code.d.ts +0 -127
- package/dist/session/connect-code.d.ts.map +0 -1
- package/dist/session/connect-code.js +0 -408
- package/dist/session/connect-code.js.map +0 -1
- package/dist/session/connect.d.ts +0 -386
- package/dist/session/connect.d.ts.map +0 -1
- package/dist/session/connect.js +0 -583
- package/dist/session/connect.js.map +0 -1
- package/dist/session/connection.d.ts +0 -135
- package/dist/session/connection.d.ts.map +0 -1
- package/dist/session/connection.js +0 -215
- package/dist/session/connection.js.map +0 -1
- package/dist/session/credentials.d.ts +0 -30
- package/dist/session/credentials.d.ts.map +0 -1
- package/dist/session/credentials.js +0 -63
- package/dist/session/credentials.js.map +0 -1
- package/dist/session/hosting.d.ts.map +0 -1
- package/dist/session/hosting.js.map +0 -1
- package/dist/session/index.d.ts +0 -29
- package/dist/session/index.d.ts.map +0 -1
- package/dist/session/index.js +0 -21
- package/dist/session/index.js.map +0 -1
- package/dist/session/pairing.d.ts +0 -97
- package/dist/session/pairing.d.ts.map +0 -1
- package/dist/session/pairing.js +0 -251
- package/dist/session/pairing.js.map +0 -1
- package/dist/session/password-strength.d.ts +0 -22
- package/dist/session/password-strength.d.ts.map +0 -1
- package/dist/session/password-strength.js +0 -130
- package/dist/session/password-strength.js.map +0 -1
- package/dist/session/places.d.ts +0 -71
- package/dist/session/places.d.ts.map +0 -1
- package/dist/session/places.js +0 -130
- package/dist/session/places.js.map +0 -1
- package/dist/session/popup.d.ts +0 -13
- package/dist/session/popup.d.ts.map +0 -1
- package/dist/session/popup.js +0 -23
- package/dist/session/popup.js.map +0 -1
- package/dist/session/stay-signed-in.d.ts +0 -49
- package/dist/session/stay-signed-in.d.ts.map +0 -1
- package/dist/session/stay-signed-in.js +0 -141
- package/dist/session/stay-signed-in.js.map +0 -1
- package/dist/storage/blob/directory.d.ts +0 -14
- package/dist/storage/blob/directory.d.ts.map +0 -1
- package/dist/storage/blob/directory.js +0 -87
- package/dist/storage/blob/directory.js.map +0 -1
- package/dist/storage/blob/memory.d.ts +0 -6
- package/dist/storage/blob/memory.d.ts.map +0 -1
- package/dist/storage/blob/memory.js +0 -20
- package/dist/storage/blob/memory.js.map +0 -1
- package/dist/storage/blob/s3.d.ts +0 -16
- package/dist/storage/blob/s3.d.ts.map +0 -1
- package/dist/storage/blob/s3.js +0 -91
- package/dist/storage/blob/s3.js.map +0 -1
- package/dist/storage/directory-access.d.ts +0 -36
- package/dist/storage/directory-access.d.ts.map +0 -1
- package/dist/storage/directory-access.js +0 -89
- package/dist/storage/directory-access.js.map +0 -1
- package/dist/storage/folder-adapter.d.ts +0 -94
- package/dist/storage/folder-adapter.d.ts.map +0 -1
- package/dist/storage/folder-adapter.js +0 -215
- package/dist/storage/folder-adapter.js.map +0 -1
- package/dist/storage/folder-reconcile.d.ts +0 -33
- package/dist/storage/folder-reconcile.d.ts.map +0 -1
- package/dist/storage/folder-reconcile.js.map +0 -1
- package/dist/storage/index.d.ts +0 -6
- package/dist/storage/index.d.ts.map +0 -1
- package/dist/storage/index.js +0 -5
- package/dist/storage/index.js.map +0 -1
- package/dist/storage/indexeddb-adapter.d.ts +0 -8
- package/dist/storage/indexeddb-adapter.d.ts.map +0 -1
- package/dist/storage/indexeddb-adapter.js +0 -103
- package/dist/storage/indexeddb-adapter.js.map +0 -1
package/dist/sync/sync-engine.js
CHANGED
|
@@ -18,18 +18,36 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { getOrAdd } from '../utils/map.js';
|
|
20
20
|
import { cidDigest, cidOfDigest } from '../utils/hash.js';
|
|
21
|
-
import { base64UrlDecode, base64UrlEncode, bytesToHex } from '../utils/encoding.js';
|
|
21
|
+
import { base64UrlDecode, base64UrlEncode, bytesToHex, utf8Encode } from '../utils/encoding.js';
|
|
22
22
|
import { createEmitter } from '../utils/events.js';
|
|
23
23
|
import { createReconciler, fingerprintOf } from './negentropy.js';
|
|
24
24
|
import { parseSyncMessage, SYNC_PROTOCOL_VERSION, } from './sync-messages.js';
|
|
25
|
-
/**
|
|
26
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Records waiting for what they depend on, past which nothing more is asked
|
|
27
|
+
* for but what they wait on: a long history fills in from below instead of
|
|
28
|
+
* being dropped and asked for again. High, since below it histories arrive
|
|
29
|
+
* as fast as the peer sends them: pausing at 1,000 made 100 records of 50
|
|
30
|
+
* edits each settle in 30 s instead of 2.
|
|
31
|
+
*/
|
|
32
|
+
const MAX_WAITING = 20_000;
|
|
33
|
+
/** Records held while what they wait on is fetched; past this the oldest is let go and asked for again later */
|
|
34
|
+
const MAX_HELD = 2 * MAX_WAITING;
|
|
27
35
|
/** Refusals remembered, so reconciling doesn't ask a peer for the same bad version every round */
|
|
28
36
|
const MAX_REFUSED = 10_000;
|
|
29
37
|
/** Records asked for, or sent, in one message */
|
|
30
38
|
const MAX_IDS_PER_REQUEST = 200;
|
|
39
|
+
/**
|
|
40
|
+
* Bytes of versions in one message, besides the count: a message of large
|
|
41
|
+
* records is held whole in memory at both ends, and a socket refuses one past
|
|
42
|
+
* its limit. One version larger than this still goes, alone.
|
|
43
|
+
*/
|
|
44
|
+
const MAX_MESSAGE_BYTES = 1024 * 1024;
|
|
45
|
+
/** How often a sender waiting for its connection to drain looks again */
|
|
46
|
+
const DRAIN_POLL_MS = 25;
|
|
31
47
|
/** `want`s in flight to one peer at a time; the rest queue */
|
|
32
48
|
const MAX_WANTS_IN_FLIGHT = 4;
|
|
49
|
+
/** Answers to one peer's `want`s waiting for room, past which more are left unanswered */
|
|
50
|
+
const MAX_ANSWERS_WAITING = 4 * MAX_WANTS_IN_FLIGHT;
|
|
33
51
|
/** Largest Negentropy message, in bytes before base64: well inside a data channel's limit */
|
|
34
52
|
const FRAME_SIZE_LIMIT = 32_000;
|
|
35
53
|
/** Collections one hello may name. Past this it is hostile, not big. */
|
|
@@ -50,36 +68,10 @@ function readHolds(value) {
|
|
|
50
68
|
}
|
|
51
69
|
/** Collection definitions first, then records by version: what others depend on comes first */
|
|
52
70
|
const rank = (e) => (e?.collection === 'sys.collection' ? -1 : (e?.seq ?? 0));
|
|
53
|
-
/**
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
*/
|
|
58
|
-
function byRecord(versions, size) {
|
|
59
|
-
const records = new Map();
|
|
60
|
-
for (const version of versions)
|
|
61
|
-
getOrAdd(records, version.key, () => []).push(version);
|
|
62
|
-
const messages = [];
|
|
63
|
-
let message = [];
|
|
64
|
-
for (const record of records.values()) {
|
|
65
|
-
if (message.length + record.length > size && message.length > 0) {
|
|
66
|
-
messages.push(message);
|
|
67
|
-
message = [];
|
|
68
|
-
}
|
|
69
|
-
// A record with more versions than fit in one message goes over several.
|
|
70
|
-
for (let i = 0; i < record.length; i += size) {
|
|
71
|
-
const part = record.slice(i, i + size);
|
|
72
|
-
if (message.length + part.length > size) {
|
|
73
|
-
messages.push(message);
|
|
74
|
-
message = [];
|
|
75
|
-
}
|
|
76
|
-
message.push(...part);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
if (message.length > 0)
|
|
80
|
-
messages.push(message);
|
|
81
|
-
return messages;
|
|
82
|
-
}
|
|
71
|
+
/** The bytes a version takes in a message, as UTF-8 JSON */
|
|
72
|
+
const wireBytes = (version) => utf8Encode(JSON.stringify(version)).length;
|
|
73
|
+
/** Whether a message already holding `bytes` in `count` versions has no room for one of `size` */
|
|
74
|
+
const full = (count, bytes, size) => count >= MAX_IDS_PER_REQUEST || (count > 0 && bytes + size > MAX_MESSAGE_BYTES);
|
|
83
75
|
/** Creates a sync engine orchestrator. */
|
|
84
76
|
export function createSyncEngine(config) {
|
|
85
77
|
const { storageProvider: storage, sendToPeer, heartbeatInterval = 30000, validate, self } = config;
|
|
@@ -91,8 +83,12 @@ export function createSyncEngine(config) {
|
|
|
91
83
|
const states = new Map();
|
|
92
84
|
/** Peers found level with this node, until it takes in or writes anything more */
|
|
93
85
|
const level = new Set();
|
|
94
|
-
/** Records that could not be judged yet,
|
|
86
|
+
/** Records that could not be judged yet, until what they wait for is admitted */
|
|
95
87
|
const waiting = new Map();
|
|
88
|
+
/** Waiting records by a version of their own chain they lack (`prev` or first): tried again once it is stored */
|
|
89
|
+
const waitingOn = new Map();
|
|
90
|
+
/** Waiting records that lack nothing of their own chain, so wait for something else: tried after every intake */
|
|
91
|
+
const waitingOther = new Set();
|
|
96
92
|
/**
|
|
97
93
|
* Refused versions, by peer and id, oldest first. An id doesn't cover the
|
|
98
94
|
* signature, so a stranger's mangled copy must not stop us fetching the real one.
|
|
@@ -102,6 +98,11 @@ export function createSyncEngine(config) {
|
|
|
102
98
|
const connections = new Map();
|
|
103
99
|
/** Where each peer's messages are up to */
|
|
104
100
|
const inOrder = new Map();
|
|
101
|
+
/** What is still being sent to each peer, so one session's versions go after another's */
|
|
102
|
+
const sending = new Map();
|
|
103
|
+
/** Answers to each peer's `want`s still waiting for room, in the order asked, and how many */
|
|
104
|
+
const answering = new Map();
|
|
105
|
+
const waitingAnswers = new Map();
|
|
105
106
|
let lastConnection = 0;
|
|
106
107
|
let intervalId = null;
|
|
107
108
|
let nextId = 1;
|
|
@@ -112,7 +113,7 @@ export function createSyncEngine(config) {
|
|
|
112
113
|
// ─── Taking in versions ────────────────────────────────────────────
|
|
113
114
|
/** Stores a version from a peer if the gatekeeper allows it; one that must wait names what it waits for */
|
|
114
115
|
const admit = async (peerId, version, intake) => {
|
|
115
|
-
const verdict = validate ? await validate(version) : { valid: true };
|
|
116
|
+
const verdict = validate ? await validate(version, peerId) : { valid: true };
|
|
116
117
|
if (!verdict.valid) {
|
|
117
118
|
if (verdict.later)
|
|
118
119
|
await holdBack(peerId, version, intake);
|
|
@@ -128,31 +129,71 @@ export function createSyncEngine(config) {
|
|
|
128
129
|
};
|
|
129
130
|
const holdBack = async (peerId, version, intake) => {
|
|
130
131
|
waiting.set(version.id, { peerId, version });
|
|
131
|
-
|
|
132
|
+
if (waiting.size > MAX_HELD)
|
|
133
|
+
letGoOldest();
|
|
132
134
|
const dependsOn = version.seq > 0 ? [version.genesis, version.prev] : [];
|
|
135
|
+
const lacking = [];
|
|
133
136
|
for (const needed of dependsOn) {
|
|
134
|
-
if (typeof needed
|
|
135
|
-
|
|
137
|
+
if (typeof needed !== 'string' || (await storage.getVersion(needed)))
|
|
138
|
+
continue;
|
|
139
|
+
lacking.push(needed);
|
|
140
|
+
// Syncing shallow, the version before is not fetched: a keeper's copy of this one comes without it.
|
|
141
|
+
if (needed === version.prev && config.shallowWith?.(peerId))
|
|
142
|
+
continue;
|
|
143
|
+
intake.firsts.set(needed, version.collection);
|
|
136
144
|
}
|
|
145
|
+
if (lacking.length === 0)
|
|
146
|
+
waitingOther.add(version.id);
|
|
147
|
+
for (const needed of lacking)
|
|
148
|
+
getOrAdd(waitingOn, needed, () => new Set()).add(version.id);
|
|
149
|
+
};
|
|
150
|
+
/** Lets the longest-waiting record go, still to be asked for: dropped and forgotten, a peer would look level without it */
|
|
151
|
+
const letGoOldest = () => {
|
|
152
|
+
const [id, held] = waiting.entries().next().value ?? [];
|
|
153
|
+
if (!id || !held)
|
|
154
|
+
return;
|
|
155
|
+
waiting.delete(id);
|
|
156
|
+
waitingOther.delete(id);
|
|
157
|
+
if (peers.has(held.peerId))
|
|
158
|
+
want(held.peerId, held.version.collection, [id]);
|
|
137
159
|
};
|
|
138
160
|
const refuse = (peerId, version, reason) => {
|
|
139
161
|
refused.add(refusal(peerId, version.id));
|
|
140
162
|
dropOldest(refused, MAX_REFUSED);
|
|
141
163
|
emit('rejected', peerId, version, reason);
|
|
142
164
|
};
|
|
143
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* Tries again what waited for the versions just stored, and what those let
|
|
167
|
+
* in in turn, then what waits for anything else; each sender hears what was
|
|
168
|
+
* stored. Each waiting record is tried once per version it lacked: trying
|
|
169
|
+
* every one on every arrival took a 1,000-edit history quadratic time.
|
|
170
|
+
*/
|
|
144
171
|
const retryWaiting = async (intake) => {
|
|
145
172
|
const stored = new Map();
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
progressed = true;
|
|
173
|
+
const tryAgain = async (id) => {
|
|
174
|
+
const held = waiting.get(id);
|
|
175
|
+
if (!held)
|
|
176
|
+
return;
|
|
177
|
+
waiting.delete(id);
|
|
178
|
+
waitingOther.delete(id);
|
|
179
|
+
if (await admit(held.peerId, held.version, intake))
|
|
154
180
|
getOrAdd(stored, held.peerId, () => []).push(id);
|
|
181
|
+
};
|
|
182
|
+
let next = 0;
|
|
183
|
+
for (;;) {
|
|
184
|
+
// Stored versions are appended to `placed` as they go in, so this follows a chain to its end.
|
|
185
|
+
for (; next < intake.placed.length; next++) {
|
|
186
|
+
const placedId = intake.placed[next].id;
|
|
187
|
+
const dependents = waitingOn.get(placedId);
|
|
188
|
+
waitingOn.delete(placedId);
|
|
189
|
+
for (const id of dependents ?? [])
|
|
190
|
+
await tryAgain(id);
|
|
155
191
|
}
|
|
192
|
+
const placedBefore = intake.placed.length;
|
|
193
|
+
for (const id of [...waitingOther])
|
|
194
|
+
await tryAgain(id);
|
|
195
|
+
if (intake.placed.length === placedBefore)
|
|
196
|
+
break;
|
|
156
197
|
}
|
|
157
198
|
for (const [peerId, ids] of stored)
|
|
158
199
|
if (peers.has(peerId))
|
|
@@ -165,7 +206,9 @@ export function createSyncEngine(config) {
|
|
|
165
206
|
const want = (peerId, collection, ids, ahead = false) => {
|
|
166
207
|
const first = ahead || collection.startsWith('sys.');
|
|
167
208
|
const state = stateOf(peerId);
|
|
168
|
-
const entries = ids
|
|
209
|
+
const entries = ids
|
|
210
|
+
.filter((id) => first || !state.queuedIds.has(id))
|
|
211
|
+
.map((id) => ({ id, collection, first }));
|
|
169
212
|
for (const { id } of entries)
|
|
170
213
|
state.queuedIds.add(id);
|
|
171
214
|
if (first)
|
|
@@ -180,6 +223,8 @@ export function createSyncEngine(config) {
|
|
|
180
223
|
if (!state)
|
|
181
224
|
return;
|
|
182
225
|
while (state.wants.size < MAX_WANTS_IN_FLIGHT && state.queuedIds.size > 0) {
|
|
226
|
+
if (waiting.size >= MAX_WAITING && !nextIsFirst(state))
|
|
227
|
+
break;
|
|
183
228
|
const batch = takeBatch(state);
|
|
184
229
|
if (!batch)
|
|
185
230
|
break;
|
|
@@ -215,9 +260,13 @@ export function createSyncEngine(config) {
|
|
|
215
260
|
/** Asks at once for the versions still waited on, unless asked, refused or already here */
|
|
216
261
|
const askFirsts = async (peerId, firsts) => {
|
|
217
262
|
const inFlight = askedOf(peerId);
|
|
263
|
+
const state = states.get(peerId);
|
|
264
|
+
// Coming anyway, a batch at a time: asking for each one alone walks a chain a link per round trip.
|
|
265
|
+
// Or here already, waiting in turn for the one below it: asked again, it comes again, and waits again.
|
|
266
|
+
const coming = (id, collection) => inFlight.has(id) || waiting.has(id) || !!state?.queuedIds.has(id) || !!state?.sessions.has(collection);
|
|
218
267
|
const byCollection = new Map();
|
|
219
268
|
for (const [id, collection] of firsts) {
|
|
220
|
-
if (
|
|
269
|
+
if (coming(id, collection) || isRefused(peerId, id) || (await storage.getVersion(id)))
|
|
221
270
|
continue;
|
|
222
271
|
getOrAdd(byCollection, collection, () => []).push(id);
|
|
223
272
|
}
|
|
@@ -227,10 +276,12 @@ export function createSyncEngine(config) {
|
|
|
227
276
|
/** Ids asked of a peer and not yet answered */
|
|
228
277
|
const askedOf = (peerId) => new Set([...(states.get(peerId)?.wants.values() ?? [])].flatMap((w) => [...w.ids]));
|
|
229
278
|
// ─── Hello ─────────────────────────────────────────────────────────
|
|
230
|
-
/**
|
|
231
|
-
const
|
|
279
|
+
/** Whether an exchange with a peer compares shallow sets: either side may say so */
|
|
280
|
+
const shallowWith = (peerId) => !!states.get(peerId)?.shallow || (config.shallowWith?.(peerId) ?? false);
|
|
281
|
+
/** A fingerprint of each collection held and kept, hex: of its shallow set in a shallow exchange */
|
|
282
|
+
const ourSums = async (holds, shallow = false) => {
|
|
232
283
|
const out = {};
|
|
233
|
-
for (const [collection, sum] of await storage.sums()) {
|
|
284
|
+
for (const [collection, sum] of await (shallow ? storage.shallowSums() : storage.sums())) {
|
|
234
285
|
if (holdsCollection(holds, collection))
|
|
235
286
|
out[collection] = bytesToHex(await fingerprintOf(sum));
|
|
236
287
|
}
|
|
@@ -238,15 +289,18 @@ export function createSyncEngine(config) {
|
|
|
238
289
|
};
|
|
239
290
|
const hello = async (peerId, reply = false) => {
|
|
240
291
|
const holds = ourHolds();
|
|
292
|
+
// Only this node's own wish: a peer that asked has its answer in kind, not the same ask back.
|
|
293
|
+
const askShallow = config.shallowWith?.(peerId) ?? false;
|
|
241
294
|
send(peerId, {
|
|
242
295
|
type: 'hello',
|
|
243
296
|
holds: holds === 'all' ? 'all' : [...holds],
|
|
244
|
-
sums: await ourSums(holds),
|
|
297
|
+
sums: await ourSums(holds, shallowWith(peerId)),
|
|
245
298
|
...(reply ? { reply } : {}),
|
|
299
|
+
...(askShallow ? { shallow: true } : {}),
|
|
246
300
|
});
|
|
247
301
|
};
|
|
248
302
|
const sayHello = (peerId, reply = false) => void hello(peerId, reply).catch(report);
|
|
249
|
-
const onHello = async (peerId, theirs, theirHolds, reply) => {
|
|
303
|
+
const onHello = async (peerId, theirs, theirHolds, reply, theirShallow) => {
|
|
250
304
|
if (typeof theirs !== 'object' || theirs === null)
|
|
251
305
|
return;
|
|
252
306
|
const names = Object.keys(theirs);
|
|
@@ -254,8 +308,9 @@ export function createSyncEngine(config) {
|
|
|
254
308
|
return;
|
|
255
309
|
const state = stateOf(peerId);
|
|
256
310
|
state.holds = readHolds(theirHolds);
|
|
311
|
+
state.shallow = theirShallow;
|
|
257
312
|
const holds = ourHolds();
|
|
258
|
-
const ours = await ourSums(holds);
|
|
313
|
+
const ours = await ourSums(holds, shallowWith(peerId));
|
|
259
314
|
// Every collection this node holds counts, even one it has nothing of yet.
|
|
260
315
|
const named = new Set([...names, ...Object.keys(ours), ...(holds === 'all' ? [] : holds)]);
|
|
261
316
|
const shared = [...named].filter((c) => holdsCollection(holds, c) && holdsCollection(state.holds, c));
|
|
@@ -263,21 +318,35 @@ export function createSyncEngine(config) {
|
|
|
263
318
|
for (const c of shared)
|
|
264
319
|
if (theirs[c] === ours[c])
|
|
265
320
|
emit('level', peerId, c, state.holds === 'all');
|
|
266
|
-
if (differing.length === 0)
|
|
321
|
+
if (differing.length === 0) {
|
|
322
|
+
// A shallow ask hears that it is level: no round of reconciling tells it so.
|
|
323
|
+
if (theirShallow && !reply)
|
|
324
|
+
await hello(peerId, true);
|
|
267
325
|
return settleIfDone(peerId);
|
|
326
|
+
}
|
|
268
327
|
if (self !== undefined && self > peerId) {
|
|
269
328
|
// The other side starts. It may not have heard from us yet.
|
|
270
329
|
if (!reply)
|
|
271
330
|
await hello(peerId, true);
|
|
272
331
|
return;
|
|
273
332
|
}
|
|
274
|
-
|
|
333
|
+
// Who may do what first, then the smallest: what a screen shows first is rarely a long log.
|
|
334
|
+
const sizes = await storage.sums();
|
|
335
|
+
const order = (c) => (c.startsWith('sys.') ? -1 : (sizes.get(c)?.count ?? 0));
|
|
336
|
+
for (const collection of differing.sort((a, b) => order(a) - order(b)))
|
|
275
337
|
await begin(peerId, collection);
|
|
276
338
|
};
|
|
277
339
|
// ─── Reconciling, as initiator ─────────────────────────────────────
|
|
278
340
|
const busy = (peerId) => {
|
|
279
341
|
const state = states.get(peerId);
|
|
280
|
-
|
|
342
|
+
if (state &&
|
|
343
|
+
(state.sessions.size > 0 || state.wants.size > 0 || state.queuedIds.size > 0 || state.sending.size > 0))
|
|
344
|
+
return true;
|
|
345
|
+
// What it sent that waits for its chain isn't taken in yet.
|
|
346
|
+
for (const held of waiting.values())
|
|
347
|
+
if (held.peerId === peerId)
|
|
348
|
+
return true;
|
|
349
|
+
return false;
|
|
281
350
|
};
|
|
282
351
|
const settleIfDone = (peerId) => {
|
|
283
352
|
if (busy(peerId))
|
|
@@ -290,6 +359,7 @@ export function createSyncEngine(config) {
|
|
|
290
359
|
id: session.id,
|
|
291
360
|
collection: session.collection,
|
|
292
361
|
message: base64UrlEncode(message),
|
|
362
|
+
...(session.shallow ? { shallow: true } : {}),
|
|
293
363
|
});
|
|
294
364
|
const begin = async (peerId, collection) => {
|
|
295
365
|
if (typeof collection !== 'string')
|
|
@@ -300,14 +370,17 @@ export function createSyncEngine(config) {
|
|
|
300
370
|
current.again = true;
|
|
301
371
|
return;
|
|
302
372
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
}
|
|
373
|
+
if (state.sending.has(collection)) {
|
|
374
|
+
state.beginAfter.add(collection);
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
const shallow = shallowWith(peerId);
|
|
378
|
+
const reconciler = createReconciler(await (shallow ? storage.shallowItems(collection) : storage.items(collection)), { initiator: true, frameSizeLimit: FRAME_SIZE_LIMIT });
|
|
307
379
|
const session = {
|
|
308
380
|
id: nextId++,
|
|
309
381
|
collection,
|
|
310
382
|
reconciler,
|
|
383
|
+
shallow,
|
|
311
384
|
rounds: 0,
|
|
312
385
|
touchedAt: Date.now(),
|
|
313
386
|
again: false,
|
|
@@ -324,7 +397,7 @@ export function createSyncEngine(config) {
|
|
|
324
397
|
*/
|
|
325
398
|
const levelIfDone = (peerId, collection) => {
|
|
326
399
|
const state = states.get(peerId);
|
|
327
|
-
if (!state || state.sessions.has(collection))
|
|
400
|
+
if (!state || state.sessions.has(collection) || state.sending.has(collection))
|
|
328
401
|
return;
|
|
329
402
|
const wanting = [...state.wants.values()].some((w) => w.collection === collection);
|
|
330
403
|
const queued = state.queued.some((q) => q.collection === collection && state.queuedIds.has(q.id));
|
|
@@ -342,13 +415,91 @@ export function createSyncEngine(config) {
|
|
|
342
415
|
// Asked for before the session goes, so the peer never looks idle in between.
|
|
343
416
|
want(peerId, session.collection, session.need.reverse());
|
|
344
417
|
state.sessions.delete(session.collection);
|
|
345
|
-
|
|
346
|
-
const have = found.filter((v) => v !== null);
|
|
347
|
-
for (const versions of byRecord(have, MAX_IDS_PER_REQUEST))
|
|
348
|
-
send(peerId, { type: 'versions', versions });
|
|
418
|
+
sendAll(peerId, state, session.collection, session.have.reverse());
|
|
349
419
|
if (session.again)
|
|
350
420
|
void begin(peerId, session.collection).catch(report);
|
|
351
421
|
};
|
|
422
|
+
/**
|
|
423
|
+
* Sends a peer versions it lacks, in order, after whatever is still going
|
|
424
|
+
* to it. Off the peer's message queue, so taking in what the peer sends
|
|
425
|
+
* never waits on it taking in what this node sends.
|
|
426
|
+
*/
|
|
427
|
+
const sendAll = (peerId, state, collection, ids) => {
|
|
428
|
+
if (ids.length === 0)
|
|
429
|
+
return;
|
|
430
|
+
const connection = connections.get(peerId);
|
|
431
|
+
state.sending.set(collection, (state.sending.get(collection) ?? 0) + 1);
|
|
432
|
+
const run = (sending.get(peerId) ?? Promise.resolve())
|
|
433
|
+
.then(() => sendPaced(peerId, connection, ids))
|
|
434
|
+
.catch(report);
|
|
435
|
+
sending.set(peerId, run);
|
|
436
|
+
void run.finally(() => {
|
|
437
|
+
if (sending.get(peerId) === run)
|
|
438
|
+
sending.delete(peerId);
|
|
439
|
+
// A peer that went, or came back on a new connection, starts afresh.
|
|
440
|
+
if (states.get(peerId) !== state)
|
|
441
|
+
return;
|
|
442
|
+
const left = (state.sending.get(collection) ?? 1) - 1;
|
|
443
|
+
if (left > 0)
|
|
444
|
+
return void state.sending.set(collection, left);
|
|
445
|
+
state.sending.delete(collection);
|
|
446
|
+
if (state.beginAfter.delete(collection))
|
|
447
|
+
return void begin(peerId, collection).catch(report);
|
|
448
|
+
levelIfDone(peerId, collection);
|
|
449
|
+
settleIfDone(peerId);
|
|
450
|
+
});
|
|
451
|
+
};
|
|
452
|
+
/**
|
|
453
|
+
* Reads versions from the store a message at a time, and waits for the
|
|
454
|
+
* connection to drain before each: what is held in memory at once is one
|
|
455
|
+
* message, however much the peer lacks. A record's versions go in the same
|
|
456
|
+
* message, in the place of its newest (spec 05 §6.3), so the store is read
|
|
457
|
+
* twice: once to learn which record each version is of, keeping only that,
|
|
458
|
+
* then a message's worth at a time to send.
|
|
459
|
+
*/
|
|
460
|
+
const sendPaced = async (peerId, connection, ids) => {
|
|
461
|
+
const open = () => connections.get(peerId) === connection && peers.has(peerId);
|
|
462
|
+
const records = new Map();
|
|
463
|
+
for (const id of ids) {
|
|
464
|
+
if (!open())
|
|
465
|
+
return;
|
|
466
|
+
const version = await storage.getVersion(id);
|
|
467
|
+
if (version)
|
|
468
|
+
getOrAdd(records, version.key, () => []).push(id);
|
|
469
|
+
}
|
|
470
|
+
let message = [];
|
|
471
|
+
let bytes = 0;
|
|
472
|
+
const flush = async () => {
|
|
473
|
+
if (message.length === 0)
|
|
474
|
+
return;
|
|
475
|
+
if (await drained(peerId, connection))
|
|
476
|
+
send(peerId, { type: 'versions', versions: message });
|
|
477
|
+
message = [];
|
|
478
|
+
bytes = 0;
|
|
479
|
+
};
|
|
480
|
+
for (const record of records.values()) {
|
|
481
|
+
if (!open())
|
|
482
|
+
return;
|
|
483
|
+
const versions = [];
|
|
484
|
+
for (const id of record) {
|
|
485
|
+
const version = await storage.getVersion(id);
|
|
486
|
+
if (version)
|
|
487
|
+
versions.push({ version, size: wireBytes(version) });
|
|
488
|
+
}
|
|
489
|
+
const size = versions.reduce((sum, one) => sum + one.size, 0);
|
|
490
|
+
// A record that doesn't fit in what is left starts a message of its own.
|
|
491
|
+
if (message.length + versions.length > MAX_IDS_PER_REQUEST || full(message.length, bytes, size))
|
|
492
|
+
await flush();
|
|
493
|
+
// One with more than a message holds goes over several.
|
|
494
|
+
for (const one of versions) {
|
|
495
|
+
if (full(message.length, bytes, one.size))
|
|
496
|
+
await flush();
|
|
497
|
+
message.push(one.version);
|
|
498
|
+
bytes += one.size;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
await flush();
|
|
502
|
+
};
|
|
352
503
|
const onReconciled = async (peerId, id, message, held) => {
|
|
353
504
|
const state = states.get(peerId);
|
|
354
505
|
const session = state && [...state.sessions.values()].find((s) => s.id === id);
|
|
@@ -386,29 +537,71 @@ export function createSyncEngine(config) {
|
|
|
386
537
|
settleIfDone(peerId);
|
|
387
538
|
};
|
|
388
539
|
// ─── Answering ─────────────────────────────────────────────────────
|
|
389
|
-
const onReconcile = async (peerId, id, collection, message) => {
|
|
540
|
+
const onReconcile = async (peerId, id, collection, message, shallow) => {
|
|
390
541
|
if (typeof collection !== 'string' || typeof message !== 'string')
|
|
391
542
|
return;
|
|
392
543
|
if (!holdsCollection(ourHolds(), collection)) {
|
|
393
544
|
send(peerId, { type: 'reconciled', id, message: '', held: false });
|
|
394
545
|
return;
|
|
395
546
|
}
|
|
396
|
-
const responder = createReconciler(await storage.items(collection), {
|
|
397
|
-
initiator: false,
|
|
398
|
-
frameSizeLimit: FRAME_SIZE_LIMIT,
|
|
399
|
-
});
|
|
547
|
+
const responder = createReconciler(await (shallow ? storage.shallowItems(collection) : storage.items(collection)), { initiator: false, frameSizeLimit: FRAME_SIZE_LIMIT });
|
|
400
548
|
const round = await responder.reconcile(base64UrlDecode(message));
|
|
401
549
|
send(peerId, { type: 'reconciled', id, message: base64UrlEncode(round.message) });
|
|
402
550
|
};
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
551
|
+
/**
|
|
552
|
+
* Answers a `want` once the connection has room, after the answers before
|
|
553
|
+
* it. Off the peer's message queue, like `sendAll`, and apart from it, so an
|
|
554
|
+
* answer never waits behind a large push. A peer that asks and doesn't read
|
|
555
|
+
* holds up only its own answers, and what waits is the ids it asked for.
|
|
556
|
+
*/
|
|
557
|
+
const serveWant = (peerId, id, ids) => {
|
|
558
|
+
const connection = connections.get(peerId);
|
|
559
|
+
const asked = ids.slice(0, MAX_IDS_PER_REQUEST).filter((v) => typeof v === 'string' && !!cidDigest(v));
|
|
560
|
+
// An honest peer has 4 in flight. Past this many waiting, one that doesn't read is asking
|
|
561
|
+
// for memory, not versions: left unanswered, an honest asker gives up on it and asks again.
|
|
562
|
+
const queued = (waitingAnswers.get(peerId) ?? 0) + 1;
|
|
563
|
+
if (queued > MAX_ANSWERS_WAITING)
|
|
564
|
+
return;
|
|
565
|
+
waitingAnswers.set(peerId, queued);
|
|
566
|
+
const run = (answering.get(peerId) ?? Promise.resolve())
|
|
567
|
+
.then(async () => {
|
|
568
|
+
if (!(await drained(peerId, connection)))
|
|
569
|
+
return;
|
|
570
|
+
const versions = [];
|
|
571
|
+
let bytes = 0;
|
|
572
|
+
// In the order asked, until a message's worth: the asker asks again for what comes after.
|
|
573
|
+
for (const v of asked) {
|
|
574
|
+
const version = await storage.getVersion(v);
|
|
575
|
+
if (!version)
|
|
576
|
+
continue;
|
|
577
|
+
const size = wireBytes(version);
|
|
578
|
+
if (full(versions.length, bytes, size))
|
|
579
|
+
break;
|
|
408
580
|
versions.push(version);
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
581
|
+
bytes += size;
|
|
582
|
+
}
|
|
583
|
+
// Always answered, even empty: the asker counts replies to know it is done.
|
|
584
|
+
if (connections.get(peerId) === connection)
|
|
585
|
+
send(peerId, { type: 'versions', id, versions });
|
|
586
|
+
})
|
|
587
|
+
.catch(report);
|
|
588
|
+
answering.set(peerId, run);
|
|
589
|
+
void run.finally(() => {
|
|
590
|
+
if (answering.get(peerId) === run)
|
|
591
|
+
answering.delete(peerId);
|
|
592
|
+
const left = (waitingAnswers.get(peerId) ?? 1) - 1;
|
|
593
|
+
if (left > 0)
|
|
594
|
+
waitingAnswers.set(peerId, left);
|
|
595
|
+
else
|
|
596
|
+
waitingAnswers.delete(peerId);
|
|
597
|
+
});
|
|
598
|
+
};
|
|
599
|
+
/** Waits until the connection has under a message's worth left to send; false once it is gone */
|
|
600
|
+
const drained = async (peerId, connection) => {
|
|
601
|
+
const open = () => connections.get(peerId) === connection && peers.has(peerId);
|
|
602
|
+
while (open() && (config.backlog?.(peerId) ?? 0) > MAX_MESSAGE_BYTES)
|
|
603
|
+
await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS));
|
|
604
|
+
return open();
|
|
412
605
|
};
|
|
413
606
|
const onVersions = async (peerId, id, versions) => {
|
|
414
607
|
// Sent unasked, because we lacked them: each still passes the gatekeeper.
|
|
@@ -418,8 +611,19 @@ export function createSyncEngine(config) {
|
|
|
418
611
|
const asked = state?.wants.get(id);
|
|
419
612
|
if (!state || !asked)
|
|
420
613
|
return;
|
|
421
|
-
|
|
614
|
+
const answered = versions.filter((v) => asked.ids.has(v?.id));
|
|
615
|
+
await admitAll(peerId, answered);
|
|
422
616
|
state.wants.delete(id);
|
|
617
|
+
// An answer that stopped at a message's worth: what came after the last sent is asked for again,
|
|
618
|
+
// ahead of the rest, where it was.
|
|
619
|
+
// One that brought none is not, so a peer lacking them is asked once.
|
|
620
|
+
const order = [...asked.ids];
|
|
621
|
+
const sent = new Set(answered.map((v) => v.id));
|
|
622
|
+
let last = order.length - 1;
|
|
623
|
+
while (last >= 0 && !sent.has(order[last]))
|
|
624
|
+
last--;
|
|
625
|
+
if (last >= 0 && last < order.length - 1)
|
|
626
|
+
want(peerId, asked.collection, order.slice(last + 1), true);
|
|
423
627
|
pump(peerId);
|
|
424
628
|
levelIfDone(peerId, asked.collection);
|
|
425
629
|
settleIfDone(peerId);
|
|
@@ -458,13 +662,13 @@ export function createSyncEngine(config) {
|
|
|
458
662
|
return;
|
|
459
663
|
switch (msg.type) {
|
|
460
664
|
case 'hello':
|
|
461
|
-
return await onHello(peerId, msg.sums, msg.holds, msg.reply === true);
|
|
665
|
+
return await onHello(peerId, msg.sums, msg.holds, msg.reply === true, msg.shallow === true);
|
|
462
666
|
case 'reconcile':
|
|
463
|
-
return await onReconcile(peerId, msg.id, msg.collection, msg.message);
|
|
667
|
+
return await onReconcile(peerId, msg.id, msg.collection, msg.message, msg.shallow === true);
|
|
464
668
|
case 'reconciled':
|
|
465
669
|
return await onReconciled(peerId, msg.id, msg.message, msg.held !== false);
|
|
466
670
|
case 'want':
|
|
467
|
-
return
|
|
671
|
+
return serveWant(peerId, msg.id, Array.isArray(msg.ids) ? msg.ids : []);
|
|
468
672
|
case 'versions':
|
|
469
673
|
return await onVersions(peerId, msg.id, Array.isArray(msg.versions) ? msg.versions : []);
|
|
470
674
|
case 'push-update':
|
|
@@ -502,6 +706,8 @@ export function createSyncEngine(config) {
|
|
|
502
706
|
clearInterval(intervalId);
|
|
503
707
|
intervalId = null;
|
|
504
708
|
states.clear();
|
|
709
|
+
// Every send still going ends: none of the connections it was for is current any more.
|
|
710
|
+
connections.clear();
|
|
505
711
|
},
|
|
506
712
|
handleMessage(peerId, message) {
|
|
507
713
|
// One peer's messages run in the order they came: a hello must not
|
|
@@ -552,6 +758,11 @@ export function createSyncEngine(config) {
|
|
|
552
758
|
};
|
|
553
759
|
}
|
|
554
760
|
/** Takes up to one want's worth of queued ids, all of one collection, off the front of the queue */
|
|
761
|
+
/** Whether the next id to ask for is one held records wait on */
|
|
762
|
+
function nextIsFirst(state) {
|
|
763
|
+
const next = state.queued.find((entry) => state.queuedIds.has(entry.id));
|
|
764
|
+
return next?.first ?? false;
|
|
765
|
+
}
|
|
555
766
|
function takeBatch(state) {
|
|
556
767
|
const ids = [];
|
|
557
768
|
let collection = null;
|
|
@@ -576,6 +787,9 @@ const newPeerState = () => ({
|
|
|
576
787
|
queued: [],
|
|
577
788
|
queuedIds: new Set(),
|
|
578
789
|
holds: 'all',
|
|
790
|
+
shallow: false,
|
|
791
|
+
sending: new Map(),
|
|
792
|
+
beginAfter: new Set(),
|
|
579
793
|
});
|
|
580
794
|
const refusal = (peerId, id) => `${peerId}\n${id}`;
|
|
581
795
|
const abandoned = (peerId, why) => new Error(`Sync with ${peerId} abandoned: ${why}`);
|